From 5a8ec7f8474f2b51623554d49a57ef957f0b1c3b Mon Sep 17 00:00:00 2001 From: Phil Dominguez <142051477+phildominguez-gsa@users.noreply.github.com> Date: Thu, 9 Nov 2023 18:16:03 -0500 Subject: [PATCH] #2676: `excel_creation.py` - Refactoring template names and paths (#2779) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Splitting out templates.py * Replacing name strings * Lint * Getting paths from excel.py --------- Co-authored-by: Phil Dominguez <“philip.dominguez@gsa.gov”> --- .../commands/historic_workbook_generator.py | 11 --------- .../workbooklib/additional_eins.py | 7 +++--- .../workbooklib/additional_ueis.py | 7 +++--- .../workbooklib/corrective_action_plan.py | 8 ++++--- .../workbooklib/excel_creation.py | 17 -------------- .../workbooklib/federal_awards.py | 9 ++++---- .../workbooklib/findings.py | 9 ++++---- .../workbooklib/findings_text.py | 6 ++--- .../workbooklib/notes_to_sefa.py | 9 ++++---- .../workbooklib/secondary_auditors.py | 10 ++++---- .../workbooklib/templates.py | 23 +++++++++++++++++++ 11 files changed, 56 insertions(+), 60 deletions(-) create mode 100644 backend/census_historical_migration/workbooklib/templates.py diff --git a/backend/census_historical_migration/management/commands/historic_workbook_generator.py b/backend/census_historical_migration/management/commands/historic_workbook_generator.py index 51b9004a4a..9916afb11e 100644 --- a/backend/census_historical_migration/management/commands/historic_workbook_generator.py +++ b/backend/census_historical_migration/management/commands/historic_workbook_generator.py @@ -36,17 +36,6 @@ # before filling in the XLSX workbooks. FieldMap = NT("FieldMap", "in_sheet in_db default type") -templates = { - "AdditionalUEIs": "additional-ueis-workbook.xlsx", - "AdditionalEINs": "additional-eins-workbook.xlsx", - "AuditFindingsText": "audit-findings-text-workbook.xlsx", - "CAP": "corrective-action-plan-workbook.xlsx", - "AuditFindings": "federal-awards-audit-findings-workbook.xlsx", - "FederalAwards": "federal-awards-workbook.xlsx", - "SEFA": "notes-to-sefa-workbook.xlsx", - "SecondaryAuditors": "secondary-auditors-workbook.xlsx", -} - def set_single_cell_range(wb, range_name, value): the_range = wb.defined_names[range_name] diff --git a/backend/census_historical_migration/workbooklib/additional_eins.py b/backend/census_historical_migration/workbooklib/additional_eins.py index 5f2300c2f7..e3be164a8d 100644 --- a/backend/census_historical_migration/workbooklib/additional_eins.py +++ b/backend/census_historical_migration/workbooklib/additional_eins.py @@ -1,14 +1,13 @@ from census_historical_migration.workbooklib.excel_creation import ( FieldMap, WorkbookFieldInDissem, - templates, set_uei, map_simple_columns, generate_dissemination_test_table, ) - - +from census_historical_migration.workbooklib.templates import sections_to_template_paths from census_historical_migration.workbooklib.census_models.census import dynamic_import +from audit.fixtures.excel import FORM_SECTIONS import openpyxl as pyxl @@ -25,7 +24,7 @@ def generate_additional_eins(dbkey, year, outfile): logger.info(f"--- generate additional eins {dbkey} {year} ---") Gen = dynamic_import("Gen", year) Eins = dynamic_import("Eins", year) - wb = pyxl.load_workbook(templates["AdditionalEINs"]) + wb = pyxl.load_workbook(sections_to_template_paths[FORM_SECTIONS.ADDITIONAL_EINS]) g = set_uei(Gen, wb, dbkey) diff --git a/backend/census_historical_migration/workbooklib/additional_ueis.py b/backend/census_historical_migration/workbooklib/additional_ueis.py index 9666f990a9..bbeff52dfb 100644 --- a/backend/census_historical_migration/workbooklib/additional_ueis.py +++ b/backend/census_historical_migration/workbooklib/additional_ueis.py @@ -1,14 +1,13 @@ from census_historical_migration.workbooklib.excel_creation import ( FieldMap, WorkbookFieldInDissem, - templates, set_uei, map_simple_columns, generate_dissemination_test_table, ) - - +from census_historical_migration.workbooklib.templates import sections_to_template_paths from census_historical_migration.workbooklib.census_models.census import dynamic_import +from audit.fixtures.excel import FORM_SECTIONS import openpyxl as pyxl @@ -25,7 +24,7 @@ def generate_additional_ueis(dbkey, year, outfile): logger.info(f"--- generate additional ueis {dbkey} {year} ---") Gen = dynamic_import("Gen", year) - wb = pyxl.load_workbook(templates["AdditionalUEIs"]) + wb = pyxl.load_workbook(sections_to_template_paths[FORM_SECTIONS.ADDITIONAL_UEIS]) g = set_uei(Gen, wb, dbkey) if int(year) >= 22: Ueis = dynamic_import("Ueis", year) diff --git a/backend/census_historical_migration/workbooklib/corrective_action_plan.py b/backend/census_historical_migration/workbooklib/corrective_action_plan.py index cdcbe5fb43..10d7dd04b2 100644 --- a/backend/census_historical_migration/workbooklib/corrective_action_plan.py +++ b/backend/census_historical_migration/workbooklib/corrective_action_plan.py @@ -1,14 +1,14 @@ from census_historical_migration.workbooklib.excel_creation import ( FieldMap, WorkbookFieldInDissem, - templates, set_uei, map_simple_columns, generate_dissemination_test_table, test_pfix, ) - +from census_historical_migration.workbooklib.templates import sections_to_template_paths from census_historical_migration.workbooklib.census_models.census import dynamic_import +from audit.fixtures.excel import FORM_SECTIONS import openpyxl as pyxl @@ -22,7 +22,9 @@ def generate_corrective_action_plan(dbkey, year, outfile): logger.info(f"--- generate corrective action plan {dbkey} {year} ---") Gen = dynamic_import("Gen", year) Captext = dynamic_import("Captext", year) - wb = pyxl.load_workbook(templates["CAP"]) + wb = pyxl.load_workbook( + sections_to_template_paths[FORM_SECTIONS.CORRECTIVE_ACTION_PLAN] + ) mappings = [ FieldMap("reference_number", "findingrefnums", "finding_ref_number", None, str), FieldMap("planned_action", "text", WorkbookFieldInDissem, None, test_pfix(3)), diff --git a/backend/census_historical_migration/workbooklib/excel_creation.py b/backend/census_historical_migration/workbooklib/excel_creation.py index 6f3bd37a17..96cf329fb7 100644 --- a/backend/census_historical_migration/workbooklib/excel_creation.py +++ b/backend/census_historical_migration/workbooklib/excel_creation.py @@ -1,6 +1,5 @@ from collections import namedtuple as NT from playhouse.shortcuts import model_to_dict -import os import sys import logging @@ -17,22 +16,6 @@ FieldMap = NT("FieldMap", "in_sheet in_db in_dissem default type") WorkbookFieldInDissem = 1000 -templates_root = "schemas/output/excel/xlsx/" -templates_raw = { - "AdditionalUEIs": "additional-ueis-workbook.xlsx", - "AdditionalEINs": "additional-eins-workbook.xlsx", - "AuditFindingsText": "audit-findings-text-workbook.xlsx", - "CAP": "corrective-action-plan-workbook.xlsx", - "AuditFindings": "federal-awards-audit-findings-workbook.xlsx", - "FederalAwards": "federal-awards-workbook.xlsx", - "SEFA": "notes-to-sefa-workbook.xlsx", - "SecondaryAuditors": "secondary-auditors-workbook.xlsx", -} - -templates = {} -for k, v in templates_raw.items(): - templates[k] = os.path.join(templates_root, v) - def test_pfix(n): def _test(o): diff --git a/backend/census_historical_migration/workbooklib/federal_awards.py b/backend/census_historical_migration/workbooklib/federal_awards.py index 7d4497c6b7..e13aebeb5b 100644 --- a/backend/census_historical_migration/workbooklib/federal_awards.py +++ b/backend/census_historical_migration/workbooklib/federal_awards.py @@ -1,16 +1,15 @@ from census_historical_migration.workbooklib.excel_creation import ( FieldMap, WorkbookFieldInDissem, - templates, set_uei, set_single_cell_range, map_simple_columns, generate_dissemination_test_table, set_range, ) - +from census_historical_migration.workbooklib.templates import sections_to_template_paths from census_historical_migration.workbooklib.census_models.census import dynamic_import - +from audit.fixtures.excel import FORM_SECTIONS from config import settings import openpyxl as pyxl @@ -197,7 +196,9 @@ def generate_federal_awards(dbkey, year, outfile): Gen = dynamic_import("Gen", year) Passthrough = dynamic_import("Passthrough", year) Cfda = dynamic_import("Cfda", year) - wb = pyxl.load_workbook(templates["FederalAwards"]) + wb = pyxl.load_workbook( + sections_to_template_paths[FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED] + ) # In sheet : in DB g = set_uei(Gen, wb, dbkey) diff --git a/backend/census_historical_migration/workbooklib/findings.py b/backend/census_historical_migration/workbooklib/findings.py index dc209ab769..c9d097c29d 100644 --- a/backend/census_historical_migration/workbooklib/findings.py +++ b/backend/census_historical_migration/workbooklib/findings.py @@ -1,14 +1,13 @@ from census_historical_migration.workbooklib.excel_creation import ( FieldMap, - templates, set_uei, map_simple_columns, generate_dissemination_test_table, set_range, ) - +from census_historical_migration.workbooklib.templates import sections_to_template_paths from census_historical_migration.workbooklib.census_models.census import dynamic_import - +from audit.fixtures.excel import FORM_SECTIONS import openpyxl as pyxl @@ -93,7 +92,9 @@ def generate_findings(dbkey, year, outfile): Gen = dynamic_import("Gen", year) Findings = dynamic_import("Findings", year) Cfda = dynamic_import("Cfda", year) - wb = pyxl.load_workbook(templates["AuditFindings"]) + wb = pyxl.load_workbook( + sections_to_template_paths[FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE] + ) g = set_uei(Gen, wb, dbkey) cfdas = Cfda.select().where(Cfda.dbkey == g.dbkey).order_by(Cfda.index) diff --git a/backend/census_historical_migration/workbooklib/findings_text.py b/backend/census_historical_migration/workbooklib/findings_text.py index d06269339d..610ea06b43 100644 --- a/backend/census_historical_migration/workbooklib/findings_text.py +++ b/backend/census_historical_migration/workbooklib/findings_text.py @@ -1,14 +1,14 @@ from census_historical_migration.workbooklib.excel_creation import ( FieldMap, WorkbookFieldInDissem, - templates, set_uei, map_simple_columns, generate_dissemination_test_table, test_pfix, ) - +from census_historical_migration.workbooklib.templates import sections_to_template_paths from census_historical_migration.workbooklib.census_models.census import dynamic_import +from audit.fixtures.excel import FORM_SECTIONS import openpyxl as pyxl @@ -29,7 +29,7 @@ def generate_findings_text(dbkey, year, outfile): logger.info(f"--- generate findings text {dbkey} {year} ---") Gen = dynamic_import("Gen", year) Findingstext = dynamic_import("Findingstext", year) - wb = pyxl.load_workbook(templates["AuditFindingsText"]) + wb = pyxl.load_workbook(sections_to_template_paths[FORM_SECTIONS.FINDINGS_TEXT]) g = set_uei(Gen, wb, dbkey) diff --git a/backend/census_historical_migration/workbooklib/notes_to_sefa.py b/backend/census_historical_migration/workbooklib/notes_to_sefa.py index cd713fec10..105b1e29df 100644 --- a/backend/census_historical_migration/workbooklib/notes_to_sefa.py +++ b/backend/census_historical_migration/workbooklib/notes_to_sefa.py @@ -1,24 +1,23 @@ from census_historical_migration.workbooklib.excel_creation import ( FieldMap, - templates, set_uei, set_single_cell_range, map_simple_columns, generate_dissemination_test_table, test_pfix, ) - +from census_historical_migration.workbooklib.templates import sections_to_template_paths from census_historical_migration.workbooklib.excel_creation import ( set_range, ) from census_historical_migration.workbooklib.census_models.census import dynamic_import - +from audit.fixtures.excel import FORM_SECTIONS import openpyxl as pyxl + import re import logging -# import unidecode logger = logging.getLogger(__name__) @@ -43,7 +42,7 @@ def generate_notes_to_sefa(dbkey, year, outfile): logger.info(f"--- generate notes to sefa {dbkey} {year}---") Gen = dynamic_import("Gen", year) Notes = dynamic_import("Notes", year) - wb = pyxl.load_workbook(templates["SEFA"]) + wb = pyxl.load_workbook(sections_to_template_paths[FORM_SECTIONS.NOTES_TO_SEFA]) g = set_uei(Gen, wb, dbkey) diff --git a/backend/census_historical_migration/workbooklib/secondary_auditors.py b/backend/census_historical_migration/workbooklib/secondary_auditors.py index 4a956427c2..6354b315fa 100644 --- a/backend/census_historical_migration/workbooklib/secondary_auditors.py +++ b/backend/census_historical_migration/workbooklib/secondary_auditors.py @@ -1,16 +1,14 @@ from census_historical_migration.workbooklib.excel_creation import ( FieldMap, - templates, set_uei, map_simple_columns, generate_dissemination_test_table, test_pfix, ) - +from census_historical_migration.workbooklib.templates import sections_to_template_paths from census_historical_migration.workbooklib.census_models.census import dynamic_import - - from census_historical_migration.workbooklib.sac_creation import add_hyphen_to_zip +from audit.fixtures.excel import FORM_SECTIONS import openpyxl as pyxl @@ -54,7 +52,9 @@ def generate_secondary_auditors(dbkey, year, outfile): logger.info(f"--- generate secondary auditors {dbkey} {year} ---") Gen = dynamic_import("Gen", year) Cpas = dynamic_import("Cpas", year) - wb = pyxl.load_workbook(templates["SecondaryAuditors"]) + wb = pyxl.load_workbook( + sections_to_template_paths[FORM_SECTIONS.SECONDARY_AUDITORS] + ) g = set_uei(Gen, wb, dbkey) diff --git a/backend/census_historical_migration/workbooklib/templates.py b/backend/census_historical_migration/workbooklib/templates.py new file mode 100644 index 0000000000..8202c15ed8 --- /dev/null +++ b/backend/census_historical_migration/workbooklib/templates.py @@ -0,0 +1,23 @@ +from audit.fixtures.excel import FORM_SECTIONS +from audit.fixtures.excel import ( + ADDITIONAL_EINS_TEMPLATE, + ADDITIONAL_UEIS_TEMPLATE, + CORRECTIVE_ACTION_PLAN_TEMPLATE, + FEDERAL_AWARDS_TEMPLATE, + FINDINGS_TEXT_TEMPLATE, + FINDINGS_UNIFORM_GUIDANCE_TEMPLATE, + NOTES_TO_SEFA_TEMPLATE, + SECONDARY_AUDITORS_TEMPLATE, +) + + +sections_to_template_paths = { + FORM_SECTIONS.ADDITIONAL_EINS: ADDITIONAL_EINS_TEMPLATE, + FORM_SECTIONS.ADDITIONAL_UEIS: ADDITIONAL_UEIS_TEMPLATE, + FORM_SECTIONS.CORRECTIVE_ACTION_PLAN: CORRECTIVE_ACTION_PLAN_TEMPLATE, + FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED: FEDERAL_AWARDS_TEMPLATE, + FORM_SECTIONS.FINDINGS_TEXT: FINDINGS_TEXT_TEMPLATE, + FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE: FINDINGS_UNIFORM_GUIDANCE_TEMPLATE, + FORM_SECTIONS.NOTES_TO_SEFA: NOTES_TO_SEFA_TEMPLATE, + FORM_SECTIONS.SECONDARY_AUDITORS: SECONDARY_AUDITORS_TEMPLATE, +}