From 3fb2d26190b46dcec3e23184ee33d808d0b2ceca Mon Sep 17 00:00:00 2001 From: rikuke <33894149+rikuke@users.noreply.github.com> Date: Tue, 6 Feb 2024 11:55:18 +0200 Subject: [PATCH] Hl 1073 csv (#2755) * feat: add new columns to export csv * feat: add missing applicant localization * fix: failing test * feat: optionally remove employee data from csv --- backend/benefit/applications/api/v1/views.py | 5 ++- .../services/applications_csv_report.py | 44 +++++++++++++++++-- .../benefit/applications/tests/conftest.py | 8 ++++ .../tests/test_applications_report.py | 18 +++++++- .../benefit/locale/en/LC_MESSAGES/django.po | 3 ++ .../benefit/locale/fi/LC_MESSAGES/django.po | 3 ++ .../benefit/locale/sv/LC_MESSAGES/django.po | 3 ++ 7 files changed, 79 insertions(+), 5 deletions(-) diff --git a/backend/benefit/applications/api/v1/views.py b/backend/benefit/applications/api/v1/views.py index 4a64f2d48f..e4025f6a58 100755 --- a/backend/benefit/applications/api/v1/views.py +++ b/backend/benefit/applications/api/v1/views.py @@ -473,9 +473,12 @@ def _csv_response( queryset: QuerySet[Application], prune_data_for_talpa: bool = False, remove_quotes: bool = False, + prune_sensitive_data: bool = True, ) -> StreamingHttpResponse: csv_service = ApplicationsCsvService( - queryset.order_by(self.APPLICATION_ORDERING), prune_data_for_talpa + queryset.order_by(self.APPLICATION_ORDERING), + prune_data_for_talpa, + prune_sensitive_data, ) response = StreamingHttpResponse( csv_service.get_csv_string_lines_generator(remove_quotes), diff --git a/backend/benefit/applications/services/applications_csv_report.py b/backend/benefit/applications/services/applications_csv_report.py index 1892d6957e..063ec6325a 100644 --- a/backend/benefit/applications/services/applications_csv_report.py +++ b/backend/benefit/applications/services/applications_csv_report.py @@ -1,6 +1,8 @@ +from typing import List + from django.utils import translation -from applications.enums import BenefitType +from applications.enums import ApplicationBatchStatus, ApplicationOrigin, BenefitType from applications.services.csv_export_base import ( CsvColumn, CsvExportBase, @@ -64,10 +66,20 @@ def getter(item): return getter -def get_benefit_type_label(benefit_type): +def get_benefit_type_label(benefit_type) -> str: return str(BenefitType(benefit_type).label) +def get_batch_status_label(batch_status: str) -> str: + if batch_status == "": + return "" + return str(ApplicationBatchStatus(batch_status).label) + + +def get_application_origin_label(application_origin: str) -> str: + return str(ApplicationOrigin(application_origin).label) + + class ApplicationsCsvService(CsvExportBase): """ Export application data for further processing in Excel and other reporting software. @@ -79,10 +91,13 @@ class ApplicationsCsvService(CsvExportBase): """ - def __init__(self, applications, prune_data_for_talpa=False): + def __init__( + self, applications, prune_data_for_talpa=False, prune_sensitive_data=False + ): self.applications = applications self.export_notes = [] self.prune_data_for_talpa = prune_data_for_talpa + self.prune_sensitive_data = prune_sensitive_data @property def CSV_COLUMNS(self): @@ -123,6 +138,16 @@ def CSV_COLUMNS(self): CsvColumn("Hakemusnumero", "application_number"), CsvColumn("Hakemusrivi", "application_row_idx"), CsvColumn("Hakemuksen tila", "status"), + CsvColumn( + "Hakemuksen tyyppi", "application_origin", get_application_origin_label + ), + CsvColumn("Hakemus saapunut", "created_at", format_datetime), + csv_default_column( + "Koonnin status", "batch.status", get_batch_status_label + ), + csv_default_column( + "Koonnin statuksen päivämäärä", "batch.modified_at", format_datetime + ), csv_default_column("Haettava lisä", "benefit_type", get_benefit_type_label), csv_default_column("Haettu alkupäivä", "start_date"), csv_default_column("Haettu päättymispäivä", "end_date"), @@ -375,12 +400,25 @@ def CSV_COLUMNS(self): columns.append(CsvColumn("Huom", get_export_notes)) + if self.prune_sensitive_data: + return self._remove_sensitive_columns(columns) + return columns MAX_AHJO_ROWS = 2 MAX_PAY_SUBSIDIES = 2 MAX_DE_MINIMIS_AIDS = 5 + def _remove_sensitive_columns(self, columns: List[CsvColumn]) -> List[CsvColumn]: + """Remove sensitive employee data from the CsvColumns list""" + sensitive_col_headings = [ + "Työntekijän etunimi", + "Työntekijän sukunimi", + "Työntekijän puhelinnumero", + "Työntekijän sähköposti", + ] + return [col for col in columns if col.heading not in sensitive_col_headings] + def get_applications(self): return self.applications diff --git a/backend/benefit/applications/tests/conftest.py b/backend/benefit/applications/tests/conftest.py index 0ae131d54f..8367e1b5a3 100755 --- a/backend/benefit/applications/tests/conftest.py +++ b/backend/benefit/applications/tests/conftest.py @@ -136,6 +136,14 @@ def pruned_applications_csv_service_with_one_application( return ApplicationsCsvService(Application.objects.filter(pk=application1.pk), True) +@pytest.fixture +def sanitized_csv_service_with_one_application(application_batch): + application1 = application_batch.applications.all().first() + return ApplicationsCsvService( + Application.objects.filter(pk=application1.pk), True, True + ) + + @pytest.fixture def applications_csv_with_no_applications(): return ApplicationsCsvService([]) diff --git a/backend/benefit/applications/tests/test_applications_report.py b/backend/benefit/applications/tests/test_applications_report.py index 1db6870c08..d62a69e712 100644 --- a/backend/benefit/applications/tests/test_applications_report.py +++ b/backend/benefit/applications/tests/test_applications_report.py @@ -301,6 +301,22 @@ def test_applications_csv_export_with_date_range(handler_api_client): ) +def test_sensitive_data_removed_csv_output(sanitized_csv_service_with_one_application): + csv_lines = split_lines_at_semicolon( + sanitized_csv_service_with_one_application.get_csv_string() + ) + + sensitive_col_headings = [ + "Työntekijän etunimi", + "Työntekijän sukunimi", + "Työntekijän puhelinnumero", + "Työntekijän sähköposti", + ] + + for col_heading in sensitive_col_headings: + assert col_heading not in csv_lines[0] + + def test_pruned_applications_csv_output( pruned_applications_csv_service_with_one_application, ): @@ -651,7 +667,7 @@ def test_applications_csv_non_ascii_characters( csv_lines = split_lines_at_semicolon( applications_csv_service_with_one_application.get_csv_string() ) - assert csv_lines[1][8] == '"test äöÄÖtest"' # string is quoted + assert csv_lines[1][12] == '"test äöÄÖtest"' # string is quoted def test_applications_csv_delimiter(applications_csv_service_with_one_application): diff --git a/backend/benefit/locale/en/LC_MESSAGES/django.po b/backend/benefit/locale/en/LC_MESSAGES/django.po index 4e2c1967a8..60baa2bc51 100644 --- a/backend/benefit/locale/en/LC_MESSAGES/django.po +++ b/backend/benefit/locale/en/LC_MESSAGES/django.po @@ -543,6 +543,9 @@ msgstr "" msgid "handler" msgstr "" +msgid "applicant" +msgstr "" + msgid "calculation" msgstr "" diff --git a/backend/benefit/locale/fi/LC_MESSAGES/django.po b/backend/benefit/locale/fi/LC_MESSAGES/django.po index 6ad344ae32..c304586833 100644 --- a/backend/benefit/locale/fi/LC_MESSAGES/django.po +++ b/backend/benefit/locale/fi/LC_MESSAGES/django.po @@ -553,6 +553,9 @@ msgstr "Laskelma on jo olemassa" msgid "handler" msgstr "käsittelijä" +msgid "applicant" +msgstr "hakija" + msgid "calculation" msgstr "laskelma" diff --git a/backend/benefit/locale/sv/LC_MESSAGES/django.po b/backend/benefit/locale/sv/LC_MESSAGES/django.po index b2546bd00b..4479e76065 100644 --- a/backend/benefit/locale/sv/LC_MESSAGES/django.po +++ b/backend/benefit/locale/sv/LC_MESSAGES/django.po @@ -548,6 +548,9 @@ msgstr "Beräkning finns redan" msgid "handler" msgstr "handläggare" +msgid "applicant" +msgstr "sökande" + msgid "calculation" msgstr "beräkning"