Hl 1073 csv (#2755)

* feat: add new columns to export csv

* feat: add missing applicant localization

* fix: failing test

* feat: optionally remove employee data from csv

backend/benefit/applications/api/v1/views.py

@@ -473,9 +473,12 @@ def _csv_response(
         queryset: QuerySet[Application],
         prune_data_for_talpa: bool = False,
         remove_quotes: bool = False,
+        prune_sensitive_data: bool = True,
     ) -> StreamingHttpResponse:
         csv_service = ApplicationsCsvService(
-            queryset.order_by(self.APPLICATION_ORDERING), prune_data_for_talpa
+            queryset.order_by(self.APPLICATION_ORDERING),
+            prune_data_for_talpa,
+            prune_sensitive_data,
         )
         response = StreamingHttpResponse(
             csv_service.get_csv_string_lines_generator(remove_quotes),

backend/benefit/applications/services/applications_csv_report.py

@@ -1,6 +1,8 @@
+from typing import List
+
 from django.utils import translation
 
-from applications.enums import BenefitType
+from applications.enums import ApplicationBatchStatus, ApplicationOrigin, BenefitType
 from applications.services.csv_export_base import (
     CsvColumn,
     CsvExportBase,
@@ -64,10 +66,20 @@ def getter(item):
     return getter
 
 
-def get_benefit_type_label(benefit_type):
+def get_benefit_type_label(benefit_type) -> str:
     return str(BenefitType(benefit_type).label)
 
 
+def get_batch_status_label(batch_status: str) -> str:
+    if batch_status == "":
+        return ""
+    return str(ApplicationBatchStatus(batch_status).label)
+
+
+def get_application_origin_label(application_origin: str) -> str:
+    return str(ApplicationOrigin(application_origin).label)
+
+
 class ApplicationsCsvService(CsvExportBase):
     """
     Export application data for further processing in Excel and other reporting software.
@@ -79,10 +91,13 @@ class ApplicationsCsvService(CsvExportBase):
 
     """
 
-    def __init__(self, applications, prune_data_for_talpa=False):
+    def __init__(
+        self, applications, prune_data_for_talpa=False, prune_sensitive_data=False
+    ):
         self.applications = applications
         self.export_notes = []
         self.prune_data_for_talpa = prune_data_for_talpa
+        self.prune_sensitive_data = prune_sensitive_data
 
     @property
     def CSV_COLUMNS(self):
@@ -123,6 +138,16 @@ def CSV_COLUMNS(self):
             CsvColumn("Hakemusnumero", "application_number"),
             CsvColumn("Hakemusrivi", "application_row_idx"),
             CsvColumn("Hakemuksen tila", "status"),
+            CsvColumn(
+                "Hakemuksen tyyppi", "application_origin", get_application_origin_label
+            ),
+            CsvColumn("Hakemus saapunut", "created_at", format_datetime),
+            csv_default_column(
+                "Koonnin status", "batch.status", get_batch_status_label
+            ),
+            csv_default_column(
+                "Koonnin statuksen päivämäärä", "batch.modified_at", format_datetime
+            ),
             csv_default_column("Haettava lisä", "benefit_type", get_benefit_type_label),
             csv_default_column("Haettu alkupäivä", "start_date"),
             csv_default_column("Haettu päättymispäivä", "end_date"),
@@ -375,12 +400,25 @@ def CSV_COLUMNS(self):
 
         columns.append(CsvColumn("Huom", get_export_notes))
 
+        if self.prune_sensitive_data:
+            return self._remove_sensitive_columns(columns)
+
         return columns
 
     MAX_AHJO_ROWS = 2
     MAX_PAY_SUBSIDIES = 2
     MAX_DE_MINIMIS_AIDS = 5
 
+    def _remove_sensitive_columns(self, columns: List[CsvColumn]) -> List[CsvColumn]:
+        """Remove sensitive employee data from the CsvColumns list"""
+        sensitive_col_headings = [
+            "Työntekijän etunimi",
+            "Työntekijän sukunimi",
+            "Työntekijän puhelinnumero",
+            "Työntekijän sähköposti",
+        ]
+        return [col for col in columns if col.heading not in sensitive_col_headings]
+
     def get_applications(self):
         return self.applications
 

backend/benefit/applications/tests/conftest.py

@@ -136,6 +136,14 @@ def pruned_applications_csv_service_with_one_application(
     return ApplicationsCsvService(Application.objects.filter(pk=application1.pk), True)
 
 
+@pytest.fixture
+def sanitized_csv_service_with_one_application(application_batch):
+    application1 = application_batch.applications.all().first()
+    return ApplicationsCsvService(
+        Application.objects.filter(pk=application1.pk), True, True
+    )
+
+
 @pytest.fixture
 def applications_csv_with_no_applications():
     return ApplicationsCsvService([])

backend/benefit/applications/tests/test_applications_report.py

@@ -301,6 +301,22 @@ def test_applications_csv_export_with_date_range(handler_api_client):
     )
 
 
+def test_sensitive_data_removed_csv_output(sanitized_csv_service_with_one_application):
+    csv_lines = split_lines_at_semicolon(
+        sanitized_csv_service_with_one_application.get_csv_string()
+    )
+
+    sensitive_col_headings = [
+        "Työntekijän etunimi",
+        "Työntekijän sukunimi",
+        "Työntekijän puhelinnumero",
+        "Työntekijän sähköposti",
+    ]
+
+    for col_heading in sensitive_col_headings:
+        assert col_heading not in csv_lines[0]
+
+
 def test_pruned_applications_csv_output(
     pruned_applications_csv_service_with_one_application,
 ):
@@ -651,7 +667,7 @@ def test_applications_csv_non_ascii_characters(
     csv_lines = split_lines_at_semicolon(
         applications_csv_service_with_one_application.get_csv_string()
     )
-    assert csv_lines[1][8] == '"test äöÄÖtest"'  # string is quoted
+    assert csv_lines[1][12] == '"test äöÄÖtest"'  # string is quoted
 
 
 def test_applications_csv_delimiter(applications_csv_service_with_one_application):

backend/benefit/locale/en/LC_MESSAGES/django.po

@@ -543,6 +543,9 @@ msgstr ""
 msgid "handler"
 msgstr ""
 
+msgid "applicant"
+msgstr ""
+
 msgid "calculation"
 msgstr ""
 

backend/benefit/locale/fi/LC_MESSAGES/django.po

@@ -553,6 +553,9 @@ msgstr "Laskelma on jo olemassa"
 msgid "handler"
 msgstr "käsittelijä"
 
+msgid "applicant"
+msgstr "hakija"
+
 msgid "calculation"
 msgstr "laskelma"
 

backend/benefit/locale/sv/LC_MESSAGES/django.po

@@ -548,6 +548,9 @@ msgstr "Beräkning finns redan"
 msgid "handler"
 msgstr "handläggare"
 
+msgid "applicant"
+msgstr "sökande"
+
 msgid "calculation"
 msgstr "beräkning"