Merge branch 'main' into snyk-upgrade-4b667552b73f225de251706f4b390969

.github/workflows/commit.yml

@@ -3,8 +3,8 @@ name: Tests on Commit
 on: [push]
 
 env:
-  PY_VERSION: "3.10"
-  POETRY_VERSION: "1.7.1"
+  PY_VERSION: "3.13"
+  POETRY_VERSION: "2.0.0"
 
 jobs:
   lint:
@@ -38,6 +38,7 @@ jobs:
         uses: abatilo/actions-poetry@v2
         with:
           poetry-version: ${{ env.POETRY_VERSION }}
+          poetry-plugins: "poetry-plugin-export"
 
       - name: Install Dependencies
         run: |
@@ -77,6 +78,7 @@ jobs:
         uses: abatilo/actions-poetry@v2
         with:
           poetry-version: ${{ env.POETRY_VERSION }}
+          poetry-plugins: "poetry-plugin-export"
       - name: Add requirement.txt
         run: |
           poetry export -f requirements.txt --output requirements.txt --without-hashes

.github/workflows/deploy.yml

@@ -12,7 +12,7 @@
           required: true
 
   env:
-    POETRY_VERSION: "1.7.1"
+    POETRY_VERSION: "2.0.0"
 
   jobs:
     deploy-staging:

.github/workflows/load_test.yml

@@ -1,7 +1,7 @@
 name: Load Test
 on:
   workflow_dispatch:
-    
+
 jobs:
   load-test:
     name: H2.0 Load Test
@@ -12,7 +12,7 @@ jobs:
       - name: Setup python
         uses: actions/setup-python@v5
         with:
-            python-version: 3.9
+            python-version: 3.13
       - name: Display Python version
         run: python -c "import sys; print(sys.version)"
       - name: Install Poetry

.github/workflows/publish.yml

@@ -7,7 +7,7 @@ on:
         required: true
 
 env:
-  POETRY_VERSION: "1.7.1"
+  POETRY_VERSION: "2.0.0"
 
 jobs:
   publish:

.python-version

@@ -1 +1 @@
-3.10.0
+3.13.0

Dockerfile

@@ -1,12 +1,12 @@
-FROM python:3.10
+FROM python:3.13
 
 WORKDIR /app
 
 COPY . /app
 
 RUN pip install --no-cache-dir -r requirements.txt
 
-ARG DEV 
+ARG DEV
 
 RUN if [ $DEV ]; \
     then pip install --no-cache-dir -r requirements-dev.txt; \
@@ -17,4 +17,4 @@ EXPOSE 8080
 ENV FLASK_APP=run.py
 
 # Run run.py when the container launches
-CMD ["flask", "run", "--host=0.0.0.0", "--port=8080"]
+CMD ["flask", "run", "--host=0.0.0.0", "--port=8080"]

Makefile

@@ -34,7 +34,7 @@ test-integration: ## Runs integration tests. Compatible with dev environment / `
 	poetry run pytest --junitxml=pytest.xml --cov=harvester ./tests/integration
 
 test-functional: ## Runs integration tests. Compatible with dev environment / `make up`
-	poetry run pytest --noconftest --junitxml=pytest.xml --cov=harvester ./tests/functional
+	poetry run pytest --noconftest --junitxml=pytest.xml ./tests/functional
 
 test: up test-unit test-integration ## Runs all tests. Compatible with dev environment / `make up`
 

app/static/package.json

@@ -25,7 +25,7 @@
   "dependencies": {
     "@uswds/uswds": "3.9.0",
     "chart.js": "^4.4.5",
-    "htmx.org": "^2.0.2",
+    "htmx.org": "^2.0.3",
     "rollup": "^4.24.0"
   },
   "devDependencies": {

harvester/__init__.py

@@ -1,12 +1,11 @@
 import logging.config
+import os
 
 from dotenv import load_dotenv
 
 from config.logger_config import LOGGING_CONFIG
 from database.interface import HarvesterDBInterface
 
-import os
-
 load_dotenv()
 
 logging.config.dictConfig(LOGGING_CONFIG)
@@ -21,5 +20,5 @@
     "password": os.getenv("HARVEST_SMTP_PASSWORD"),
     "default_sender": os.getenv("HARVEST_SMTP_SENDER"),
     "base_url": os.getenv("REDIRECT_URI").rsplit("/", 1)[0],
-    "recipient": os.getenv("HARVEST_SMTP_RECIPIENT")
+    "recipient": os.getenv("HARVEST_SMTP_RECIPIENT"),
 }

harvester/harvest.py

@@ -16,12 +16,11 @@
 
 sys.path.insert(1, "/".join(os.path.realpath(__file__).split("/")[0:-2]))
 
-from harvester import SMTP_CONFIG
 import smtplib
-from email.mime.text import MIMEText
 from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
 
-from harvester import HarvesterDBInterface, db_interface
+from harvester import SMTP_CONFIG, HarvesterDBInterface, db_interface
 from harvester.exceptions import (
     CompareException,
     DCATUSToCKANException,
@@ -73,7 +72,7 @@ class HarvestSource:
             "schema_type",
             "source_type",
             "id",  # db guuid
-            "notification_emails"
+            "notification_emails",
         ],
         repr=False,
     )
@@ -162,7 +161,6 @@ def internal_records_to_id_hash(self, records: list[dict]) -> None:
             )
 
     def get_record_identifier(self, record: dict) -> str:
-
         record_id = "identifier" if self.schema_type.startswith("dcatus") else "url"
 
         if record_id not in record:
@@ -181,7 +179,6 @@ def external_records_to_id_hash(self, records: list[dict]) -> None:
         logger.info("converting harvest records to id: hash")
         for record in records:
             try:
-
                 identifier = self.get_record_identifier(record)
 
                 if self.source_type == "document":
@@ -196,7 +193,7 @@ def external_records_to_id_hash(self, records: list[dict]) -> None:
             except Exception as e:
                 # TODO: do something with 'e'
                 raise ExtractExternalException(
-                    f"{self.title} {self.url} failed to convert to id:hash",
+                    f"{self.name} {self.url} failed to convert to id:hash",
                     self.job_id,
                 )
 
@@ -248,7 +245,7 @@ def compare(self) -> None:
         except Exception as e:
             # TODO: do something with 'e'
             raise CompareException(
-                f"{self.title} {self.url} failed to run compare. exiting.",
+                f"{self.name} {self.url} failed to run compare. exiting.",
                 self.job_id,
             )
 
@@ -402,45 +399,45 @@ def report(self) -> None:
             self.send_notification_emails(actual_results_action)
 
     def send_notification_emails(self, results: dict) -> None:
-        try:
-            job_url = f'{SMTP_CONFIG["base_url"]}/harvest_job/{self.job_id}'
-
-            subject = "Harvest Job Completed"
-            body = (
-                f"The harvest job ({self.job_id}) has been successfully completed.\n"
-                f"You can view the details here: {job_url}\n\n"
-                "Summary of the job:\n"
-                f"- Records Added: {results['create']}\n"
-                f"- Records Updated: {results['update']}\n"
-                f"- Records Deleted: {results['delete']}\n"
-                f"- Records Ignored: {results[None]}\n\n"
-                "====\n"
-                "You received this email because you subscribed to harvester updates.\n"
-                "Please do not reply to this email, as it is not monitored."
-            )
-            support_recipient = SMTP_CONFIG.get("recipient")
-            user_recipients = self.notification_emails
-            all_recipients = [support_recipient] + user_recipients
+        job_url = f'{SMTP_CONFIG["base_url"]}/harvest_job/{self.job_id}'
+
+        subject = "Harvest Job Completed"
+        body = (
+            f"The harvest job ({self.job_id}) has been successfully completed.\n"
+            f"You can view the details here: {job_url}\n\n"
+            "Summary of the job:\n"
+            f"- Records Added: {results['create']}\n"
+            f"- Records Updated: {results['update']}\n"
+            f"- Records Deleted: {results['delete']}\n"
+            f"- Records Ignored: {results[None]}\n\n"
+            "====\n"
+            "You received this email because you subscribed to harvester updates.\n"
+            "Please do not reply to this email, as it is not monitored."
+        )
+        support_recipient = SMTP_CONFIG.get("recipient")
+        user_recipients = self.notification_emails
+        all_recipients = [support_recipient] + user_recipients
 
+        msg = MIMEMultipart()
+        msg["From"] = SMTP_CONFIG["default_sender"]
+        msg["Reply-To"] = "[email protected]"
+        msg["Subject"] = subject
+        msg.attach(MIMEText(body, "plain"))
+
+        try:
             with smtplib.SMTP(SMTP_CONFIG["server"], SMTP_CONFIG["port"]) as server:
                 if SMTP_CONFIG["use_tls"]:
                     server.starttls()
                 server.login(SMTP_CONFIG["username"], SMTP_CONFIG["password"])
 
                 for recipient in all_recipients:
-                    msg = MIMEMultipart()
-                    msg["From"] = SMTP_CONFIG["default_sender"]
                     msg["To"] = recipient
-                    msg["Reply-To"] = "[email protected]"
-                    msg["Subject"] = subject
-                    msg.attach(MIMEText(body, "plain"))
-
-                    server.sendmail(SMTP_CONFIG["default_sender"], [recipient],
-                                    msg.as_string())
+                    server.sendmail(
+                        SMTP_CONFIG["default_sender"], [recipient], msg.as_string()
+                    )
                     logger.info(f"Notification email sent to: {recipient}")
-
         except Exception as e:
-            logger.error(f"Error preparing or sending notification emails: {e}")
+            logger.error(f"Failed to send notification email: {e}")
 
 
 @dataclass
@@ -565,7 +562,6 @@ def status(self, value) -> None:
         self._status = value
 
     def transform(self) -> None:
-
         data = {
             "file": self.metadata["content"],
             "reader": self.reader_map[self.harvest_source.schema_type],
@@ -584,7 +580,11 @@ def transform(self) -> None:
                 self.harvest_source.internal_records_lookup_table[self.identifier],
             )
 
+        record_id = self.harvest_source.internal_records_lookup_table[self.identifier]
         if 200 <= resp.status_code < 300:
+            logger.info(
+                f"successfully transformed record: {self.identifier} db id: {record_id}"
+            )
             self.transformed_data = json.loads(data["writerOutput"])
 
     def validate(self) -> None: