From 2e68b1af666bd0fbf77adeeb2c90a9ae4f327655 Mon Sep 17 00:00:00 2001 From: Christopher Hedrick Date: Wed, 12 Feb 2025 12:13:35 -0500 Subject: [PATCH] ran lint commands --- app/routes.py | 13 ++++--- database/interface.py | 27 +++++++-------- database/models.py | 1 + migrations/env.py | 34 ++++++++----------- ...c04b66d4d5b9_add_notification_frequency.py | 20 +++++++---- .../test_harvest_job_full_flow.py | 2 +- 6 files changed, 50 insertions(+), 47 deletions(-) diff --git a/app/routes.py b/app/routes.py index faa25747..dcbddd7e 100644 --- a/app/routes.py +++ b/app/routes.py @@ -299,7 +299,7 @@ def make_new_source_contract(form): "frequency": form.frequency.data, "schema_type": form.schema_type.data, "source_type": form.source_type.data, - "notification_frequency": form.notification_frequency.data + "notification_frequency": form.notification_frequency.data, } @@ -754,12 +754,15 @@ def download_harvest_errors_by_job(job_id, error_type): [ error.id, identifier, - json.loads(source_raw).get("title", None) if source_raw else None, + ( + json.loads(source_raw).get("title", None) + if source_raw + else None + ), error.harvest_record_id, error.type, error.message, - error.date_created - + error.date_created, ] for error, identifier, source_raw in db.get_harvest_record_errors_by_job( job_id, paginate=False @@ -773,7 +776,7 @@ def download_harvest_errors_by_job(job_id, error_type): "harvest_record_id", "record_error_type", "message", - "date_created" + "date_created", ] ] diff --git a/database/interface.py b/database/interface.py index c261cb4f..5d12b429 100644 --- a/database/interface.py +++ b/database/interface.py @@ -340,17 +340,17 @@ def get_harvest_job_errors_by_job(self, job_id: str) -> list[dict]: @paginate def get_harvest_record_errors_by_job(self, job_id: str, **kwargs): """ - Retrieves harvest record errors for a given job. + Retrieves harvest record errors for a given job. - This function fetches all records where the harvest status is 'error' and - belongs to the specified job. The query returns a tuple containing: - - HarvestRecordError object - - identifier (retrieved from HarvestRecord) - - source_raw (retrieved from HarvestRecord, containing 'title') + This function fetches all records where the harvest status is 'error' and + belongs to the specified job. The query returns a tuple containing: + - HarvestRecordError object + - identifier (retrieved from HarvestRecord) + - source_raw (retrieved from HarvestRecord, containing 'title') - Returns: - Query: A SQLAlchemy Query object that, when executed, yields tuples of: - (HarvestRecordError, identifier, source_raw). + Returns: + Query: A SQLAlchemy Query object that, when executed, yields tuples of: + (HarvestRecordError, identifier, source_raw). """ subquery = ( self.db.query(HarvestRecord.id) @@ -360,12 +360,11 @@ def get_harvest_record_errors_by_job(self, job_id: str, **kwargs): ) query = ( self.db.query( - HarvestRecordError, - HarvestRecord.identifier, - HarvestRecord.source_raw + HarvestRecordError, HarvestRecord.identifier, HarvestRecord.source_raw + ) + .join( + HarvestRecord, HarvestRecord.id == HarvestRecordError.harvest_record_id ) - .join(HarvestRecord, - HarvestRecord.id == HarvestRecordError.harvest_record_id) .filter(HarvestRecord.id.in_(select(subquery))) ) return query diff --git a/database/models.py b/database/models.py index 317d0e3f..e79b0636 100644 --- a/database/models.py +++ b/database/models.py @@ -79,6 +79,7 @@ class HarvestSource(db.Model): nullable=False, ) + class HarvestJob(db.Model): __tablename__ = "harvest_job" diff --git a/migrations/env.py b/migrations/env.py index 4c970927..54a60716 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -1,9 +1,8 @@ import logging from logging.config import fileConfig -from flask import current_app - from alembic import context +from flask import current_app # this is the Alembic Config object, which provides # access to the values within the .ini file in use. @@ -12,32 +11,31 @@ # Interpret the config file for Python logging. # This line sets up loggers basically. fileConfig(config.config_file_name) -logger = logging.getLogger('alembic.env') +logger = logging.getLogger("alembic.env") def get_engine(): try: # this works with Flask-SQLAlchemy<3 and Alchemical - return current_app.extensions['migrate'].db.get_engine() + return current_app.extensions["migrate"].db.get_engine() except (TypeError, AttributeError): # this works with Flask-SQLAlchemy>=3 - return current_app.extensions['migrate'].db.engine + return current_app.extensions["migrate"].db.engine def get_engine_url(): try: - return get_engine().url.render_as_string(hide_password=False).replace( - '%', '%%') + return get_engine().url.render_as_string(hide_password=False).replace("%", "%%") except AttributeError: - return str(get_engine().url).replace('%', '%%') + return str(get_engine().url).replace("%", "%%") # add your model's MetaData object here # for 'autogenerate' support # from myapp import mymodel # target_metadata = mymodel.Base.metadata -config.set_main_option('sqlalchemy.url', get_engine_url()) -target_db = current_app.extensions['migrate'].db +config.set_main_option("sqlalchemy.url", get_engine_url()) +target_db = current_app.extensions["migrate"].db # other values from the config, defined by the needs of env.py, # can be acquired: @@ -46,7 +44,7 @@ def get_engine_url(): def get_metadata(): - if hasattr(target_db, 'metadatas'): + if hasattr(target_db, "metadatas"): return target_db.metadatas[None] return target_db.metadata @@ -64,9 +62,7 @@ def run_migrations_offline(): """ url = config.get_main_option("sqlalchemy.url") - context.configure( - url=url, target_metadata=get_metadata(), literal_binds=True - ) + context.configure(url=url, target_metadata=get_metadata(), literal_binds=True) with context.begin_transaction(): context.run_migrations() @@ -84,13 +80,13 @@ def run_migrations_online(): # when there are no changes to the schema # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html def process_revision_directives(context, revision, directives): - if getattr(config.cmd_opts, 'autogenerate', False): + if getattr(config.cmd_opts, "autogenerate", False): script = directives[0] if script.upgrade_ops.is_empty(): directives[:] = [] - logger.info('No changes in schema detected.') + logger.info("No changes in schema detected.") - conf_args = current_app.extensions['migrate'].configure_args + conf_args = current_app.extensions["migrate"].configure_args if conf_args.get("process_revision_directives") is None: conf_args["process_revision_directives"] = process_revision_directives @@ -98,9 +94,7 @@ def process_revision_directives(context, revision, directives): with connectable.connect() as connection: context.configure( - connection=connection, - target_metadata=get_metadata(), - **conf_args + connection=connection, target_metadata=get_metadata(), **conf_args ) with context.begin_transaction(): diff --git a/migrations/versions/c04b66d4d5b9_add_notification_frequency.py b/migrations/versions/c04b66d4d5b9_add_notification_frequency.py index 25694b14..dbca3d0e 100644 --- a/migrations/versions/c04b66d4d5b9_add_notification_frequency.py +++ b/migrations/versions/c04b66d4d5b9_add_notification_frequency.py @@ -5,12 +5,12 @@ Create Date: 2025-02-07 09:55:49.013498 """ -from alembic import op -import sqlalchemy as sa +import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. -revision = 'c04b66d4d5b9' +revision = "c04b66d4d5b9" down_revision = None branch_labels = None depends_on = None @@ -18,15 +18,21 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('harvest_source', schema=None) as batch_op: - batch_op.add_column(sa.Column('notification_frequency', sa.Enum('on_error', 'always', name='notification_frequency'), nullable=False)) + with op.batch_alter_table("harvest_source", schema=None) as batch_op: + batch_op.add_column( + sa.Column( + "notification_frequency", + sa.Enum("on_error", "always", name="notification_frequency"), + nullable=False, + ) + ) # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('harvest_source', schema=None) as batch_op: - batch_op.drop_column('notification_frequency') + with op.batch_alter_table("harvest_source", schema=None) as batch_op: + batch_op.drop_column("notification_frequency") # ### end Alembic commands ### diff --git a/tests/integration/harvest_job_flows/test_harvest_job_full_flow.py b/tests/integration/harvest_job_flows/test_harvest_job_full_flow.py index 0d1821b1..5a83f531 100644 --- a/tests/integration/harvest_job_flows/test_harvest_job_full_flow.py +++ b/tests/integration/harvest_job_flows/test_harvest_job_full_flow.py @@ -1,6 +1,6 @@ import json import os -from unittest.mock import patch, MagicMock +from unittest.mock import MagicMock, patch from harvester.harvest import harvest_job_starter from harvester.utils.general_utils import download_file