diff --git a/crawler/wpull_plugin.py b/crawler/wpull_plugin.py index 171bc61..a7ae1cc 100644 --- a/crawler/wpull_plugin.py +++ b/crawler/wpull_plugin.py @@ -70,9 +70,19 @@ def deactivate(self): def init_db(self): db_alias = "crawler" + # Django doesn't easily support dynamically adding an additional + # database once settings have been initialized. We have to manually + # define certain database options that are typically initialized + # with defaults by Django: + # https://github.com/django/django/blob/4d32ebcd57340aa8de2d6d31613f1646dc6391f6/django/db/utils.py#L159 connections.databases[db_alias] = { "ENGINE": "django.db.backends.sqlite3", "NAME": self.db_filename, + "AUTOCOMMIT": True, + "CONN_HEALTH_CHECKS": False, + "CONN_MAX_AGE": 0, + "OPTIONS": {}, + "TIME_ZONE": None, } call_command("migrate", database=db_alias, app_label="crawler", run_syncdb=True) diff --git a/requirements/base.txt b/requirements/base.txt index 43b4ead..196ab4f 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,14 +1,14 @@ -beautifulsoup4==4.12.2 -click==8.0.4 -cssselect==1.1.0 -Django==3.2.25 -django-click==2.3.0 -django-debug-toolbar==3.2.4 -django-filter==21.1 -django-modelcluster==5.3 -djangorestframework==3.15.1 -djangorestframework-csv==2.1.1 -whitenoise==5.3.0 +beautifulsoup4==4.12.3 +click==8.1.7 +cssselect==1.2.0 +Django==4.2.15 +django-click==2.4.0 +django-debug-toolbar==4.4.6 +django-filter==24.3 +django-modelcluster==6.3 +djangorestframework==3.15.2 +djangorestframework-csv==3.0.0 +whitenoise==6.7.0 # Ensure libxml2 is loaded dynamically; see # https://html5-parser.readthedocs.io/en/latest/#unix diff --git a/requirements/gunicorn.txt b/requirements/gunicorn.txt index 9d41f26..4afe40e 100644 --- a/requirements/gunicorn.txt +++ b/requirements/gunicorn.txt @@ -1 +1 @@ -gunicorn==20.1.0 +gunicorn==23.0.0 diff --git a/requirements/test.txt b/requirements/test.txt index 723f534..82e4c7f 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,2 +1,2 @@ black -coverage==6.4.3 +coverage==7.6.1