From 48e30a10a6369f7e42ddd342d775894a4a7ba9d2 Mon Sep 17 00:00:00 2001 From: Andy Chosak Date: Fri, 16 Aug 2024 15:14:41 -0400 Subject: [PATCH 1/2] Bump Django to version 4.2, bump other packages --- requirements/base.txt | 22 +++++++++++----------- requirements/gunicorn.txt | 2 +- requirements/test.txt | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 43b4ead..196ab4f 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,14 +1,14 @@ -beautifulsoup4==4.12.2 -click==8.0.4 -cssselect==1.1.0 -Django==3.2.25 -django-click==2.3.0 -django-debug-toolbar==3.2.4 -django-filter==21.1 -django-modelcluster==5.3 -djangorestframework==3.15.1 -djangorestframework-csv==2.1.1 -whitenoise==5.3.0 +beautifulsoup4==4.12.3 +click==8.1.7 +cssselect==1.2.0 +Django==4.2.15 +django-click==2.4.0 +django-debug-toolbar==4.4.6 +django-filter==24.3 +django-modelcluster==6.3 +djangorestframework==3.15.2 +djangorestframework-csv==3.0.0 +whitenoise==6.7.0 # Ensure libxml2 is loaded dynamically; see # https://html5-parser.readthedocs.io/en/latest/#unix diff --git a/requirements/gunicorn.txt b/requirements/gunicorn.txt index 9d41f26..4afe40e 100644 --- a/requirements/gunicorn.txt +++ b/requirements/gunicorn.txt @@ -1 +1 @@ -gunicorn==20.1.0 +gunicorn==23.0.0 diff --git a/requirements/test.txt b/requirements/test.txt index 723f534..82e4c7f 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,2 +1,2 @@ black -coverage==6.4.3 +coverage==7.6.1 From 4cb5b631f63e98b60fc1aa5b52895082dc30c74e Mon Sep 17 00:00:00 2001 From: Andy Chosak Date: Fri, 16 Aug 2024 16:34:48 -0400 Subject: [PATCH 2/2] Properly add dynamic database for SQLite crawl --- crawler/wpull_plugin.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/crawler/wpull_plugin.py b/crawler/wpull_plugin.py index 171bc61..a7ae1cc 100644 --- a/crawler/wpull_plugin.py +++ b/crawler/wpull_plugin.py @@ -70,9 +70,19 @@ def deactivate(self): def init_db(self): db_alias = "crawler" + # Django doesn't easily support dynamically adding an additional + # database once settings have been initialized. We have to manually + # define certain database options that are typically initialized + # with defaults by Django: + # https://github.com/django/django/blob/4d32ebcd57340aa8de2d6d31613f1646dc6391f6/django/db/utils.py#L159 connections.databases[db_alias] = { "ENGINE": "django.db.backends.sqlite3", "NAME": self.db_filename, + "AUTOCOMMIT": True, + "CONN_HEALTH_CHECKS": False, + "CONN_MAX_AGE": 0, + "OPTIONS": {}, + "TIME_ZONE": None, } call_command("migrate", database=db_alias, app_label="crawler", run_syncdb=True)