diff --git a/.github/workflows/commit.yml b/.github/workflows/commit.yml index 7a65f06e..09a976eb 100644 --- a/.github/workflows/commit.yml +++ b/.github/workflows/commit.yml @@ -34,7 +34,7 @@ jobs: - name: test run: make build test - name: test extension - run: make test_extension + run: make test-extension - name: cypress-artifacts uses: actions/upload-artifact@v3 if: failure() diff --git a/.github/workflows/snyk.yml b/.github/workflows/snyk.yml index 45efc806..6ec44123 100644 --- a/.github/workflows/snyk.yml +++ b/.github/workflows/snyk.yml @@ -2,6 +2,9 @@ name: Check for Snyk Vulnerabilities on: # yamllint disable-line rule:truthy + pull_request: + branches: + - main workflow_dispatch: schedule: - cron: '0 12 * * *' # every day at 12pm UTC @@ -13,10 +16,11 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - - name: Set up Python 3.9 - uses: actions/setup-python@v4 + - name: Set up Python 3.10 + uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.10.14 + cache: 'pip' - name: Display Python version run: python -c "import sys; print(sys.version)" - name: Install Dependencies @@ -57,7 +61,7 @@ jobs: # Fail so that PR is created exit 1 - name: Create Pull Request - if: ${{ failure() }} + if: ${{ failure() && github.event_name == 'schedule' }} id: scpr uses: peter-evans/create-pull-request@v5 with: diff --git a/.snyk b/.snyk index a75fbfcc..36fcacac 100644 --- a/.snyk +++ b/.snyk @@ -28,7 +28,7 @@ ignore: reason: >- Upgrade path is complex, Issue tracked in github: https://github.com/GSA/data.gov/issues/4217 - expires: 2024-06-01T16:20:58.017Z + expires: 2024-09-30T16:20:58.017Z created: 2023-02-15T16:20:58.023Z SNYK-PYTHON-FLASK-5490129: - '*': @@ -66,19 +66,19 @@ ignore: - '*': reason: >- Not affecting us since no debugger is enabled in cloud.gov apps - expires: 2024-06-31T16:20:58.017Z + expires: 2024-09-30T16:20:58.017Z SNYK-PYTHON-CRYPTOGRAPHY-7161587: - '*': reason: >- No remediation available yet. Issue tracked in github: https://github.com/GSA/data.gov/issues/4781 - expires: 2024-06-31T16:20:58.017Z + expires: 2024-09-30T16:20:58.017Z SNYK-PYTHON-PYOPENSSL-7161590: - '*': reason: >- No remediation available yet. Issue tracked in github: https://github.com/GSA/data.gov/issues/4782 - expires: 2024-06-31T16:20:58.017Z + expires: 2024-09-30T16:20:58.017Z patch: {} # specify the directories or files to be excludeed from import: exclude: diff --git a/Makefile b/Makefile index 15ea8f05..bc9200ee 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,12 @@ build: clean: docker compose down -v --remove-orphans +clear-solr-volume: + # Destructive + docker stop $(shell docker volume rm catalogdatagov_solr_data 2>&1 | cut -d "[" -f2 | cut -d "]" -f1) + docker rm $(shell docker volume rm catalogdatagov_solr_data 2>&1 | cut -d "[" -f2 | cut -d "]" -f1) + docker volume rm catalogdatagov_solr_data + cypress: # Turn on local system, and open cypress in interactive mode docker compose up -d && cd e2e && npm install && npm run test @@ -17,33 +23,27 @@ debug: # so that we have interactive console access for the debugger. docker compose stop ckan ; docker compose run --service-ports ckan -requirements: - docker compose run --rm -T ckan /app/bin/requirements.sh - lint: flake8 . --count --show-source --statistics +requirements: + docker compose run --rm -T ckan /app/bin/requirements.sh + restart: docker compose restart ckan -test-build: - docker compose -f docker-compose.yml -f docker-compose.test.yml build - test: docker compose -f docker-compose.yml -f docker-compose.test.yml up --abort-on-container-exit test -test_extension: - docker compose run --rm -T ckan pytest --cov=ckanext.datagov_inventory --disable-warnings /app/ckanext/datagov_inventory/tests/ - -up: - docker compose up $(ARGS) +test-build: + docker compose -f docker-compose.yml -f docker-compose.test.yml build -clear-solr-volume: - # Destructive - docker stop $(shell docker volume rm catalogdatagov_solr_data 2>&1 | cut -d "[" -f2 | cut -d "]" -f1) - docker rm $(shell docker volume rm catalogdatagov_solr_data 2>&1 | cut -d "[" -f2 | cut -d "]" -f1) - docker volume rm catalogdatagov_solr_data +test-extension: + docker compose run --rm -T ckan pytest --cov=ckanext.datagov_inventory --disable-warnings /app/ckanext/datagov_inventory/tests/ unlock-solr-volume: # Corruptible docker compose run solr /bin/bash -c "rm -rf /var/solr/data/ckan/data/index/write.lock" + +up: + docker compose up $(ARGS) \ No newline at end of file diff --git a/config/ckan.ini b/config/ckan.ini index 2c8a2a0a..1615cf53 100644 --- a/config/ckan.ini +++ b/config/ckan.ini @@ -37,6 +37,8 @@ beaker.session.secret = $CKAN___BEAKER__SESSION__SECRET beaker.session.type=ext:database beaker.session.cookie_expires=true +beaker.session.secure = True +beaker.session.samesite = Strict #beaker.session.url = $CKAN___BEAKER__SESSION__URL # 900 seconds = 15 mins beaker.session.timeout=900 diff --git a/docker-compose.test.yml b/docker-compose.test.yml index ff8316d6..3dd4686f 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -1,4 +1,3 @@ -version: '3' services: ckan: environment: diff --git a/docker-compose.yml b/docker-compose.yml index 003ac9c7..4b333000 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: '3' services: ckan: image: datagov/inventory-app:2.10.4 diff --git a/requirements-dev.txt b/requirements-dev.txt index 5d07002d..7737e814 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -22,4 +22,5 @@ Sphinx==3.3.0 sphinx-rtd-theme==0.4.3 pygments>=2.7.4 # not directly required, pinned by Snyk to avoid a vulnerability ipython>=8.10.0 # not directly required, pinned by Snyk to avoid a vulnerability -setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability +zipp>=3.19.1 # not directly required, pinned by Snyk to avoid a vulnerability +setuptools>=70.0.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/requirements.in.txt b/requirements.in.txt index 9386666b..2829cae8 100644 --- a/requirements.in.txt +++ b/requirements.in.txt @@ -78,14 +78,12 @@ python-dateutil>=2.8.2 # (most likely for snyk) itsdangerous==2.0.1 MarkupSafe==2.0.1 -certifi>=2022.12.7 -setuptools==68.0.0 +setuptools>=70.0.0 wheel==0.42.0 # avoid ImportError error https://github.com/GSA/data.gov/issues/4396 importlib-resources<6.0 gevent>=23.9.0 -urllib3~=1.26.17 cryptography>=42.0.4 pip>=23.3 jinja2>=3.1.4 @@ -95,3 +93,6 @@ pyparsing # need to avoid solr missing module error on cloud.gov # duplicate Werkzeug[watchdog]==2.0.3 requrements from ckan list # so that snyk doesn't complain about the version mismatch Werkzeug==2.0.3 +urllib3>=1.26.19 + +certifi>=2024.7.4 diff --git a/requirements.txt b/requirements.txt index f74c1208..5dd56995 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,9 +5,9 @@ Babel==2.10.3 Beaker==1.11.0 bleach==5.0.1 blinker==1.5 -boto3==1.34.111 -botocore==1.34.111 -certifi==2024.2.2 +boto3==1.34.144 +botocore==1.34.144 +certifi==2024.7.4 cffi==1.16.0 chardet==5.2.0 charset-normalizer==3.3.2 @@ -19,10 +19,10 @@ ckanext-googleanalyticsbasic==0.2.1 ckanext-s3filestore @ git+https://github.com/keitaroinc/ckanext-s3filestore.git@caf88c0352ffe7b4432d3d55ddfb0a71249ceddd ckanext-saml2auth @ git+https://github.com/GSA/ckanext-saml2auth.git@387cfc1c6a7619f670bf387384f2634516de5844 ckanext-usmetadata==0.3.2 --e git+https://github.com/ckan/ckanext-xloader.git@dd475e70255da1e46b93e2193bf3e20f2e694db5#egg=ckanext_xloader +-e git+https://github.com/ckan/ckanext-xloader.git@3a865acd843bf7e2a7e2eb3c4e983816eddb6658#egg=ckanext_xloader ckantoolkit==0.0.7 click==8.1.3 -cryptography==42.0.7 +cryptography==42.0.8 defusedxml==0.7.1 dominate==2.7.0 elementpath==4.4.0 @@ -38,7 +38,7 @@ greenlet==2.0.2 gunicorn==22.0.0 html5lib==1.1 idna==3.7 -ijson==3.2.3 +ijson==3.3.0 importlib-resources==5.13.0 itsdangerous==2.0.1 Jinja2==3.1.4 @@ -52,15 +52,15 @@ Mako==1.3.5 Markdown==3.4.1 MarkupSafe==2.0.1 messytables==0.15.2 -mypy==1.10.0 +mypy==1.10.1 mypy-extensions==1.0.0 -newrelic==9.9.1 +newrelic==9.12.0 nose==1.3.7 -openpyxl==3.1.2 -packaging==24.0 +openpyxl==3.1.5 +packaging==24.1 passlib==1.7.4 pika==1.3.2 -pip==24.0 +pip==24.1 polib==1.1.1 psycopg2==2.9.3 pycparser==2.22 @@ -75,13 +75,13 @@ pytz==2024.1 pytz-deprecation-shim==0.1.0.post0 PyUtilib==6.0.0 PyYAML==6.0.1 -redis==5.0.4 -requests==2.32.2 +redis==5.0.7 +requests==2.32.3 rfc3987==1.3.8 rq==1.11.0 -s3transfer==0.10.1 +s3transfer==0.10.2 sansjson==0.3.0 -setuptools==68.0.0 +setuptools==70.1.0 simplejson==3.18.0 six==1.16.0 SQLAlchemy==1.4.41 @@ -94,8 +94,8 @@ tzdata==2024.1 tzlocal==4.2 unicodecsv==0.14.1 Unidecode==1.0.22 -urllib3==1.26.18 -watchdog==4.0.0 +urllib3==2.2.2 +watchdog==4.0.1 webassets==2.0 webencodings==0.5.1 Werkzeug==2.0.3 diff --git a/setup.py b/setup.py index 61be93e8..d3325bed 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ 'Development Status :: 4 - Beta', # Pick your license as you wish (should match "license" above) - 'License :: OSI Approved :: GNU Affero General Public License'\ + 'License :: OSI Approved :: GNU Affero General Public License' ' v3 or later (AGPLv3+)', # Specify the Python versions you support here. In particular, ensure