From ed9023ea59be444db07c5f6cbc37a5c6fd9d631a Mon Sep 17 00:00:00 2001 From: R Max Espinoza Date: Tue, 2 Jul 2024 21:27:50 +0200 Subject: [PATCH] fix test requirements and simplified tox config --- .dockerignore | 5 ++- .github/workflows/builds.yml | 2 +- .github/workflows/checks.yml | 2 +- .github/workflows/tests.yml | 15 +++++++-- Dockerfile | 16 ++++++++++ VERSION | 2 +- docker-compose.yaml | 20 ++++++++++++ requirements-tests.txt | 4 +-- tests/test_scrapy_redis.py | 2 +- tests/test_spiders.py | 12 +++++++- tox.ini | 60 +++++++++++++++--------------------- 11 files changed, 94 insertions(+), 46 deletions(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yaml diff --git a/.dockerignore b/.dockerignore index e89a57d8..6203c75b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -40,4 +40,7 @@ nosetests.xml .pydevproject # JetBrains PyCharm IDE -/.idea/ \ No newline at end of file +/.idea/ + +.venv +.tags diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index ac9fbd59..834ff7a2 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -25,5 +25,5 @@ jobs: env: TOXENV: build run: | - pip install -U tox + pip install -r requirements-tests.txt tox diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 75e707e5..eb51abc6 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -25,5 +25,5 @@ jobs: env: TOXENV: ${{ matrix.env }} run: | - pip install -U tox + pip install -r requirements-tests.txt tox diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6e2ac27e..902bedd9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,6 +12,15 @@ jobs: matrix: python-version: ["3.11.3"] + services: + redis: + image: redis + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: - uses: actions/checkout@v2 @@ -21,8 +30,10 @@ jobs: python-version: ${{ matrix.python-version }} - name: Run pytest - env: + env: + REDIS_HOST: redis TOXENV: pytest + TOX_TESTENV_PASSENV: REDIS_HOST run: | - pip install -U tox + pip install -r requirements-tests.txt tox diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..c76f90d3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.11-slim + +# Set working directory +WORKDIR /app + +# Install tox and dependencies (replace 'your-requirements.txt' with your actual file) +COPY requirements.txt . +COPY requirements-tests.txt . +RUN pip install -r requirements.txt -r requirements-tests.txt + +# Copy your project code +COPY . . + +# Run Tox tests +CMD ["tox"] + diff --git a/VERSION b/VERSION index b09a54cb..f38fc539 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.7.3 \ No newline at end of file +0.7.3 diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 00000000..dd2c37e9 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,20 @@ +version: '3.8' + +services: + python: + build: . + command: tox -e security,flake8,pytest + environment: + REDIS_HOST: redis # Use service name for hostname within docker network + REDIS_PORT: 6379 + TOX_TESTENV_PASSENV: "REDIS_HOST REDIS_PORT" + volumes: + - ./:/app # Mount your project directory into the container + depends_on: + - redis + + redis: + image: redis:6.2-alpine + ports: + - "6379:6379" # Map Redis port to host port + diff --git a/requirements-tests.txt b/requirements-tests.txt index 72c6790b..1ce8f1a3 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -1,6 +1,6 @@ # This packages are required to run all the tests. flake8 mock -pytest +pytest>=6.0,<7 pytest-cov -tox +tox>=3.0,<4 diff --git a/tests/test_scrapy_redis.py b/tests/test_scrapy_redis.py index 045f2fcf..f5db4e40 100644 --- a/tests/test_scrapy_redis.py +++ b/tests/test_scrapy_redis.py @@ -15,7 +15,7 @@ # allow test settings from environment -REDIS_HOST = os.environ.get('REDIST_HOST', 'localhost') +REDIS_HOST = os.environ.get('REDIS_HOST', 'localhost') REDIS_PORT = int(os.environ.get('REDIS_PORT', 6379)) diff --git a/tests/test_spiders.py b/tests/test_spiders.py index a49aedcf..1dce5cbd 100644 --- a/tests/test_spiders.py +++ b/tests/test_spiders.py @@ -1,5 +1,6 @@ import contextlib import mock +import os import pytest from scrapy import signals @@ -12,6 +13,10 @@ ) +REDIS_HOST = os.environ.get('REDIS_HOST', 'localhost') +REDIS_PORT = int(os.environ.get('REDIS_PORT', 6379)) + + @contextlib.contextmanager def flushall(server): try: @@ -29,7 +34,10 @@ class MyCrawlSpider(RedisCrawlSpider): def get_crawler(**kwargs): - return mock.Mock(settings=Settings(), **kwargs) + return mock.Mock(settings=Settings({ + "REDIS_HOST": REDIS_HOST, + "REDIS_PORT": REDIS_PORT, + }), **kwargs) class TestRedisMixin_setup_redis(object): @@ -124,6 +132,8 @@ def test_consume_urls_from_redis(start_urls_as_zset, start_urls_as_set, spider_c redis_key = 'start:urls' crawler = get_crawler() crawler.settings.setdict({ + 'REDIS_HOST': REDIS_HOST, + 'REDIS_PORT': REDIS_PORT, 'REDIS_START_URLS_KEY': redis_key, 'REDIS_START_URLS_AS_ZSET': start_urls_as_zset, 'REDIS_START_URLS_AS_SET': start_urls_as_set, diff --git a/tox.ini b/tox.ini index 03e98439..cfbf3941 100644 --- a/tox.ini +++ b/tox.ini @@ -1,12 +1,14 @@ [tox] -envlist = security,flake8,py{38,39,310,311}-scrapy{26,27,28,29}-redis{42,43,44,45} -minversion = 1.7.0 +envlist = + security + flake8 + py{38,39,310,311}-scrapy{26,27,28,29}-redis{42,43,44,45,46,50} +minversion = 3.0.0 [base] deps = - scrapy>=2.6 - redis>=4.2 - six>=1.5.2 + -r requirements-tests.txt + -r requirements.txt [testenv] basepython = @@ -16,26 +18,26 @@ basepython = py311: python3.11 deps = {[base]deps} - mock - pytest - pytest-cov + scrapy26: scrapy~=2.6.0 + scrapy27: scrapy~=2.7.0 + scrapy28: scrapy~=2.8.0 + scrapy29: scrapy~=2.9.0 + scrapy210: scrapy~=2.10.0 + scrapy211: scrapy~=2.11.0 + redis42: redis~=4.2.0 + redis43: redis~=4.3.0 + redis44: redis~=4.4.0 + redis45: redis~=4.5.0 + redis46: redis~=4.6.0 + redis50: redis~=5.0.0 commands = - scrapy26: pip install scrapy==2.6.3 - scrapy27: pip install scrapy==2.7.1 - scrapy28: pip install scrapy==2.8.0 - scrapy29: pip install scrapy==2.9.0 - redis42: pip install redis==4.2.0 - redis43: pip install redis==4.3.6 - redis44: pip install redis==4.4.4 - redis45: pip install redis==4.5.5 - pip install . python -m pytest # --cov-report term --cov=scrapy_redis [testenv:flake8] -basepython = python3.11 +basepython = + python3.11 deps = {[base]deps} - flake8 # https://github.com/tholo/pytest-flake8/issues/81 commands = flake8 --ignore=W503,E265,E731 docs/ tests/ @@ -46,31 +48,17 @@ deps = commands = bandit -r -c .bandit.yml src/ tests/ -[testenv:pylint] -basepython = python3.11 -deps = - {[base]deps} - pylint==2.12.2 -commands = - pylint setup.py docs/ src/ tests/ - [testenv:pytest] basepython = python3.11 deps = {[testenv]deps} - scrapy==2.6.1 - redis==4.2.2 -allowlist_externals = sudo commands = - sudo apt-get update - sudo apt-get install -y redis - sudo systemctl start redis-server - pip install . python -m pytest --cov-report term --cov=scrapy_redis [testenv:build] basepython=python3.11 deps = {[base]deps} -commands = - pip install . + build +commands = + python -m build