diff --git a/.bandit.yml b/.bandit.yml index 7fcde04..4f60a02 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -1,2 +1,6 @@ skips: - B101 +- B311 +- B320 +- B410 +exclude_dirs: ['tests'] diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..2417f2e --- /dev/null +++ b/.flake8 @@ -0,0 +1,16 @@ +[flake8] +max-line-length = 99 +ignore = + W503 + # too many leading '#' for block comment + E266 + E704 +exclude = + .git + .tox + venv* + + # pending revision + docs/conf.py +per-file-ignores = + cssselect/__init__.py:F401 diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000..02d8fcb --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,2 @@ +# applying pre-commit hooks to the project +7e8be052f6c49034d94571151b0d53d5d64717ce \ No newline at end of file diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 98b07ad..6a1568d 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -7,18 +7,9 @@ jobs: strategy: matrix: include: - - python-version: 3.12 - env: - TOXENV: black - - python-version: 3.12 - env: - TOXENV: flake8 - python-version: 3.12 env: TOXENV: pylint - - python-version: 3.12 - env: - TOXENV: security - python-version: 3.12 env: TOXENV: typing @@ -40,3 +31,9 @@ jobs: pip install -U pip pip install -U tox tox + + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pre-commit/action@v3.0.0 diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000..6860bdb --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,2 @@ +[settings] +profile = black \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..a27d3db --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,18 @@ +repos: +- repo: https://github.com/PyCQA/bandit + rev: 1.7.8 + hooks: + - id: bandit + args: [-r, -c, .bandit.yml] +- repo: https://github.com/PyCQA/flake8 + rev: 7.0.0 + hooks: + - id: flake8 +- repo: https://github.com/psf/black.git + rev: 24.3.0 + hooks: + - id: black +- repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index adb017d..a6efa85 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,17 @@ -[tool.black] -line-length = 119 -include = '\.py$' +[tool.isort] +profile = "black" +multi_line_output = 3 + +[tool.mypy] +check_untyped_defs = true +ignore_missing_imports = true +no_warn_no_return = true [[tool.mypy.overrides]] module = "queuelib.tests.*" allow_untyped_defs = true allow_untyped_calls = true check_untyped_defs = false + +[tool.black] +target-version = ["py38", "py39", "py310", "py311", "py312"] \ No newline at end of file diff --git a/queuelib/__init__.py b/queuelib/__init__.py index d89e7ac..263c513 100644 --- a/queuelib/__init__.py +++ b/queuelib/__init__.py @@ -1,5 +1,5 @@ __version__ = "1.6.2" -from queuelib.queue import FifoDiskQueue, LifoDiskQueue from queuelib.pqueue import PriorityQueue +from queuelib.queue import FifoDiskQueue, LifoDiskQueue from queuelib.rrqueue import RoundRobinQueue diff --git a/queuelib/pqueue.py b/queuelib/pqueue.py index 8ea8488..3a31cc4 100644 --- a/queuelib/pqueue.py +++ b/queuelib/pqueue.py @@ -27,7 +27,9 @@ class PriorityQueue: """ - def __init__(self, qfactory: Callable[[int], BaseQueue], startprios: Iterable[int] = ()) -> None: + def __init__( + self, qfactory: Callable[[int], BaseQueue], startprios: Iterable[int] = () + ) -> None: self.queues = {} self.qfactory = qfactory for p in startprios: diff --git a/queuelib/queue.py b/queuelib/queue.py index e2a82e7..c63da68 100644 --- a/queuelib/queue.py +++ b/queuelib/queue.py @@ -6,7 +6,7 @@ from abc import abstractmethod from collections import deque from contextlib import suppress -from typing import Any, Optional, Deque, BinaryIO, Literal, cast, Dict +from typing import Any, BinaryIO, Deque, Dict, Literal, Optional, cast class _BaseQueueMeta(type): @@ -15,7 +15,9 @@ class _BaseQueueMeta(type): """ def __instancecheck__(cls, instance: Any) -> bool: - return cls.__subclasscheck__(type(instance)) # pylint: disable=no-value-for-parameter + return cls.__subclasscheck__( # pylint: disable=no-value-for-parameter + type(instance) + ) def __subclasscheck__(cls, subclass: Any) -> bool: return ( diff --git a/queuelib/rrqueue.py b/queuelib/rrqueue.py index 7cccc85..d3c8bc5 100644 --- a/queuelib/rrqueue.py +++ b/queuelib/rrqueue.py @@ -24,7 +24,11 @@ class RoundRobinQueue: first and the next queue for that key is then popped. """ - def __init__(self, qfactory: Callable[[Hashable], BaseQueue], start_domains: Iterable[Hashable] = ()) -> None: + def __init__( + self, + qfactory: Callable[[Hashable], BaseQueue], + start_domains: Iterable[Hashable] = (), + ) -> None: self.queues = {} self.qfactory = qfactory for key in start_domains: diff --git a/queuelib/tests/test_pqueue.py b/queuelib/tests/test_pqueue.py index 22aff68..aa9567e 100644 --- a/queuelib/tests/test_pqueue.py +++ b/queuelib/tests/test_pqueue.py @@ -1,11 +1,12 @@ import os + from queuelib.pqueue import PriorityQueue from queuelib.queue import ( - FifoMemoryQueue, - LifoMemoryQueue, FifoDiskQueue, - LifoDiskQueue, + FifoMemoryQueue, FifoSQLiteQueue, + LifoDiskQueue, + LifoMemoryQueue, LifoSQLiteQueue, ) from queuelib.tests import QueuelibTestCase, track_closed @@ -174,25 +175,33 @@ def test_reopen_with_prio(self): self.assertEqual(q2.close(), []) -class FifoDiskPriorityQueueTest(PQueueTestMixin, FifoTestMixin, DiskTestMixin, QueuelibTestCase): +class FifoDiskPriorityQueueTest( + PQueueTestMixin, FifoTestMixin, DiskTestMixin, QueuelibTestCase +): def qfactory(self, prio): path = os.path.join(self.qdir, str(prio)) return track_closed(FifoDiskQueue)(path) -class LifoDiskPriorityQueueTest(PQueueTestMixin, LifoTestMixin, DiskTestMixin, QueuelibTestCase): +class LifoDiskPriorityQueueTest( + PQueueTestMixin, LifoTestMixin, DiskTestMixin, QueuelibTestCase +): def qfactory(self, prio): path = os.path.join(self.qdir, str(prio)) return track_closed(LifoDiskQueue)(path) -class FifoSQLitePriorityQueueTest(PQueueTestMixin, FifoTestMixin, DiskTestMixin, QueuelibTestCase): +class FifoSQLitePriorityQueueTest( + PQueueTestMixin, FifoTestMixin, DiskTestMixin, QueuelibTestCase +): def qfactory(self, prio): path = os.path.join(self.qdir, str(prio)) return track_closed(FifoSQLiteQueue)(path) -class LifoSQLitePriorityQueueTest(PQueueTestMixin, LifoTestMixin, DiskTestMixin, QueuelibTestCase): +class LifoSQLitePriorityQueueTest( + PQueueTestMixin, LifoTestMixin, DiskTestMixin, QueuelibTestCase +): def qfactory(self, prio): path = os.path.join(self.qdir, str(prio)) return track_closed(LifoSQLiteQueue)(path) diff --git a/queuelib/tests/test_queue.py b/queuelib/tests/test_queue.py index e198bf7..355eb18 100644 --- a/queuelib/tests/test_queue.py +++ b/queuelib/tests/test_queue.py @@ -1,18 +1,18 @@ -import os import glob +import os from abc import abstractmethod +from typing import Any, List, Optional from unittest import mock -from typing import Any, Optional, List import pytest from queuelib.queue import ( BaseQueue, - FifoMemoryQueue, - LifoMemoryQueue, FifoDiskQueue, - LifoDiskQueue, + FifoMemoryQueue, FifoSQLiteQueue, + LifoDiskQueue, + LifoMemoryQueue, LifoSQLiteQueue, ) from queuelib.tests import QueuelibTestCase @@ -215,7 +215,9 @@ def test_peek_lifo(self): class PersistentTestMixin: chunksize = 100000 - @pytest.mark.xfail(reason="Reenable once Scrapy.squeues stops extending from this testsuite") + @pytest.mark.xfail( + reason="Reenable once Scrapy.squeues stops extending from this testsuite" + ) def test_non_bytes_raises_typeerror(self): q = self.queue() self.assertRaises(TypeError, q.push, 0) @@ -280,7 +282,9 @@ def queue(self): return LifoMemoryQueue() -class FifoDiskQueueTest(FifoTestMixin, PersistentTestMixin, QueueTestMixin, QueuelibTestCase): +class FifoDiskQueueTest( + FifoTestMixin, PersistentTestMixin, QueueTestMixin, QueuelibTestCase +): def queue(self): return FifoDiskQueue(self.qpath, chunksize=self.chunksize) @@ -325,7 +329,9 @@ class ChunkSize4FifoDiskQueueTest(FifoDiskQueueTest): chunksize = 4 -class LifoDiskQueueTest(LifoTestMixin, PersistentTestMixin, QueueTestMixin, QueuelibTestCase): +class LifoDiskQueueTest( + LifoTestMixin, PersistentTestMixin, QueueTestMixin, QueuelibTestCase +): def queue(self): return LifoDiskQueue(self.qpath) @@ -342,11 +348,15 @@ def test_file_size_shrinks(self): assert os.path.getsize(self.qpath), size -class FifoSQLiteQueueTest(FifoTestMixin, PersistentTestMixin, QueueTestMixin, QueuelibTestCase): +class FifoSQLiteQueueTest( + FifoTestMixin, PersistentTestMixin, QueueTestMixin, QueuelibTestCase +): def queue(self): return FifoSQLiteQueue(self.qpath) -class LifoSQLiteQueueTest(LifoTestMixin, PersistentTestMixin, QueueTestMixin, QueuelibTestCase): +class LifoSQLiteQueueTest( + LifoTestMixin, PersistentTestMixin, QueueTestMixin, QueuelibTestCase +): def queue(self): return LifoSQLiteQueue(self.qpath) diff --git a/queuelib/tests/test_rrqueue.py b/queuelib/tests/test_rrqueue.py index d00a531..606826b 100644 --- a/queuelib/tests/test_rrqueue.py +++ b/queuelib/tests/test_rrqueue.py @@ -1,15 +1,14 @@ import os -from queuelib.rrqueue import RoundRobinQueue from queuelib.queue import ( - FifoMemoryQueue, - LifoMemoryQueue, FifoDiskQueue, - LifoDiskQueue, + FifoMemoryQueue, FifoSQLiteQueue, + LifoDiskQueue, + LifoMemoryQueue, LifoSQLiteQueue, ) - +from queuelib.rrqueue import RoundRobinQueue from queuelib.tests import QueuelibTestCase, track_closed @@ -127,25 +126,33 @@ def test_nonserializable_object_many_pop(self): self.assertEqual(self.q.close(), []) -class FifoDiskRRQueueTest(RRQueueTestMixin, FifoTestMixin, DiskTestMixin, QueuelibTestCase): +class FifoDiskRRQueueTest( + RRQueueTestMixin, FifoTestMixin, DiskTestMixin, QueuelibTestCase +): def qfactory(self, key): path = os.path.join(self.qdir, str(key)) return track_closed(FifoDiskQueue)(path) -class LifoDiskRRQueueTest(RRQueueTestMixin, LifoTestMixin, DiskTestMixin, QueuelibTestCase): +class LifoDiskRRQueueTest( + RRQueueTestMixin, LifoTestMixin, DiskTestMixin, QueuelibTestCase +): def qfactory(self, key): path = os.path.join(self.qdir, str(key)) return track_closed(LifoDiskQueue)(path) -class FifoSQLiteRRQueueTest(RRQueueTestMixin, FifoTestMixin, DiskTestMixin, QueuelibTestCase): +class FifoSQLiteRRQueueTest( + RRQueueTestMixin, FifoTestMixin, DiskTestMixin, QueuelibTestCase +): def qfactory(self, key): path = os.path.join(self.qdir, str(key)) return track_closed(FifoSQLiteQueue)(path) -class LifoSQLiteRRQueueTest(RRQueueTestMixin, LifoTestMixin, DiskTestMixin, QueuelibTestCase): +class LifoSQLiteRRQueueTest( + RRQueueTestMixin, LifoTestMixin, DiskTestMixin, QueuelibTestCase +): def qfactory(self, key): path = os.path.join(self.qdir, str(key)) return track_closed(LifoSQLiteQueue)(path) diff --git a/setup.py b/setup.py index fa55ed8..86c7d51 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -from setuptools import setup, find_packages +from setuptools import find_packages, setup setup( name="queuelib", diff --git a/tox.ini b/tox.ini index 0cb805e..5e1e824 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py, pypy3, black, flake8, pylint, security, typing, twinecheck +envlist = py, pylint, typing, twinecheck, pre-commit [testenv] deps = @@ -14,20 +14,6 @@ deps = commands = py.test --cov=queuelib --cov-report=xml --cov-report=term --cov-report=html {posargs:queuelib} -[testenv:black] -basepython = python3 -deps = - black==23.11.0 -commands = - black --check {posargs:queuelib setup.py} - -[testenv:flake8] -basepython = python3 -deps = - flake8==6.1.0 -commands = - flake8 {posargs:queuelib setup.py} - [testenv:pylint] basepython = python3 deps = @@ -36,13 +22,6 @@ deps = commands = pylint {posargs:queuelib setup.py} -[testenv:security] -basepython = python3 -deps = - bandit==1.7.5 -commands = - bandit -r -c .bandit.yml {posargs:queuelib setup.py} - [testenv:typing] basepython = python3 deps = @@ -59,3 +38,8 @@ deps = commands = python -m build --sdist twine check dist/* + +[testenv:pre-commit] +deps = pre-commit +commands = pre-commit run --all-files --show-diff-on-failure +skip_install = true