Merge pull request #65 from emarondan/adding-pre-commit

Adding pre commit

.bandit.yml

@@ -1,2 +1,6 @@
 skips:
 - B101
+- B311
+- B320
+- B410
+exclude_dirs: ['tests']

.flake8

@@ -0,0 +1,16 @@
+[flake8]
+max-line-length = 99
+ignore =
+    W503
+    # too many leading '#' for block comment
+    E266
+    E704
+exclude =
+    .git
+    .tox
+    venv*
+
+    # pending revision
+    docs/conf.py
+per-file-ignores =
+    cssselect/__init__.py:F401

.git-blame-ignore-revs

@@ -0,0 +1,2 @@
+# applying pre-commit hooks to the project
+7e8be052f6c49034d94571151b0d53d5d64717ce

.github/workflows/checks.yml

@@ -7,18 +7,9 @@ jobs:
     strategy:
       matrix:
         include:
-        - python-version: 3.12
-          env:
-            TOXENV: black
-        - python-version: 3.12
-          env:
-            TOXENV: flake8
         - python-version: 3.12
           env:
             TOXENV: pylint
-        - python-version: 3.12
-          env:
-            TOXENV: security
         - python-version: 3.12
           env:
             TOXENV: typing
@@ -40,3 +31,9 @@ jobs:
         pip install -U pip
         pip install -U tox
         tox
+
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: pre-commit/[email protected]

.isort.cfg

@@ -0,0 +1,2 @@
+[settings]
+profile = black

.pre-commit-config.yaml

@@ -0,0 +1,18 @@
+repos:
+- repo: https://github.com/PyCQA/bandit
+  rev: 1.7.8
+  hooks:
+  - id: bandit
+    args: [-r, -c, .bandit.yml]
+- repo: https://github.com/PyCQA/flake8
+  rev: 7.0.0
+  hooks:
+  - id: flake8
+- repo: https://github.com/psf/black.git
+  rev: 24.3.0
+  hooks:
+  - id: black
+- repo: https://github.com/pycqa/isort
+  rev: 5.13.2
+  hooks:
+  - id: isort

pyproject.toml

@@ -1,9 +1,17 @@
-[tool.black]
-line-length = 119
-include = '\.py$'
+[tool.isort]
+profile = "black"
+multi_line_output = 3
+
+[tool.mypy]
+check_untyped_defs = true
+ignore_missing_imports = true
+no_warn_no_return = true
 
 [[tool.mypy.overrides]]
 module = "queuelib.tests.*"
 allow_untyped_defs = true
 allow_untyped_calls = true
 check_untyped_defs = false
+
+[tool.black]
+target-version = ["py38", "py39", "py310", "py311", "py312"]

queuelib/__init__.py

@@ -1,5 +1,5 @@
 __version__ = "1.6.2"
 
-from queuelib.queue import FifoDiskQueue, LifoDiskQueue
 from queuelib.pqueue import PriorityQueue
+from queuelib.queue import FifoDiskQueue, LifoDiskQueue
 from queuelib.rrqueue import RoundRobinQueue

queuelib/pqueue.py

@@ -27,7 +27,9 @@ class PriorityQueue:
 
     """
 
-    def __init__(self, qfactory: Callable[[int], BaseQueue], startprios: Iterable[int] = ()) -> None:
+    def __init__(
+        self, qfactory: Callable[[int], BaseQueue], startprios: Iterable[int] = ()
+    ) -> None:
         self.queues = {}
         self.qfactory = qfactory
         for p in startprios:

queuelib/queue.py

@@ -6,7 +6,7 @@
 from abc import abstractmethod
 from collections import deque
 from contextlib import suppress
-from typing import Any, Optional, Deque, BinaryIO, Literal, cast, Dict
+from typing import Any, BinaryIO, Deque, Dict, Literal, Optional, cast
 
 
 class _BaseQueueMeta(type):
@@ -15,7 +15,9 @@ class _BaseQueueMeta(type):
     """
 
     def __instancecheck__(cls, instance: Any) -> bool:
-        return cls.__subclasscheck__(type(instance))  # pylint: disable=no-value-for-parameter
+        return cls.__subclasscheck__(  # pylint: disable=no-value-for-parameter
+            type(instance)
+        )
 
     def __subclasscheck__(cls, subclass: Any) -> bool:
         return (

queuelib/rrqueue.py

@@ -24,7 +24,11 @@ class RoundRobinQueue:
     first and the next queue for that key is then popped.
     """
 
-    def __init__(self, qfactory: Callable[[Hashable], BaseQueue], start_domains: Iterable[Hashable] = ()) -> None:
+    def __init__(
+        self,
+        qfactory: Callable[[Hashable], BaseQueue],
+        start_domains: Iterable[Hashable] = (),
+    ) -> None:
         self.queues = {}
         self.qfactory = qfactory
         for key in start_domains:

queuelib/tests/test_pqueue.py

@@ -1,11 +1,12 @@
 import os
+
 from queuelib.pqueue import PriorityQueue
 from queuelib.queue import (
-    FifoMemoryQueue,
-    LifoMemoryQueue,
     FifoDiskQueue,
-    LifoDiskQueue,
+    FifoMemoryQueue,
     FifoSQLiteQueue,
+    LifoDiskQueue,
+    LifoMemoryQueue,
     LifoSQLiteQueue,
 )
 from queuelib.tests import QueuelibTestCase, track_closed
@@ -174,25 +175,33 @@ def test_reopen_with_prio(self):
         self.assertEqual(q2.close(), [])
 
 
-class FifoDiskPriorityQueueTest(PQueueTestMixin, FifoTestMixin, DiskTestMixin, QueuelibTestCase):
+class FifoDiskPriorityQueueTest(
+    PQueueTestMixin, FifoTestMixin, DiskTestMixin, QueuelibTestCase
+):
     def qfactory(self, prio):
         path = os.path.join(self.qdir, str(prio))
         return track_closed(FifoDiskQueue)(path)
 
 
-class LifoDiskPriorityQueueTest(PQueueTestMixin, LifoTestMixin, DiskTestMixin, QueuelibTestCase):
+class LifoDiskPriorityQueueTest(
+    PQueueTestMixin, LifoTestMixin, DiskTestMixin, QueuelibTestCase
+):
     def qfactory(self, prio):
         path = os.path.join(self.qdir, str(prio))
         return track_closed(LifoDiskQueue)(path)
 
 
-class FifoSQLitePriorityQueueTest(PQueueTestMixin, FifoTestMixin, DiskTestMixin, QueuelibTestCase):
+class FifoSQLitePriorityQueueTest(
+    PQueueTestMixin, FifoTestMixin, DiskTestMixin, QueuelibTestCase
+):
     def qfactory(self, prio):
         path = os.path.join(self.qdir, str(prio))
         return track_closed(FifoSQLiteQueue)(path)
 
 
-class LifoSQLitePriorityQueueTest(PQueueTestMixin, LifoTestMixin, DiskTestMixin, QueuelibTestCase):
+class LifoSQLitePriorityQueueTest(
+    PQueueTestMixin, LifoTestMixin, DiskTestMixin, QueuelibTestCase
+):
     def qfactory(self, prio):
         path = os.path.join(self.qdir, str(prio))
         return track_closed(LifoSQLiteQueue)(path)

queuelib/tests/test_queue.py

@@ -1,18 +1,18 @@
-import os
 import glob
+import os
 from abc import abstractmethod
+from typing import Any, List, Optional
 from unittest import mock
-from typing import Any, Optional, List
 
 import pytest
 
 from queuelib.queue import (
     BaseQueue,
-    FifoMemoryQueue,
-    LifoMemoryQueue,
     FifoDiskQueue,
-    LifoDiskQueue,
+    FifoMemoryQueue,
     FifoSQLiteQueue,
+    LifoDiskQueue,
+    LifoMemoryQueue,
     LifoSQLiteQueue,
 )
 from queuelib.tests import QueuelibTestCase
@@ -215,7 +215,9 @@ def test_peek_lifo(self):
 class PersistentTestMixin:
     chunksize = 100000
 
-    @pytest.mark.xfail(reason="Reenable once Scrapy.squeues stops extending from this testsuite")
+    @pytest.mark.xfail(
+        reason="Reenable once Scrapy.squeues stops extending from this testsuite"
+    )
     def test_non_bytes_raises_typeerror(self):
         q = self.queue()
         self.assertRaises(TypeError, q.push, 0)
@@ -280,7 +282,9 @@ def queue(self):
         return LifoMemoryQueue()
 
 
-class FifoDiskQueueTest(FifoTestMixin, PersistentTestMixin, QueueTestMixin, QueuelibTestCase):
+class FifoDiskQueueTest(
+    FifoTestMixin, PersistentTestMixin, QueueTestMixin, QueuelibTestCase
+):
     def queue(self):
         return FifoDiskQueue(self.qpath, chunksize=self.chunksize)
 
@@ -325,7 +329,9 @@ class ChunkSize4FifoDiskQueueTest(FifoDiskQueueTest):
     chunksize = 4
 
 
-class LifoDiskQueueTest(LifoTestMixin, PersistentTestMixin, QueueTestMixin, QueuelibTestCase):
+class LifoDiskQueueTest(
+    LifoTestMixin, PersistentTestMixin, QueueTestMixin, QueuelibTestCase
+):
     def queue(self):
         return LifoDiskQueue(self.qpath)
 
@@ -342,11 +348,15 @@ def test_file_size_shrinks(self):
         assert os.path.getsize(self.qpath), size
 
 
-class FifoSQLiteQueueTest(FifoTestMixin, PersistentTestMixin, QueueTestMixin, QueuelibTestCase):
+class FifoSQLiteQueueTest(
+    FifoTestMixin, PersistentTestMixin, QueueTestMixin, QueuelibTestCase
+):
     def queue(self):
         return FifoSQLiteQueue(self.qpath)
 
 
-class LifoSQLiteQueueTest(LifoTestMixin, PersistentTestMixin, QueueTestMixin, QueuelibTestCase):
+class LifoSQLiteQueueTest(
+    LifoTestMixin, PersistentTestMixin, QueueTestMixin, QueuelibTestCase
+):
     def queue(self):
         return LifoSQLiteQueue(self.qpath)

queuelib/tests/test_rrqueue.py

@@ -1,15 +1,14 @@
 import os
 
-from queuelib.rrqueue import RoundRobinQueue
 from queuelib.queue import (
-    FifoMemoryQueue,
-    LifoMemoryQueue,
     FifoDiskQueue,
-    LifoDiskQueue,
+    FifoMemoryQueue,
     FifoSQLiteQueue,
+    LifoDiskQueue,
+    LifoMemoryQueue,
     LifoSQLiteQueue,
 )
-
+from queuelib.rrqueue import RoundRobinQueue
 from queuelib.tests import QueuelibTestCase, track_closed
 
 
@@ -127,25 +126,33 @@ def test_nonserializable_object_many_pop(self):
         self.assertEqual(self.q.close(), [])
 
 
-class FifoDiskRRQueueTest(RRQueueTestMixin, FifoTestMixin, DiskTestMixin, QueuelibTestCase):
+class FifoDiskRRQueueTest(
+    RRQueueTestMixin, FifoTestMixin, DiskTestMixin, QueuelibTestCase
+):
     def qfactory(self, key):
         path = os.path.join(self.qdir, str(key))
         return track_closed(FifoDiskQueue)(path)
 
 
-class LifoDiskRRQueueTest(RRQueueTestMixin, LifoTestMixin, DiskTestMixin, QueuelibTestCase):
+class LifoDiskRRQueueTest(
+    RRQueueTestMixin, LifoTestMixin, DiskTestMixin, QueuelibTestCase
+):
     def qfactory(self, key):
         path = os.path.join(self.qdir, str(key))
         return track_closed(LifoDiskQueue)(path)
 
 
-class FifoSQLiteRRQueueTest(RRQueueTestMixin, FifoTestMixin, DiskTestMixin, QueuelibTestCase):
+class FifoSQLiteRRQueueTest(
+    RRQueueTestMixin, FifoTestMixin, DiskTestMixin, QueuelibTestCase
+):
     def qfactory(self, key):
         path = os.path.join(self.qdir, str(key))
         return track_closed(FifoSQLiteQueue)(path)
 
 
-class LifoSQLiteRRQueueTest(RRQueueTestMixin, LifoTestMixin, DiskTestMixin, QueuelibTestCase):
+class LifoSQLiteRRQueueTest(
+    RRQueueTestMixin, LifoTestMixin, DiskTestMixin, QueuelibTestCase
+):
     def qfactory(self, key):
         path = os.path.join(self.qdir, str(key))
         return track_closed(LifoSQLiteQueue)(path)

setup.py

@@ -1,4 +1,4 @@
-from setuptools import setup, find_packages
+from setuptools import find_packages, setup
 
 setup(
     name="queuelib",

tox.ini

@@ -4,7 +4,7 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py, pypy3, black, flake8, pylint, security, typing, twinecheck
+envlist = py, pylint, typing, twinecheck, pre-commit
 
 [testenv]
 deps =
@@ -14,20 +14,6 @@ deps =
 commands =
     py.test --cov=queuelib --cov-report=xml --cov-report=term --cov-report=html {posargs:queuelib}
 
-[testenv:black]
-basepython = python3
-deps =
-    black==23.11.0
-commands =
-    black --check {posargs:queuelib setup.py}
-
-[testenv:flake8]
-basepython = python3
-deps =
-    flake8==6.1.0
-commands =
-    flake8 {posargs:queuelib setup.py}
-
 [testenv:pylint]
 basepython = python3
 deps =
@@ -36,13 +22,6 @@ deps =
 commands =
     pylint {posargs:queuelib setup.py}
 
-[testenv:security]
-basepython = python3
-deps =
-    bandit==1.7.5
-commands =
-    bandit -r -c .bandit.yml {posargs:queuelib setup.py}
-
 [testenv:typing]
 basepython = python3
 deps =
@@ -59,3 +38,8 @@ deps =
 commands =
     python -m build --sdist
     twine check dist/*
+
+[testenv:pre-commit]
+deps = pre-commit
+commands = pre-commit run --all-files --show-diff-on-failure
+skip_install = true