diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..e8900e8 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,21 @@ +## Generic requirements in order to contribute: + +* One Pull Request per fix/feature/change/... +* Keep the amount of commits per PR as small as possible: if for any reason, you need to fix your commit after the pull request, please squash the changes in one single commit (or tell us why not) +* Always make sure it is mergeable in the default branch +* Any major changes adding a functionality should be disabled by default in the configuration. + + +#### What does it do? + +If it fixes an existing issue, please use GitHub syntax: `#` + +#### Questions + +- [ ] Does it require a DB change? +- [ ] Are you using it in production? + +#### Release Type: +- [ ] Major +- [ ] Minor +- [X] Patch diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000..1634886 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,19 @@ +name: pre-commit +on: + pull_request: + push: + branches: [master] +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: | + pip install reorder-python-imports + pip install pyupgrade + pip install black + pip install pip-audit + - uses: pre-commit/action@v2.0.3 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..18a674a --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,41 @@ +ci: + autoupdate_schedule: monthly +repos: + - repo: https://github.com/asottile/pyupgrade + rev: v3.3.1 + hooks: + - id: pyupgrade + args: ["--py37-plus"] + - repo: https://github.com/asottile/reorder-python-imports + rev: v3.12.0 + hooks: + - id: reorder-python-imports + - repo: https://github.com/psf/black + rev: 23.9.1 + hooks: + - id: black + - repo: https://github.com/asottile/blacken-docs + rev: 1.13.0 + hooks: + - id: blacken-docs + - repo: https://github.com/PyCQA/flake8 + rev: 6.1.0 + hooks: + - id: flake8 + additional_dependencies: + - flake8-bugbear + - flake8-implicit-str-concat + args: ["--max-line-length=125"] + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: fix-byte-order-marker + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + args: ['--maxkb=800'] + - repo: https://github.com/pypa/pip-audit + rev: v2.7.3 + hooks: + - id: pip-audit diff --git a/docs/conf.py b/docs/conf.py index 6566932..0f4d46a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -2,31 +2,29 @@ # # For the full list of built-in configuration values, see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html - # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information -project = 'RSTORE' +project = "RSTORE" copyright = "Computer Incident Response Center Luxembourg (CIRCL) " author = "Cédric Bonhomme " -release = '0.1.0' +release = "0.1.0" # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration extensions = [] -templates_path = ['_templates'] -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] - +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output -html_theme = 'sphinx_book_theme' -html_static_path = ['_static'] -html_title = 'RSTORE' +html_theme = "sphinx_book_theme" +html_static_path = ["_static"] +html_title = "RSTORE" html_theme_options = { "path_to_docs": "docs", "repository_url": "https://github.com/scandale-project/RSTORE", diff --git a/docs/installation.rst b/docs/installation.rst index ca48ff1..f65ee0e 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -25,7 +25,7 @@ In order to compile kvrocks, you will need a few packages: .. code-block:: bash - git clone --recursive https://github.com/apache/incubator-kvrocks.git kvrocks + git clone --recursive https://github.com/apache/kvrocks.git cd kvrocks git checkout v2.9.0 ./x.py build diff --git a/instance/sample.py b/instance/sample.py index 40c786f..5e2afb0 100644 --- a/instance/sample.py +++ b/instance/sample.py @@ -3,8 +3,4 @@ API_URL = "http://127.0.0.1:8000" AUTHENTICATION_REQUIRED = True -USERS = { - "admin": { - "password": "Password1234!" - } -} +USERS = {"admin": {"password": "Password1234!"}} diff --git a/poetry.lock b/poetry.lock index d8cd644..534e9a5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -138,6 +138,17 @@ files = [ {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"}, ] +[[package]] +name = "cfgv" +version = "3.4.0" +description = "Validate configuration and produce human readable error messages." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9"}, + {file = "cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560"}, +] + [[package]] name = "charset-normalizer" version = "3.3.2" @@ -273,6 +284,17 @@ files = [ {file = "deepmerge-1.1.1.tar.gz", hash = "sha256:53a489dc9449636e480a784359ae2aab3191748c920649551c8e378622f0eca4"}, ] +[[package]] +name = "distlib" +version = "0.3.8" +description = "Distribution utilities" +optional = false +python-versions = "*" +files = [ + {file = "distlib-0.3.8-py2.py3-none-any.whl", hash = "sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784"}, + {file = "distlib-0.3.8.tar.gz", hash = "sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64"}, +] + [[package]] name = "dnspython" version = "2.6.1" @@ -375,6 +397,22 @@ typer = ">=0.12.3" [package.extras] standard = ["fastapi", "uvicorn[standard] (>=0.15.0)"] +[[package]] +name = "filelock" +version = "3.15.4" +description = "A platform independent file lock." +optional = false +python-versions = ">=3.8" +files = [ + {file = "filelock-3.15.4-py3-none-any.whl", hash = "sha256:6ca1fffae96225dab4c6eaf1c4f4f28cd2568d3ec2a44e15a08520504de468e7"}, + {file = "filelock-3.15.4.tar.gz", hash = "sha256:2207938cbc1844345cb01a5a95524dae30f0ce089eba5b00378295a17e3e90cb"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8.0.1)", "pytest (>=7.4.3)", "pytest-asyncio (>=0.21)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)", "virtualenv (>=20.26.2)"] +typing = ["typing-extensions (>=4.8)"] + [[package]] name = "flake8" version = "7.1.0" @@ -495,6 +533,20 @@ cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] +[[package]] +name = "identify" +version = "2.6.0" +description = "File identification library for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "identify-2.6.0-py2.py3-none-any.whl", hash = "sha256:e79ae4406387a9d300332b5fd366d8994f1525e8414984e1a59e058b2eda2dd0"}, + {file = "identify-2.6.0.tar.gz", hash = "sha256:cb171c685bdc31bcc4c1734698736a7d5b6c8bf2e0c15117f4d469c8640ae5cf"}, +] + +[package.extras] +license = ["ukkonen"] + [[package]] name = "idna" version = "3.7" @@ -764,6 +816,17 @@ files = [ {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] +[[package]] +name = "nodeenv" +version = "1.9.1" +description = "Node.js virtual environment builder" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9"}, + {file = "nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f"}, +] + [[package]] name = "packaging" version = "24.1" @@ -786,6 +849,40 @@ files = [ {file = "picobox-4.0.0.tar.gz", hash = "sha256:114da1b5606b2f615e8b0eb68d04198ad9de75af5adbcf5b36fe4f664ab927b6"}, ] +[[package]] +name = "platformdirs" +version = "4.2.2" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.2-py3-none-any.whl", hash = "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee"}, + {file = "platformdirs-4.2.2.tar.gz", hash = "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] +type = ["mypy (>=1.8)"] + +[[package]] +name = "pre-commit" +version = "3.8.0" +description = "A framework for managing and maintaining multi-language pre-commit hooks." +optional = false +python-versions = ">=3.9" +files = [ + {file = "pre_commit-3.8.0-py2.py3-none-any.whl", hash = "sha256:9a90a53bf82fdd8778d58085faf8d83df56e40dfe18f45b19446e26bf1b3a63f"}, + {file = "pre_commit-3.8.0.tar.gz", hash = "sha256:8bb6494d4a20423842e198980c9ecf9f96607a07ea29549e180eef9ae80fe7af"}, +] + +[package.dependencies] +cfgv = ">=2.0.0" +identify = ">=1.0.0" +nodeenv = ">=0.11.1" +pyyaml = ">=5.1" +virtualenv = ">=20.10.0" + [[package]] name = "pybtex" version = "0.24.0" @@ -1717,6 +1814,26 @@ async-timeout = {version = ">=4.0.3", markers = "python_full_version < \"3.11.3\ hiredis = ["hiredis (>=1.0.0,<3.0.0)"] ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)"] +[[package]] +name = "virtualenv" +version = "20.26.3" +description = "Virtual Python Environment builder" +optional = false +python-versions = ">=3.7" +files = [ + {file = "virtualenv-20.26.3-py3-none-any.whl", hash = "sha256:8cc4a31139e796e9a7de2cd5cf2489de1217193116a8fd42328f1bd65f434589"}, + {file = "virtualenv-20.26.3.tar.gz", hash = "sha256:4c43a2a236279d9ea36a0d76f98d84bd6ca94ac4e0f4a3b9d46d05e10fea542a"}, +] + +[package.dependencies] +distlib = ">=0.3.7,<1" +filelock = ">=3.12.2,<4" +platformdirs = ">=3.9.1,<5" + +[package.extras] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2,!=7.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] +test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8)", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10)"] + [[package]] name = "watchfiles" version = "0.22.0" @@ -1888,4 +2005,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "fefc92fc324f54dd27b91d189b3fe2f8161fa1adfb4d0a622cf8a8f5a266fd84" +content-hash = "cb54994a391c154f33623bbcf4eef701e40155ff56372391bd1e5b2cff6cf472" diff --git a/pyproject.toml b/pyproject.toml index 2b517aa..e8bb107 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ homepage = "https://github.com/scandale-project/RSTORE" repository = "https://github.com/scandale-project/RSTORE" documentation = "https://rstore.readthedocs.io" -keywords = ["observations", "evidences", "kvrocks", "api"] +keywords = ["observations", "evidences", "kvrocks", "api", "scanning", "csirt"] classifiers = [ "Environment :: Console", @@ -35,6 +35,7 @@ uvicorn = "^0.30.4" [tool.poetry.group.dev.dependencies] mypy = "^1.11.1" flake8 = "^7.1.0" +pre-commit = "^3.8.0" [tool.poetry.group.docs] optional = true diff --git a/rstore/api.py b/rstore/api.py index 006155c..056bbdb 100644 --- a/rstore/api.py +++ b/rstore/api.py @@ -23,6 +23,7 @@ http_security = Depends(security) + def verification(creds: HTTPBasicCredentials = http_security): """Verify the credentials via HTTP Basic Authentication method.""" if not config.AUTHENTICATION_REQUIRED: @@ -38,6 +39,7 @@ def verification(creds: HTTPBasicCredentials = http_security): headers={"WWW-Authenticate": "Basic"}, ) + def custom_openapi() -> Dict[str, Any]: if app.openapi_schema: return app.openapi_schema @@ -64,6 +66,7 @@ def custom_openapi() -> Dict[str, Any]: app.openapi_schema = openapi_schema return app.openapi_schema + app.openapi = custom_openapi diff --git a/rstore/schemas.py b/rstore/schemas.py index baadfec..e69de29 100644 --- a/rstore/schemas.py +++ b/rstore/schemas.py @@ -1,28 +0,0 @@ -import uuid - -from pydantic import BaseModel - -class Payload(BaseModel): - raw: str - - -class Meta(BaseModel): - uuid: str - ts: int - type: str - - -class ScanData(BaseModel): - version: str - format: str - meta: Meta - payload: Payload - - class Config: - from_attributes = True - -class ItemBase(ScanData): - scan_data: ScanData - - class Config: - from_attributes = True \ No newline at end of file