From a3de8dff070cbc895e5e44ff6860af96091f2cd1 Mon Sep 17 00:00:00 2001 From: Nicola Coretti Date: Wed, 15 Jan 2025 14:43:05 +0100 Subject: [PATCH 01/19] Relock dependencies --- poetry.lock | 242 ++++--------------------------------------------- pyproject.toml | 2 +- 2 files changed, 20 insertions(+), 224 deletions(-) diff --git a/poetry.lock b/poetry.lock index dd4f489a..afd86a6b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "alabaster" @@ -11,28 +11,6 @@ files = [ {file = "alabaster-0.7.16.tar.gz", hash = "sha256:75a8b99c28a5dad50dd7f8ccdd447a121ddb3892da9e53d1ca5cca3106d58d65"}, ] -[[package]] -name = "anyio" -version = "4.8.0" -description = "High level compatibility layer for multiple asynchronous event loop implementations" -optional = false -python-versions = ">=3.9" -files = [ - {file = "anyio-4.8.0-py3-none-any.whl", hash = "sha256:b5011f270ab5eb0abf13385f851315585cc37ef330dd88e27ec3d34d651fd47a"}, - {file = "anyio-4.8.0.tar.gz", hash = "sha256:1d9fe889df5212298c0c0723fa20479d1b94883a2df44bd3897aa91083316f7a"}, -] - -[package.dependencies] -exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} -idna = ">=2.8" -sniffio = ">=1.1" -typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} - -[package.extras] -doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx_rtd_theme"] -test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21)"] -trio = ["trio (>=0.26.1)"] - [[package]] name = "argcomplete" version = "3.5.3" @@ -61,25 +39,6 @@ files = [ [package.dependencies] typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""} -[[package]] -name = "attrs" -version = "24.3.0" -description = "Classes Without Boilerplate" -optional = false -python-versions = ">=3.8" -files = [ - {file = "attrs-24.3.0-py3-none-any.whl", hash = "sha256:ac96cd038792094f438ad1f6ff80837353805ac950cd2aa0e0625ef19850c308"}, - {file = "attrs-24.3.0.tar.gz", hash = "sha256:8f5c07333d543103541ba7be0e2ce16eeee8130cb0b3f9238ab904ce1e85baff"}, -] - -[package.extras] -benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"] -tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] - [[package]] name = "babel" version = "2.16.0" @@ -671,62 +630,42 @@ websockets = ["websocket-client (>=1.3.0)"] [[package]] name = "docutils" -version = "0.20.1" +version = "0.21.2" description = "Docutils -- Python Documentation Utilities" optional = false -python-versions = ">=3.7" +python-versions = ">=3.9" files = [ - {file = "docutils-0.20.1-py3-none-any.whl", hash = "sha256:96f387a2c5562db4476f09f13bbab2192e764cac08ebbf3a34a95d9b1e4a59d6"}, - {file = "docutils-0.20.1.tar.gz", hash = "sha256:f08a4e276c3a1583a86dce3e34aba3fe04d02bba2dd51ed16106244e8a923e3b"}, + {file = "docutils-0.21.2-py3-none-any.whl", hash = "sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2"}, + {file = "docutils-0.21.2.tar.gz", hash = "sha256:3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f"}, ] -[[package]] -name = "exasol-bucketfs" -version = "1.0.0" -description = "BucketFS utilities for the Python programming language" -optional = false -python-versions = "<4.0,>=3.9" -files = [ - {file = "exasol_bucketfs-1.0.0-py3-none-any.whl", hash = "sha256:14db00aecbc0568e2da4558349931b004020e4846ea5257ae5b1b5ddfe638357"}, - {file = "exasol_bucketfs-1.0.0.tar.gz", hash = "sha256:286a07f62f7becffee241bad97a5af9dc715a9d8a2203ff4b12a3f01e15f925d"}, -] - -[package.dependencies] -attrs = ">=23.2.0" -exasol-saas-api = ">=0.3.0" -httpx = ">=0.27.0" -joblib = ">=1.0.1" -requests = ">=2.24.0" -typeguard = ">=4.3.0" - [[package]] name = "exasol-error-reporting" -version = "0.4.0" +version = "0.5.0" description = "Exasol Python Error Reporting" optional = false -python-versions = ">=3.8,<4.0" +python-versions = "<4.0,>=3.9" files = [ - {file = "exasol_error_reporting-0.4.0-py3-none-any.whl", hash = "sha256:851aed9fd95bc3c6a566ba174a8052897442bfe503c9b8858faab26e3d6f0153"}, - {file = "exasol_error_reporting-0.4.0.tar.gz", hash = "sha256:a0bc5056aa23df97eb47d2da95d112d9b2dcd5295016fdefac2fc7aa5e7eea19"}, + {file = "exasol_error_reporting-0.5.0-py3-none-any.whl", hash = "sha256:17a5e6cd862e79796ae9acd62f7d0dd13c485ad4a4d7467bb7f8d57c1c213d5e"}, + {file = "exasol_error_reporting-0.5.0.tar.gz", hash = "sha256:5051f9a0dc9dbfc2dfa808d6b88c7595fa2688aa11a2b51c0895db00fe3e41bb"}, ] [[package]] name = "exasol-integration-test-docker-environment" -version = "3.2.0" +version = "3.3.0" description = "Integration Test Docker Environment for Exasol" optional = false -python-versions = "<4,>=3.8" +python-versions = "<4,>=3.9" files = [ - {file = "exasol_integration_test_docker_environment-3.2.0-py3-none-any.whl", hash = "sha256:b0fc41a70b73ec5ad43171e2c8fcf76a54a1eb31befa09e6a3214af55f1c93fc"}, - {file = "exasol_integration_test_docker_environment-3.2.0.tar.gz", hash = "sha256:47b11dde66be0149d54cfa6d7eebc4665bd996fccb22f64ce6489a07cec7b331"}, + {file = "exasol_integration_test_docker_environment-3.3.0-py3-none-any.whl", hash = "sha256:36ad2d1f0074277a3375fd0a417c69bf58f59f0a8efbcabf6aea6285498e4a80"}, + {file = "exasol_integration_test_docker_environment-3.3.0.tar.gz", hash = "sha256:5d1ea0b27b747cf348408cc9f12e9659b07c38ab21b547a207d1222d73d5e0e6"}, ] [package.dependencies] click = ">=7.0" docker = {version = ">=4.0.0,<7.0.0 || >7.0.0", markers = "sys_platform != \"win32\""} -docutils = "<=0.20.1" -exasol-bucketfs = ">=0.6.0,<2.0.0" -exasol-error-reporting = ">=0.4.0,<0.5.0" +docutils = ">=0.21.2" +exasol-error-reporting = ">=0.4.0" fabric = ">=3.0.1,<4.0.0" gitpython = ">=2.1.0" humanfriendly = ">=4.18" @@ -742,35 +681,15 @@ requests = ">=2.21.0" simplejson = ">=3.16.0" "stopwatch.py" = ">=1.0.0" -[[package]] -name = "exasol-saas-api" -version = "0.7.0" -description = "API enabling Python applications connecting to Exasol database SaaS instances and using their SaaS services" -optional = false -python-versions = "<4.0,>=3.8.0" -files = [ - {file = "exasol_saas_api-0.7.0-py3-none-any.whl", hash = "sha256:7ffe1a05aa419099bcafa3984af5f750dc2234c8b18170ccda5336b95bac7c09"}, - {file = "exasol_saas_api-0.7.0.tar.gz", hash = "sha256:8d69780cdc876dc206797fea5b2f964a06248f0a087b611ae06ac3646f84a846"}, -] - -[package.dependencies] -attrs = ">=21.3.0" -httpx = ">=0.20.0,<0.28.0" -ifaddr = ">=0.2.0,<0.3.0" -python-dateutil = ">=2.8.0,<3.0.0" -requests = ">=2.31.0,<3.0.0" -tenacity = ">=8.2.3,<9.0.0" -types-requests = ">=2.31.0.6,<3.0.0.0" - [[package]] name = "exasol-toolbox" -version = "0.19.0" +version = "0.20.0" description = "Your one-stop solution for managing all standard tasks and core workflows of your Python project." optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "exasol_toolbox-0.19.0-py3-none-any.whl", hash = "sha256:bc29492a552d64d921b5a70985f9ca1821a3eda15ceb382d5750dd574ef94ec3"}, - {file = "exasol_toolbox-0.19.0.tar.gz", hash = "sha256:32a9bfa867c1f24e113a06c026f4f660fea63fb563805983b084caae2a7e44bd"}, + {file = "exasol_toolbox-0.20.0-py3-none-any.whl", hash = "sha256:c90e805dbaf0d2ec96a1fab44a00adedb55043179201f2218e10fc8ec811ce11"}, + {file = "exasol_toolbox-0.20.0.tar.gz", hash = "sha256:f7e0cf6e346b7b3af59a007a274dc1ef48195f03eba87ad6ebf8eb95740fb1ae"}, ] [package.dependencies] @@ -1086,63 +1005,6 @@ files = [ [package.dependencies] typing-extensions = ">=3.10.0.0" -[[package]] -name = "h11" -version = "0.14.0" -description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" -optional = false -python-versions = ">=3.7" -files = [ - {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, - {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, -] - -[[package]] -name = "httpcore" -version = "1.0.7" -description = "A minimal low-level HTTP client." -optional = false -python-versions = ">=3.8" -files = [ - {file = "httpcore-1.0.7-py3-none-any.whl", hash = "sha256:a3fff8f43dc260d5bd363d9f9cf1830fa3a458b332856f34282de498ed420edd"}, - {file = "httpcore-1.0.7.tar.gz", hash = "sha256:8551cb62a169ec7162ac7be8d4817d561f60e08eaa485234898414bb5a8a0b4c"}, -] - -[package.dependencies] -certifi = "*" -h11 = ">=0.13,<0.15" - -[package.extras] -asyncio = ["anyio (>=4.0,<5.0)"] -http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] -trio = ["trio (>=0.22.0,<1.0)"] - -[[package]] -name = "httpx" -version = "0.27.2" -description = "The next generation HTTP client." -optional = false -python-versions = ">=3.8" -files = [ - {file = "httpx-0.27.2-py3-none-any.whl", hash = "sha256:7bb2708e112d8fdd7829cd4243970f0c223274051cb35ee80c03301ee29a3df0"}, - {file = "httpx-0.27.2.tar.gz", hash = "sha256:f7c2be1d2f3c3c3160d441802406b206c2b76f5947b11115e6df10c6c65e66c2"}, -] - -[package.dependencies] -anyio = "*" -certifi = "*" -httpcore = "==1.*" -idna = "*" -sniffio = "*" - -[package.extras] -brotli = ["brotli", "brotlicffi"] -cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] -http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] -zstd = ["zstandard (>=0.18.0)"] - [[package]] name = "humanfriendly" version = "10.0" @@ -1185,17 +1047,6 @@ files = [ [package.extras] all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] -[[package]] -name = "ifaddr" -version = "0.2.0" -description = "Cross-platform network interface and IP address enumeration library" -optional = false -python-versions = "*" -files = [ - {file = "ifaddr-0.2.0-py3-none-any.whl", hash = "sha256:085e0305cfe6f16ab12d72e2024030f5d52674afad6911bb1eee207177b8a748"}, - {file = "ifaddr-0.2.0.tar.gz", hash = "sha256:cc0cbfcaabf765d44595825fb96a99bb12c79716b73b44330ea38ee2b0c4aed4"}, -] - [[package]] name = "imagesize" version = "1.4.1" @@ -1322,17 +1173,6 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] -[[package]] -name = "joblib" -version = "1.4.2" -description = "Lightweight pipelining with Python functions" -optional = false -python-versions = ">=3.8" -files = [ - {file = "joblib-1.4.2-py3-none-any.whl", hash = "sha256:06d478d5674cbc267e7496a410ee875abd68e4340feff4490bcb7afb88060ae6"}, - {file = "joblib-1.4.2.tar.gz", hash = "sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e"}, -] - [[package]] name = "jsonpickle" version = "4.0.1" @@ -2523,17 +2363,6 @@ files = [ {file = "smmap-5.0.2.tar.gz", hash = "sha256:26ea65a03958fa0c8a1c7e8c7a58fdc77221b8910f6be2131affade476898ad5"}, ] -[[package]] -name = "sniffio" -version = "1.3.1" -description = "Sniff out which async library your code is running under" -optional = false -python-versions = ">=3.7" -files = [ - {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, - {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, -] - [[package]] name = "snowballstemmer" version = "2.2.0" @@ -2978,25 +2807,6 @@ files = [ arrow = ["pyarrow (>=1.0,<8.1.0)"] numpy = ["numpy (>=1.19.0)"] -[[package]] -name = "typeguard" -version = "4.4.1" -description = "Run-time type checker for Python" -optional = false -python-versions = ">=3.9" -files = [ - {file = "typeguard-4.4.1-py3-none-any.whl", hash = "sha256:9324ec07a27ec67fc54a9c063020ca4c0ae6abad5e9f0f9804ca59aee68c6e21"}, - {file = "typeguard-4.4.1.tar.gz", hash = "sha256:0d22a89d00b453b47c49875f42b6601b961757541a2e1e0ef517b6e24213c21b"}, -] - -[package.dependencies] -importlib-metadata = {version = ">=3.6", markers = "python_version < \"3.10\""} -typing-extensions = ">=4.10.0" - -[package.extras] -doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme (>=1.3.0)"] -test = ["coverage[toml] (>=7)", "mypy (>=1.2.0)", "pytest (>=7)"] - [[package]] name = "typer" version = "0.15.1" @@ -3014,20 +2824,6 @@ rich = ">=10.11.0" shellingham = ">=1.3.0" typing-extensions = ">=3.7.4.3" -[[package]] -name = "types-requests" -version = "2.32.0.20241016" -description = "Typing stubs for requests" -optional = false -python-versions = ">=3.8" -files = [ - {file = "types-requests-2.32.0.20241016.tar.gz", hash = "sha256:0d9cad2f27515d0e3e3da7134a1b6f28fb97129d86b867f24d9c726452634d95"}, - {file = "types_requests-2.32.0.20241016-py3-none-any.whl", hash = "sha256:4195d62d6d3e043a4eaaf08ff8a62184584d2e8684e9d2aa178c7915a7da3747"}, -] - -[package.dependencies] -urllib3 = ">=2" - [[package]] name = "typing-extensions" version = "4.12.2" @@ -3256,4 +3052,4 @@ turbodbc = ["turbodbc"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0" -content-hash = "a54f4e080b16d1d13cba50b40181ac9c001af24028f95814225ccd78de06fb02" +content-hash = "d59daf4cb19f5ebb6b8ee1caea2073cde00c0a11a350ecfb8ebd620c379e0492" diff --git a/pyproject.toml b/pyproject.toml index a85ec291..98e27f6b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,7 +94,7 @@ exasol-integration-test-docker-environment = "^3.1.0" pytest-history = ">=0.2.0" pyodbc = ">=4.0.34,<6" pytest-exasol-itde = ">=0.2.0,<1" -exasol-toolbox = "^0.19.0" +exasol-toolbox = "^0.20.0" [tool.poetry.extras] pyodbc = ["pyodbc"] From cb5444e8406bcd0da0897b58c052ffb8bd27862d Mon Sep 17 00:00:00 2001 From: Nicola Coretti Date: Wed, 15 Jan 2025 15:04:45 +0100 Subject: [PATCH 02/19] Update workflows --- .github/workflows/build-and-publish.yml | 38 ++++ .github/workflows/cd.yml | 24 +++ .github/workflows/check-release-tag.yml | 21 +++ .github/workflows/checks.yml | 165 ++++++++++++++++++ .github/workflows/ci-cd.yml | 118 ------------- .github/workflows/ci.yaml | 114 ------------ .github/workflows/ci.yml | 24 +++ .../workflows/{gh-pages.yaml => gh-pages.yml} | 25 ++- .github/workflows/matrix-all.yml | 30 ++++ .github/workflows/matrix-exasol.yml | 30 ++++ .github/workflows/matrix-python.yml | 30 ++++ .github/workflows/merge-gate.yml | 30 ++++ .github/workflows/pr-merge.yml | 25 +++ .github/workflows/report.yml | 55 ++++++ .github/workflows/slow-checks.yml | 46 +++++ noxconfig.py | 6 +- noxfile.py | 24 +++ 17 files changed, 558 insertions(+), 247 deletions(-) create mode 100644 .github/workflows/build-and-publish.yml create mode 100644 .github/workflows/cd.yml create mode 100644 .github/workflows/check-release-tag.yml create mode 100644 .github/workflows/checks.yml delete mode 100644 .github/workflows/ci-cd.yml delete mode 100644 .github/workflows/ci.yaml create mode 100644 .github/workflows/ci.yml rename .github/workflows/{gh-pages.yaml => gh-pages.yml} (53%) create mode 100644 .github/workflows/matrix-all.yml create mode 100644 .github/workflows/matrix-exasol.yml create mode 100644 .github/workflows/matrix-python.yml create mode 100644 .github/workflows/merge-gate.yml create mode 100644 .github/workflows/pr-merge.yml create mode 100644 .github/workflows/report.yml create mode 100644 .github/workflows/slow-checks.yml diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml new file mode 100644 index 00000000..dffc2999 --- /dev/null +++ b/.github/workflows/build-and-publish.yml @@ -0,0 +1,38 @@ +name: Build & Publish + +on: + workflow_call: + secrets: + PYPI_TOKEN: + required: true + +jobs: + + cd-job: + name: Continuous Delivery + runs-on: ubuntu-latest + steps: + + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@0.20.0 + + - name: Build Artifacts + run: poetry build + + - name: PyPi Release + env: + POETRY_HTTP_BASIC_PYPI_USERNAME: "__token__" + POETRY_HTTP_BASIC_PYPI_PASSWORD: "${{ secrets.PYPI_TOKEN }}" + run: poetry publish + + - name: GitHub Release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: > + gh release create ${GITHUB_REF_NAME} + --title ${GITHUB_REF_NAME} + --notes-file ./doc/changes/changes_${GITHUB_REF_NAME}.md + dist/* diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 00000000..11673c95 --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,24 @@ +name: CD + +on: + push: + tags: + - '**' + +jobs: + + check-tag-version-job: + name: Check Release Tag + uses: ./.github/workflows/check-release-tag.yml + + cd-job: + name: Continuous Delivery + uses: ./.github/workflows/build-and-publish.yml + secrets: + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} + + publish-docs: + needs: [ cd-job ] + name: Publish Documentation + uses: ./.github/workflows/gh-pages.yml + diff --git a/.github/workflows/check-release-tag.yml b/.github/workflows/check-release-tag.yml new file mode 100644 index 00000000..31989d34 --- /dev/null +++ b/.github/workflows/check-release-tag.yml @@ -0,0 +1,21 @@ +name: Check Release Tag + +on: workflow_call + +jobs: + + check-tag-version-job: + + name: Check Tag Version + runs-on: ubuntu-latest + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@0.20.0 + + - name: Check Tag Version + # make sure the pushed/created tag matched the project version + run: "[[ `poetry version --short` == ${{ github.ref_name }} ]]" diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 00000000..3aebac24 --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,165 @@ +name: Checks + +on: + workflow_call: + secrets: + ALTERNATIVE_GITHUB_TOKEN: + required: false + +jobs: + + Version-Check: + name: Version + runs-on: ubuntu-latest + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@0.20.0 + + - name: Check Version(s) + run: | + poetry run version-check `poetry run python -c "from noxconfig import PROJECT_CONFIG; print(PROJECT_CONFIG.version_file)"` + + Documentation: + name: Docs + needs: [ Version-Check ] + runs-on: ubuntu-latest + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@0.20.0 + + - name: Build Documentation + run: | + poetry run python -m nox -s docs:build + + build-matrix: + name: Generate Build Matrix + uses: ./.github/workflows/matrix-python.yml + + Lint: + name: Linting (Python-${{ matrix.python-version }}) + needs: [ Version-Check, build-matrix ] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@0.20.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Run lint + run: poetry run nox -s lint:code + + - name: Upload Artifacts + uses: actions/upload-artifact@v4.4.0 + with: + name: lint-python${{ matrix.python-version }} + path: .lint.txt + include-hidden-files: true + + Type-Check: + name: Type Checking (Python-${{ matrix.python-version }}) + needs: [ Version-Check, build-matrix ] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@0.20.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Run type-check + run: poetry run nox -s lint:typing + + Security: + name: Security Checks (Python-${{ matrix.python-version }}) + needs: [ Version-Check, build-matrix ] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@0.20.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Run security linter + run: poetry run nox -s lint:security + + - name: Upload Artifacts + uses: actions/upload-artifact@v4.4.0 + with: + name: security-python${{ matrix.python-version }} + path: .security.json + include-hidden-files: true + + Format: + name: Format Check + runs-on: ubuntu-latest + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@0.20.0 + with: + python-version: "3.9" + + - name: Run format check + run: poetry run nox -s project:format + + Tests: + name: Unit-Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}}) + needs: [ Documentation, Lint, Type-Check, Security, Format, build-matrix ] + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ secrets.ALTERNATIVE_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@0.20.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Run Tests and Collect Coverage + run: poetry run nox -s test:unit -- -- --coverage + + - name: Upload Artifacts + uses: actions/upload-artifact@v4.4.0 + with: + name: coverage-python${{ matrix.python-version }}-fast + path: .coverage + include-hidden-files: true diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml deleted file mode 100644 index 1e22b282..00000000 --- a/.github/workflows/ci-cd.yml +++ /dev/null @@ -1,118 +0,0 @@ -name: CI-CD - -on: - push: - branches: - - 'master' - tags: - - '**' - -jobs: - - docs: - runs-on: ubuntu-latest - name: Build Documentation - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Python & Poetry Environment - uses: ./.github/actions/python-environment - with: - python-version: 3.9 - - - name: Build Documentation - run: | - poetry run python -m nox -s docs:build - - checks: - runs-on: ubuntu-latest - name: Project Checks (Python-${{ matrix.python }}) - strategy: - fail-fast: false - matrix: - python: - - "3.9" - - "3.10" - steps: - - name: Fetch sqlalchemy_exasol code from repository - uses: actions/checkout@v4 - - - name: Install via apt - run: sudo apt-get install unixodbc unixodbc-dev libboost-date-time-dev libboost-locale-dev libboost-system-dev - - - name: Setup Python & Poetry Environment - uses: ./.github/actions/python-environment - with: - python-version: ${{ matrix.python }} - - - name: Project Checks (Python-${{ matrix.python }}) - run: poetry run nox -s project:check - - tests: - runs-on: ubuntu-latest - needs: [docs, checks] - strategy: - fail-fast: false - matrix: - python: - - "3.9" - - "3.10" - connector: - - pyodbc - - turbodbc - - websocket - exasol_version: - - 7.1.17 - - name: Integration Tests (Python-${{ matrix.python }}, Connector-${{ matrix.connector }}, Exasol-${{ matrix.exasol_version }}) - - steps: - - - name: Fetch sqlalchemy_exasol code from repository - uses: actions/checkout@v4 - - - name: Install via apt - run: sudo apt-get install unixodbc unixodbc-dev libboost-date-time-dev libboost-locale-dev libboost-system-dev - - - name: Setup Python & Poetry Environment - uses: ./.github/actions/python-environment - with: - python-version: ${{ matrix.python }} - - - name: Install python project dependencies including turbodbc - run: poetry install --all-extras - if: ${{ matrix.connector == 'turbodbc' }} - - - name: Run Test for Python ${{ matrix.python }} using ${{ matrix.connector }} - run: | - poetry run nox -s test:unit - poetry run nox -s test:integration -- -- --connector ${{ matrix.connector }} --db-version ${{ matrix.exasol_version }} - - upload_to_pypi: - runs-on: ubuntu-latest - needs: tests - name: Build & Upload Package [PYPI] - if: startsWith(github.event.ref, 'refs/tags') - strategy: - matrix: - python: [ 3.9 ] - - steps: - - name: Fetch sqlalchemy_exasol code from repository - uses: actions/checkout@v4 - - - name: Fetch all tags - run: git fetch origin +refs/tags/*:refs/tags/* - - - name: Setup Python & Poetry Environment - uses: ./.github/actions/python-environment - with: - python-version: ${{ matrix.python }} - - - name: Build and push package to PYPI - env: - POETRY_HTTP_BASIC_PYPI_USERNAME: "__token__" - POETRY_HTTP_BASIC_PYPI_PASSWORD: "${{ secrets.pypi_token }}" - run: poetry run nox -s release diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml deleted file mode 100644 index e2d7fb1f..00000000 --- a/.github/workflows/ci.yaml +++ /dev/null @@ -1,114 +0,0 @@ -name: CI - -on: - pull_request: - schedule: - # “At 00:00 on every 7th day-of-month from 1 through 31.” (https://crontab.guru) - - cron: "0 0 1/7 * *" - -jobs: - - docs: - runs-on: ubuntu-latest - name: Build Documentation - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Python & Poetry Environment - uses: ./.github/actions/python-environment - with: - python-version: 3.9 - - - name: Build Documentation - run: | - poetry run python -m nox -s docs:build - - checks: - runs-on: ubuntu-latest - name: Project Checks (Python-${{ matrix.python }}) - strategy: - fail-fast: false - matrix: - python: - - "3.9" - - "3.10" - steps: - - name: Fetch sqlalchemy_exasol code from repository - uses: actions/checkout@v4 - - - name: Install via apt - run: sudo apt-get install unixodbc unixodbc-dev libboost-date-time-dev libboost-locale-dev libboost-system-dev - - - name: Setup Python & Poetry Environment - uses: ./.github/actions/python-environment - with: - python-version: ${{ matrix.python }} - - - name: Project Checks (Python-${{ matrix.python }}) - run: poetry run nox -s project:check - - tests: - runs-on: ubuntu-latest - needs: [docs, checks] - strategy: - fail-fast: false - matrix: - python: - - "3.9" - - "3.10" - connector: - - pyodbc - - turbodbc - - websocket - exasol_version: - - 7.1.17 - - name: Integration Tests (Python-${{ matrix.python }}, Connector-${{ matrix.connector }}, Exasol-${{ matrix.exasol_version }}) - - steps: - - - name: Fetch sqlalchemy_exasol code from repository - uses: actions/checkout@v4 - - - name: Install via apt - run: sudo apt-get install unixodbc unixodbc-dev libboost-date-time-dev libboost-locale-dev libboost-system-dev - - - name: Setup Python & Poetry Environment - uses: ./.github/actions/python-environment - with: - python-version: ${{ matrix.python }} - - - name: Install python project dependencies including turbodbc - run: poetry install --all-extras - if: ${{ matrix.connector == 'turbodbc' }} - - - name: Run Test for Python ${{ matrix.python }} using ${{ matrix.connector }} - run: | - poetry run nox -s test:unit - poetry run nox -s test:integration -- -- --connector ${{ matrix.connector }} --db-version ${{ matrix.exasol_version }} - - build_package: - runs-on: ubuntu-latest - name: Build Package - needs: tests - strategy: - matrix: - python: [ 3.9 ] - - steps: - - - name: Fetch sqlalchemy_exasol code from repository - uses: actions/checkout@v4 - - - name: Fetch all tags - run: git fetch origin +refs/tags/*:refs/tags/* - - - name: Setup Python & Poetry Environment - uses: ./.github/actions/python-environment - with: - python-version: ${{ matrix.python }} - - - name: Build sdist and wheel packages - run: poetry build diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..e02cb553 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,24 @@ +name: CI + +on: + push: + branches-ignore: + - "github-pages/*" + - "gh-pages/*" + - "main" + - "master" + pull_request: + types: [opened, reopened] + schedule: + # “At 00:00 on every 7th day-of-month from 1 through 31.” (https://crontab.guru) + - cron: "0 0 1/7 * *" + +jobs: + + CI: + uses: ./.github/workflows/merge-gate.yml + secrets: inherit + + Metrics: + needs: [ CI ] + uses: ./.github/workflows/report.yml diff --git a/.github/workflows/gh-pages.yaml b/.github/workflows/gh-pages.yml similarity index 53% rename from .github/workflows/gh-pages.yaml rename to .github/workflows/gh-pages.yml index 9056e536..698fd3c7 100644 --- a/.github/workflows/gh-pages.yaml +++ b/.github/workflows/gh-pages.yml @@ -1,34 +1,31 @@ -name: GH-Pages +name: Publish Documentation + on: - push: - branches: - - master - tags: - - "*" - workflow_dispatch: + workflow_call: + workflow_dispatch: jobs: - build-and-deploy: + + documentation-job: runs-on: ubuntu-latest + steps: - - name: Checkout + - name: SCM Checkout uses: actions/checkout@v4 with: fetch-depth: 0 - name: Setup Python & Poetry Environment - uses: ./.github/actions/python-environment + uses: exasol/python-toolbox/.github/actions/python-environment@0.20.0 - name: Build Documentation run: | - poetry run python -m nox -vvv -s docs:multiversion - touch .html-documentation/.nojekyll + poetry run nox -s docs:multiversion - name: Deploy - uses: JamesIves/github-pages-deploy-action@v4.5.0 + uses: JamesIves/github-pages-deploy-action@v4.6.0 with: branch: gh-pages folder: .html-documentation git-config-name: Github Action git-config-email: opensource@exasol.com - diff --git a/.github/workflows/matrix-all.yml b/.github/workflows/matrix-all.yml new file mode 100644 index 00000000..eeeb379c --- /dev/null +++ b/.github/workflows/matrix-all.yml @@ -0,0 +1,30 @@ +name: Build Matrix (All Versions) + +on: + workflow_call: + outputs: + matrix: + description: "Generates the all versions build matrix" + value: ${{ jobs.all_versions.outputs.matrix }} + +jobs: + all_versions: + + runs-on: ubuntu-latest + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@0.20.0 + + - name: Generate matrix + run: poetry run nox -s matrix:all + + - id: set-matrix + run: | + echo "matrix=$(poetry run nox -s matrix:all)" >> $GITHUB_OUTPUT + + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/.github/workflows/matrix-exasol.yml b/.github/workflows/matrix-exasol.yml new file mode 100644 index 00000000..7d11ef5e --- /dev/null +++ b/.github/workflows/matrix-exasol.yml @@ -0,0 +1,30 @@ +name: Build Matrix (Exasol) + +on: + workflow_call: + outputs: + matrix: + description: "Generates the exasol version build matrix" + value: ${{ jobs.exasol_versions.outputs.matrix }} + +jobs: + exasol_versions: + + runs-on: ubuntu-latest + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@0.20.0 + + - name: Generate matrix + run: poetry run nox -s matrix:exasol + + - id: set-matrix + run: | + echo "matrix=$(poetry run nox -s matrix:exasol)" >> $GITHUB_OUTPUT + + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/.github/workflows/matrix-python.yml b/.github/workflows/matrix-python.yml new file mode 100644 index 00000000..58a618a8 --- /dev/null +++ b/.github/workflows/matrix-python.yml @@ -0,0 +1,30 @@ +name: Build Matrix (Python) + +on: + workflow_call: + outputs: + matrix: + description: "Generates the python version build matrix" + value: ${{ jobs.python_versions.outputs.matrix }} + +jobs: + python_versions: + + runs-on: ubuntu-latest + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@0.20.0 + + - name: Generate matrix + run: poetry run nox -s matrix:python + + - id: set-matrix + run: | + echo "matrix=$(poetry run nox -s matrix:python)" >> $GITHUB_OUTPUT + + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/.github/workflows/merge-gate.yml b/.github/workflows/merge-gate.yml new file mode 100644 index 00000000..f9e7aa2b --- /dev/null +++ b/.github/workflows/merge-gate.yml @@ -0,0 +1,30 @@ +name: Merge-Gate + +on: + workflow_call: + secrets: + ALTERNATIVE_GITHUB_TOKEN: + required: false + +jobs: + + fast-checks: + name: Fast + uses: ./.github/workflows/checks.yml + + slow-checks: + name: Slow + uses: ./.github/workflows/slow-checks.yml + + # This job ensures inputs have been executed successfully. + approve-merge: + name: Allow Merge + runs-on: ubuntu-latest + # If you need additional jobs to be part of the merge gate, add them below + needs: [ fast-checks, slow-checks ] + + # Each job requires a step, so we added this dummy step. + steps: + - name: Approve + run: | + echo "Merge Approved" diff --git a/.github/workflows/pr-merge.yml b/.github/workflows/pr-merge.yml new file mode 100644 index 00000000..e4be0103 --- /dev/null +++ b/.github/workflows/pr-merge.yml @@ -0,0 +1,25 @@ +name: PR-Merge + +on: + push: + branches: + - 'main' + - 'master' + +jobs: + + # This job can be removed if certain preconditions are met. See + # https://exasol.github.io/python-toolbox/user_guide/workflows.html#pr-merge-workflow + + ci-job: + name: Checks + uses: ./.github/workflows/checks.yml + secrets: inherit + + publish-docs: + name: Publish Documentation + uses: ./.github/workflows/gh-pages.yml + + metrics: + needs: [ ci-job ] + uses: ./.github/workflows/report.yml diff --git a/.github/workflows/report.yml b/.github/workflows/report.yml new file mode 100644 index 00000000..a84f9fec --- /dev/null +++ b/.github/workflows/report.yml @@ -0,0 +1,55 @@ +name: Status Report + +on: + workflow_call: + secrets: + ALTERNATIVE_GITHUB_TOKEN: + required: false + +jobs: + + report: + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ secrets.ALTERNATIVE_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@0.20.0 + + - name: Download Artifacts + uses: actions/download-artifact@v4.1.8 + with: + path: ./artifacts + + - name: Copy Artifacts into Root Folder + working-directory: ./artifacts + run: | + poetry run coverage combine --keep coverage-python3.9*/.coverage + cp .coverage ../ + cp lint-python3.9/.lint.txt ../ + cp security-python3.9/.security.json ../ + + - name: Generate Report + run: poetry run nox -s project:report -- -- --format json | tee metrics.json + + - name: Upload Artifacts + uses: actions/upload-artifact@v4.4.0 + with: + name: metrics.json + path: metrics.json + + - name: Generate GitHub Summary + run: | + echo -e "# Summary\n" >> $GITHUB_STEP_SUMMARY + poetry run nox -s project:report -- -- --format markdown >> $GITHUB_STEP_SUMMARY + echo -e "\n\n# Coverage\n" >> $GITHUB_STEP_SUMMARY + poetry run coverage report -- --format markdown >> $GITHUB_STEP_SUMMARY + echo -e "\n\n# Static Code Analysis\n" >> $GITHUB_STEP_SUMMARY + cat .lint.txt >> $GITHUB_STEP_SUMMARY + poetry run tbx security pretty-print .security.json >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/slow-checks.yml b/.github/workflows/slow-checks.yml new file mode 100644 index 00000000..efb95c02 --- /dev/null +++ b/.github/workflows/slow-checks.yml @@ -0,0 +1,46 @@ +name: Slow-Checks + +on: + workflow_call: + secrets: + ALTERNATIVE_GITHUB_TOKEN: + required: false + +jobs: + + build-matrix: + name: Generate Build Matrix + uses: ./.github/workflows/matrix-all.yml + + Tests: + name: Integration-Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}}) + needs: [ build-matrix ] + runs-on: ubuntu-latest + # Even though the environment "manual-approval" will be created automatically, + # it still needs to be configured to require interactive review. + # See project settings on GitHub (Settings / Environments / manual-approval). + environment: manual-approval + env: + GITHUB_TOKEN: ${{ secrets.ALTERNATIVE_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@0.20.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Run Tests and Collect Coverage + run: poetry run nox -s test:integration -- -- --coverage --db-version ${{ matrix.exasol-version }} + + - name: Upload Artifacts + uses: actions/upload-artifact@v4.4.0 + with: + name: coverage-python${{ matrix.python-version }}-slow + path: .coverage + include-hidden-files: true diff --git a/noxconfig.py b/noxconfig.py index b46bd9c7..8681ef2b 100644 --- a/noxconfig.py +++ b/noxconfig.py @@ -42,12 +42,16 @@ def post_integration_tests_hook(self, session, config, context): class Config: root: Path = Path(__file__).parent doc: Path = Path(__file__).parent / "doc" - version_file: Path = Path(__file__).parent / "sqlalchemy_exasol" / "version.py" + version_file: Path = Path(__file__).parent / \ + "sqlalchemy_exasol" / "version.py" path_filters: Iterable[str] = ( "dist", ".eggs", "venv", ) + connectors = ["pyodbc", "turbodbc", "websocket"] + python_versions = ["3.9", "3.10", "3.11", "3.12", "3.13"] + exasol_versions = ["7.1.17"] plugins = [StartDB, StopDB] diff --git a/noxfile.py b/noxfile.py index b78620ad..2a327cb7 100644 --- a/noxfile.py +++ b/noxfile.py @@ -390,3 +390,27 @@ def check_links(session: Session) -> None: ) # fmt: on + +def _connector_matrix(config: Config): + CONNECTORS = ['websocket'] + attr = "connectors" + connectors = getattr(config, attr, CONNECTORS) + if not hasattr(config, attr): + _log.warning( + "Config does not contain '%s' setting. Using default: %s", + attr, + CONNECTORS, + ) + return {"connector": exasol_versions} + + +@nox.session(name="matrix:all", python=False) +def full_matrix(session: Session) -> None: + """Output the full build matrix for Python & Exasol versions as JSON.""" + from exasol.toolbox.nox._ci import _python_matrix + from exasol.toolbox.nox._ci import _exasol_matrix + matrix = _python_matrix(PROJECT_CONFIG) + matrix.update(_exasol_matrix(PROJECT_CONFIG)) + matrix.update(_connector_matrix(PROJECT_CONFIG)) + print(json.dumps(matrix)) + From 7a124017878f162cfb51f66f622dd6008ecf34a7 Mon Sep 17 00:00:00 2001 From: Nicola Coretti Date: Wed, 15 Jan 2025 15:18:16 +0100 Subject: [PATCH 03/19] Make all matricies available --- noxfile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 2a327cb7..9ca8d37a 100644 --- a/noxfile.py +++ b/noxfile.py @@ -389,7 +389,6 @@ def check_links(session: Session) -> None: open_docs, ) -# fmt: on def _connector_matrix(config: Config): CONNECTORS = ['websocket'] @@ -403,6 +402,7 @@ def _connector_matrix(config: Config): ) return {"connector": exasol_versions} +from exasol.toolbox.nox._ci import python_matrix, exasol_matrix @nox.session(name="matrix:all", python=False) def full_matrix(session: Session) -> None: @@ -414,3 +414,4 @@ def full_matrix(session: Session) -> None: matrix.update(_connector_matrix(PROJECT_CONFIG)) print(json.dumps(matrix)) +# fmt: on From a9028547ca91b03664fd1e21859288ec135a035b Mon Sep 17 00:00:00 2001 From: Nicola Coretti Date: Wed, 15 Jan 2025 15:18:35 +0100 Subject: [PATCH 04/19] Apply code formatter --- noxconfig.py | 3 +-- noxfile.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/noxconfig.py b/noxconfig.py index 8681ef2b..31376c76 100644 --- a/noxconfig.py +++ b/noxconfig.py @@ -42,8 +42,7 @@ def post_integration_tests_hook(self, session, config, context): class Config: root: Path = Path(__file__).parent doc: Path = Path(__file__).parent / "doc" - version_file: Path = Path(__file__).parent / \ - "sqlalchemy_exasol" / "version.py" + version_file: Path = Path(__file__).parent / "sqlalchemy_exasol" / "version.py" path_filters: Iterable[str] = ( "dist", ".eggs", diff --git a/noxfile.py b/noxfile.py index 9ca8d37a..e9474efe 100644 --- a/noxfile.py +++ b/noxfile.py @@ -402,13 +402,19 @@ def _connector_matrix(config: Config): ) return {"connector": exasol_versions} -from exasol.toolbox.nox._ci import python_matrix, exasol_matrix +from exasol.toolbox.nox._ci import ( + exasol_matrix, + python_matrix, +) + @nox.session(name="matrix:all", python=False) def full_matrix(session: Session) -> None: """Output the full build matrix for Python & Exasol versions as JSON.""" - from exasol.toolbox.nox._ci import _python_matrix - from exasol.toolbox.nox._ci import _exasol_matrix + from exasol.toolbox.nox._ci import ( + _exasol_matrix, + _python_matrix, + ) matrix = _python_matrix(PROJECT_CONFIG) matrix.update(_exasol_matrix(PROJECT_CONFIG)) matrix.update(_connector_matrix(PROJECT_CONFIG)) From 378a7a59dc5e544d36e83ec07da5290f8e9da8ac Mon Sep 17 00:00:00 2001 From: Nicola Coretti Date: Wed, 15 Jan 2025 15:21:12 +0100 Subject: [PATCH 05/19] Fix matrix:all target --- noxfile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/noxfile.py b/noxfile.py index e9474efe..5bc2343f 100644 --- a/noxfile.py +++ b/noxfile.py @@ -411,6 +411,7 @@ def _connector_matrix(config: Config): @nox.session(name="matrix:all", python=False) def full_matrix(session: Session) -> None: """Output the full build matrix for Python & Exasol versions as JSON.""" + from noxconfig import PROJECT_CONFIG from exasol.toolbox.nox._ci import ( _exasol_matrix, _python_matrix, From 05e168a60f5b6aed1d6bff4eaf071fcdb244225d Mon Sep 17 00:00:00 2001 From: Nicola Coretti Date: Wed, 15 Jan 2025 15:22:45 +0100 Subject: [PATCH 06/19] Fixed format --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 5bc2343f..bea0fab5 100644 --- a/noxfile.py +++ b/noxfile.py @@ -411,11 +411,11 @@ def _connector_matrix(config: Config): @nox.session(name="matrix:all", python=False) def full_matrix(session: Session) -> None: """Output the full build matrix for Python & Exasol versions as JSON.""" - from noxconfig import PROJECT_CONFIG from exasol.toolbox.nox._ci import ( _exasol_matrix, _python_matrix, ) + from noxconfig import PROJECT_CONFIG matrix = _python_matrix(PROJECT_CONFIG) matrix.update(_exasol_matrix(PROJECT_CONFIG)) matrix.update(_connector_matrix(PROJECT_CONFIG)) From 059ca06a080eb3c17cdc3acaaecda64017c02216 Mon Sep 17 00:00:00 2001 From: Nicola Coretti Date: Wed, 15 Jan 2025 15:24:44 +0100 Subject: [PATCH 07/19] Fix connectors matrix --- noxfile.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index bea0fab5..7eac8eb5 100644 --- a/noxfile.py +++ b/noxfile.py @@ -400,7 +400,7 @@ def _connector_matrix(config: Config): attr, CONNECTORS, ) - return {"connector": exasol_versions} + return {"connector": connectors} from exasol.toolbox.nox._ci import ( exasol_matrix, @@ -411,6 +411,8 @@ def _connector_matrix(config: Config): @nox.session(name="matrix:all", python=False) def full_matrix(session: Session) -> None: """Output the full build matrix for Python & Exasol versions as JSON.""" + import json + from exasol.toolbox.nox._ci import ( _exasol_matrix, _python_matrix, From d88dd53c2849be45b43fd741f96171fe71403424 Mon Sep 17 00:00:00 2001 From: Nicola Coretti Date: Wed, 15 Jan 2025 15:36:03 +0100 Subject: [PATCH 08/19] Rework nox tasks --- noxfile.py | 49 ++++--------------------------------------------- 1 file changed, 4 insertions(+), 45 deletions(-) diff --git a/noxfile.py b/noxfile.py index 7eac8eb5..ec71d9bf 100644 --- a/noxfile.py +++ b/noxfile.py @@ -105,7 +105,7 @@ def start_db(session: Session) -> None: def parser() -> ArgumentParser: p = ArgumentParser( - usage="nox -s start-db -- [-h] [--db-version]", + usage="nox -s db:start -- [-h] [--db-version]", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) p.add_argument( @@ -158,7 +158,7 @@ def sqlalchemy_tests(session: Session) -> None: def parser() -> ArgumentParser: p = ArgumentParser( - usage="nox -s sqla-tests -- [-h] [--connector]", + usage="nox -s test:sqla -- [-h] [--connector]", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) p.add_argument( @@ -199,7 +199,7 @@ def exasol_tests(session: Session) -> None: def parser() -> ArgumentParser: p = ArgumentParser( - usage="nox -s exasol-tests -- [-h] [--connector]", + usage="nox -s test:exasol -- [-h] [--connector]", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) p.add_argument( @@ -236,7 +236,7 @@ def integration_tests(session: Session) -> None: def parser() -> ArgumentParser: p = ArgumentParser( - usage="nox -s integration-tests -- [-h] [--connector] [--db-version]", + usage="nox -s test:integration -- [-h] [--connector] [--db-version]", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) p.add_argument( @@ -272,47 +272,6 @@ def parser() -> ArgumentParser: session.notify(find_session_runner(session, "db:stop")) -@nox.session(python=False) -def release(session: Session) -> None: - """Release a sqlalchemy-exasol package. For more details append '-- -h'""" - - def create_parser() -> ArgumentParser: - p = ArgumentParser( - "Release a pypi package", - usage="nox -s release -- [-h] [-d]", - ) - p.add_argument("-d", "--dry-run", action="store_true", help="just do a dry run") - return p - - args = [] - parser = create_parser() - cli_args = parser.parse_args(session.posargs) - if cli_args.dry_run: - args.append("--dry-run") - - version_file = version_from_python_module(Settings.VERSION_FILE) - module_version = version_from_poetry() - git_version = version_from_string(list(tags())[-1]) - - if not (module_version == git_version == version_file): - session.error( - f"Versions out of sync, version file: {version_file}, poetry: {module_version}, tag: {git_version}." - ) - - session.run( - "poetry", - "build", - external=True, - ) - - session.run( - "poetry", - "publish", - *args, - external=True, - ) - - @nox.session(name="test:skipped", python=False) def report_skipped(session: Session) -> None: """ From 6036b95aaaec6a3f854ccb0e77d336f7e6939b18 Mon Sep 17 00:00:00 2001 From: Nicola Coretti Date: Wed, 15 Jan 2025 15:36:15 +0100 Subject: [PATCH 09/19] Remove coverage --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 3aebac24..330a469d 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -155,7 +155,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Run Tests and Collect Coverage - run: poetry run nox -s test:unit -- -- --coverage + run: poetry run nox -s test:unit - name: Upload Artifacts uses: actions/upload-artifact@v4.4.0 From 27e43b3b59894e2a1dde53a5bfbf914a3ccb455c Mon Sep 17 00:00:00 2001 From: Nicola Coretti Date: Wed, 15 Jan 2025 15:42:09 +0100 Subject: [PATCH 10/19] Remove Settings class and replace with PROJECT_CONFIG object --- noxconfig.py | 3 +++ noxfile.py | 38 +++++++++++++++----------------------- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/noxconfig.py b/noxconfig.py index 31376c76..a4d235ce 100644 --- a/noxconfig.py +++ b/noxconfig.py @@ -48,6 +48,9 @@ class Config: ".eggs", "venv", ) + environment_name = "test" + db_port = 8563 + bucketfs_port = 2580 connectors = ["pyodbc", "turbodbc", "websocket"] python_versions = ["3.9", "3.10", "3.11", "3.12", "3.13"] exasol_versions = ["7.1.17"] diff --git a/noxfile.py b/noxfile.py index ec71d9bf..91c88fd4 100644 --- a/noxfile.py +++ b/noxfile.py @@ -41,13 +41,7 @@ nox.options.sessions = ["project:fix"] -class Settings: - CONNECTORS = ("pyodbc", "turbodbc", "websocket") - ENVIRONMENT_NAME = "test" - DB_PORT = 8563 - BUCKETFS_PORT = 2580 - VERSION_FILE = PROJECT_ROOT / "sqlalchemy_exasol" / "version.py" - DB_VERSIONS = ("7.1.17",) +from noxconfig import PROJECT_CONFIG def find_session_runner(session: Session, name: str) -> SessionRunner: @@ -83,7 +77,6 @@ def check(session: Session) -> None: ) from exasol.toolbox.nox._shared import _context from exasol.toolbox.nox._test import _coverage - from noxconfig import PROJECT_CONFIG context = _context(session, coverage=True) py_files = [f"{file}" for file in _python_files(PROJECT_CONFIG.root)] @@ -110,8 +103,8 @@ def parser() -> ArgumentParser: ) p.add_argument( "--db-version", - choices=Settings.DB_VERSIONS, - default=Settings.DB_VERSIONS[0], + choices=PROJECT_CONFIG.exasol_versions, + default=PROJECT_CONFIG.exasol_versions[0], help="which will be used", ) return p @@ -121,11 +114,11 @@ def start(db_version: str) -> None: "itde", "spawn-test-environment", "--environment-name", - f"{Settings.ENVIRONMENT_NAME}", + f"{PROJECT_CONFIG.environment_name}", "--database-port-forward", - f"{Settings.DB_PORT}", + f"{PROJECT_CONFIG.db_port}", "--bucketfs-port-forward", - f"{Settings.BUCKETFS_PORT}", + f"{PROJECT_CONFIG.bucketfs_port}", "--docker-db-image-version", db_version, "--db-mem-size", @@ -163,8 +156,8 @@ def parser() -> ArgumentParser: ) p.add_argument( "--connector", - choices=Settings.CONNECTORS, - default=Settings.CONNECTORS[0], + choices=PROJECT_CONFIG.connectors, + default=PROJECT_CONFIG.connectors[0], help="which will be used", ) return p @@ -204,8 +197,8 @@ def parser() -> ArgumentParser: ) p.add_argument( "--connector", - choices=Settings.CONNECTORS, - default=Settings.CONNECTORS[0], + choices=PROJECT_CONFIG.connectors, + default=PROJECT_CONFIG.connectors[0], help="which will be used", ) return p @@ -241,14 +234,14 @@ def parser() -> ArgumentParser: ) p.add_argument( "--connector", - choices=Settings.CONNECTORS, - default=Settings.CONNECTORS[0], + choices=PROJECT_CONFIG.connectors, + default=PROJECT_CONFIG.connectors[0], help="which will be used", ) p.add_argument( "--db-version", - choices=Settings.DB_VERSIONS, - default=Settings.DB_VERSIONS[0], + choices=PROJECT_CONFIG.exasol_versions, + default=PROJECT_CONFIG.exasol_versions[0], help="which will be used", ) return p @@ -280,7 +273,7 @@ def report_skipped(session: Session) -> None: Attention: This task expects a running test database (db-start). """ with TemporaryDirectory() as tmp_dir: - for connector in Settings.CONNECTORS: + for connector in PROJECT_CONFIG.connectors: report = Path(tmp_dir) / f"test-report{connector}.json" with odbcconfig(ODBC_DRIVER) as (config, env): session.run( @@ -376,7 +369,6 @@ def full_matrix(session: Session) -> None: _exasol_matrix, _python_matrix, ) - from noxconfig import PROJECT_CONFIG matrix = _python_matrix(PROJECT_CONFIG) matrix.update(_exasol_matrix(PROJECT_CONFIG)) matrix.update(_connector_matrix(PROJECT_CONFIG)) From dc3be567fc62950dec754ad20226ee8fe4c8cde7 Mon Sep 17 00:00:00 2001 From: Nicola Coretti Date: Wed, 15 Jan 2025 15:44:01 +0100 Subject: [PATCH 11/19] Remove unused code --- noxfile.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/noxfile.py b/noxfile.py index 91c88fd4..b7324586 100644 --- a/noxfile.py +++ b/noxfile.py @@ -75,10 +75,6 @@ def check(session: Session) -> None: _pylint, _type_check, ) - from exasol.toolbox.nox._shared import _context - from exasol.toolbox.nox._test import _coverage - - context = _context(session, coverage=True) py_files = [f"{file}" for file in _python_files(PROJECT_CONFIG.root)] _version(session, Mode.Check, PROJECT_CONFIG.version_file) _code_format(session, Mode.Check, py_files) From 187c95b45584798f85223f715a893954bc51d13d Mon Sep 17 00:00:00 2001 From: Nicola Coretti Date: Wed, 15 Jan 2025 15:55:42 +0100 Subject: [PATCH 12/19] Add coverage support --- .github/workflows/checks.yml | 2 +- noxfile.py | 29 ++++++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 330a469d..3aebac24 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -155,7 +155,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Run Tests and Collect Coverage - run: poetry run nox -s test:unit + run: poetry run nox -s test:unit -- -- --coverage - name: Upload Artifacts uses: actions/upload-artifact@v4.4.0 diff --git a/noxfile.py b/noxfile.py index b7324586..217a19ec 100644 --- a/noxfile.py +++ b/noxfile.py @@ -75,6 +75,7 @@ def check(session: Session) -> None: _pylint, _type_check, ) + py_files = [f"{file}" for file in _python_files(PROJECT_CONFIG.root)] _version(session, Mode.Check, PROJECT_CONFIG.version_file) _code_format(session, Mode.Check, py_files) @@ -132,6 +133,18 @@ def stop_db(session: Session) -> None: session.run("docker", "kill", "db_container_test", external=True) +def _coverage_command(): + base_command = ["poetry", "run"] + coverage_command = [ + "coverage", + "run", + "-a", + f"--rcfile={PROJECT_ROOT / 'pyproject.toml'}", + "-m", + ] + return coverage_command + + @nox.session(name="test:sqla", python=False) def sqlalchemy_tests(session: Session) -> None: """ @@ -162,6 +175,7 @@ def parser() -> ArgumentParser: args = parser().parse_args(session.posargs) connector = args.connector session.run( + *_coverage_command(), "pytest", "--dropfirst", "--db", @@ -203,6 +217,7 @@ def parser() -> ArgumentParser: args = parser().parse_args(session.posargs) connector = args.connector session.run( + *_coverage_command(), "pytest", "--dropfirst", "--db", @@ -216,7 +231,11 @@ def parser() -> ArgumentParser: @nox.session(name="test:regression", python=False) def regression_tests(session: Session) -> None: """Run regression tests""" - session.run("pytest", f"{PROJECT_ROOT / 'test' / 'integration' / 'regression'}") + session.run( + *_coverage_command(), + "pytest", + f"{PROJECT_ROOT / 'test' / 'integration' / 'regression'}", + ) @nox.session(name="test:integration", python=False) @@ -240,8 +259,16 @@ def parser() -> ArgumentParser: default=PROJECT_CONFIG.exasol_versions[0], help="which will be used", ) + p.add_argument( + "--coverage", + action="store_true", + help="This is only here for compatibility. Coverage will always be collected.", + ) return p + coverage_file = PROJECT_ROOT / ".coverage" + coverage_file.unlink(missing_ok=True) + args = parser().parse_args(session.posargs) session.notify( find_session_runner(session, "db:start"), From c700a5f3e54d1685d04fbe8670f704716e26f0ee Mon Sep 17 00:00:00 2001 From: Nicola Coretti Date: Wed, 15 Jan 2025 16:02:16 +0100 Subject: [PATCH 13/19] Add connector to matrix name --- .github/workflows/slow-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/slow-checks.yml b/.github/workflows/slow-checks.yml index efb95c02..59bf6f69 100644 --- a/.github/workflows/slow-checks.yml +++ b/.github/workflows/slow-checks.yml @@ -13,7 +13,7 @@ jobs: uses: ./.github/workflows/matrix-all.yml Tests: - name: Integration-Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}}) + name: Integration-Tests (Connector: ${{matrix.connector}}, Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}}) needs: [ build-matrix ] runs-on: ubuntu-latest # Even though the environment "manual-approval" will be created automatically, From 80714128576ac3b940ecad498f38adb189a78b0c Mon Sep 17 00:00:00 2001 From: Nicola Coretti Date: Wed, 15 Jan 2025 16:06:49 +0100 Subject: [PATCH 14/19] Update coverage command --- noxfile.py | 1 - 1 file changed, 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 217a19ec..787269ef 100644 --- a/noxfile.py +++ b/noxfile.py @@ -134,7 +134,6 @@ def stop_db(session: Session) -> None: def _coverage_command(): - base_command = ["poetry", "run"] coverage_command = [ "coverage", "run", From 2d729fbb5ba2c26c56228845add4e1fcd86d38db Mon Sep 17 00:00:00 2001 From: Nicola Coretti Date: Wed, 15 Jan 2025 16:09:27 +0100 Subject: [PATCH 15/19] Rework connector name in build --- .github/workflows/slow-checks.yml | 2 +- noxfile.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/slow-checks.yml b/.github/workflows/slow-checks.yml index 59bf6f69..8836a9a7 100644 --- a/.github/workflows/slow-checks.yml +++ b/.github/workflows/slow-checks.yml @@ -13,7 +13,7 @@ jobs: uses: ./.github/workflows/matrix-all.yml Tests: - name: Integration-Tests (Connector: ${{matrix.connector}}, Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}}) + name: Integration-Tests (${{matrix.connector}}, Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}}) needs: [ build-matrix ] runs-on: ubuntu-latest # Even though the environment "manual-approval" will be created automatically, diff --git a/noxfile.py b/noxfile.py index 787269ef..43b0a25c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -393,7 +393,7 @@ def full_matrix(session: Session) -> None: ) matrix = _python_matrix(PROJECT_CONFIG) matrix.update(_exasol_matrix(PROJECT_CONFIG)) - matrix.update(_connector_matrix(PROJECT_CONFIG)) + matrix.update(_connector_matrix(PROJECT_CONFIG)) print(json.dumps(matrix)) # fmt: on From 71bd6e9b1611743f1056300632f11d28e1fff2ea Mon Sep 17 00:00:00 2001 From: Nicola Coretti Date: Wed, 15 Jan 2025 16:18:48 +0100 Subject: [PATCH 16/19] Fix workflow --- .github/workflows/slow-checks.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/slow-checks.yml b/.github/workflows/slow-checks.yml index 8836a9a7..ab1cacf9 100644 --- a/.github/workflows/slow-checks.yml +++ b/.github/workflows/slow-checks.yml @@ -35,6 +35,9 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Install via apt + run: sudo apt-get install unixodbc unixodbc-dev libboost-date-time-dev libboost-locale-dev libboost-system-dev + - name: Run Tests and Collect Coverage run: poetry run nox -s test:integration -- -- --coverage --db-version ${{ matrix.exasol-version }} From 99d44240167fb7787c50a9791a18d22e7d7550bc Mon Sep 17 00:00:00 2001 From: Nicola Coretti Date: Thu, 16 Jan 2025 08:47:02 +0100 Subject: [PATCH 17/19] Fix artifact upload(s) --- .github/workflows/slow-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/slow-checks.yml b/.github/workflows/slow-checks.yml index ab1cacf9..6135b7fa 100644 --- a/.github/workflows/slow-checks.yml +++ b/.github/workflows/slow-checks.yml @@ -44,6 +44,6 @@ jobs: - name: Upload Artifacts uses: actions/upload-artifact@v4.4.0 with: - name: coverage-python${{ matrix.python-version }}-slow + name: coverage-python${{ matrix.python-version }}-${{matrix.connector}}-slow path: .coverage include-hidden-files: true From c35c38cf06657f6de4a472f8e5eaec7475529bcc Mon Sep 17 00:00:00 2001 From: Nicola Coretti Date: Thu, 16 Jan 2025 09:48:03 +0100 Subject: [PATCH 18/19] Add report tasks --- noxfile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/noxfile.py b/noxfile.py index 43b0a25c..ab4bb2e8 100644 --- a/noxfile.py +++ b/noxfile.py @@ -87,6 +87,7 @@ def check(session: Session) -> None: lint, type_check, ) +from exasol.toolbox.nox._metrics import report @nox.session(name="db:start", python=False) From 949ba6dafc3deee6610fabe4dc1197cc841919ac Mon Sep 17 00:00:00 2001 From: Nicola Coretti Date: Thu, 16 Jan 2025 10:04:24 +0100 Subject: [PATCH 19/19] Fix naming in for unit-test runs --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 3aebac24..4fcc9b38 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -136,7 +136,7 @@ jobs: run: poetry run nox -s project:format Tests: - name: Unit-Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}}) + name: Unit-Tests (Python-${{ matrix.python-version }}) needs: [ Documentation, Lint, Type-Check, Security, Format, build-matrix ] runs-on: ubuntu-latest env: