From 965fb0825426edf0e4607dd85305c6f4b6c0c9f8 Mon Sep 17 00:00:00 2001 From: Elvis Pranskevichus Date: Fri, 18 Oct 2024 15:08:02 -0700 Subject: [PATCH 1/2] Fix release workflow Install Kerberos deps when testing wheels and make cibuildwheel tests run properly. --- .github/workflows/install-krb5.sh | 42 ++++++++++-- .github/workflows/install-postgres.sh | 93 +++++++++++++++------------ .github/workflows/release.yml | 29 ++++++--- .github/workflows/tests.yml | 24 +++---- asyncpg/_testbase/__init__.py | 15 ++--- asyncpg/cluster.py | 57 +++++++++++++--- pyproject.toml | 7 +- tests/test_connect.py | 10 ++- 8 files changed, 191 insertions(+), 86 deletions(-) diff --git a/.github/workflows/install-krb5.sh b/.github/workflows/install-krb5.sh index 093b8519..bdb5744d 100755 --- a/.github/workflows/install-krb5.sh +++ b/.github/workflows/install-krb5.sh @@ -1,10 +1,42 @@ #!/bin/bash set -Eexuo pipefail +shopt -s nullglob -if [ "$RUNNER_OS" == "Linux" ]; then - # Assume Ubuntu since this is the only Linux used in CI. - sudo apt-get update - sudo apt-get install -y --no-install-recommends \ - libkrb5-dev krb5-user krb5-kdc krb5-admin-server +if [[ $OSTYPE == linux* ]]; then + if [ "$(id -u)" = "0" ]; then + SUDO= + else + SUDO=sudo + fi + + if [ -e /etc/os-release ]; then + source /etc/os-release + elif [ -e /etc/centos-release ]; then + ID="centos" + VERSION_ID=$(cat /etc/centos-release | cut -f3 -d' ' | cut -f1 -d.) + else + echo "install-krb5.sh: cannot determine which Linux distro this is" >&2 + exit 1 + fi + + if [ "${ID}" = "debian" -o "${ID}" = "ubuntu" ]; then + export DEBIAN_FRONTEND=noninteractive + + $SUDO apt-get update + $SUDO apt-get install -y --no-install-recommends \ + libkrb5-dev krb5-user krb5-kdc krb5-admin-server + elif [ "${ID}" = "almalinux" ]; then + $SUDO dnf install -y krb5-server krb5-workstation krb5-libs krb5-devel + elif [ "${ID}" = "centos" ]; then + $SUDO yum install -y krb5-server krb5-workstation krb5-libs krb5-devel + elif [ "${ID}" = "alpine" ]; then + $SUDO apk add krb5 krb5-server krb5-dev + else + echo "install-krb5.sh: Unsupported linux distro: ${distro}" >&2 + exit 1 + fi +else + echo "install-krb5.sh: unsupported OS: ${OSTYPE}" >&2 + exit 1 fi diff --git a/.github/workflows/install-postgres.sh b/.github/workflows/install-postgres.sh index 4ffbb4d6..733c7033 100755 --- a/.github/workflows/install-postgres.sh +++ b/.github/workflows/install-postgres.sh @@ -3,51 +3,60 @@ set -Eexuo pipefail shopt -s nullglob -PGVERSION=${PGVERSION:-12} +if [[ $OSTYPE == linux* ]]; then + PGVERSION=${PGVERSION:-12} -if [ -e /etc/os-release ]; then - source /etc/os-release -elif [ -e /etc/centos-release ]; then - ID="centos" - VERSION_ID=$(cat /etc/centos-release | cut -f3 -d' ' | cut -f1 -d.) -else - echo "install-postgres.sh: cannot determine which Linux distro this is" >&2 - exit 1 -fi + if [ -e /etc/os-release ]; then + source /etc/os-release + elif [ -e /etc/centos-release ]; then + ID="centos" + VERSION_ID=$(cat /etc/centos-release | cut -f3 -d' ' | cut -f1 -d.) + else + echo "install-postgres.sh: cannot determine which Linux distro this is" >&2 + exit 1 + fi + + if [ "${ID}" = "debian" -o "${ID}" = "ubuntu" ]; then + export DEBIAN_FRONTEND=noninteractive -if [ "${ID}" = "debian" -o "${ID}" = "ubuntu" ]; then - export DEBIAN_FRONTEND=noninteractive - - apt-get install -y --no-install-recommends curl gnupg ca-certificates - curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - - mkdir -p /etc/apt/sources.list.d/ - echo "deb https://apt.postgresql.org/pub/repos/apt/ ${VERSION_CODENAME}-pgdg main" \ - >> /etc/apt/sources.list.d/pgdg.list - apt-get update - apt-get install -y --no-install-recommends \ - "postgresql-${PGVERSION}" \ - "postgresql-contrib-${PGVERSION}" -elif [ "${ID}" = "almalinux" ]; then - yum install -y \ - "postgresql-server" \ - "postgresql-devel" \ - "postgresql-contrib" -elif [ "${ID}" = "centos" ]; then - el="EL-${VERSION_ID%.*}-$(arch)" - baseurl="https://download.postgresql.org/pub/repos/yum/reporpms" - yum install -y "${baseurl}/${el}/pgdg-redhat-repo-latest.noarch.rpm" - if [ ${VERSION_ID%.*} -ge 8 ]; then - dnf -qy module disable postgresql + apt-get install -y --no-install-recommends curl gnupg ca-certificates + curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - + mkdir -p /etc/apt/sources.list.d/ + echo "deb https://apt.postgresql.org/pub/repos/apt/ ${VERSION_CODENAME}-pgdg main" \ + >> /etc/apt/sources.list.d/pgdg.list + apt-get update + apt-get install -y --no-install-recommends \ + "postgresql-${PGVERSION}" \ + "postgresql-contrib-${PGVERSION}" + elif [ "${ID}" = "almalinux" ]; then + yum install -y \ + "postgresql-server" \ + "postgresql-devel" \ + "postgresql-contrib" + elif [ "${ID}" = "centos" ]; then + el="EL-${VERSION_ID%.*}-$(arch)" + baseurl="https://download.postgresql.org/pub/repos/yum/reporpms" + yum install -y "${baseurl}/${el}/pgdg-redhat-repo-latest.noarch.rpm" + if [ ${VERSION_ID%.*} -ge 8 ]; then + dnf -qy module disable postgresql + fi + yum install -y \ + "postgresql${PGVERSION}-server" \ + "postgresql${PGVERSION}-contrib" + ln -s "/usr/pgsql-${PGVERSION}/bin/pg_config" "/usr/local/bin/pg_config" + elif [ "${ID}" = "alpine" ]; then + apk add shadow postgresql postgresql-dev postgresql-contrib + else + echo "install-postgres.sh: unsupported Linux distro: ${distro}" >&2 + exit 1 fi - yum install -y \ - "postgresql${PGVERSION}-server" \ - "postgresql${PGVERSION}-contrib" - ln -s "/usr/pgsql-${PGVERSION}/bin/pg_config" "/usr/local/bin/pg_config" -elif [ "${ID}" = "alpine" ]; then - apk add shadow postgresql postgresql-dev postgresql-contrib + + useradd -m -s /bin/bash apgtest + +elif [[ $OSTYPE == darwin* ]]; then + brew install postgresql + else - echo "install-postgres.sh: Unsupported distro: ${distro}" >&2 + echo "install-postgres.sh: unsupported OS: ${OSTYPE}" >&2 exit 1 fi - -useradd -m -s /bin/bash apgtest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a32a3aeb..5ea543eb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,8 +39,8 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: dist - path: dist/ + name: dist-version + path: dist/VERSION build-sdist: needs: validate-release-request @@ -67,7 +67,7 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: dist + name: dist-sdist path: dist/*.tar.* build-wheels-matrix: @@ -127,9 +127,19 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: dist + name: dist-wheels-${{ matrix.only }} path: wheelhouse/*.whl + merge-artifacts: + runs-on: ubuntu-latest + needs: [build-sdist, build-wheels] + steps: + - name: Merge Artifacts + uses: actions/upload-artifact/merge@v4 + with: + name: dist + delete-merged: true + publish-docs: needs: [build-sdist, build-wheels] runs-on: ubuntu-latest @@ -180,6 +190,12 @@ jobs: needs: [build-sdist, build-wheels, publish-docs] runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/asyncpg + permissions: + id-token: write + steps: - uses: actions/checkout@v4 with: @@ -223,7 +239,4 @@ jobs: - name: Upload to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: - user: __token__ - password: ${{ secrets.PYPI_TOKEN }} - # password: ${{ secrets.TEST_PYPI_TOKEN }} - # repository_url: https://test.pypi.org/legacy/ + attestations: true diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ce06e7f5..a4869312 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -48,28 +48,28 @@ jobs: missing_version_ok: yes version_file: asyncpg/_version.py version_line_pattern: | - __version__\s*=\s*(?:['"])([[:PEP440:]])(?:['"]) + __version__(?:\s*:\s*typing\.Final)?\s*=\s*(?:['"])([[:PEP440:]])(?:['"]) - name: Setup PostgreSQL - if: steps.release.outputs.version == 0 && matrix.os == 'macos-latest' + if: "!steps.release.outputs.is_release && matrix.os == 'macos-latest'" run: | brew install postgresql - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 - if: steps.release.outputs.version == 0 + if: "!steps.release.outputs.is_release" with: python-version: ${{ matrix.python-version }} - name: Install Python Deps - if: steps.release.outputs.version == 0 + if: "!steps.release.outputs.is_release" run: | - .github/workflows/install-krb5.sh + [ "$RUNNER_OS" = "Linux" ] && .github/workflows/install-krb5.sh python -m pip install -U pip setuptools wheel python -m pip install -e .[test] - name: Test - if: steps.release.outputs.version == 0 + if: "!steps.release.outputs.is_release" env: LOOP_IMPL: ${{ matrix.loop }} run: | @@ -103,10 +103,10 @@ jobs: missing_version_ok: yes version_file: asyncpg/_version.py version_line_pattern: | - __version__\s*=\s*(?:['"])([[:PEP440:]])(?:['"]) + __version__(?:\s*:\s*typing\.Final)?\s*=\s*(?:['"])([[:PEP440:]])(?:['"]) - name: Set up PostgreSQL - if: steps.release.outputs.version == 0 + if: "!steps.release.outputs.is_release" env: PGVERSION: ${{ matrix.postgres-version }} DISTRO_NAME: focal @@ -118,19 +118,19 @@ jobs: - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 - if: steps.release.outputs.version == 0 + if: "!steps.release.outputs.is_release" with: python-version: "3.x" - name: Install Python Deps - if: steps.release.outputs.version == 0 + if: "!steps.release.outputs.is_release" run: | - .github/workflows/install-krb5.sh + [ "$RUNNER_OS" = "Linux" ] && .github/workflows/install-krb5.sh python -m pip install -U pip setuptools wheel python -m pip install -e .[test] - name: Test - if: steps.release.outputs.version == 0 + if: "!steps.release.outputs.is_release" env: PGVERSION: ${{ matrix.postgres-version }} run: | diff --git a/asyncpg/_testbase/__init__.py b/asyncpg/_testbase/__init__.py index 2d785dac..95775e11 100644 --- a/asyncpg/_testbase/__init__.py +++ b/asyncpg/_testbase/__init__.py @@ -226,13 +226,6 @@ def _init_cluster(ClusterCls, cluster_kwargs, initdb_options=None): return cluster -def _start_cluster(ClusterCls, cluster_kwargs, server_settings, - initdb_options=None): - cluster = _init_cluster(ClusterCls, cluster_kwargs, initdb_options) - cluster.start(port='dynamic', server_settings=server_settings) - return cluster - - def _get_initdb_options(initdb_options=None): if not initdb_options: initdb_options = {} @@ -256,8 +249,12 @@ def _init_default_cluster(initdb_options=None): _default_cluster = pg_cluster.RunningCluster() else: _default_cluster = _init_cluster( - pg_cluster.TempCluster, cluster_kwargs={}, - initdb_options=_get_initdb_options(initdb_options)) + pg_cluster.TempCluster, + cluster_kwargs={ + "data_dir_suffix": ".apgtest", + }, + initdb_options=_get_initdb_options(initdb_options), + ) return _default_cluster diff --git a/asyncpg/cluster.py b/asyncpg/cluster.py index 4467cc2a..606c2eae 100644 --- a/asyncpg/cluster.py +++ b/asyncpg/cluster.py @@ -9,9 +9,11 @@ import os import os.path import platform +import random import re import shutil import socket +import string import subprocess import sys import tempfile @@ -45,6 +47,29 @@ def find_available_port(): sock.close() +def _world_readable_mkdtemp(suffix=None, prefix=None, dir=None): + name = "".join(random.choices(string.ascii_lowercase, k=8)) + if dir is None: + dir = tempfile.gettempdir() + if prefix is None: + prefix = tempfile.gettempprefix() + if suffix is None: + suffix = "" + fn = os.path.join(dir, prefix + name + suffix) + os.mkdir(fn, 0o755) + return fn + + +def _mkdtemp(suffix=None, prefix=None, dir=None): + if _system == 'Windows' and os.environ.get("GITHUB_ACTIONS"): + # Due to mitigations introduced in python/cpython#118486 + # when Python runs in a session created via an SSH connection + # tempfile.mkdtemp creates directories that are not accessible. + return _world_readable_mkdtemp(suffix, prefix, dir) + else: + return tempfile.mkdtemp(suffix, prefix, dir) + + class ClusterError(Exception): pass @@ -122,9 +147,13 @@ def init(self, **settings): else: extra_args = [] + os.makedirs(self._data_dir, exist_ok=True) process = subprocess.run( [self._pg_ctl, 'init', '-D', self._data_dir] + extra_args, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd=self._data_dir, + ) output = process.stdout @@ -199,7 +228,10 @@ def start(self, wait=60, *, server_settings={}, **opts): process = subprocess.run( [self._pg_ctl, 'start', '-D', self._data_dir, '-o', ' '.join(extra_args)], - stdout=stdout, stderr=subprocess.STDOUT) + stdout=stdout, + stderr=subprocess.STDOUT, + cwd=self._data_dir, + ) if process.returncode != 0: if process.stderr: @@ -218,7 +250,10 @@ def start(self, wait=60, *, server_settings={}, **opts): self._daemon_process = \ subprocess.Popen( [self._postgres, '-D', self._data_dir, *extra_args], - stdout=stdout, stderr=subprocess.STDOUT) + stdout=stdout, + stderr=subprocess.STDOUT, + cwd=self._data_dir, + ) self._daemon_pid = self._daemon_process.pid @@ -232,7 +267,10 @@ def reload(self): process = subprocess.run( [self._pg_ctl, 'reload', '-D', self._data_dir], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=self._data_dir, + ) stderr = process.stderr @@ -245,7 +283,10 @@ def stop(self, wait=60): process = subprocess.run( [self._pg_ctl, 'stop', '-D', self._data_dir, '-t', str(wait), '-m', 'fast'], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=self._data_dir, + ) stderr = process.stderr @@ -583,9 +624,9 @@ class TempCluster(Cluster): def __init__(self, *, data_dir_suffix=None, data_dir_prefix=None, data_dir_parent=None, pg_config_path=None): - self._data_dir = tempfile.mkdtemp(suffix=data_dir_suffix, - prefix=data_dir_prefix, - dir=data_dir_parent) + self._data_dir = _mkdtemp(suffix=data_dir_suffix, + prefix=data_dir_prefix, + dir=data_dir_parent) super().__init__(self._data_dir, pg_config_path=pg_config_path) diff --git a/pyproject.toml b/pyproject.toml index 4bb9e8f0..dabb7d8b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ gssauth = [ test = [ 'flake8~=6.1', 'flake8-pyi~=24.1.0', + 'distro~=1.9.0', 'uvloop>=0.15.3; platform_system != "Windows" and python_version < "3.14.0"', 'gssapi; platform_system == "Linux"', 'k5test; platform_system == "Linux"', @@ -75,13 +76,17 @@ build-frontend = "build" test-extras = "test" [tool.cibuildwheel.macos] +before-all = ".github/workflows/install-postgres.sh" test-command = "python {project}/tests/__init__.py" [tool.cibuildwheel.windows] test-command = "python {project}\\tests\\__init__.py" [tool.cibuildwheel.linux] -before-all = ".github/workflows/install-postgres.sh" +before-all = """ + .github/workflows/install-postgres.sh \ + && .github/workflows/install-krb5.sh \ + """ test-command = """\ PY=`which python` \ && chmod -R go+rX "$(dirname $(dirname $(dirname $PY)))" \ diff --git a/tests/test_connect.py b/tests/test_connect.py index 517f05f9..0037ee5e 100644 --- a/tests/test_connect.py +++ b/tests/test_connect.py @@ -24,6 +24,8 @@ import warnings import weakref +import distro + import asyncpg from asyncpg import _testbase as tb from asyncpg import connection as pg_connection @@ -388,6 +390,10 @@ async def test_auth_md5_unsupported(self, _): await self.connect(user='md5_user', password=CORRECT_PASSWORD) +@unittest.skipIf( + distro.id() == "alpine", + "Alpine Linux ships PostgreSQL without GSS auth support", +) class TestGssAuthentication(BaseTestAuthentication): @classmethod def setUpClass(cls): @@ -426,10 +432,11 @@ def setup_cluster(cls): cls.start_cluster( cls.cluster, server_settings=cls.get_server_settings()) - async def test_auth_gssapi(self): + async def test_auth_gssapi_ok(self): conn = await self.connect(user=self.realm.user_princ) await conn.close() + async def test_auth_gssapi_bad_srvname(self): # Service name mismatch. with self.assertRaisesRegex( exceptions.InternalClientError, @@ -437,6 +444,7 @@ async def test_auth_gssapi(self): ): await self.connect(user=self.realm.user_princ, krbsrvname='wrong') + async def test_auth_gssapi_bad_user(self): # Credentials mismatch. with self.assertRaisesRegex( exceptions.InvalidAuthorizationSpecificationError, From abe1c63f241f6f37bf286c92cc81342c928cdb93 Mon Sep 17 00:00:00 2001 From: Elvis Pranskevichus Date: Fri, 18 Oct 2024 13:25:30 -0700 Subject: [PATCH 2/2] asyncpg v0.30.0 Support Python 3.13 and PostgreSQL 17. Improvements ============ * Implement GSSAPI authentication (by @eltoder in 1d4e5680 for #1122) * Implement SSPI authentication (by @eltoder in 1aab2094 for #1128) * Add initial typings (by @bryanforbes in d42432bf for #1127) * Allow building with Cython 3 (by @musicinmybrain in 258d8a95 for #1101) * docs: fix connection pool close call (#1125) (by @paulovitorweb in e8488149 for #1125) * Add support for the `sslnegotiation` parameter (by @elprans in afdb05c7 for #1187) * Test and build on Python 3.13 (by @elprans in 3aa98944 for #1188) * Support PostgreSQL 17 (by @elprans in cee97e1a for #1189) (by @MeggyCal in aa2d0e69 for #1185) * Add `fetchmany` to execute many *and* return rows (by @rossmacarthur in 73f2209d for #1175) * Add `connect` kwarg to Pool to better support GCP's CloudSQL (by @d1manson in 3ee19baa for #1170) * Allow customizing connection state reset (#1191) (by @elprans in f6ec755c for #1191) Fixes ===== * s/quote/quote_plus/ in the note about DSN part quoting (by @elprans in 1194a8a6 for #1151) * Use asyncio.run() instead of run_until_complete() (by @eltoder in 9fcddfc1 for #1140) * Require async_timeout for python < 3.11 (#1177) (by @Pliner in 327f2a7a for #1177) * Allow testing with uvloop on Python 3.12 (#1182) (by @musicinmybrain in 597fe541 for #1182) * Mark pool-wrapped connection coroutine methods as coroutines (by @elprans in 636420b1 for #1134) * handle `None` parameters in `copy_from_query`, returning `NULL` (by @fobispotc in 259d16e5 for #1180) * fix: return the pool from _async_init__ if it's already initialized (#1104) (by @guacs in 7dc58728 for #1104) * Replace obsolete, unsafe `Py_TRASHCAN_SAFE_BEGIN/END` (#1150) (by @musicinmybrain in 11101c6e for #1150) --- asyncpg/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/asyncpg/_version.py b/asyncpg/_version.py index 383fe4d2..245eee7e 100644 --- a/asyncpg/_version.py +++ b/asyncpg/_version.py @@ -14,4 +14,4 @@ import typing -__version__: typing.Final = '0.30.0.dev0' +__version__: typing.Final = '0.30.0'