From 3b9c92261441056fc172db4afeaba8d79588db34 Mon Sep 17 00:00:00 2001 From: tarsil Date: Wed, 29 Nov 2023 14:51:17 +0000 Subject: [PATCH 1/4] Improve sql ssl options --- databasez/backends/aiopg.py | 2 +- databasez/backends/asyncmy.py | 2 +- databasez/backends/mssql.py | 2 +- databasez/backends/mysql.py | 2 +- databasez/backends/postgres.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/databasez/backends/aiopg.py b/databasez/backends/aiopg.py index c6129b7..5c947c7 100644 --- a/databasez/backends/aiopg.py +++ b/databasez/backends/aiopg.py @@ -56,7 +56,7 @@ def _get_connection_kwargs(self) -> dict: if max_size is not None: kwargs["maxsize"] = int(max_size) if ssl is not None: - kwargs["ssl"] = {"true": True, "false": False}[ssl.lower()] + kwargs["ssl"] = {"true": True, "false": False}.get(ssl.lower(), ssl.lower()) for key, value in self._options.items(): # Coerce 'min_size' and 'max_size' for consistency. diff --git a/databasez/backends/asyncmy.py b/databasez/backends/asyncmy.py index 15c1554..3538413 100644 --- a/databasez/backends/asyncmy.py +++ b/databasez/backends/asyncmy.py @@ -48,7 +48,7 @@ def _get_connection_kwargs(self) -> dict: if pool_recycle is not None: kwargs["pool_recycle"] = int(pool_recycle) if ssl is not None: - kwargs["ssl"] = {"true": True, "false": False}[ssl.lower()] + kwargs["ssl"] = {"true": True, "false": False}.get(ssl.lower(), ssl.lower()) for key, value in self._options.items(): # Coerce 'min_size' and 'max_size' for consistency. diff --git a/databasez/backends/mssql.py b/databasez/backends/mssql.py index 4d1b7dd..f817616 100644 --- a/databasez/backends/mssql.py +++ b/databasez/backends/mssql.py @@ -51,7 +51,7 @@ def _get_connection_kwargs(self) -> dict: if pool_recycle is not None: kwargs["pool_recycle"] = int(pool_recycle) if ssl is not None: - kwargs["ssl"] = {"true": True, "false": False}[ssl.lower()] + kwargs["ssl"] = {"true": True, "false": False}.get(ssl.lower(), ssl.lower()) kwargs.update( { diff --git a/databasez/backends/mysql.py b/databasez/backends/mysql.py index b1799b9..d7c2222 100644 --- a/databasez/backends/mysql.py +++ b/databasez/backends/mysql.py @@ -48,7 +48,7 @@ def _get_connection_kwargs(self) -> dict: if pool_recycle is not None: kwargs["pool_recycle"] = int(pool_recycle) if ssl is not None: - kwargs["ssl"] = {"true": True, "false": False}[ssl.lower()] + kwargs["ssl"] = {"true": True, "false": False}.get(ssl.lower(), ssl.lower()) for key, value in self._options.items(): # Coerce 'min_size' and 'max_size' for consistency. diff --git a/databasez/backends/postgres.py b/databasez/backends/postgres.py index a75cd95..cb92ba9 100644 --- a/databasez/backends/postgres.py +++ b/databasez/backends/postgres.py @@ -51,7 +51,7 @@ def _get_connection_kwargs(self) -> dict: if max_size is not None: kwargs["max_size"] = int(max_size) if ssl is not None: - kwargs["ssl"] = {"true": True, "false": False}[ssl.lower()] + kwargs["ssl"] = {"true": True, "false": False}.get(ssl.lower(), ssl.lower()) kwargs.update(self._options) From f113043d8cbef6ddccfb0a91f70d1811051569c2 Mon Sep 17 00:00:00 2001 From: tarsil Date: Wed, 29 Nov 2023 15:17:47 +0000 Subject: [PATCH 2/4] Sanitize password for urlsplit --- .github/workflows/test-suite.yml | 4 ++-- databasez/core.py | 19 +++++++++++++++++-- docker-compose.yml | 4 ++-- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index a0424c3..c6044c2 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -24,7 +24,7 @@ jobs: env: MYSQL_USER: username MYSQL_PASSWORD: password - MYSQL_ROOT_PASSWORD: password + MYSQL_ROOT_PASSWORD: password*1123456879# MYSQL_DATABASE: testsuite ports: - 3306:3306 @@ -34,7 +34,7 @@ jobs: image: postgres:14 env: POSTGRES_USER: username - POSTGRES_PASSWORD: password + POSTGRES_PASSWORD: password*1123456879# POSTGRES_DB: testsuite ports: - 5432:5432 diff --git a/databasez/core.py b/databasez/core.py index db32b1c..89f76c4 100644 --- a/databasez/core.py +++ b/databasez/core.py @@ -6,9 +6,10 @@ import weakref from contextvars import ContextVar from types import TracebackType -from urllib.parse import SplitResult, parse_qsl, unquote, urlencode, urlsplit +from urllib.parse import SplitResult, parse_qsl, quote_plus, unquote, urlencode, urlsplit from sqlalchemy import text +from sqlalchemy.engine import URL, make_url from sqlalchemy.sql import ClauseElement from databasez.importer import import_from_string @@ -568,10 +569,24 @@ def __init__(self, url: typing.Union[str, "DatabaseURL"]): f"Invalid type for DatabaseURL. Expected str or DatabaseURL, got {type(url)}" ) + def _sanitize_password(self, url: URL) -> URL: + """ + Making sure all the passwords are allowed. + """ + password = url.password + if not password or password is None: + return url + + quoted_password = quote_plus(password) + url = url._replace(password=quoted_password) + return url + @property def components(self) -> SplitResult: if not hasattr(self, "_components"): - self._components = urlsplit(self._url) + raw_url = make_url(self._url) + url = self._sanitize_password(raw_url) + self._components = urlsplit(url.render_as_string(hide_password=False)) return self._components @property diff --git a/docker-compose.yml b/docker-compose.yml index d2a1d75..347d10a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: environment: POSTGRES_HOST_AUTH_METHOD: trust POSTGRES_USER: "postgres" - POSTGRES_PASSWORD: "postgres" + POSTGRES_PASSWORD: "password*1123456879#" POSTGRES_DB: "testsuite" expose: - "5432" @@ -20,7 +20,7 @@ services: MYSQL_USER: "mysql" MYSQL_PASSWORD: "mysql" MYSQL_DATABASE: "testsuite" - MYSQL_ROOT_PASSWORD: "password" + MYSQL_ROOT_PASSWORD: "password*1123456879#" expose: - "3306" ports: From c38689498e0a625ca15beb167442aa3918262932 Mon Sep 17 00:00:00 2001 From: tarsil Date: Wed, 29 Nov 2023 15:27:09 +0000 Subject: [PATCH 3/4] Update CI --- .github/dependbot.yml | 12 ++++++++---- .github/workflows/test-suite.yml | 18 +++++++++--------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/.github/dependbot.yml b/.github/dependbot.yml index b9038ca..273ba55 100644 --- a/.github/dependbot.yml +++ b/.github/dependbot.yml @@ -1,10 +1,14 @@ version: 2 updates: - - package-ecosystem: "pip" + - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "monthly" - - package-ecosystem: "github-actions" + interval: "daily" + commit-message: + prefix: ⬆ + - package-ecosystem: "pip" directory: "/" schedule: - interval: monthly + interval: "daily" + commit-message: + prefix: ⬆ diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index c6044c2..61bc1ba 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -23,8 +23,8 @@ jobs: image: mysql:5.7 env: MYSQL_USER: username - MYSQL_PASSWORD: password - MYSQL_ROOT_PASSWORD: password*1123456879# + MYSQL_PASSWORD: passwsss*1348394# + MYSQL_ROOT_PASSWORD: password MYSQL_DATABASE: testsuite ports: - 3306:3306 @@ -34,7 +34,7 @@ jobs: image: postgres:14 env: POSTGRES_USER: username - POSTGRES_PASSWORD: password*1123456879# + POSTGRES_PASSWORD: passwsss*1348394# POSTGRES_DB: testsuite ports: - 5432:5432 @@ -80,12 +80,12 @@ jobs: TEST_DATABASE_URLS: | sqlite:///testsuite, sqlite+aiosqlite:///testsuite, - mysql://username:password@localhost:3306/testsuite, - mysql+aiomysql://username:password@localhost:3306/testsuite, - mysql+asyncmy://username:password@localhost:3306/testsuite, - postgresql://username:password@localhost:5432/testsuite, - postgresql+aiopg://username:password@127.0.0.1:5432/testsuite, - postgresql+asyncpg://username:password@localhost:5432/testsuite, + mysql://username:passwsss*1348394#@localhost:3306/testsuite, + mysql+aiomysql://username:passwsss*1348394#@localhost:3306/testsuite, + mysql+asyncmy://username:passwsss*1348394#@localhost:3306/testsuite, + postgresql://username:passwsss*1348394#@localhost:5432/testsuite, + postgresql+aiopg://username:passwsss*1348394#@127.0.0.1:5432/testsuite, + postgresql+asyncpg://username:passwsss*1348394#@localhost:5432/testsuite, mssql://sa:Mssql123mssql-@localhost:1433/master?driver=ODBC+Driver+17+for+SQL+Server, mssql+pyodbc://sa:Mssql123mssql-@localhost:1433/master?driver=ODBC+Driver+17+for+SQL+Server, mssql+aioodbc://sa:Mssql123mssql-@localhost:1433/master?driver=ODBC+Driver+17+for+SQL+Server From dd19d1423dc2ba3183e561769283e9a6c0c64b20 Mon Sep 17 00:00:00 2001 From: tarsil Date: Wed, 29 Nov 2023 15:40:28 +0000 Subject: [PATCH 4/4] Fix tests --- databasez/core.py | 5 +++-- docker-compose.yml | 6 +++--- tests/test_databases.py | 8 ++++++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/databasez/core.py b/databasez/core.py index 89f76c4..a7f6519 100644 --- a/databasez/core.py +++ b/databasez/core.py @@ -569,7 +569,8 @@ def __init__(self, url: typing.Union[str, "DatabaseURL"]): f"Invalid type for DatabaseURL. Expected str or DatabaseURL, got {type(url)}" ) - def _sanitize_password(self, url: URL) -> URL: + @classmethod + def _sanitize_password(cls, url: URL) -> URL: """ Making sure all the passwords are allowed. """ @@ -585,7 +586,7 @@ def _sanitize_password(self, url: URL) -> URL: def components(self) -> SplitResult: if not hasattr(self, "_components"): raw_url = make_url(self._url) - url = self._sanitize_password(raw_url) + url = DatabaseURL._sanitize_password(raw_url) self._components = urlsplit(url.render_as_string(hide_password=False)) return self._components diff --git a/docker-compose.yml b/docker-compose.yml index 347d10a..b4edef2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: environment: POSTGRES_HOST_AUTH_METHOD: trust POSTGRES_USER: "postgres" - POSTGRES_PASSWORD: "password*1123456879#" + POSTGRES_PASSWORD: "passwsss*1348394#" POSTGRES_DB: "testsuite" expose: - "5432" @@ -19,8 +19,8 @@ services: environment: MYSQL_USER: "mysql" MYSQL_PASSWORD: "mysql" - MYSQL_DATABASE: "testsuite" - MYSQL_ROOT_PASSWORD: "password*1123456879#" + MYSQL_DATABASE: "passwsss*1348394#" + MYSQL_ROOT_PASSWORD: "password" expose: - "3306" ports: diff --git a/tests/test_databases.py b/tests/test_databases.py index 23c6b61..128175d 100644 --- a/tests/test_databases.py +++ b/tests/test_databases.py @@ -10,6 +10,7 @@ import pytest import sqlalchemy +from sqlalchemy.engine import URL, make_url from databasez import Database, DatabaseURL @@ -17,9 +18,12 @@ DATABASE_URLS = [url.strip() for url in os.environ["TEST_DATABASE_URLS"].split(",")] + DATABASE_CONFIG_URLS = [] for value in DATABASE_URLS: - spliter = urlsplit(value) + raw_url = make_url(value) + url: URL = DatabaseURL._sanitize_password(raw_url) + spliter = urlsplit(url.render_as_string(hide_password=False)) DATABASE_CONFIG_URLS.append( { "connection": { @@ -39,7 +43,7 @@ class AsyncMock(MagicMock): async def __call__(self, *args, **kwargs): - return super(AsyncMock, self).__call__(*args, **kwargs) + return super().__call__(*args, **kwargs) class MyEpochType(sqlalchemy.types.TypeDecorator):