From 733625e2bc72141c06c8da3ceb3409399bf100c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89loi=20Rivard?= Date: Thu, 9 Jan 2025 18:16:27 +0100 Subject: [PATCH] feat: SQL migrations with flask-alembic --- canaille/__init__.py | 3 +- canaille/app/commands.py | 2 +- canaille/backends/__init__.py | 8 +- canaille/backends/ldap/backend.py | 6 +- canaille/backends/memory/backend.py | 2 +- canaille/backends/sql/backend.py | 53 +-- canaille/backends/sql/configuration.py | 11 + .../sql/migrations/1736443094_init.py | 313 ++++++++++++++++++ .../sql/migrations/1736443538_0_0_58.py | 76 +++++ .../backends/sql/migrations/script.py.mako | 29 ++ canaille/commands.py | 7 +- canaille/migrations/script.py.mako | 26 ++ demo/demoapp.py | 2 +- doc/conf.py | 1 + doc/tutorial/databases.rst | 13 + pyproject.toml | 3 + tests/backends/sql/fixtures.py | 2 +- tests/backends/sql/test_alembic.py | 4 + tests/backends/test_backends.py | 2 +- tests/conftest.py | 4 +- uv.lock | 46 +++ 21 files changed, 579 insertions(+), 34 deletions(-) create mode 100644 canaille/backends/sql/migrations/1736443094_init.py create mode 100644 canaille/backends/sql/migrations/1736443538_0_0_58.py create mode 100644 canaille/backends/sql/migrations/script.py.mako create mode 100644 canaille/migrations/script.py.mako create mode 100644 tests/backends/sql/test_alembic.py diff --git a/canaille/__init__.py b/canaille/__init__.py index 945d14c7..91ad01e0 100644 --- a/canaille/__init__.py +++ b/canaille/__init__.py @@ -86,6 +86,7 @@ def create_app( config: dict = None, validate: bool = True, backend=None, + init_backend=None, env_file: str = None, env_prefix: str = "", ): @@ -119,7 +120,7 @@ def create_app( sentry_sdk = setup_sentry(app) try: setup_logging(app) - backend = setup_backend(app, backend) + backend = setup_backend(app, backend, init_backend) setup_features(app) setup_flask_converters(app) setup_blueprints(app) diff --git a/canaille/app/commands.py b/canaille/app/commands.py index 31bd391f..dc49a8bf 100644 --- a/canaille/app/commands.py +++ b/canaille/app/commands.py @@ -54,7 +54,7 @@ def install(): from canaille.app.installation import install try: - install(current_app.config) + install(current_app) except ConfigurationException as exc: # pragma: no cover print(exc) diff --git a/canaille/backends/__init__.py b/canaille/backends/__init__.py index fe4b2207..59d3d5b8 100644 --- a/canaille/backends/__init__.py +++ b/canaille/backends/__init__.py @@ -63,7 +63,7 @@ def __init__(self, config): def instance(cls): return cls._instance - def init_app(self, app): + def init_app(self, app, init_backend=None): @app.before_request def before_request(): return self.setup() @@ -79,7 +79,7 @@ def session(self, *args, **kwargs): self.teardown() @classmethod - def install(self, config): + def install(self, app): """Prepare the database to host canaille data.""" raise NotImplementedError() @@ -203,7 +203,7 @@ def register_models(self): models.register(getattr(backend_models, model_name)) -def setup_backend(app, backend=None): +def setup_backend(app, backend=None, init_backend=None): if not backend: prefix = "CANAILLE_" available_backends_names = [ @@ -224,7 +224,7 @@ def setup_backend(app, backend=None): module, f"{backend_name.title()}Backend", None ) or getattr(module, f"{backend_name.upper()}Backend", None) backend = backend_class(app.config) - backend.init_app(app) + backend.init_app(app, init_backend) with app.app_context(): g.backend = backend diff --git a/canaille/backends/ldap/backend.py b/canaille/backends/ldap/backend.py index f213c0ef..2ab45eab 100644 --- a/canaille/backends/ldap/backend.py +++ b/canaille/backends/ldap/backend.py @@ -61,9 +61,9 @@ def __init__(self, config): setup_ldap_models(config) @classmethod - def install(cls, config): - cls.setup_schemas(config) - with cls(config).session(): + def install(cls, app): + cls.setup_schemas(app.config) + with cls(app.config).session(): models.Token.install() models.AuthorizationCode.install() models.Client.install() diff --git a/canaille/backends/memory/backend.py b/canaille/backends/memory/backend.py index 46c2fc5a..34f6747b 100644 --- a/canaille/backends/memory/backend.py +++ b/canaille/backends/memory/backend.py @@ -41,7 +41,7 @@ def attribute_index(self, model, attribute="id"): ) @classmethod - def install(cls, config): + def install(cls, app): pass def setup(self): diff --git a/canaille/backends/sql/backend.py b/canaille/backends/sql/backend.py index 429d644f..fb3c5eaa 100644 --- a/canaille/backends/sql/backend.py +++ b/canaille/backends/sql/backend.py @@ -1,6 +1,8 @@ import datetime +from pathlib import Path from flask import current_app +from flask_alembic import Alembic from sqlalchemy import create_engine from sqlalchemy import or_ from sqlalchemy import select @@ -15,14 +17,6 @@ Base = declarative_base() -def db_session(db_uri=None, init=False): - engine = create_engine(db_uri, echo=False, future=True) - if init: - Base.metadata.create_all(engine) - session = Session(engine) - return session - - class SQLModelEncoder(ModelEncoder): def default(self, obj): if isinstance(obj, Password): @@ -31,24 +25,45 @@ def default(self, obj): class SQLBackend(Backend): + engine = None db_session = None json_encoder = SQLModelEncoder + alembic = None + + def __init__(self, config): + super().__init__(config) + SQLBackend.engine = create_engine( + self.config["CANAILLE_SQL"]["DATABASE_URI"], echo=False, future=True + ) + SQLBackend.alembic = Alembic(metadatas=Base.metadata, engines=SQLBackend.engine) @classmethod - def install(cls, config): # pragma: no cover - engine = create_engine( - config["CANAILLE_SQL"]["DATABASE_URI"], - echo=False, - future=True, + def install(cls, app): # pragma: no cover + cls.init_alembic(app) + SQLBackend.alembic.upgrade() + + @classmethod + def init_alembic(cls, app): + app.config["ALEMBIC"] = { + "script_location": str(Path(__file__).resolve().parent / "migrations"), + } + SQLBackend.alembic.init_app(app) + + def init_app(self, app, init_backend=None): + super().init_app(app) + self.init_alembic(app) + init_backend = ( + app.config["CANAILLE_SQL"]["AUTO_MIGRATE"] + if init_backend is None + else init_backend ) - Base.metadata.create_all(engine) + if init_backend: # pragma: no cover + with app.app_context(): + self.alembic.upgrade() - def setup(self, init=False): + def setup(self): if not self.db_session: - self.db_session = db_session( - self.config["CANAILLE_SQL"]["DATABASE_URI"], - init=init, - ) + self.db_session = Session(SQLBackend.engine) def teardown(self): pass diff --git a/canaille/backends/sql/configuration.py b/canaille/backends/sql/configuration.py index 160be45d..894eefde 100644 --- a/canaille/backends/sql/configuration.py +++ b/canaille/backends/sql/configuration.py @@ -22,3 +22,14 @@ class SQLSettings(BaseModel): Defines password hashing scheme in SQL database. examples : "mssql2000", "ldap_salted_sha1", "pbkdf2_sha512" """ + + AUTO_MIGRATE: bool = True + """Whether to automatically apply database migrations. + + If :data:`True`, database migrations will be automatically applied when Canaille web application is launched. + If :data:`False`, migrations must be applied manually with ``canaille db upgrade``. + + .. note:: + + When running the CLI, migrations will never be applied. + """ diff --git a/canaille/backends/sql/migrations/1736443094_init.py b/canaille/backends/sql/migrations/1736443094_init.py new file mode 100644 index 00000000..6bb9e5f3 --- /dev/null +++ b/canaille/backends/sql/migrations/1736443094_init.py @@ -0,0 +1,313 @@ +"""initial migration + +Represents the state of the database in version 0.0.56 + +Revision ID: 1736443094 +Revises: +Create Date: 2025-01-09 18:18:14.276914 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +import sqlalchemy_utils.types.password +from alembic import op + +import canaille.backends.sql.utils + +# revision identifiers, used by Alembic. +revision: str = "1736443094" +down_revision: str | None = None +branch_labels: str | Sequence[str] | None = ("default",) +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "client", + sa.Column("id", sa.String(), nullable=False), + sa.Column( + "created", + canaille.backends.sql.utils.TZDateTime(timezone=True), + nullable=True, + ), + sa.Column( + "last_modified", + canaille.backends.sql.utils.TZDateTime(timezone=True), + nullable=True, + ), + sa.Column("description", sa.String(), nullable=True), + sa.Column("preconsent", sa.Boolean(), nullable=True), + sa.Column("post_logout_redirect_uris", sa.JSON(), nullable=True), + sa.Column("client_id", sa.String(), nullable=True), + sa.Column("client_secret", sa.String(), nullable=True), + sa.Column( + "client_id_issued_at", + canaille.backends.sql.utils.TZDateTime(timezone=True), + nullable=True, + ), + sa.Column( + "client_secret_expires_at", + canaille.backends.sql.utils.TZDateTime(timezone=True), + nullable=True, + ), + sa.Column("client_name", sa.String(), nullable=True), + sa.Column("contacts", sa.JSON(), nullable=True), + sa.Column("client_uri", sa.String(), nullable=True), + sa.Column("redirect_uris", sa.JSON(), nullable=True), + sa.Column("logo_uri", sa.String(), nullable=True), + sa.Column("grant_types", sa.JSON(), nullable=True), + sa.Column("response_types", sa.JSON(), nullable=True), + sa.Column("scope", sa.JSON(), nullable=True), + sa.Column("tos_uri", sa.String(), nullable=True), + sa.Column("policy_uri", sa.String(), nullable=True), + sa.Column("jwks_uri", sa.String(), nullable=True), + sa.Column("jwk", sa.String(), nullable=True), + sa.Column("token_endpoint_auth_method", sa.String(), nullable=True), + sa.Column("software_id", sa.String(), nullable=True), + sa.Column("software_version", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "group", + sa.Column("id", sa.String(), nullable=False), + sa.Column( + "created", + canaille.backends.sql.utils.TZDateTime(timezone=True), + nullable=True, + ), + sa.Column( + "last_modified", + canaille.backends.sql.utils.TZDateTime(timezone=True), + nullable=True, + ), + sa.Column("display_name", sa.String(), nullable=False), + sa.Column("description", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "user", + sa.Column("id", sa.String(), nullable=False), + sa.Column( + "created", + canaille.backends.sql.utils.TZDateTime(timezone=True), + nullable=True, + ), + sa.Column( + "last_modified", + canaille.backends.sql.utils.TZDateTime(timezone=True), + nullable=True, + ), + sa.Column("user_name", sa.String(), nullable=False), + sa.Column( + "password", + sqlalchemy_utils.types.password.PasswordType(max_length=4096), + nullable=True, + ), + sa.Column("preferred_language", sa.String(), nullable=True), + sa.Column("family_name", sa.String(), nullable=True), + sa.Column("given_name", sa.String(), nullable=True), + sa.Column("formatted_name", sa.String(), nullable=True), + sa.Column("display_name", sa.String(), nullable=True), + sa.Column("emails", sa.JSON(), nullable=True), + sa.Column("phone_numbers", sa.JSON(), nullable=True), + sa.Column("formatted_address", sa.String(), nullable=True), + sa.Column("street", sa.String(), nullable=True), + sa.Column("postal_code", sa.String(), nullable=True), + sa.Column("locality", sa.String(), nullable=True), + sa.Column("region", sa.String(), nullable=True), + sa.Column("photo", sa.LargeBinary(), nullable=True), + sa.Column("profile_url", sa.String(), nullable=True), + sa.Column("employee_number", sa.String(), nullable=True), + sa.Column("department", sa.String(), nullable=True), + sa.Column("title", sa.String(), nullable=True), + sa.Column("organization", sa.String(), nullable=True), + sa.Column( + "lock_date", + canaille.backends.sql.utils.TZDateTime(timezone=True), + nullable=True, + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("user_name"), + ) + op.create_table( + "authorization_code", + sa.Column("id", sa.String(), nullable=False), + sa.Column( + "created", + canaille.backends.sql.utils.TZDateTime(timezone=True), + nullable=True, + ), + sa.Column( + "last_modified", + canaille.backends.sql.utils.TZDateTime(timezone=True), + nullable=True, + ), + sa.Column("authorization_code_id", sa.String(), nullable=True), + sa.Column("code", sa.String(), nullable=True), + sa.Column("client_id", sa.String(), nullable=False), + sa.Column("subject_id", sa.String(), nullable=False), + sa.Column("redirect_uri", sa.String(), nullable=True), + sa.Column("response_type", sa.String(), nullable=True), + sa.Column("scope", sa.JSON(), nullable=True), + sa.Column("nonce", sa.String(), nullable=True), + sa.Column( + "issue_date", + canaille.backends.sql.utils.TZDateTime(timezone=True), + nullable=True, + ), + sa.Column("lifetime", sa.Integer(), nullable=True), + sa.Column("challenge", sa.String(), nullable=True), + sa.Column("challenge_method", sa.String(), nullable=True), + sa.Column( + "revokation_date", + canaille.backends.sql.utils.TZDateTime(timezone=True), + nullable=True, + ), + sa.ForeignKeyConstraint( + ["client_id"], + ["client.id"], + ), + sa.ForeignKeyConstraint( + ["subject_id"], + ["user.id"], + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "client_audience_association_table", + sa.Column("audience_id", sa.String(), nullable=True), + sa.Column("client_id", sa.String(), nullable=True), + sa.ForeignKeyConstraint( + ["audience_id"], + ["client.id"], + ), + sa.ForeignKeyConstraint( + ["client_id"], + ["client.id"], + ), + sa.PrimaryKeyConstraint("audience_id", "client_id"), + ) + op.create_table( + "consent", + sa.Column("id", sa.String(), nullable=False), + sa.Column( + "created", + canaille.backends.sql.utils.TZDateTime(timezone=True), + nullable=True, + ), + sa.Column( + "last_modified", + canaille.backends.sql.utils.TZDateTime(timezone=True), + nullable=True, + ), + sa.Column("consent_id", sa.String(), nullable=True), + sa.Column("subject_id", sa.String(), nullable=False), + sa.Column("client_id", sa.String(), nullable=False), + sa.Column("scope", sa.JSON(), nullable=True), + sa.Column( + "issue_date", + canaille.backends.sql.utils.TZDateTime(timezone=True), + nullable=True, + ), + sa.Column( + "revokation_date", + canaille.backends.sql.utils.TZDateTime(timezone=True), + nullable=True, + ), + sa.ForeignKeyConstraint( + ["client_id"], + ["client.id"], + ), + sa.ForeignKeyConstraint( + ["subject_id"], + ["user.id"], + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "membership_association_table", + sa.Column("user_id", sa.String(), nullable=False), + sa.Column("group_id", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["group_id"], + ["group.id"], + ), + sa.ForeignKeyConstraint( + ["user_id"], + ["user.id"], + ), + sa.PrimaryKeyConstraint("user_id", "group_id"), + ) + op.create_table( + "token", + sa.Column("id", sa.String(), nullable=False), + sa.Column( + "created", + canaille.backends.sql.utils.TZDateTime(timezone=True), + nullable=True, + ), + sa.Column( + "last_modified", + canaille.backends.sql.utils.TZDateTime(timezone=True), + nullable=True, + ), + sa.Column("token_id", sa.String(), nullable=True), + sa.Column("access_token", sa.String(), nullable=True), + sa.Column("client_id", sa.String(), nullable=False), + sa.Column("subject_id", sa.String(), nullable=False), + sa.Column("type", sa.String(), nullable=True), + sa.Column("refresh_token", sa.String(), nullable=True), + sa.Column("scope", sa.JSON(), nullable=True), + sa.Column( + "issue_date", + canaille.backends.sql.utils.TZDateTime(timezone=True), + nullable=True, + ), + sa.Column("lifetime", sa.Integer(), nullable=True), + sa.Column( + "revokation_date", + canaille.backends.sql.utils.TZDateTime(timezone=True), + nullable=True, + ), + sa.ForeignKeyConstraint( + ["client_id"], + ["client.id"], + ), + sa.ForeignKeyConstraint( + ["subject_id"], + ["user.id"], + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "token_audience_association_table", + sa.Column("token_id", sa.String(), nullable=True), + sa.Column("client_id", sa.String(), nullable=True), + sa.ForeignKeyConstraint( + ["client_id"], + ["client.id"], + ), + sa.ForeignKeyConstraint( + ["token_id"], + ["token.id"], + ), + sa.PrimaryKeyConstraint("token_id", "client_id"), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("token_audience_association_table") + op.drop_table("token") + op.drop_table("membership_association_table") + op.drop_table("consent") + op.drop_table("client_audience_association_table") + op.drop_table("authorization_code") + op.drop_table("user") + op.drop_table("group") + op.drop_table("client") + # ### end Alembic commands ### diff --git a/canaille/backends/sql/migrations/1736443538_0_0_58.py b/canaille/backends/sql/migrations/1736443538_0_0_58.py new file mode 100644 index 00000000..9b04784b --- /dev/null +++ b/canaille/backends/sql/migrations/1736443538_0_0_58.py @@ -0,0 +1,76 @@ +"""0.0.58 + +Revision ID: 1736443538 +Revises: 1736443094 +Create Date: 2025-01-09 18:25:38.443578 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +import canaille.backends.sql.utils + +# revision identifiers, used by Alembic. +revision: str = "1736443538" +down_revision: str | None = "1736443094" +branch_labels: str | Sequence[str] | None = () +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("token") as batch_op: + batch_op.alter_column("subject_id", existing_type=sa.VARCHAR(), nullable=True) + + op.add_column( + "user", + sa.Column( + "password_last_update", + canaille.backends.sql.utils.TZDateTime(timezone=True), + nullable=True, + ), + ) + op.add_column( + "user", sa.Column("_password_failure_timestamps", sa.JSON(), nullable=True) + ) + op.add_column( + "user", + sa.Column( + "last_otp_login", + canaille.backends.sql.utils.TZDateTime(timezone=True), + nullable=True, + ), + ) + op.add_column("user", sa.Column("secret_token", sa.String(), nullable=True)) + op.add_column("user", sa.Column("hotp_counter", sa.Integer(), nullable=True)) + op.add_column("user", sa.Column("one_time_password", sa.String(), nullable=True)) + op.add_column( + "user", + sa.Column( + "one_time_password_emission_date", + canaille.backends.sql.utils.TZDateTime(timezone=True), + nullable=True, + ), + ) + with op.batch_alter_table("user") as batch_op: + batch_op.create_unique_constraint("uq_user_secret_token", ["secret_token"]) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("user") as batch_op: + batch_op.drop_constraint("uq_user_secret_token", type_="unique") + op.drop_column("user", "one_time_password_emission_date") + op.drop_column("user", "one_time_password") + op.drop_column("user", "hotp_counter") + op.drop_column("user", "secret_token") + op.drop_column("user", "last_otp_login") + op.drop_column("user", "_password_failure_timestamps") + op.drop_column("user", "password_last_update") + with op.batch_alter_table("token") as batch_op: + batch_op.alter_column("subject_id", existing_type=sa.VARCHAR(), nullable=False) + # ### end Alembic commands ### diff --git a/canaille/backends/sql/migrations/script.py.mako b/canaille/backends/sql/migrations/script.py.mako new file mode 100644 index 00000000..669173c0 --- /dev/null +++ b/canaille/backends/sql/migrations/script.py.mako @@ -0,0 +1,29 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from collections.abc import Sequence + +import sqlalchemy as sa +import sqlalchemy_utils.types.password +from alembic import op + +import canaille.backends.sql.utils +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/canaille/commands.py b/canaille/commands.py index 3f62e540..28e3a76e 100644 --- a/canaille/commands.py +++ b/canaille/commands.py @@ -12,9 +12,14 @@ version = importlib.metadata.version("canaille") +def create_cli_app(): # pragma: no cover + # Force the non-application of migrations + return create_app(init_backend=False) + + @click.group( cls=FlaskGroup, - create_app=create_app, + create_app=create_cli_app, add_version_option=False, add_default_commands=False, ) diff --git a/canaille/migrations/script.py.mako b/canaille/migrations/script.py.mako new file mode 100644 index 00000000..fbc4b07d --- /dev/null +++ b/canaille/migrations/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/demo/demoapp.py b/demo/demoapp.py index 5c0544ad..b3a61e90 100644 --- a/demo/demoapp.py +++ b/demo/demoapp.py @@ -15,7 +15,7 @@ def populate(app): from canaille.core.populate import fake_users with app.app_context(): - app.backend.install(app.config) + app.backend.install(app) with app.backend.session(): if app.backend.query(models.User): return diff --git a/doc/conf.py b/doc/conf.py index 06754e76..16e90b84 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -61,6 +61,7 @@ def __getattr__(cls, name): "python": ("https://docs.python.org/3", None), "authlib": ("https://docs.authlib.org/en/latest", None), "flask": ("https://flask.palletsprojects.com", None), + "flask-alembic": ("https://flask-alembic.readthedocs.io/en/latest", None), "flask-babel": ("https://python-babel.github.io/flask-babel", None), "flask-wtf": ("https://flask-wtf.readthedocs.io", None), "pydantic": ("https://docs.pydantic.dev/latest", None), diff --git a/doc/tutorial/databases.rst b/doc/tutorial/databases.rst index 55d66a24..f76f4fe9 100644 --- a/doc/tutorial/databases.rst +++ b/doc/tutorial/databases.rst @@ -18,6 +18,9 @@ SQL Canaille can use any database supported by `SQLAlchemy `_, such as sqlite, postgresql or mariadb. +Configuration +------------- + It is used when the ``CANAILLE_SQL`` configuration parameter is defined. For instance: .. code-block:: toml @@ -28,6 +31,16 @@ It is used when the ``CANAILLE_SQL`` configuration parameter is defined. For ins You can find more details on the SQL configuration in the :class:`dedicated section `. +Migrations +---------- + +By default, migrations are applied when you run the web application. +You can disable this behavior with the :attr:`~canaille.backends.sql.configuration.SQLSettings.AUTO_MIGRATE` setting. +Migrations are not automatically applied with the use of the CLI though. + +Migrations are done with :doc:`flask-alembic `, that provides a dedicated CLI to manually tune migrations. +You can check the :doc:`flask-alembic documentation ` and the ``canaille db`` command line if you are in trouble. + LDAP ==== diff --git a/pyproject.toml b/pyproject.toml index b4fdec8b..895e1d62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,7 @@ sentry = [ ] sqlite = [ + "flask-alembic>=3.1.1", "passlib >= 1.7.4", "sqlalchemy >= 2.0.23", "sqlalchemy-json >= 0.7.0", @@ -73,6 +74,7 @@ sqlite = [ ] postgresql = [ + "flask-alembic>=3.1.1", "passlib >= 1.7.4", "sqlalchemy[postgresql-psycopg2binary] >= 2.0.23", "sqlalchemy-json >= 0.7.0", @@ -80,6 +82,7 @@ postgresql = [ ] mysql = [ + "flask-alembic>=3.1.1", "passlib >= 1.7.4", "sqlalchemy[mysql-connector] >= 2.0.23", "sqlalchemy-json >= 0.7.0", diff --git a/tests/backends/sql/fixtures.py b/tests/backends/sql/fixtures.py index e980807f..6269e5da 100644 --- a/tests/backends/sql/fixtures.py +++ b/tests/backends/sql/fixtures.py @@ -16,5 +16,5 @@ def sql_backend(sqlalchemy_configuration): config_obj = settings_factory(sqlalchemy_configuration) config_dict = config_obj.model_dump() backend = SQLBackend(config_dict) - with backend.session(init=True): + with backend.session(): yield backend diff --git a/tests/backends/sql/test_alembic.py b/tests/backends/sql/test_alembic.py new file mode 100644 index 00000000..e2deb34f --- /dev/null +++ b/tests/backends/sql/test_alembic.py @@ -0,0 +1,4 @@ +def test_migrations(app, backend): + """Test downgrading back to the first revision, and then re-apply all migrations.""" + backend.alembic.downgrade("base") + backend.alembic.upgrade() diff --git a/tests/backends/test_backends.py b/tests/backends/test_backends.py index 2312b03a..c33ccef0 100644 --- a/tests/backends/test_backends.py +++ b/tests/backends/test_backends.py @@ -5,7 +5,7 @@ def test_required_methods(testclient): with pytest.raises(NotImplementedError): - Backend.install(config=None) + Backend.install(app=None) with pytest.raises(NotImplementedError): Backend.validate({}) diff --git a/tests/conftest.py b/tests/conftest.py index 680e7c08..a1d2d0f5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -176,8 +176,10 @@ def jinja_cache_directory(tmp_path_factory): @pytest.fixture def app(configuration, backend, jinja_cache_directory): app = create_app(configuration, backend=backend) + # caches the Jinja compiled files for faster unit test execution app.jinja_env.bytecode_cache = FileSystemBytecodeCache(jinja_cache_directory) - backend.install(app.config) + with app.app_context(): + backend.install(app) with app.test_request_context(): yield app diff --git a/uv.lock b/uv.lock index 9faa3393..d1649ba1 100644 --- a/uv.lock +++ b/uv.lock @@ -28,6 +28,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl", hash = "sha256:fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b", size = 13929 }, ] +[[package]] +name = "alembic" +version = "1.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mako" }, + { name = "sqlalchemy" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/00/1e/8cb8900ba1b6360431e46fb7a89922916d3a1b017a8908a7c0499cc7e5f6/alembic-1.14.0.tar.gz", hash = "sha256:b00892b53b3642d0b8dbedba234dbf1924b69be83a9a769d5a624b01094e304b", size = 1916172 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/06/8b505aea3d77021b18dcbd8133aa1418f1a1e37e432a465b14c46b2c0eaa/alembic-1.14.0-py3-none-any.whl", hash = "sha256:99bd884ca390466db5e27ffccff1d179ec5c05c965cfefc0607e69f9e411cb25", size = 233482 }, +] + [[package]] name = "annotated-types" version = "0.7.0" @@ -145,6 +159,7 @@ ldap = [ { name = "python-ldap" }, ] mysql = [ + { name = "flask-alembic" }, { name = "passlib" }, { name = "sqlalchemy", extra = ["mysql-connector"] }, { name = "sqlalchemy-json" }, @@ -159,6 +174,7 @@ otp = [ { name = "qrcode" }, ] postgresql = [ + { name = "flask-alembic" }, { name = "passlib" }, { name = "sqlalchemy", extra = ["postgresql-psycopg2binary"] }, { name = "sqlalchemy-json" }, @@ -175,6 +191,7 @@ sms = [ { name = "smpplib" }, ] sqlite = [ + { name = "flask-alembic" }, { name = "passlib" }, { name = "sqlalchemy" }, { name = "sqlalchemy-json" }, @@ -228,6 +245,9 @@ requires-dist = [ { name = "authlib", marker = "extra == 'scim'", specifier = ">=1.3.0" }, { name = "email-validator", marker = "extra == 'front'", specifier = ">=2.0.0" }, { name = "flask", specifier = ">=3.0.0" }, + { name = "flask-alembic", marker = "extra == 'mysql'", specifier = ">=3.1.1" }, + { name = "flask-alembic", marker = "extra == 'postgresql'", specifier = ">=3.1.1" }, + { name = "flask-alembic", marker = "extra == 'sqlite'", specifier = ">=3.1.1" }, { name = "flask-babel", marker = "extra == 'front'", specifier = ">=4.0.0" }, { name = "flask-themer", marker = "extra == 'front'", specifier = ">=2.0.0" }, { name = "flask-wtf", specifier = ">=1.2.1" }, @@ -672,6 +692,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/af/47/93213ee66ef8fae3b93b3e29206f6b251e65c97bd91d8e1c5596ef15af0a/flask-3.1.0-py3-none-any.whl", hash = "sha256:d667207822eb83f1c4b50949b1623c8fc8d51f2341d65f72e1a1815397551136", size = 102979 }, ] +[[package]] +name = "flask-alembic" +version = "3.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "alembic" }, + { name = "flask" }, + { name = "sqlalchemy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2f/08/84d5a3155bef151190bf665bc31389971daf9e82271086439bc8f8d3a7a5/flask_alembic-3.1.1.tar.gz", hash = "sha256:358a0ad8f74e2969273a8d89feaf5842f65cc909064b51fde4b51415790d92f4", size = 11943 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/32/9ecc1df856fd6ba96276393d374c8e7592c89bc1e5861c1c66d6a81b68f8/flask_alembic-3.1.1-py3-none-any.whl", hash = "sha256:1d0cda58518d4332d8563da555bee03107fe2169c7157f61a2e0759d0150209b", size = 12070 }, +] + [[package]] name = "flask-babel" version = "4.0.0" @@ -1028,6 +1062,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ba/b2/6a22fb5c0885da3b00e116aee81f0b829ec9ac8f736cd414b4a09413fc7d/lxml-5.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:6e91cf736959057f7aac7adfc83481e03615a8e8dd5758aa1d95ea69e8931dba", size = 3487557 }, ] +[[package]] +name = "mako" +version = "1.3.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5f/d9/8518279534ed7dace1795d5a47e49d5299dd0994eed1053996402a8902f9/mako-1.3.8.tar.gz", hash = "sha256:577b97e414580d3e088d47c2dbbe9594aa7a5146ed2875d4dfa9075af2dd3cc8", size = 392069 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/bf/7a6a36ce2e4cafdfb202752be68850e22607fccd692847c45c1ae3c17ba6/Mako-1.3.8-py3-none-any.whl", hash = "sha256:42f48953c7eb91332040ff567eb7eea69b22e7a4affbc5ba8e845e8f730f6627", size = 78569 }, +] + [[package]] name = "markupsafe" version = "3.0.2"