From 1c69449179c59decada485d99ac28b8144f979d1 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 28 Sep 2023 09:08:42 -0400 Subject: [PATCH 1/5] Bail during start-up if there are old background updates. --- synapse/storage/prepare_database.py | 32 ++++++++++++++++++++++++++++- synapse/storage/schema/__init__.py | 11 ++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index 31501fd57331..927525327725 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -32,7 +32,11 @@ from synapse.config.homeserver import HomeServerConfig from synapse.storage.database import LoggingDatabaseConnection, LoggingTransaction from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine -from synapse.storage.schema import SCHEMA_COMPAT_VERSION, SCHEMA_VERSION +from synapse.storage.schema import ( + BACKGROUND_UPDATES_COMPAT_VERSION, + SCHEMA_COMPAT_VERSION, + SCHEMA_VERSION, +) from synapse.storage.types import Cursor logger = logging.getLogger(__name__) @@ -80,6 +84,9 @@ class _SchemaState: applied_deltas: Collection[str] = attr.ib(factory=tuple) """Any delta files for `current_version` which have already been applied""" + background_updates: Collection[Tuple[str, int]] = attr.ib(factory=tuple) + """Any (pending) updates in the `background_updates` table.""" + upgraded: bool = attr.ib(default=False) """Whether the current state was reached by applying deltas. @@ -359,6 +366,7 @@ def _upgrade_existing_database( """ if is_empty: assert not current_schema_state.applied_deltas + assert not current_schema_state.background_updates else: assert config @@ -413,6 +421,24 @@ def _upgrade_existing_database( start_ver += 1 logger.debug("applied_delta_files: %s", current_schema_state.applied_deltas) + logger.debug( + "pending background_updates: %s", + (name for name, ordering in current_schema_state.background_updates), + ) + + # Bail if there are any pending background updates from before the background schema compat version. + for update_name, ordering in sorted( + current_schema_state.background_updates, key=lambda b: b[1] + ): + # ordering is an int based on when the background update was added: + # + # (schema version when added * 100) + (schema delta when added). + update_schema_version = ordering // 100 + if update_schema_version < BACKGROUND_UPDATES_COMPAT_VERSION: + raise UpgradeDatabaseException( + "Database has old pending background updates for version %d: %s" + % (update_schema_version, update_name) + ) if isinstance(database_engine, PostgresEngine): specific_engine_extension = ".postgres" @@ -705,10 +731,14 @@ def _get_or_create_schema_state( ) applied_deltas = tuple(d for d, in txn) + txn.execute("SELECT update_name, ordering FROM background_updates") + background_Updates = tuple(txn) + return _SchemaState( current_version=current_version, compat_version=compat_version, applied_deltas=applied_deltas, + background_updates=background_Updates, upgraded=upgraded, ) diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index 5b50bd66bcb3..3bf9eeeca90c 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -133,3 +133,14 @@ This value is stored in the database, and checked on startup. If the value in the database is greater than SCHEMA_VERSION, then Synapse will refuse to start. """ + +BACKGROUND_UPDATES_COMPAT_VERSION = ( + # The replace_stream_ordering_column from 6001 must have run. + 61 +) +"""Limit on how far the syanpse can be rolled forward without breaking db compat + +This value is checked on startup against any pending background updates. If there +are any pending background updates less than BACKGROUND_UPDATES_COMPAT_VERSION, then +Synapse will refuse to start. +""" From e1813f21ce3be29c3e14dd490b7cc5c885887bbe Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 28 Sep 2023 09:17:33 -0400 Subject: [PATCH 2/5] Add warning about interaction with full dumps. --- synapse/storage/schema/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index 3bf9eeeca90c..eb461a151cb7 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -143,4 +143,7 @@ This value is checked on startup against any pending background updates. If there are any pending background updates less than BACKGROUND_UPDATES_COMPAT_VERSION, then Synapse will refuse to start. + +In order to work with *new* databases this *must* be smaller than the latest full +dump of the database. """ From c26d6ffd3fcd700533c5a30badf94b106259f8c5 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 28 Sep 2023 09:57:24 -0400 Subject: [PATCH 3/5] Newsfragment --- changelog.d/16397.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/16397.bugfix diff --git a/changelog.d/16397.bugfix b/changelog.d/16397.bugfix new file mode 100644 index 000000000000..e4b27221a56e --- /dev/null +++ b/changelog.d/16397.bugfix @@ -0,0 +1 @@ +Enforce that old background updates have run when starting Synapse. From 75576c151c57d27f2667e023171d9200f6c9f6e2 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 28 Sep 2023 10:13:23 -0400 Subject: [PATCH 4/5] Fix-up types. --- synapse/storage/prepare_database.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index 927525327725..598ded33b5e1 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -25,6 +25,7 @@ Optional, TextIO, Tuple, + cast, ) import attr @@ -732,7 +733,7 @@ def _get_or_create_schema_state( applied_deltas = tuple(d for d, in txn) txn.execute("SELECT update_name, ordering FROM background_updates") - background_Updates = tuple(txn) + background_Updates = cast(Tuple[Tuple[str, int], ...], tuple(txn)) return _SchemaState( current_version=current_version, From aae2a3867102275c619b2d3d324fd0d3fc801020 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Mon, 16 Oct 2023 16:03:51 -0400 Subject: [PATCH 5/5] Documentation. --- docs/development/database_schema.md | 31 ++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/docs/development/database_schema.md b/docs/development/database_schema.md index 37a06acc1282..9fedebda6163 100644 --- a/docs/development/database_schema.md +++ b/docs/development/database_schema.md @@ -25,20 +25,37 @@ updated. They work as follows: * The Synapse codebase defines a constant `synapse.storage.schema.SCHEMA_VERSION` which represents the expectations made about the database by that version. For - example, as of Synapse v1.36, this is `59`. + example, as of Synapse v1.36, this is `59`. This version should be incremented + whenever a backwards-incompatible change is made to the database format (normally + via a `delta` file) + + * The Synapse codebase defines a constant `synapse.storage.schema.SCHEMA_COMPAT_VERSION` + which represents the minimum database versions the current code supports. + Whenever the Synapse code is updated to assume backwards-incompatible changes + made to the database format, `synapse.storage.schema.SCHEMA_COMPAT_VERSION` is also updated + so that administrators can not accidentally roll back to a too-old version of Synapse. - * The database stores a "compatibility version" in + The database stores a "compatibility version" in `schema_compat_version.compat_version` which defines the `SCHEMA_VERSION` of the oldest version of Synapse which will work with the database. On startup, if `compat_version` is found to be newer than `SCHEMA_VERSION`, Synapse will refuse to start. - Synapse automatically updates this field from - `synapse.storage.schema.SCHEMA_COMPAT_VERSION`. + Synapse automatically updates `schema_compat_version.compat_version` from + `synapse.storage.schema.SCHEMA_COMPAT_VERSION` during start-up. - * Whenever a backwards-incompatible change is made to the database format (normally - via a `delta` file), `synapse.storage.schema.SCHEMA_COMPAT_VERSION` is also updated - so that administrators can not accidentally roll back to a too-old version of Synapse. + * The Synapse codebase defines a constant `synapse.storage.schema.BACKGROUND_UPDATES_COMPAT_VERSION` + which represents the earliest supported background updates. + + On startup, if there exists any background update (via the + `background_updates.ordering` column) older than `BACKGROUND_UPDATES_COMPAT_VERSION`, + Synpase will refuse to start. + + This is useful for adding delta files which assume background updates have + finished; overall maintenance of Synapse (by allowing for removal of code + supporting old background updates); among other things. + + `BACKGROUND_UPDATES_COMPAT_VERSION` must be < the latest [full schema dump](#full-schema-dumps). Generally, the goal is to maintain compatibility with at least one or two previous releases of Synapse, so any substantial change tends to require multiple releases and a