From 78259ce955c74d023d2f887e9cadfee66805254c Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Wed, 20 Nov 2024 18:08:18 +0100 Subject: [PATCH 01/14] Fix ORM vs migration files inconsistencies There have been some inconsistences between ORM and migration files but it doesn't fail in tests. This is an attempt to fix the inconsistency and also have it fail in tests --- .../versions/0033_3_0_0_add_tables_for_backfill.py | 8 ++++++-- .../0036_3_0_0_add_name_field_to_dataset_model.py | 4 +--- .../0040_3_0_0_add_exception_reason_and_logical_date_.py | 2 +- airflow/models/__init__.py | 1 - airflow/models/asset.py | 2 +- docs/apache-airflow/img/airflow_erd.sha256 | 2 +- scripts/in_container/run_generate_migration.sh | 4 +++- tests_common/test_utils/db.py | 2 ++ 8 files changed, 15 insertions(+), 10 deletions(-) diff --git a/airflow/migrations/versions/0033_3_0_0_add_tables_for_backfill.py b/airflow/migrations/versions/0033_3_0_0_add_tables_for_backfill.py index bcbaa8b89a0ca..5ac391413e946 100644 --- a/airflow/migrations/versions/0033_3_0_0_add_tables_for_backfill.py +++ b/airflow/migrations/versions/0033_3_0_0_add_tables_for_backfill.py @@ -46,10 +46,10 @@ def upgrade(): op.create_table( "backfill", sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), - sa.Column("dag_id", sa.String(length=250), nullable=True), + sa.Column("dag_id", sa.String(length=250), nullable=False), sa.Column("from_date", airflow.utils.sqlalchemy.UtcDateTime(timezone=True), nullable=False), sa.Column("to_date", airflow.utils.sqlalchemy.UtcDateTime(timezone=True), nullable=False), - sa.Column("dag_run_conf", sqlalchemy_jsonfield.jsonfield.JSONField(), nullable=True), + sa.Column("dag_run_conf", sqlalchemy_jsonfield.jsonfield.JSONField(), nullable=False), sa.Column("is_paused", sa.Boolean(), nullable=True), sa.Column("max_active_runs", sa.Integer(), nullable=False), sa.Column("created_at", airflow.utils.sqlalchemy.UtcDateTime(timezone=True), nullable=False), @@ -65,6 +65,10 @@ def upgrade(): sa.Column("sort_ordinal", sa.Integer(), nullable=False), sa.PrimaryKeyConstraint("id", name=op.f("backfill_dag_run_pkey")), sa.UniqueConstraint("backfill_id", "dag_run_id", name="ix_bdr_backfill_id_dag_run_id"), + sa.ForeignKeyConstraint( + ["backfill_id"], ["backfill.id"], name="bdr_backfill_fkey", ondelete="cascade" + ), + sa.ForeignKeyConstraint(["dag_run_id"], ["dag_run.id"], name="bdr_dag_run_fkey", ondelete="set null"), ) diff --git a/airflow/migrations/versions/0036_3_0_0_add_name_field_to_dataset_model.py b/airflow/migrations/versions/0036_3_0_0_add_name_field_to_dataset_model.py index 353dcbf0f8fdd..2676176692a81 100644 --- a/airflow/migrations/versions/0036_3_0_0_add_name_field_to_dataset_model.py +++ b/airflow/migrations/versions/0036_3_0_0_add_name_field_to_dataset_model.py @@ -61,9 +61,7 @@ def upgrade(): # Add 'name' column. Set it to nullable for now. with op.batch_alter_table("dataset", schema=None) as batch_op: batch_op.add_column(sa.Column("name", _STRING_COLUMN_TYPE)) - batch_op.add_column( - sa.Column("group", _STRING_COLUMN_TYPE, default=str, server_default="", nullable=False) - ) + batch_op.add_column(sa.Column("group", _STRING_COLUMN_TYPE, default="", nullable=False)) # Fill name from uri column. with Session(bind=op.get_bind()) as session: session.execute(sa.text("update dataset set name=uri")) diff --git a/airflow/migrations/versions/0040_3_0_0_add_exception_reason_and_logical_date_.py b/airflow/migrations/versions/0040_3_0_0_add_exception_reason_and_logical_date_.py index c4f96fb0ebf35..ab1a060af1d40 100644 --- a/airflow/migrations/versions/0040_3_0_0_add_exception_reason_and_logical_date_.py +++ b/airflow/migrations/versions/0040_3_0_0_add_exception_reason_and_logical_date_.py @@ -42,7 +42,7 @@ def upgrade(): """Apply Add exception_reason and logical_date to BackfillDagRun.""" with op.batch_alter_table("backfill", schema=None) as batch_op: - batch_op.add_column(sa.Column("reprocess_behavior", sa.String(length=250), nullable=True)) + batch_op.add_column(sa.Column("reprocess_behavior", sa.String(length=250), nullable=False)) with op.batch_alter_table("backfill_dag_run", schema=None) as batch_op: batch_op.add_column(sa.Column("exception_reason", sa.String(length=250), nullable=True)) batch_op.add_column(sa.Column("logical_date", UtcDateTime(timezone=True), nullable=False)) diff --git a/airflow/models/__init__.py b/airflow/models/__init__.py index 6d8803410532a..731c7d60da4a3 100644 --- a/airflow/models/__init__.py +++ b/airflow/models/__init__.py @@ -63,7 +63,6 @@ def import_all_models(): import airflow.models.serialized_dag import airflow.models.taskinstancehistory import airflow.models.tasklog - import airflow.providers.fab.auth_manager.models def __getattr__(name): diff --git a/airflow/models/asset.py b/airflow/models/asset.py index d47986a85e560..c3c61b2785c88 100644 --- a/airflow/models/asset.py +++ b/airflow/models/asset.py @@ -119,7 +119,7 @@ class AssetAliasModel(Base): ), "mysql", ), - default=str, + default="", nullable=False, ) diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index 6b004f91298b9..c2c67e6872f58 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1 @@ -aa9e2e5b2a52af1e92bc876727ad5e8958e291315096fc5249a9afa2c21a5d06 \ No newline at end of file +2f8a9fc0bb125d3103c89af0afd0c33c041055243dcd6bc3de9682e544e180a2 \ No newline at end of file diff --git a/scripts/in_container/run_generate_migration.sh b/scripts/in_container/run_generate_migration.sh index 50a6985513247..46560c9112272 100755 --- a/scripts/in_container/run_generate_migration.sh +++ b/scripts/in_container/run_generate_migration.sh @@ -20,5 +20,7 @@ cd "${AIRFLOW_SOURCES}" || exit 1 cd "airflow" || exit 1 -airflow db reset +airflow db reset -y +airflow db downgrade -n 2.10.3 -y +airflow db migrate -r heads alembic revision --autogenerate -m "${@}" diff --git a/tests_common/test_utils/db.py b/tests_common/test_utils/db.py index 9b8f2646dd6fa..c684174da0d8e 100644 --- a/tests_common/test_utils/db.py +++ b/tests_common/test_utils/db.py @@ -74,6 +74,8 @@ def initial_db_init(): from airflow.www.extensions.init_auth_manager import get_auth_manager db.resetdb() + db.downgrade(to_revision="044f740568ec") + db.upgradedb(to_revision="head") _bootstrap_dagbag() # minimal app to add roles flask_app = Flask(__name__) From f42acfd759c2fb066b6493dec964e504872309be Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Thu, 21 Nov 2024 15:42:16 +0100 Subject: [PATCH 02/14] fix for mysql and postgres --- ..._log_table_and_change_event_name_length.py | 4 +- .../0028_3_0_0_drop_ab_user_id_foreign_key.py | 14 ++++++ .../0032_3_0_0_drop_execution_date_unique.py | 14 +++++- ..._add_uuid_primary_key_to_task_instance_.py | 46 +++++++++++-------- ...0_0_remove_pickled_data_from_xcom_table.py | 5 +- docs/apache-airflow/img/airflow_erd.sha256 | 2 +- tests_common/test_utils/db.py | 2 +- 7 files changed, 61 insertions(+), 26 deletions(-) diff --git a/airflow/migrations/versions/0010_2_9_0_add_run_id_to_audit_log_table_and_change_event_name_length.py b/airflow/migrations/versions/0010_2_9_0_add_run_id_to_audit_log_table_and_change_event_name_length.py index 44bef77578b65..22b9c4337811b 100644 --- a/airflow/migrations/versions/0010_2_9_0_add_run_id_to_audit_log_table_and_change_event_name_length.py +++ b/airflow/migrations/versions/0010_2_9_0_add_run_id_to_audit_log_table_and_change_event_name_length.py @@ -59,8 +59,8 @@ def downgrade(): if conn.dialect.name == "mssql": with op.batch_alter_table("log") as batch_op: batch_op.drop_index("idx_log_event") - batch_op.alter_column("event", type_=sa.String(30), nullable=False) + batch_op.alter_column("event", type_=sa.String(30)) batch_op.create_index("idx_log_event", ["event"]) else: with op.batch_alter_table("log") as batch_op: - batch_op.alter_column("event", type_=sa.String(30), nullable=False) + batch_op.alter_column("event", type_=sa.String(30)) diff --git a/airflow/migrations/versions/0028_3_0_0_drop_ab_user_id_foreign_key.py b/airflow/migrations/versions/0028_3_0_0_drop_ab_user_id_foreign_key.py index f88aaa014bb3a..8a9c77042e95f 100644 --- a/airflow/migrations/versions/0028_3_0_0_drop_ab_user_id_foreign_key.py +++ b/airflow/migrations/versions/0028_3_0_0_drop_ab_user_id_foreign_key.py @@ -45,6 +45,13 @@ def upgrade(): with op.batch_alter_table("task_instance_note", schema=None) as batch_op: batch_op.drop_constraint("task_instance_note_user_fkey", type_="foreignkey") + if op.get_bind().dialect.name == "mysql": + with op.batch_alter_table("dag_run_note", schema=None) as batch_op: + batch_op.drop_index("dag_run_note_user_fkey") + + with op.batch_alter_table("task_instance_note", schema=None) as batch_op: + batch_op.drop_index("task_instance_note_user_fkey") + def downgrade(): """Unapply Drop ab_user.id foreign key.""" @@ -53,3 +60,10 @@ def downgrade(): with op.batch_alter_table("dag_run_note", schema=None) as batch_op: batch_op.create_foreign_key("dag_run_note_user_fkey", "ab_user", ["user_id"], ["id"]) + + if op.get_bind().dialect.name == "mysql": + with op.batch_alter_table("task_instance_note", schema=None) as batch_op: + batch_op.create_index("task_instance_note_user_fkey", ["user_id"], unique=False) + + with op.batch_alter_table("dag_run_note", schema=None) as batch_op: + batch_op.create_index("dag_run_note_user_fkey", ["user_id"], unique=False) diff --git a/airflow/migrations/versions/0032_3_0_0_drop_execution_date_unique.py b/airflow/migrations/versions/0032_3_0_0_drop_execution_date_unique.py index b76bf209bd412..c782991881079 100644 --- a/airflow/migrations/versions/0032_3_0_0_drop_execution_date_unique.py +++ b/airflow/migrations/versions/0032_3_0_0_drop_execution_date_unique.py @@ -44,14 +44,24 @@ def upgrade(): with op.batch_alter_table("dag_run", schema=None) as batch_op: - batch_op.alter_column("execution_date", new_column_name="logical_date", existing_type=sa.TIMESTAMP) + batch_op.alter_column( + "execution_date", + new_column_name="logical_date", + existing_type=sa.TIMESTAMP, + existing_nullable=False, + ) with op.batch_alter_table("dag_run", schema=None) as batch_op: batch_op.drop_constraint("dag_run_dag_id_execution_date_key", type_="unique") def downgrade(): with op.batch_alter_table("dag_run", schema=None) as batch_op: - batch_op.alter_column("logical_date", new_column_name="execution_date", existing_type=sa.TIMESTAMP) + batch_op.alter_column( + "logical_date", + new_column_name="execution_date", + existing_type=sa.TIMESTAMP, + existing_nullable=False, + ) with op.batch_alter_table("dag_run", schema=None) as batch_op: batch_op.create_unique_constraint( "dag_run_dag_id_execution_date_key", diff --git a/airflow/migrations/versions/0042_3_0_0_add_uuid_primary_key_to_task_instance_.py b/airflow/migrations/versions/0042_3_0_0_add_uuid_primary_key_to_task_instance_.py index 2abd2116f989a..41cfddc9cef0b 100644 --- a/airflow/migrations/versions/0042_3_0_0_add_uuid_primary_key_to_task_instance_.py +++ b/airflow/migrations/versions/0042_3_0_0_add_uuid_primary_key_to_task_instance_.py @@ -167,6 +167,32 @@ def _get_type_id_column(dialect_name: str) -> sa.types.TypeEngine: return sa.String(36) +def create_foreign_keys(): + for fk in ti_fk_constraints: + if fk["table"] in ["task_instance_history", "task_map"]: + continue + with op.batch_alter_table(fk["table"]) as batch_op: + batch_op.create_foreign_key( + constraint_name=fk["fk"], + referent_table=ti_table, + local_cols=ti_fk_cols, + remote_cols=ti_fk_cols, + ondelete="CASCADE", + ) + for fk in ti_fk_constraints: + if fk["table"] not in ["task_instance_history", "task_map"]: + continue + with op.batch_alter_table(fk["table"]) as batch_op: + batch_op.create_foreign_key( + constraint_name=fk["fk"], + referent_table=ti_table, + local_cols=ti_fk_cols, + remote_cols=ti_fk_cols, + ondelete="CASCADE", + onupdate="CASCADE", + ) + + def upgrade(): """Add UUID primary key to task instance table.""" conn = op.get_bind() @@ -232,15 +258,7 @@ def upgrade(): batch_op.create_primary_key("task_instance_pkey", ["id"]) # Create foreign key constraints - for fk in ti_fk_constraints: - with op.batch_alter_table(fk["table"]) as batch_op: - batch_op.create_foreign_key( - constraint_name=fk["fk"], - referent_table=ti_table, - local_cols=ti_fk_cols, - remote_cols=ti_fk_cols, - ondelete="CASCADE", - ) + create_foreign_keys() def downgrade(): @@ -270,12 +288,4 @@ def downgrade(): batch_op.create_primary_key("task_instance_pkey", ti_fk_cols) # Re-add foreign key constraints - for fk in ti_fk_constraints: - with op.batch_alter_table(fk["table"]) as batch_op: - batch_op.create_foreign_key( - constraint_name=fk["fk"], - referent_table=ti_table, - local_cols=ti_fk_cols, - remote_cols=ti_fk_cols, - ondelete="CASCADE", - ) + create_foreign_keys() diff --git a/airflow/migrations/versions/0049_3_0_0_remove_pickled_data_from_xcom_table.py b/airflow/migrations/versions/0049_3_0_0_remove_pickled_data_from_xcom_table.py index 2b19827b6ae4c..ebe8e37a71ff0 100644 --- a/airflow/migrations/versions/0049_3_0_0_remove_pickled_data_from_xcom_table.py +++ b/airflow/migrations/versions/0049_3_0_0_remove_pickled_data_from_xcom_table.py @@ -101,9 +101,9 @@ def upgrade(): op.execute( """ ALTER TABLE xcom - ALTER COLUMN value TYPE JSONB + ALTER COLUMN value TYPE JSON USING CASE - WHEN value IS NOT NULL THEN CAST(CONVERT_FROM(value, 'UTF8') AS JSONB) + WHEN value IS NOT NULL THEN CAST(CONVERT_FROM(value, 'UTF8') AS JSON) ELSE NULL END """ @@ -136,6 +136,7 @@ def upgrade(): # Drop the old `value_old` column with op.batch_alter_table("xcom", schema=None) as batch_op: batch_op.drop_column("value_old") + op.drop_table("_xcom_archive") def downgrade(): diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index c2c67e6872f58..d27f5eda6ce03 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1 @@ -2f8a9fc0bb125d3103c89af0afd0c33c041055243dcd6bc3de9682e544e180a2 \ No newline at end of file +b12f6811bb7a340362e4b8774b3bb81db28a7d0258564b3431bd537368554cc3 \ No newline at end of file diff --git a/tests_common/test_utils/db.py b/tests_common/test_utils/db.py index c684174da0d8e..8303cbd2e8dba 100644 --- a/tests_common/test_utils/db.py +++ b/tests_common/test_utils/db.py @@ -74,7 +74,7 @@ def initial_db_init(): from airflow.www.extensions.init_auth_manager import get_auth_manager db.resetdb() - db.downgrade(to_revision="044f740568ec") + db.downgrade(to_revision="5f2621c13b39") db.upgradedb(to_revision="head") _bootstrap_dagbag() # minimal app to add roles From 3318cabd257fdbde203a96064a1f67c4090325b9 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Fri, 22 Nov 2024 09:18:47 +0100 Subject: [PATCH 03/14] fixup! fix for mysql and postgres --- tests_common/test_utils/db.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests_common/test_utils/db.py b/tests_common/test_utils/db.py index 8303cbd2e8dba..c0ca875b84d2f 100644 --- a/tests_common/test_utils/db.py +++ b/tests_common/test_utils/db.py @@ -73,9 +73,12 @@ def initial_db_init(): from airflow.www.extensions.init_appbuilder import init_appbuilder from airflow.www.extensions.init_auth_manager import get_auth_manager + from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS + db.resetdb() - db.downgrade(to_revision="5f2621c13b39") - db.upgradedb(to_revision="head") + if AIRFLOW_V_3_0_PLUS: + db.downgrade(to_revision="5f2621c13b39") + db.upgradedb(to_revision="head") _bootstrap_dagbag() # minimal app to add roles flask_app = Flask(__name__) From 5e28d1310d602df7da2c1be1c014ca69974c2648 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Fri, 22 Nov 2024 12:11:56 +0100 Subject: [PATCH 04/14] fix for sqlite --- .../0041_3_0_0_rename_dataset_as_asset.py | 18 +----- ..._0_0_create_fks_as_datasets_are_renamed.py | 55 +++++++++++++++++++ ...add_uuid_primary_key_to_task_instance_.py} | 4 +- ...044_3_0_0_remove_scheduler_lock_column.py} | 0 ...py => 0045_3_0_0__drop_task_fail_table.py} | 0 ...0_add_last_heartbeat_at_directly_to_ti.py} | 0 ...ing.py => 0047_3_0_0_drop_dag_pickling.py} | 0 ...ng.py => 0048_3_0_0_add_dag_versioning.py} | 0 ...0049_3_0_0_add_trigger_asset_reference.py} | 0 ..._0_remove_pickled_data_from_xcom_table.py} | 0 docs/apache-airflow/img/airflow_erd.sha256 | 2 +- docs/apache-airflow/migrations-ref.rst | 4 +- 12 files changed, 63 insertions(+), 20 deletions(-) create mode 100644 airflow/migrations/versions/0042_3_0_0_create_fks_as_datasets_are_renamed.py rename airflow/migrations/versions/{0042_3_0_0_add_uuid_primary_key_to_task_instance_.py => 0043_3_0_0_add_uuid_primary_key_to_task_instance_.py} (99%) rename airflow/migrations/versions/{0043_3_0_0_remove_scheduler_lock_column.py => 0044_3_0_0_remove_scheduler_lock_column.py} (100%) rename airflow/migrations/versions/{0044_3_0_0__drop_task_fail_table.py => 0045_3_0_0__drop_task_fail_table.py} (100%) rename airflow/migrations/versions/{0045_3_0_0_add_last_heartbeat_at_directly_to_ti.py => 0046_3_0_0_add_last_heartbeat_at_directly_to_ti.py} (100%) rename airflow/migrations/versions/{0046_3_0_0_drop_dag_pickling.py => 0047_3_0_0_drop_dag_pickling.py} (100%) rename airflow/migrations/versions/{0047_3_0_0_add_dag_versioning.py => 0048_3_0_0_add_dag_versioning.py} (100%) rename airflow/migrations/versions/{0048_3_0_0_add_trigger_asset_reference.py => 0049_3_0_0_add_trigger_asset_reference.py} (100%) rename airflow/migrations/versions/{0049_3_0_0_remove_pickled_data_from_xcom_table.py => 0050_3_0_0_remove_pickled_data_from_xcom_table.py} (100%) diff --git a/airflow/migrations/versions/0041_3_0_0_rename_dataset_as_asset.py b/airflow/migrations/versions/0041_3_0_0_rename_dataset_as_asset.py index 03836503efe62..746ca46ec76f3 100644 --- a/airflow/migrations/versions/0041_3_0_0_rename_dataset_as_asset.py +++ b/airflow/migrations/versions/0041_3_0_0_rename_dataset_as_asset.py @@ -285,13 +285,6 @@ def upgrade(): unique=False, ) - batch_op.create_foreign_key( - constraint_name="toar_asset_fkey", - referent_table="asset", - local_cols=["asset_id"], - remote_cols=["id"], - ondelete="CASCADE", - ) batch_op.create_foreign_key( constraint_name="toar_dag_id_fkey", referent_table="dag", @@ -321,13 +314,6 @@ def upgrade(): unique=False, ) - batch_op.create_foreign_key( - constraint_name="adrq_asset_fkey", - referent_table="asset", - local_cols=["asset_id"], - remote_cols=["id"], - ondelete="CASCADE", - ) batch_op.create_foreign_key( constraint_name="adrq_dag_fkey", referent_table="dag", @@ -565,7 +551,7 @@ def downgrade(): with op.batch_alter_table("task_outlet_dataset_reference", schema=None) as batch_op: batch_op.alter_column("asset_id", new_column_name="dataset_id", type_=sa.Integer(), nullable=False) - batch_op.drop_constraint("toar_asset_fkey", type_="foreignkey") + # batch_op.drop_constraint("toar_asset_fkey", type_="foreignkey") batch_op.drop_constraint("toar_dag_id_fkey", type_="foreignkey") _rename_index( @@ -600,7 +586,7 @@ def downgrade(): with op.batch_alter_table("dataset_dag_run_queue", schema=None) as batch_op: batch_op.alter_column("asset_id", new_column_name="dataset_id", type_=sa.Integer(), nullable=False) - batch_op.drop_constraint("adrq_asset_fkey", type_="foreignkey") + # batch_op.drop_constraint("adrq_asset_fkey", type_="foreignkey") batch_op.drop_constraint("adrq_dag_fkey", type_="foreignkey") _rename_pk_constraint( diff --git a/airflow/migrations/versions/0042_3_0_0_create_fks_as_datasets_are_renamed.py b/airflow/migrations/versions/0042_3_0_0_create_fks_as_datasets_are_renamed.py new file mode 100644 index 0000000000000..c7729ed99c3ff --- /dev/null +++ b/airflow/migrations/versions/0042_3_0_0_create_fks_as_datasets_are_renamed.py @@ -0,0 +1,55 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +create foreign key constraints for assets. + +Revision ID: c4a1639f0f67 +Revises: 05234396c6fc +Create Date: 2024-11-22 09:49:41.813016 + +""" + +from __future__ import annotations + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "c4a1639f0f67" +down_revision = "05234396c6fc" +branch_labels = None +depends_on = None +airflow_version = "3.0.0" + + +def upgrade(): + """Apply create foreign key constraints for assets.""" + with op.batch_alter_table("asset_dag_run_queue", schema=None) as batch_op: + batch_op.create_foreign_key("adrq_asset_fkey", "asset", ["asset_id"], ["id"], ondelete="CASCADE") + + with op.batch_alter_table("task_outlet_asset_reference", schema=None) as batch_op: + batch_op.create_foreign_key("toar_asset_fkey", "asset", ["asset_id"], ["id"], ondelete="CASCADE") + + +def downgrade(): + """Unapply create foreign key constraints for assets.""" + with op.batch_alter_table("task_outlet_asset_reference", schema=None) as batch_op: + batch_op.drop_constraint("toar_asset_fkey", type_="foreignkey") + + with op.batch_alter_table("asset_dag_run_queue", schema=None) as batch_op: + batch_op.drop_constraint("adrq_asset_fkey", type_="foreignkey") diff --git a/airflow/migrations/versions/0042_3_0_0_add_uuid_primary_key_to_task_instance_.py b/airflow/migrations/versions/0043_3_0_0_add_uuid_primary_key_to_task_instance_.py similarity index 99% rename from airflow/migrations/versions/0042_3_0_0_add_uuid_primary_key_to_task_instance_.py rename to airflow/migrations/versions/0043_3_0_0_add_uuid_primary_key_to_task_instance_.py index 41cfddc9cef0b..75c008798a794 100644 --- a/airflow/migrations/versions/0042_3_0_0_add_uuid_primary_key_to_task_instance_.py +++ b/airflow/migrations/versions/0043_3_0_0_add_uuid_primary_key_to_task_instance_.py @@ -20,7 +20,7 @@ Add UUID primary key to ``task_instance`` table. Revision ID: d59cbbef95eb -Revises: 05234396c6fc +Revises: c4a1639f0f67 Create Date: 2024-10-21 22:39:12.394079 """ @@ -33,7 +33,7 @@ # revision identifiers, used by Alembic. revision = "d59cbbef95eb" -down_revision = "05234396c6fc" +down_revision = "c4a1639f0f67" branch_labels = "None" depends_on = None airflow_version = "3.0.0" diff --git a/airflow/migrations/versions/0043_3_0_0_remove_scheduler_lock_column.py b/airflow/migrations/versions/0044_3_0_0_remove_scheduler_lock_column.py similarity index 100% rename from airflow/migrations/versions/0043_3_0_0_remove_scheduler_lock_column.py rename to airflow/migrations/versions/0044_3_0_0_remove_scheduler_lock_column.py diff --git a/airflow/migrations/versions/0044_3_0_0__drop_task_fail_table.py b/airflow/migrations/versions/0045_3_0_0__drop_task_fail_table.py similarity index 100% rename from airflow/migrations/versions/0044_3_0_0__drop_task_fail_table.py rename to airflow/migrations/versions/0045_3_0_0__drop_task_fail_table.py diff --git a/airflow/migrations/versions/0045_3_0_0_add_last_heartbeat_at_directly_to_ti.py b/airflow/migrations/versions/0046_3_0_0_add_last_heartbeat_at_directly_to_ti.py similarity index 100% rename from airflow/migrations/versions/0045_3_0_0_add_last_heartbeat_at_directly_to_ti.py rename to airflow/migrations/versions/0046_3_0_0_add_last_heartbeat_at_directly_to_ti.py diff --git a/airflow/migrations/versions/0046_3_0_0_drop_dag_pickling.py b/airflow/migrations/versions/0047_3_0_0_drop_dag_pickling.py similarity index 100% rename from airflow/migrations/versions/0046_3_0_0_drop_dag_pickling.py rename to airflow/migrations/versions/0047_3_0_0_drop_dag_pickling.py diff --git a/airflow/migrations/versions/0047_3_0_0_add_dag_versioning.py b/airflow/migrations/versions/0048_3_0_0_add_dag_versioning.py similarity index 100% rename from airflow/migrations/versions/0047_3_0_0_add_dag_versioning.py rename to airflow/migrations/versions/0048_3_0_0_add_dag_versioning.py diff --git a/airflow/migrations/versions/0048_3_0_0_add_trigger_asset_reference.py b/airflow/migrations/versions/0049_3_0_0_add_trigger_asset_reference.py similarity index 100% rename from airflow/migrations/versions/0048_3_0_0_add_trigger_asset_reference.py rename to airflow/migrations/versions/0049_3_0_0_add_trigger_asset_reference.py diff --git a/airflow/migrations/versions/0049_3_0_0_remove_pickled_data_from_xcom_table.py b/airflow/migrations/versions/0050_3_0_0_remove_pickled_data_from_xcom_table.py similarity index 100% rename from airflow/migrations/versions/0049_3_0_0_remove_pickled_data_from_xcom_table.py rename to airflow/migrations/versions/0050_3_0_0_remove_pickled_data_from_xcom_table.py diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index d27f5eda6ce03..1fe2044c93a36 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1 @@ -b12f6811bb7a340362e4b8774b3bb81db28a7d0258564b3431bd537368554cc3 \ No newline at end of file +7173ce2b98bc53467274ebe89bd3d4c8e7672f7b40b3f22989139ece45741f8f \ No newline at end of file diff --git a/docs/apache-airflow/migrations-ref.rst b/docs/apache-airflow/migrations-ref.rst index 88a6079d6b6c5..582055210eb84 100644 --- a/docs/apache-airflow/migrations-ref.rst +++ b/docs/apache-airflow/migrations-ref.rst @@ -53,7 +53,9 @@ Here's the list of all the Database Migrations that are executed via when you ru +-------------------------+------------------+-------------------+--------------------------------------------------------------+ | ``486ac7936b78`` | ``d59cbbef95eb`` | ``3.0.0`` | remove scheduler_lock column. | +-------------------------+------------------+-------------------+--------------------------------------------------------------+ -| ``d59cbbef95eb`` | ``05234396c6fc`` | ``3.0.0`` | Add UUID primary key to ``task_instance`` table. | +| ``d59cbbef95eb`` | ``c4a1639f0f67`` | ``3.0.0`` | Add UUID primary key to ``task_instance`` table. | ++-------------------------+------------------+-------------------+--------------------------------------------------------------+ +| ``c4a1639f0f67`` | ``05234396c6fc`` | ``3.0.0`` | create foreign key constraints for assets. | +-------------------------+------------------+-------------------+--------------------------------------------------------------+ | ``05234396c6fc`` | ``3a8972ecb8f9`` | ``3.0.0`` | Rename dataset as asset. | +-------------------------+------------------+-------------------+--------------------------------------------------------------+ From 4c6e9837c5e75c5fc43502c7f449df942524bdcc Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Fri, 22 Nov 2024 13:37:02 +0100 Subject: [PATCH 05/14] fixup! fix for sqlite --- .../versions/0032_3_0_0_drop_execution_date_unique.py | 7 ++++--- docs/apache-airflow/img/airflow_erd.sha256 | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/airflow/migrations/versions/0032_3_0_0_drop_execution_date_unique.py b/airflow/migrations/versions/0032_3_0_0_drop_execution_date_unique.py index c782991881079..a08682afca422 100644 --- a/airflow/migrations/versions/0032_3_0_0_drop_execution_date_unique.py +++ b/airflow/migrations/versions/0032_3_0_0_drop_execution_date_unique.py @@ -31,9 +31,10 @@ from __future__ import annotations -import sqlalchemy as sa from alembic import op +from airflow.utils.sqlalchemy import UtcDateTime + # revision identifiers, used by Alembic. revision = "1cdc775ca98f" down_revision = "a2c32e6c7729" @@ -47,7 +48,7 @@ def upgrade(): batch_op.alter_column( "execution_date", new_column_name="logical_date", - existing_type=sa.TIMESTAMP, + existing_type=UtcDateTime, existing_nullable=False, ) with op.batch_alter_table("dag_run", schema=None) as batch_op: @@ -59,7 +60,7 @@ def downgrade(): batch_op.alter_column( "logical_date", new_column_name="execution_date", - existing_type=sa.TIMESTAMP, + existing_type=UtcDateTime, existing_nullable=False, ) with op.batch_alter_table("dag_run", schema=None) as batch_op: diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index 1fe2044c93a36..62d88dfe5a18b 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1 @@ -7173ce2b98bc53467274ebe89bd3d4c8e7672f7b40b3f22989139ece45741f8f \ No newline at end of file +a81bc0639419e66abacc09d95177df593b10d8944708212ea2a96a51070b7b31 \ No newline at end of file From fec265fe180a70ce5da071c2a82bbcb0b65c2844 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Sun, 24 Nov 2024 09:40:36 +0100 Subject: [PATCH 06/14] fixup! fixup! fix for sqlite --- .../versions/0041_3_0_0_rename_dataset_as_asset.py | 2 -- .../0050_3_0_0_remove_pickled_data_from_xcom_table.py | 7 ++++--- docs/apache-airflow/img/airflow_erd.sha256 | 2 +- tests/utils/test_db.py | 2 ++ 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/airflow/migrations/versions/0041_3_0_0_rename_dataset_as_asset.py b/airflow/migrations/versions/0041_3_0_0_rename_dataset_as_asset.py index 746ca46ec76f3..55dbcdb2c840e 100644 --- a/airflow/migrations/versions/0041_3_0_0_rename_dataset_as_asset.py +++ b/airflow/migrations/versions/0041_3_0_0_rename_dataset_as_asset.py @@ -551,7 +551,6 @@ def downgrade(): with op.batch_alter_table("task_outlet_dataset_reference", schema=None) as batch_op: batch_op.alter_column("asset_id", new_column_name="dataset_id", type_=sa.Integer(), nullable=False) - # batch_op.drop_constraint("toar_asset_fkey", type_="foreignkey") batch_op.drop_constraint("toar_dag_id_fkey", type_="foreignkey") _rename_index( @@ -586,7 +585,6 @@ def downgrade(): with op.batch_alter_table("dataset_dag_run_queue", schema=None) as batch_op: batch_op.alter_column("asset_id", new_column_name="dataset_id", type_=sa.Integer(), nullable=False) - # batch_op.drop_constraint("adrq_asset_fkey", type_="foreignkey") batch_op.drop_constraint("adrq_dag_fkey", type_="foreignkey") _rename_pk_constraint( diff --git a/airflow/migrations/versions/0050_3_0_0_remove_pickled_data_from_xcom_table.py b/airflow/migrations/versions/0050_3_0_0_remove_pickled_data_from_xcom_table.py index ebe8e37a71ff0..ef357824e9efb 100644 --- a/airflow/migrations/versions/0050_3_0_0_remove_pickled_data_from_xcom_table.py +++ b/airflow/migrations/versions/0050_3_0_0_remove_pickled_data_from_xcom_table.py @@ -101,9 +101,9 @@ def upgrade(): op.execute( """ ALTER TABLE xcom - ALTER COLUMN value TYPE JSON + ALTER COLUMN value TYPE JSONB USING CASE - WHEN value IS NOT NULL THEN CAST(CONVERT_FROM(value, 'UTF8') AS JSON) + WHEN value IS NOT NULL THEN CAST(CONVERT_FROM(value, 'UTF8') AS JSONB) ELSE NULL END """ @@ -136,7 +136,6 @@ def upgrade(): # Drop the old `value_old` column with op.batch_alter_table("xcom", schema=None) as batch_op: batch_op.drop_column("value_old") - op.drop_table("_xcom_archive") def downgrade(): @@ -181,3 +180,5 @@ def downgrade(): with op.batch_alter_table("xcom", schema=None) as batch_op: batch_op.drop_column("value_old") + + op.drop_table("_xcom_archive", if_exists=True) diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index 62d88dfe5a18b..9eeef16072c00 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1 @@ -a81bc0639419e66abacc09d95177df593b10d8944708212ea2a96a51070b7b31 \ No newline at end of file +65f26e091cabd5c67f45abb682d1ef4fe63cd0d701b5729e5cf95d9cf2599c6a \ No newline at end of file diff --git a/tests/utils/test_db.py b/tests/utils/test_db.py index 5312fcacaaa82..2290fd2cb9891 100644 --- a/tests/utils/test_db.py +++ b/tests/utils/test_db.py @@ -96,6 +96,8 @@ def test_database_schema_and_sqlalchemy_model_are_in_sync(self): lambda t: (t[0] == "remove_table" and t[1].name == "sqlite_sequence"), # fab version table lambda t: (t[0] == "remove_table" and t[1].name == "alembic_version_fab"), + # Ignore _xcom_archive table + lambda t: (t[0] == "remove_table" and t[1].name == "_xcom_archive"), ] for ignore in ignores: From d2f715420d89bb02298e2d62f84e8ce5d6505310 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Sun, 24 Nov 2024 21:21:09 +0100 Subject: [PATCH 07/14] use TIMESTAMP from db_types --- .../0032_3_0_0_drop_execution_date_unique.py | 6 +- docs/apache-airflow/img/airflow_erd.sha256 | 2 +- docs/apache-airflow/img/airflow_erd.svg | 1769 ++++++++--------- 3 files changed, 886 insertions(+), 891 deletions(-) diff --git a/airflow/migrations/versions/0032_3_0_0_drop_execution_date_unique.py b/airflow/migrations/versions/0032_3_0_0_drop_execution_date_unique.py index a08682afca422..399cc8aff91f3 100644 --- a/airflow/migrations/versions/0032_3_0_0_drop_execution_date_unique.py +++ b/airflow/migrations/versions/0032_3_0_0_drop_execution_date_unique.py @@ -33,7 +33,7 @@ from alembic import op -from airflow.utils.sqlalchemy import UtcDateTime +from airflow.migrations.db_types import TIMESTAMP # revision identifiers, used by Alembic. revision = "1cdc775ca98f" @@ -48,7 +48,7 @@ def upgrade(): batch_op.alter_column( "execution_date", new_column_name="logical_date", - existing_type=UtcDateTime, + existing_type=TIMESTAMP(timezone=True), existing_nullable=False, ) with op.batch_alter_table("dag_run", schema=None) as batch_op: @@ -60,7 +60,7 @@ def downgrade(): batch_op.alter_column( "logical_date", new_column_name="execution_date", - existing_type=UtcDateTime, + existing_type=TIMESTAMP(timezone=True), existing_nullable=False, ) with op.batch_alter_table("dag_run", schema=None) as batch_op: diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index 9eeef16072c00..0886222b9e0e3 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1 @@ -65f26e091cabd5c67f45abb682d1ef4fe63cd0d701b5729e5cf95d9cf2599c6a \ No newline at end of file +9f404e900b4d166d170d878fc00cad8e4397a4288167eebef1d77281a881e92d \ No newline at end of file diff --git a/docs/apache-airflow/img/airflow_erd.svg b/docs/apache-airflow/img/airflow_erd.svg index fd06e1810f731..cd1e5aac103be 100644 --- a/docs/apache-airflow/img/airflow_erd.svg +++ b/docs/apache-airflow/img/airflow_erd.svg @@ -4,11 +4,11 @@ - + %3 - + log @@ -653,24 +653,24 @@ dagrun_asset_event - -dagrun_asset_event - -dag_run_id - - [INTEGER] - NOT NULL - -event_id - - [INTEGER] - NOT NULL + +dagrun_asset_event + +dag_run_id + + [INTEGER] + NOT NULL + +event_id + + [INTEGER] + NOT NULL asset_event--dagrun_asset_event - -0..N + +0..N 1 @@ -713,687 +713,687 @@ task_instance - -task_instance - -id - - [UUID] - NOT NULL - -custom_operator_name - - [VARCHAR(1000)] - -dag_id - - [VARCHAR(250)] - NOT NULL - -dag_version_id - - [UUID] - -duration - - [DOUBLE_PRECISION] - -end_date - - [TIMESTAMP] - -executor - - [VARCHAR(1000)] - -executor_config - - [BYTEA] - -external_executor_id - - [VARCHAR(250)] - -hostname - - [VARCHAR(1000)] - -last_heartbeat_at - - [TIMESTAMP] - -map_index - - [INTEGER] - NOT NULL - -max_tries - - [INTEGER] - -next_kwargs - - [JSON] - -next_method - - [VARCHAR(1000)] - -operator - - [VARCHAR(1000)] - -pid - - [INTEGER] - -pool - - [VARCHAR(256)] - NOT NULL - -pool_slots - - [INTEGER] - NOT NULL - -priority_weight - - [INTEGER] - -queue - - [VARCHAR(256)] - -queued_by_job_id - - [INTEGER] - -queued_dttm - - [TIMESTAMP] - -rendered_map_index - - [VARCHAR(250)] - -run_id - - [VARCHAR(250)] - NOT NULL - -start_date - - [TIMESTAMP] - -state - - [VARCHAR(20)] - -task_display_name - - [VARCHAR(2000)] - -task_id - - [VARCHAR(250)] - NOT NULL - -trigger_id - - [INTEGER] - -trigger_timeout - - [TIMESTAMP] - -try_number - - [INTEGER] - -unixname - - [VARCHAR(1000)] - -updated_at - - [TIMESTAMP] + +task_instance + +id + + [UUID] + NOT NULL + +custom_operator_name + + [VARCHAR(1000)] + +dag_id + + [VARCHAR(250)] + NOT NULL + +dag_version_id + + [UUID] + +duration + + [DOUBLE_PRECISION] + +end_date + + [TIMESTAMP] + +executor + + [VARCHAR(1000)] + +executor_config + + [BYTEA] + +external_executor_id + + [VARCHAR(250)] + +hostname + + [VARCHAR(1000)] + +last_heartbeat_at + + [TIMESTAMP] + +map_index + + [INTEGER] + NOT NULL + +max_tries + + [INTEGER] + +next_kwargs + + [JSON] + +next_method + + [VARCHAR(1000)] + +operator + + [VARCHAR(1000)] + +pid + + [INTEGER] + +pool + + [VARCHAR(256)] + NOT NULL + +pool_slots + + [INTEGER] + NOT NULL + +priority_weight + + [INTEGER] + +queue + + [VARCHAR(256)] + +queued_by_job_id + + [INTEGER] + +queued_dttm + + [TIMESTAMP] + +rendered_map_index + + [VARCHAR(250)] + +run_id + + [VARCHAR(250)] + NOT NULL + +start_date + + [TIMESTAMP] + +state + + [VARCHAR(20)] + +task_display_name + + [VARCHAR(2000)] + +task_id + + [VARCHAR(250)] + NOT NULL + +trigger_id + + [INTEGER] + +trigger_timeout + + [TIMESTAMP] + +try_number + + [INTEGER] + +unixname + + [VARCHAR(1000)] + +updated_at + + [TIMESTAMP] trigger--task_instance - -0..N + +0..N {0,1} task_reschedule - -task_reschedule - -id - - [INTEGER] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -duration - - [INTEGER] - NOT NULL - -end_date - - [TIMESTAMP] - NOT NULL - -map_index - - [INTEGER] - NOT NULL - -reschedule_date - - [TIMESTAMP] - NOT NULL - -run_id - - [VARCHAR(250)] - NOT NULL - -start_date - - [TIMESTAMP] - NOT NULL - -task_id - - [VARCHAR(250)] - NOT NULL - -try_number - - [INTEGER] - NOT NULL + +task_reschedule + +id + + [INTEGER] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +duration + + [INTEGER] + NOT NULL + +end_date + + [TIMESTAMP] + NOT NULL + +map_index + + [INTEGER] + NOT NULL + +reschedule_date + + [TIMESTAMP] + NOT NULL + +run_id + + [VARCHAR(250)] + NOT NULL + +start_date + + [TIMESTAMP] + NOT NULL + +task_id + + [VARCHAR(250)] + NOT NULL + +try_number + + [INTEGER] + NOT NULL task_instance--task_reschedule - -0..N -1 + +0..N +1 task_instance--task_reschedule - -0..N -1 + +0..N +1 task_instance--task_reschedule - -0..N -1 + +0..N +1 task_instance--task_reschedule - -0..N -1 + +0..N +1 rendered_task_instance_fields - -rendered_task_instance_fields - -dag_id - - [VARCHAR(250)] - NOT NULL - -map_index - - [INTEGER] - NOT NULL - -run_id - - [VARCHAR(250)] - NOT NULL - -task_id - - [VARCHAR(250)] - NOT NULL - -k8s_pod_yaml - - [JSON] - -rendered_fields - - [JSON] - NOT NULL + +rendered_task_instance_fields + +dag_id + + [VARCHAR(250)] + NOT NULL + +map_index + + [INTEGER] + NOT NULL + +run_id + + [VARCHAR(250)] + NOT NULL + +task_id + + [VARCHAR(250)] + NOT NULL + +k8s_pod_yaml + + [JSON] + +rendered_fields + + [JSON] + NOT NULL task_instance--rendered_task_instance_fields - -0..N -1 + +0..N +1 task_instance--rendered_task_instance_fields - -0..N -1 + +0..N +1 task_instance--rendered_task_instance_fields - -0..N -1 + +0..N +1 task_instance--rendered_task_instance_fields - -0..N -1 + +0..N +1 task_map - -task_map - -dag_id - - [VARCHAR(250)] - NOT NULL - -map_index - - [INTEGER] - NOT NULL - -run_id - - [VARCHAR(250)] - NOT NULL - -task_id - - [VARCHAR(250)] - NOT NULL - -keys - - [JSON] - -length - - [INTEGER] - NOT NULL + +task_map + +dag_id + + [VARCHAR(250)] + NOT NULL + +map_index + + [INTEGER] + NOT NULL + +run_id + + [VARCHAR(250)] + NOT NULL + +task_id + + [VARCHAR(250)] + NOT NULL + +keys + + [JSON] + +length + + [INTEGER] + NOT NULL task_instance--task_map - -0..N -1 + +0..N +1 task_instance--task_map - -0..N -1 + +0..N +1 task_instance--task_map - -0..N -1 + +0..N +1 task_instance--task_map - -0..N -1 + +0..N +1 xcom - -xcom - -dag_run_id - - [INTEGER] - NOT NULL - -key - - [VARCHAR(512)] - NOT NULL - -map_index - - [INTEGER] - NOT NULL - -task_id - - [VARCHAR(250)] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -run_id - - [VARCHAR(250)] - NOT NULL - -timestamp - - [TIMESTAMP] - NOT NULL - -value - - [JSONB] + +xcom + +dag_run_id + + [INTEGER] + NOT NULL + +key + + [VARCHAR(512)] + NOT NULL + +map_index + + [INTEGER] + NOT NULL + +task_id + + [VARCHAR(250)] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +run_id + + [VARCHAR(250)] + NOT NULL + +timestamp + + [TIMESTAMP] + NOT NULL + +value + + [JSONB] task_instance--xcom - -0..N -1 + +0..N +1 task_instance--xcom - -0..N -1 + +0..N +1 task_instance--xcom - -0..N -1 + +0..N +1 task_instance--xcom - -0..N -1 + +0..N +1 task_instance_note - -task_instance_note - -dag_id - - [VARCHAR(250)] - NOT NULL - -map_index - - [INTEGER] - NOT NULL - -run_id - - [VARCHAR(250)] - NOT NULL - -task_id - - [VARCHAR(250)] - NOT NULL - -content - - [VARCHAR(1000)] - -created_at - - [TIMESTAMP] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL - -user_id - - [VARCHAR(128)] + +task_instance_note + +dag_id + + [VARCHAR(250)] + NOT NULL + +map_index + + [INTEGER] + NOT NULL + +run_id + + [VARCHAR(250)] + NOT NULL + +task_id + + [VARCHAR(250)] + NOT NULL + +content + + [VARCHAR(1000)] + +created_at + + [TIMESTAMP] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL + +user_id + + [VARCHAR(128)] task_instance--task_instance_note - -0..N -1 + +0..N +1 task_instance--task_instance_note - -0..N -1 + +0..N +1 task_instance--task_instance_note - -0..N -1 + +0..N +1 task_instance--task_instance_note - -0..N -1 + +0..N +1 task_instance_history - -task_instance_history - -id - - [INTEGER] - NOT NULL - -custom_operator_name - - [VARCHAR(1000)] - -dag_id - - [VARCHAR(250)] - NOT NULL - -dag_version_id - - [UUID] - -duration - - [DOUBLE_PRECISION] - -end_date - - [TIMESTAMP] - -executor - - [VARCHAR(1000)] - -executor_config - - [BYTEA] - -external_executor_id - - [VARCHAR(250)] - -hostname - - [VARCHAR(1000)] - -map_index - - [INTEGER] - NOT NULL - -max_tries - - [INTEGER] - -next_kwargs - - [JSON] - -next_method - - [VARCHAR(1000)] - -operator - - [VARCHAR(1000)] - -pid - - [INTEGER] - -pool - - [VARCHAR(256)] - NOT NULL - -pool_slots - - [INTEGER] - NOT NULL - -priority_weight - - [INTEGER] - -queue - - [VARCHAR(256)] - -queued_by_job_id - - [INTEGER] - -queued_dttm - - [TIMESTAMP] - -rendered_map_index - - [VARCHAR(250)] - -run_id - - [VARCHAR(250)] - NOT NULL - -start_date - - [TIMESTAMP] - -state - - [VARCHAR(20)] - -task_display_name - - [VARCHAR(2000)] - -task_id - - [VARCHAR(250)] - NOT NULL - -trigger_id - - [INTEGER] - -trigger_timeout - - [TIMESTAMP] - -try_number - - [INTEGER] - NOT NULL - -unixname - - [VARCHAR(1000)] - -updated_at - - [TIMESTAMP] + +task_instance_history + +id + + [INTEGER] + NOT NULL + +custom_operator_name + + [VARCHAR(1000)] + +dag_id + + [VARCHAR(250)] + NOT NULL + +dag_version_id + + [UUID] + +duration + + [DOUBLE_PRECISION] + +end_date + + [TIMESTAMP] + +executor + + [VARCHAR(1000)] + +executor_config + + [BYTEA] + +external_executor_id + + [VARCHAR(250)] + +hostname + + [VARCHAR(1000)] + +map_index + + [INTEGER] + NOT NULL + +max_tries + + [INTEGER] + +next_kwargs + + [JSON] + +next_method + + [VARCHAR(1000)] + +operator + + [VARCHAR(1000)] + +pid + + [INTEGER] + +pool + + [VARCHAR(256)] + NOT NULL + +pool_slots + + [INTEGER] + NOT NULL + +priority_weight + + [INTEGER] + +queue + + [VARCHAR(256)] + +queued_by_job_id + + [INTEGER] + +queued_dttm + + [TIMESTAMP] + +rendered_map_index + + [VARCHAR(250)] + +run_id + + [VARCHAR(250)] + NOT NULL + +start_date + + [TIMESTAMP] + +state + + [VARCHAR(20)] + +task_display_name + + [VARCHAR(2000)] + +task_id + + [VARCHAR(250)] + NOT NULL + +trigger_id + + [INTEGER] + +trigger_timeout + + [TIMESTAMP] + +try_number + + [INTEGER] + NOT NULL + +unixname + + [VARCHAR(1000)] + +updated_at + + [TIMESTAMP] task_instance--task_instance_history - -0..N -1 + +0..N +1 task_instance--task_instance_history - -0..N -1 + +0..N +1 task_instance--task_instance_history - -0..N -1 + +0..N +1 task_instance--task_instance_history - -0..N -1 + +0..N +1 @@ -1645,322 +1645,317 @@ dag_version--task_instance - -0..N -{0,1} + +0..N +{0,1} - + -dag_code - -dag_code - -id - - [UUID] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -dag_version_id - - [UUID] - NOT NULL - -fileloc - - [VARCHAR(2000)] - NOT NULL - -last_updated - - [TIMESTAMP] - NOT NULL - -source_code - - [TEXT] - NOT NULL - -source_code_hash - - [VARCHAR(32)] - NOT NULL +dag_run + +dag_run + +id + + [INTEGER] + NOT NULL + +backfill_id + + [INTEGER] + +clear_number + + [INTEGER] + NOT NULL + +conf + + [BYTEA] + +creating_job_id + + [INTEGER] + +dag_id + + [VARCHAR(250)] + NOT NULL + +dag_version_id + + [UUID] + +data_interval_end + + [TIMESTAMP] + +data_interval_start + + [TIMESTAMP] + +end_date + + [TIMESTAMP] + +external_trigger + + [BOOLEAN] + +last_scheduling_decision + + [TIMESTAMP] + +log_template_id + + [INTEGER] + +logical_date + + [TIMESTAMP] + NOT NULL + +queued_at + + [TIMESTAMP] + +run_id + + [VARCHAR(250)] + NOT NULL + +run_type + + [VARCHAR(50)] + NOT NULL + +start_date + + [TIMESTAMP] + +state + + [VARCHAR(50)] + +triggered_by + + [VARCHAR(50)] + +updated_at + + [TIMESTAMP] - + -dag_version--dag_code - -0..N -1 +dag_version--dag_run + +0..N +{0,1} - + -dag_run - -dag_run - -id - - [INTEGER] - NOT NULL - -backfill_id - - [INTEGER] - -clear_number - - [INTEGER] - NOT NULL - -conf - - [BYTEA] - -creating_job_id - - [INTEGER] - -dag_id - - [VARCHAR(250)] - NOT NULL - -dag_version_id - - [UUID] - -data_interval_end - - [TIMESTAMP] - -data_interval_start - - [TIMESTAMP] - -end_date - - [TIMESTAMP] - -external_trigger - - [BOOLEAN] - -last_scheduling_decision - - [TIMESTAMP] - -log_template_id - - [INTEGER] - -logical_date - - [TIMESTAMP] - NOT NULL - -queued_at - - [TIMESTAMP] - -run_id - - [VARCHAR(250)] - NOT NULL - -run_type - - [VARCHAR(50)] - NOT NULL - -start_date - - [TIMESTAMP] - -state - - [VARCHAR(50)] - -triggered_by - - [VARCHAR(50)] - -updated_at - - [TIMESTAMP] +dag_code + +dag_code + +id + + [UUID] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +dag_version_id + + [UUID] + NOT NULL + +fileloc + + [VARCHAR(2000)] + NOT NULL + +fileloc_hash + + [BIGINT] + NOT NULL + +source_code + + [TEXT] + NOT NULL - + -dag_version--dag_run - -0..N -{0,1} +dag_version--dag_code + +0..N +1 serialized_dag - -serialized_dag - -id - - [UUID] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -dag_hash - - [VARCHAR(32)] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -dag_version_id - - [UUID] - NOT NULL - -data - - [JSON] - -data_compressed - - [BYTEA] - -processor_subdir - - [VARCHAR(2000)] + +serialized_dag + +id + + [UUID] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +dag_hash + + [VARCHAR(32)] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +dag_version_id + + [UUID] + NOT NULL + +data + + [JSON] + +data_compressed + + [BYTEA] + +processor_subdir + + [VARCHAR(2000)] dag_version--serialized_dag - -0..N -1 + +0..N +1 dag_run--dagrun_asset_event - -0..N -1 + +0..N +1 dag_run--task_instance - -0..N -1 + +0..N +1 dag_run--task_instance - -0..N -1 + +0..N +1 backfill_dag_run - -backfill_dag_run - -id - - [INTEGER] - NOT NULL - -backfill_id - - [INTEGER] - NOT NULL - -dag_run_id - - [INTEGER] - -exception_reason - - [VARCHAR(250)] - -logical_date - - [TIMESTAMP] - NOT NULL - -sort_ordinal - - [INTEGER] - NOT NULL + +backfill_dag_run + +id + + [INTEGER] + NOT NULL + +backfill_id + + [INTEGER] + NOT NULL + +dag_run_id + + [INTEGER] + +exception_reason + + [VARCHAR(250)] + +logical_date + + [TIMESTAMP] + NOT NULL + +sort_ordinal + + [INTEGER] + NOT NULL dag_run--backfill_dag_run - -0..N -{0,1} + +0..N +{0,1} dag_run_note - -dag_run_note - -dag_run_id - - [INTEGER] - NOT NULL - -content - - [VARCHAR(1000)] - -created_at - - [TIMESTAMP] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL - -user_id - - [VARCHAR(128)] + +dag_run_note + +dag_run_id + + [INTEGER] + NOT NULL + +content + + [VARCHAR(1000)] + +created_at + + [TIMESTAMP] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL + +user_id + + [VARCHAR(128)] dag_run--dag_run_note - -1 -1 + +1 +1 dag_run--task_reschedule - -0..N -1 + +0..N +1 dag_run--task_reschedule - -0..N -1 + +0..N +1 @@ -1991,9 +1986,9 @@ log_template--dag_run - -0..N -{0,1} + +0..N +{0,1} @@ -2057,15 +2052,15 @@ backfill--dag_run - -0..N -{0,1} + +0..N +{0,1} backfill--backfill_dag_run - -0..N + +0..N 1 @@ -2186,28 +2181,28 @@ ab_user_role - -ab_user_role - -id - - [INTEGER] - NOT NULL - -role_id - - [INTEGER] - -user_id - - [INTEGER] + +ab_user_role + +id + + [INTEGER] + NOT NULL + +role_id + + [INTEGER] + +user_id + + [INTEGER] ab_user--ab_user_role - -0..N -{0,1} + +0..N +{0,1} @@ -2297,28 +2292,28 @@ ab_permission_view_role - -ab_permission_view_role - -id - - [INTEGER] - NOT NULL - -permission_view_id - - [INTEGER] - -role_id - - [INTEGER] + +ab_permission_view_role + +id + + [INTEGER] + NOT NULL + +permission_view_id + + [INTEGER] + +role_id + + [INTEGER] ab_permission_view--ab_permission_view_role - -0..N -{0,1} + +0..N +{0,1} @@ -2362,16 +2357,16 @@ ab_role--ab_user_role - -0..N -{0,1} + +0..N +{0,1} ab_role--ab_permission_view_role - -0..N -{0,1} + +0..N +{0,1} From 5cf3c0b38d7d772d6f827709453fc831d64a350b Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Mon, 25 Nov 2024 18:14:30 +0100 Subject: [PATCH 08/14] skip_archive should not delete _xcom_archive tables since that was created by migration --- tests/utils/test_db_cleanup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/utils/test_db_cleanup.py b/tests/utils/test_db_cleanup.py index 7dca5815904e2..531480d376155 100644 --- a/tests/utils/test_db_cleanup.py +++ b/tests/utils/test_db_cleanup.py @@ -277,11 +277,13 @@ def test__cleanup_table(self, table_name, date_add_kwargs, expected_to_delete, e @pytest.mark.parametrize( "skip_archive, expected_archives", - [pytest.param(True, 0, id="skip_archive"), pytest.param(False, 1, id="do_archive")], + [pytest.param(True, 1, id="skip_archive"), pytest.param(False, 2, id="do_archive")], ) def test__skip_archive(self, skip_archive, expected_archives): """ Verify that running cleanup_table with drops the archives when requested. + + Archived tables from DB migration should be kept when skip_archive is True. """ base_date = pendulum.DateTime(2022, 1, 1, tzinfo=pendulum.timezone("UTC")) num_tis = 10 From 5b2e18eb374529ff6ad90dcc2df3a42e66e0ef9a Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Mon, 25 Nov 2024 20:07:18 +0100 Subject: [PATCH 09/14] fix conflicts --- docs/apache-airflow/img/airflow_erd.sha256 | 2 +- docs/apache-airflow/img/airflow_erd.svg | 1769 ++++++++++---------- 2 files changed, 888 insertions(+), 883 deletions(-) diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index 0886222b9e0e3..eb6ddd02289ec 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1 @@ -9f404e900b4d166d170d878fc00cad8e4397a4288167eebef1d77281a881e92d \ No newline at end of file +0bfb0c5a54b321be706055a05d05ba2f9309b0a52e801aed485a7f816c2a75bd \ No newline at end of file diff --git a/docs/apache-airflow/img/airflow_erd.svg b/docs/apache-airflow/img/airflow_erd.svg index cd1e5aac103be..fd06e1810f731 100644 --- a/docs/apache-airflow/img/airflow_erd.svg +++ b/docs/apache-airflow/img/airflow_erd.svg @@ -4,11 +4,11 @@ - + %3 - + log @@ -653,24 +653,24 @@ dagrun_asset_event - -dagrun_asset_event - -dag_run_id - - [INTEGER] - NOT NULL - -event_id - - [INTEGER] - NOT NULL + +dagrun_asset_event + +dag_run_id + + [INTEGER] + NOT NULL + +event_id + + [INTEGER] + NOT NULL asset_event--dagrun_asset_event - -0..N + +0..N 1 @@ -713,687 +713,687 @@ task_instance - -task_instance - -id - - [UUID] - NOT NULL - -custom_operator_name - - [VARCHAR(1000)] - -dag_id - - [VARCHAR(250)] - NOT NULL - -dag_version_id - - [UUID] - -duration - - [DOUBLE_PRECISION] - -end_date - - [TIMESTAMP] - -executor - - [VARCHAR(1000)] - -executor_config - - [BYTEA] - -external_executor_id - - [VARCHAR(250)] - -hostname - - [VARCHAR(1000)] - -last_heartbeat_at - - [TIMESTAMP] - -map_index - - [INTEGER] - NOT NULL - -max_tries - - [INTEGER] - -next_kwargs - - [JSON] - -next_method - - [VARCHAR(1000)] - -operator - - [VARCHAR(1000)] - -pid - - [INTEGER] - -pool - - [VARCHAR(256)] - NOT NULL - -pool_slots - - [INTEGER] - NOT NULL - -priority_weight - - [INTEGER] - -queue - - [VARCHAR(256)] - -queued_by_job_id - - [INTEGER] - -queued_dttm - - [TIMESTAMP] - -rendered_map_index - - [VARCHAR(250)] - -run_id - - [VARCHAR(250)] - NOT NULL - -start_date - - [TIMESTAMP] - -state - - [VARCHAR(20)] - -task_display_name - - [VARCHAR(2000)] - -task_id - - [VARCHAR(250)] - NOT NULL - -trigger_id - - [INTEGER] - -trigger_timeout - - [TIMESTAMP] - -try_number - - [INTEGER] - -unixname - - [VARCHAR(1000)] - -updated_at - - [TIMESTAMP] + +task_instance + +id + + [UUID] + NOT NULL + +custom_operator_name + + [VARCHAR(1000)] + +dag_id + + [VARCHAR(250)] + NOT NULL + +dag_version_id + + [UUID] + +duration + + [DOUBLE_PRECISION] + +end_date + + [TIMESTAMP] + +executor + + [VARCHAR(1000)] + +executor_config + + [BYTEA] + +external_executor_id + + [VARCHAR(250)] + +hostname + + [VARCHAR(1000)] + +last_heartbeat_at + + [TIMESTAMP] + +map_index + + [INTEGER] + NOT NULL + +max_tries + + [INTEGER] + +next_kwargs + + [JSON] + +next_method + + [VARCHAR(1000)] + +operator + + [VARCHAR(1000)] + +pid + + [INTEGER] + +pool + + [VARCHAR(256)] + NOT NULL + +pool_slots + + [INTEGER] + NOT NULL + +priority_weight + + [INTEGER] + +queue + + [VARCHAR(256)] + +queued_by_job_id + + [INTEGER] + +queued_dttm + + [TIMESTAMP] + +rendered_map_index + + [VARCHAR(250)] + +run_id + + [VARCHAR(250)] + NOT NULL + +start_date + + [TIMESTAMP] + +state + + [VARCHAR(20)] + +task_display_name + + [VARCHAR(2000)] + +task_id + + [VARCHAR(250)] + NOT NULL + +trigger_id + + [INTEGER] + +trigger_timeout + + [TIMESTAMP] + +try_number + + [INTEGER] + +unixname + + [VARCHAR(1000)] + +updated_at + + [TIMESTAMP] trigger--task_instance - -0..N + +0..N {0,1} task_reschedule - -task_reschedule - -id - - [INTEGER] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -duration - - [INTEGER] - NOT NULL - -end_date - - [TIMESTAMP] - NOT NULL - -map_index - - [INTEGER] - NOT NULL - -reschedule_date - - [TIMESTAMP] - NOT NULL - -run_id - - [VARCHAR(250)] - NOT NULL - -start_date - - [TIMESTAMP] - NOT NULL - -task_id - - [VARCHAR(250)] - NOT NULL - -try_number - - [INTEGER] - NOT NULL + +task_reschedule + +id + + [INTEGER] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +duration + + [INTEGER] + NOT NULL + +end_date + + [TIMESTAMP] + NOT NULL + +map_index + + [INTEGER] + NOT NULL + +reschedule_date + + [TIMESTAMP] + NOT NULL + +run_id + + [VARCHAR(250)] + NOT NULL + +start_date + + [TIMESTAMP] + NOT NULL + +task_id + + [VARCHAR(250)] + NOT NULL + +try_number + + [INTEGER] + NOT NULL task_instance--task_reschedule - -0..N -1 + +0..N +1 task_instance--task_reschedule - -0..N -1 + +0..N +1 task_instance--task_reschedule - -0..N -1 + +0..N +1 task_instance--task_reschedule - -0..N -1 + +0..N +1 rendered_task_instance_fields - -rendered_task_instance_fields - -dag_id - - [VARCHAR(250)] - NOT NULL - -map_index - - [INTEGER] - NOT NULL - -run_id - - [VARCHAR(250)] - NOT NULL - -task_id - - [VARCHAR(250)] - NOT NULL - -k8s_pod_yaml - - [JSON] - -rendered_fields - - [JSON] - NOT NULL + +rendered_task_instance_fields + +dag_id + + [VARCHAR(250)] + NOT NULL + +map_index + + [INTEGER] + NOT NULL + +run_id + + [VARCHAR(250)] + NOT NULL + +task_id + + [VARCHAR(250)] + NOT NULL + +k8s_pod_yaml + + [JSON] + +rendered_fields + + [JSON] + NOT NULL task_instance--rendered_task_instance_fields - -0..N -1 + +0..N +1 task_instance--rendered_task_instance_fields - -0..N -1 + +0..N +1 task_instance--rendered_task_instance_fields - -0..N -1 + +0..N +1 task_instance--rendered_task_instance_fields - -0..N -1 + +0..N +1 task_map - -task_map - -dag_id - - [VARCHAR(250)] - NOT NULL - -map_index - - [INTEGER] - NOT NULL - -run_id - - [VARCHAR(250)] - NOT NULL - -task_id - - [VARCHAR(250)] - NOT NULL - -keys - - [JSON] - -length - - [INTEGER] - NOT NULL + +task_map + +dag_id + + [VARCHAR(250)] + NOT NULL + +map_index + + [INTEGER] + NOT NULL + +run_id + + [VARCHAR(250)] + NOT NULL + +task_id + + [VARCHAR(250)] + NOT NULL + +keys + + [JSON] + +length + + [INTEGER] + NOT NULL task_instance--task_map - -0..N -1 + +0..N +1 task_instance--task_map - -0..N -1 + +0..N +1 task_instance--task_map - -0..N -1 + +0..N +1 task_instance--task_map - -0..N -1 + +0..N +1 xcom - -xcom - -dag_run_id - - [INTEGER] - NOT NULL - -key - - [VARCHAR(512)] - NOT NULL - -map_index - - [INTEGER] - NOT NULL - -task_id - - [VARCHAR(250)] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -run_id - - [VARCHAR(250)] - NOT NULL - -timestamp - - [TIMESTAMP] - NOT NULL - -value - - [JSONB] + +xcom + +dag_run_id + + [INTEGER] + NOT NULL + +key + + [VARCHAR(512)] + NOT NULL + +map_index + + [INTEGER] + NOT NULL + +task_id + + [VARCHAR(250)] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +run_id + + [VARCHAR(250)] + NOT NULL + +timestamp + + [TIMESTAMP] + NOT NULL + +value + + [JSONB] task_instance--xcom - -0..N -1 + +0..N +1 task_instance--xcom - -0..N -1 + +0..N +1 task_instance--xcom - -0..N -1 + +0..N +1 task_instance--xcom - -0..N -1 + +0..N +1 task_instance_note - -task_instance_note - -dag_id - - [VARCHAR(250)] - NOT NULL - -map_index - - [INTEGER] - NOT NULL - -run_id - - [VARCHAR(250)] - NOT NULL - -task_id - - [VARCHAR(250)] - NOT NULL - -content - - [VARCHAR(1000)] - -created_at - - [TIMESTAMP] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL - -user_id - - [VARCHAR(128)] + +task_instance_note + +dag_id + + [VARCHAR(250)] + NOT NULL + +map_index + + [INTEGER] + NOT NULL + +run_id + + [VARCHAR(250)] + NOT NULL + +task_id + + [VARCHAR(250)] + NOT NULL + +content + + [VARCHAR(1000)] + +created_at + + [TIMESTAMP] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL + +user_id + + [VARCHAR(128)] task_instance--task_instance_note - -0..N -1 + +0..N +1 task_instance--task_instance_note - -0..N -1 + +0..N +1 task_instance--task_instance_note - -0..N -1 + +0..N +1 task_instance--task_instance_note - -0..N -1 + +0..N +1 task_instance_history - -task_instance_history - -id - - [INTEGER] - NOT NULL - -custom_operator_name - - [VARCHAR(1000)] - -dag_id - - [VARCHAR(250)] - NOT NULL - -dag_version_id - - [UUID] - -duration - - [DOUBLE_PRECISION] - -end_date - - [TIMESTAMP] - -executor - - [VARCHAR(1000)] - -executor_config - - [BYTEA] - -external_executor_id - - [VARCHAR(250)] - -hostname - - [VARCHAR(1000)] - -map_index - - [INTEGER] - NOT NULL - -max_tries - - [INTEGER] - -next_kwargs - - [JSON] - -next_method - - [VARCHAR(1000)] - -operator - - [VARCHAR(1000)] - -pid - - [INTEGER] - -pool - - [VARCHAR(256)] - NOT NULL - -pool_slots - - [INTEGER] - NOT NULL - -priority_weight - - [INTEGER] - -queue - - [VARCHAR(256)] - -queued_by_job_id - - [INTEGER] - -queued_dttm - - [TIMESTAMP] - -rendered_map_index - - [VARCHAR(250)] - -run_id - - [VARCHAR(250)] - NOT NULL - -start_date - - [TIMESTAMP] - -state - - [VARCHAR(20)] - -task_display_name - - [VARCHAR(2000)] - -task_id - - [VARCHAR(250)] - NOT NULL - -trigger_id - - [INTEGER] - -trigger_timeout - - [TIMESTAMP] - -try_number - - [INTEGER] - NOT NULL - -unixname - - [VARCHAR(1000)] - -updated_at - - [TIMESTAMP] + +task_instance_history + +id + + [INTEGER] + NOT NULL + +custom_operator_name + + [VARCHAR(1000)] + +dag_id + + [VARCHAR(250)] + NOT NULL + +dag_version_id + + [UUID] + +duration + + [DOUBLE_PRECISION] + +end_date + + [TIMESTAMP] + +executor + + [VARCHAR(1000)] + +executor_config + + [BYTEA] + +external_executor_id + + [VARCHAR(250)] + +hostname + + [VARCHAR(1000)] + +map_index + + [INTEGER] + NOT NULL + +max_tries + + [INTEGER] + +next_kwargs + + [JSON] + +next_method + + [VARCHAR(1000)] + +operator + + [VARCHAR(1000)] + +pid + + [INTEGER] + +pool + + [VARCHAR(256)] + NOT NULL + +pool_slots + + [INTEGER] + NOT NULL + +priority_weight + + [INTEGER] + +queue + + [VARCHAR(256)] + +queued_by_job_id + + [INTEGER] + +queued_dttm + + [TIMESTAMP] + +rendered_map_index + + [VARCHAR(250)] + +run_id + + [VARCHAR(250)] + NOT NULL + +start_date + + [TIMESTAMP] + +state + + [VARCHAR(20)] + +task_display_name + + [VARCHAR(2000)] + +task_id + + [VARCHAR(250)] + NOT NULL + +trigger_id + + [INTEGER] + +trigger_timeout + + [TIMESTAMP] + +try_number + + [INTEGER] + NOT NULL + +unixname + + [VARCHAR(1000)] + +updated_at + + [TIMESTAMP] task_instance--task_instance_history - -0..N -1 + +0..N +1 task_instance--task_instance_history - -0..N -1 + +0..N +1 task_instance--task_instance_history - -0..N -1 + +0..N +1 task_instance--task_instance_history - -0..N -1 + +0..N +1 @@ -1645,317 +1645,322 @@ dag_version--task_instance - -0..N -{0,1} + +0..N +{0,1} - + -dag_run - -dag_run - -id - - [INTEGER] - NOT NULL - -backfill_id - - [INTEGER] - -clear_number - - [INTEGER] - NOT NULL - -conf - - [BYTEA] - -creating_job_id - - [INTEGER] - -dag_id - - [VARCHAR(250)] - NOT NULL - -dag_version_id - - [UUID] - -data_interval_end - - [TIMESTAMP] - -data_interval_start - - [TIMESTAMP] - -end_date - - [TIMESTAMP] - -external_trigger - - [BOOLEAN] - -last_scheduling_decision - - [TIMESTAMP] - -log_template_id - - [INTEGER] - -logical_date - - [TIMESTAMP] - NOT NULL - -queued_at - - [TIMESTAMP] - -run_id - - [VARCHAR(250)] - NOT NULL - -run_type - - [VARCHAR(50)] - NOT NULL - -start_date - - [TIMESTAMP] - -state - - [VARCHAR(50)] - -triggered_by - - [VARCHAR(50)] - -updated_at - - [TIMESTAMP] +dag_code + +dag_code + +id + + [UUID] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +dag_version_id + + [UUID] + NOT NULL + +fileloc + + [VARCHAR(2000)] + NOT NULL + +last_updated + + [TIMESTAMP] + NOT NULL + +source_code + + [TEXT] + NOT NULL + +source_code_hash + + [VARCHAR(32)] + NOT NULL - + -dag_version--dag_run - -0..N -{0,1} +dag_version--dag_code + +0..N +1 - + -dag_code - -dag_code - -id - - [UUID] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -dag_version_id - - [UUID] - NOT NULL - -fileloc - - [VARCHAR(2000)] - NOT NULL - -fileloc_hash - - [BIGINT] - NOT NULL - -source_code - - [TEXT] - NOT NULL +dag_run + +dag_run + +id + + [INTEGER] + NOT NULL + +backfill_id + + [INTEGER] + +clear_number + + [INTEGER] + NOT NULL + +conf + + [BYTEA] + +creating_job_id + + [INTEGER] + +dag_id + + [VARCHAR(250)] + NOT NULL + +dag_version_id + + [UUID] + +data_interval_end + + [TIMESTAMP] + +data_interval_start + + [TIMESTAMP] + +end_date + + [TIMESTAMP] + +external_trigger + + [BOOLEAN] + +last_scheduling_decision + + [TIMESTAMP] + +log_template_id + + [INTEGER] + +logical_date + + [TIMESTAMP] + NOT NULL + +queued_at + + [TIMESTAMP] + +run_id + + [VARCHAR(250)] + NOT NULL + +run_type + + [VARCHAR(50)] + NOT NULL + +start_date + + [TIMESTAMP] + +state + + [VARCHAR(50)] + +triggered_by + + [VARCHAR(50)] + +updated_at + + [TIMESTAMP] - + -dag_version--dag_code - -0..N -1 +dag_version--dag_run + +0..N +{0,1} serialized_dag - -serialized_dag - -id - - [UUID] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -dag_hash - - [VARCHAR(32)] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -dag_version_id - - [UUID] - NOT NULL - -data - - [JSON] - -data_compressed - - [BYTEA] - -processor_subdir - - [VARCHAR(2000)] + +serialized_dag + +id + + [UUID] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +dag_hash + + [VARCHAR(32)] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +dag_version_id + + [UUID] + NOT NULL + +data + + [JSON] + +data_compressed + + [BYTEA] + +processor_subdir + + [VARCHAR(2000)] dag_version--serialized_dag - -0..N -1 + +0..N +1 dag_run--dagrun_asset_event - -0..N -1 + +0..N +1 dag_run--task_instance - -0..N -1 + +0..N +1 dag_run--task_instance - -0..N -1 + +0..N +1 backfill_dag_run - -backfill_dag_run - -id - - [INTEGER] - NOT NULL - -backfill_id - - [INTEGER] - NOT NULL - -dag_run_id - - [INTEGER] - -exception_reason - - [VARCHAR(250)] - -logical_date - - [TIMESTAMP] - NOT NULL - -sort_ordinal - - [INTEGER] - NOT NULL + +backfill_dag_run + +id + + [INTEGER] + NOT NULL + +backfill_id + + [INTEGER] + NOT NULL + +dag_run_id + + [INTEGER] + +exception_reason + + [VARCHAR(250)] + +logical_date + + [TIMESTAMP] + NOT NULL + +sort_ordinal + + [INTEGER] + NOT NULL dag_run--backfill_dag_run - -0..N -{0,1} + +0..N +{0,1} dag_run_note - -dag_run_note - -dag_run_id - - [INTEGER] - NOT NULL - -content - - [VARCHAR(1000)] - -created_at - - [TIMESTAMP] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL - -user_id - - [VARCHAR(128)] + +dag_run_note + +dag_run_id + + [INTEGER] + NOT NULL + +content + + [VARCHAR(1000)] + +created_at + + [TIMESTAMP] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL + +user_id + + [VARCHAR(128)] dag_run--dag_run_note - -1 -1 + +1 +1 dag_run--task_reschedule - -0..N -1 + +0..N +1 dag_run--task_reschedule - -0..N -1 + +0..N +1 @@ -1986,9 +1991,9 @@ log_template--dag_run - -0..N -{0,1} + +0..N +{0,1} @@ -2052,15 +2057,15 @@ backfill--dag_run - -0..N -{0,1} + +0..N +{0,1} backfill--backfill_dag_run - -0..N + +0..N 1 @@ -2181,28 +2186,28 @@ ab_user_role - -ab_user_role - -id - - [INTEGER] - NOT NULL - -role_id - - [INTEGER] - -user_id - - [INTEGER] + +ab_user_role + +id + + [INTEGER] + NOT NULL + +role_id + + [INTEGER] + +user_id + + [INTEGER] ab_user--ab_user_role - -0..N -{0,1} + +0..N +{0,1} @@ -2292,28 +2297,28 @@ ab_permission_view_role - -ab_permission_view_role - -id - - [INTEGER] - NOT NULL - -permission_view_id - - [INTEGER] - -role_id - - [INTEGER] + +ab_permission_view_role + +id + + [INTEGER] + NOT NULL + +permission_view_id + + [INTEGER] + +role_id + + [INTEGER] ab_permission_view--ab_permission_view_role - -0..N -{0,1} + +0..N +{0,1} @@ -2357,16 +2362,16 @@ ab_role--ab_user_role - -0..N -{0,1} + +0..N +{0,1} ab_role--ab_permission_view_role - -0..N -{0,1} + +0..N +{0,1} From b601e6b5159fb427811b2e67d8dd798a1ed976f9 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Tue, 26 Nov 2024 12:06:07 +0100 Subject: [PATCH 10/14] fixup! fix conflicts --- .../versions/0050_3_0_0_remove_pickled_data_from_xcom_table.py | 2 -- docs/apache-airflow/img/airflow_erd.sha256 | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/airflow/migrations/versions/0050_3_0_0_remove_pickled_data_from_xcom_table.py b/airflow/migrations/versions/0050_3_0_0_remove_pickled_data_from_xcom_table.py index ef357824e9efb..2b19827b6ae4c 100644 --- a/airflow/migrations/versions/0050_3_0_0_remove_pickled_data_from_xcom_table.py +++ b/airflow/migrations/versions/0050_3_0_0_remove_pickled_data_from_xcom_table.py @@ -180,5 +180,3 @@ def downgrade(): with op.batch_alter_table("xcom", schema=None) as batch_op: batch_op.drop_column("value_old") - - op.drop_table("_xcom_archive", if_exists=True) diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index eb6ddd02289ec..dc9ef5688c2b6 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1 @@ -0bfb0c5a54b321be706055a05d05ba2f9309b0a52e801aed485a7f816c2a75bd \ No newline at end of file +7e6189547c880bdbd8760f9b7d90919d3e1d08ef2477210dcf68a4edf8d0bcea \ No newline at end of file From 2140c9c6f271d78c7c46c4c94ab47f36e0af5b76 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Tue, 26 Nov 2024 17:11:35 +0100 Subject: [PATCH 11/14] drop _xcom_archive table if it exists --- .../versions/0050_3_0_0_remove_pickled_data_from_xcom_table.py | 1 + docs/apache-airflow/img/airflow_erd.sha256 | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/airflow/migrations/versions/0050_3_0_0_remove_pickled_data_from_xcom_table.py b/airflow/migrations/versions/0050_3_0_0_remove_pickled_data_from_xcom_table.py index 2b19827b6ae4c..0f1f8b4c72ab4 100644 --- a/airflow/migrations/versions/0050_3_0_0_remove_pickled_data_from_xcom_table.py +++ b/airflow/migrations/versions/0050_3_0_0_remove_pickled_data_from_xcom_table.py @@ -180,3 +180,4 @@ def downgrade(): with op.batch_alter_table("xcom", schema=None) as batch_op: batch_op.drop_column("value_old") + op.drop_table("_xcom_archive", if_exists=True) diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index dc9ef5688c2b6..d2e227826fa53 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1 @@ -7e6189547c880bdbd8760f9b7d90919d3e1d08ef2477210dcf68a4edf8d0bcea \ No newline at end of file +b3073ec4d411d6bf9aa6388840c7dbb52ed0ca40817d3e4bd76c7ce7f298ea74 \ No newline at end of file From 918d5e1f4a8c9fc514c54340d5dc06c688d4f057 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Thu, 28 Nov 2024 08:58:01 +0100 Subject: [PATCH 12/14] use sql for dropping xcom_archive table --- .../versions/0050_3_0_0_remove_pickled_data_from_xcom_table.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airflow/migrations/versions/0050_3_0_0_remove_pickled_data_from_xcom_table.py b/airflow/migrations/versions/0050_3_0_0_remove_pickled_data_from_xcom_table.py index 0f1f8b4c72ab4..ed3379ef51b1f 100644 --- a/airflow/migrations/versions/0050_3_0_0_remove_pickled_data_from_xcom_table.py +++ b/airflow/migrations/versions/0050_3_0_0_remove_pickled_data_from_xcom_table.py @@ -180,4 +180,5 @@ def downgrade(): with op.batch_alter_table("xcom", schema=None) as batch_op: batch_op.drop_column("value_old") - op.drop_table("_xcom_archive", if_exists=True) + + op.execute(sa.text("DROP TABLE IF EXISTS _xcom_archive")) From bd6e5c1ccefa57ed6b74297a5497661166c2a650 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Thu, 28 Nov 2024 09:09:24 +0100 Subject: [PATCH 13/14] fix conflicts --- docs/apache-airflow/img/airflow_erd.sha256 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index d2e227826fa53..7391e9c47de85 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1 @@ -b3073ec4d411d6bf9aa6388840c7dbb52ed0ca40817d3e4bd76c7ce7f298ea74 \ No newline at end of file +4b6f0aec7b0cb8a2e78a5483b69d38f396102693095623d901163f0aa32e8dfd \ No newline at end of file From 9ce384a046e9c14f1778ff1a76b55209c51db4a9 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Fri, 29 Nov 2024 15:22:13 +0100 Subject: [PATCH 14/14] remove added migration file and make it work in one file --- .../0041_3_0_0_rename_dataset_as_asset.py | 11 ++-- ...add_uuid_primary_key_to_task_instance_.py} | 4 +- ..._0_0_create_fks_as_datasets_are_renamed.py | 55 ------------------- ...043_3_0_0_remove_scheduler_lock_column.py} | 0 ...py => 0044_3_0_0__drop_task_fail_table.py} | 0 ...0_add_last_heartbeat_at_directly_to_ti.py} | 0 ...ing.py => 0046_3_0_0_drop_dag_pickling.py} | 0 ...ng.py => 0047_3_0_0_add_dag_versioning.py} | 0 ...0048_3_0_0_add_trigger_asset_reference.py} | 0 ..._0_remove_pickled_data_from_xcom_table.py} | 0 docs/apache-airflow/img/airflow_erd.sha256 | 2 +- docs/apache-airflow/migrations-ref.rst | 4 +- 12 files changed, 11 insertions(+), 65 deletions(-) rename airflow/migrations/versions/{0043_3_0_0_add_uuid_primary_key_to_task_instance_.py => 0042_3_0_0_add_uuid_primary_key_to_task_instance_.py} (99%) delete mode 100644 airflow/migrations/versions/0042_3_0_0_create_fks_as_datasets_are_renamed.py rename airflow/migrations/versions/{0044_3_0_0_remove_scheduler_lock_column.py => 0043_3_0_0_remove_scheduler_lock_column.py} (100%) rename airflow/migrations/versions/{0045_3_0_0__drop_task_fail_table.py => 0044_3_0_0__drop_task_fail_table.py} (100%) rename airflow/migrations/versions/{0046_3_0_0_add_last_heartbeat_at_directly_to_ti.py => 0045_3_0_0_add_last_heartbeat_at_directly_to_ti.py} (100%) rename airflow/migrations/versions/{0047_3_0_0_drop_dag_pickling.py => 0046_3_0_0_drop_dag_pickling.py} (100%) rename airflow/migrations/versions/{0048_3_0_0_add_dag_versioning.py => 0047_3_0_0_add_dag_versioning.py} (100%) rename airflow/migrations/versions/{0049_3_0_0_add_trigger_asset_reference.py => 0048_3_0_0_add_trigger_asset_reference.py} (100%) rename airflow/migrations/versions/{0050_3_0_0_remove_pickled_data_from_xcom_table.py => 0049_3_0_0_remove_pickled_data_from_xcom_table.py} (100%) diff --git a/airflow/migrations/versions/0041_3_0_0_rename_dataset_as_asset.py b/airflow/migrations/versions/0041_3_0_0_rename_dataset_as_asset.py index 55dbcdb2c840e..6c7b20f381629 100644 --- a/airflow/migrations/versions/0041_3_0_0_rename_dataset_as_asset.py +++ b/airflow/migrations/versions/0041_3_0_0_rename_dataset_as_asset.py @@ -210,7 +210,7 @@ def upgrade(): columns=["dag_id"], unique=False, ) - + with op.batch_alter_table("dag_schedule_asset_alias_reference", schema=None) as batch_op: batch_op.create_foreign_key( constraint_name="dsaar_asset_alias_fkey", referent_table="asset_alias", @@ -284,7 +284,8 @@ def upgrade(): columns=["dag_id"], unique=False, ) - + with op.batch_alter_table("task_outlet_asset_reference", schema=None) as batch_op: + batch_op.create_foreign_key("toar_asset_fkey", "asset", ["asset_id"], ["id"], ondelete="CASCADE") batch_op.create_foreign_key( constraint_name="toar_dag_id_fkey", referent_table="dag", @@ -313,7 +314,8 @@ def upgrade(): columns=["target_dag_id"], unique=False, ) - + with op.batch_alter_table("asset_dag_run_queue", schema=None) as batch_op: + batch_op.create_foreign_key("adrq_asset_fkey", "asset", ["asset_id"], ["id"], ondelete="CASCADE") batch_op.create_foreign_key( constraint_name="adrq_dag_fkey", referent_table="dag", @@ -550,7 +552,7 @@ def downgrade(): with op.batch_alter_table("task_outlet_dataset_reference", schema=None) as batch_op: batch_op.alter_column("asset_id", new_column_name="dataset_id", type_=sa.Integer(), nullable=False) - + batch_op.drop_constraint("toar_asset_fkey", type_="foreignkey") batch_op.drop_constraint("toar_dag_id_fkey", type_="foreignkey") _rename_index( @@ -585,6 +587,7 @@ def downgrade(): with op.batch_alter_table("dataset_dag_run_queue", schema=None) as batch_op: batch_op.alter_column("asset_id", new_column_name="dataset_id", type_=sa.Integer(), nullable=False) + batch_op.drop_constraint("adrq_asset_fkey", type_="foreignkey") batch_op.drop_constraint("adrq_dag_fkey", type_="foreignkey") _rename_pk_constraint( diff --git a/airflow/migrations/versions/0043_3_0_0_add_uuid_primary_key_to_task_instance_.py b/airflow/migrations/versions/0042_3_0_0_add_uuid_primary_key_to_task_instance_.py similarity index 99% rename from airflow/migrations/versions/0043_3_0_0_add_uuid_primary_key_to_task_instance_.py rename to airflow/migrations/versions/0042_3_0_0_add_uuid_primary_key_to_task_instance_.py index 75c008798a794..41cfddc9cef0b 100644 --- a/airflow/migrations/versions/0043_3_0_0_add_uuid_primary_key_to_task_instance_.py +++ b/airflow/migrations/versions/0042_3_0_0_add_uuid_primary_key_to_task_instance_.py @@ -20,7 +20,7 @@ Add UUID primary key to ``task_instance`` table. Revision ID: d59cbbef95eb -Revises: c4a1639f0f67 +Revises: 05234396c6fc Create Date: 2024-10-21 22:39:12.394079 """ @@ -33,7 +33,7 @@ # revision identifiers, used by Alembic. revision = "d59cbbef95eb" -down_revision = "c4a1639f0f67" +down_revision = "05234396c6fc" branch_labels = "None" depends_on = None airflow_version = "3.0.0" diff --git a/airflow/migrations/versions/0042_3_0_0_create_fks_as_datasets_are_renamed.py b/airflow/migrations/versions/0042_3_0_0_create_fks_as_datasets_are_renamed.py deleted file mode 100644 index c7729ed99c3ff..0000000000000 --- a/airflow/migrations/versions/0042_3_0_0_create_fks_as_datasets_are_renamed.py +++ /dev/null @@ -1,55 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" -create foreign key constraints for assets. - -Revision ID: c4a1639f0f67 -Revises: 05234396c6fc -Create Date: 2024-11-22 09:49:41.813016 - -""" - -from __future__ import annotations - -from alembic import op - -# revision identifiers, used by Alembic. -revision = "c4a1639f0f67" -down_revision = "05234396c6fc" -branch_labels = None -depends_on = None -airflow_version = "3.0.0" - - -def upgrade(): - """Apply create foreign key constraints for assets.""" - with op.batch_alter_table("asset_dag_run_queue", schema=None) as batch_op: - batch_op.create_foreign_key("adrq_asset_fkey", "asset", ["asset_id"], ["id"], ondelete="CASCADE") - - with op.batch_alter_table("task_outlet_asset_reference", schema=None) as batch_op: - batch_op.create_foreign_key("toar_asset_fkey", "asset", ["asset_id"], ["id"], ondelete="CASCADE") - - -def downgrade(): - """Unapply create foreign key constraints for assets.""" - with op.batch_alter_table("task_outlet_asset_reference", schema=None) as batch_op: - batch_op.drop_constraint("toar_asset_fkey", type_="foreignkey") - - with op.batch_alter_table("asset_dag_run_queue", schema=None) as batch_op: - batch_op.drop_constraint("adrq_asset_fkey", type_="foreignkey") diff --git a/airflow/migrations/versions/0044_3_0_0_remove_scheduler_lock_column.py b/airflow/migrations/versions/0043_3_0_0_remove_scheduler_lock_column.py similarity index 100% rename from airflow/migrations/versions/0044_3_0_0_remove_scheduler_lock_column.py rename to airflow/migrations/versions/0043_3_0_0_remove_scheduler_lock_column.py diff --git a/airflow/migrations/versions/0045_3_0_0__drop_task_fail_table.py b/airflow/migrations/versions/0044_3_0_0__drop_task_fail_table.py similarity index 100% rename from airflow/migrations/versions/0045_3_0_0__drop_task_fail_table.py rename to airflow/migrations/versions/0044_3_0_0__drop_task_fail_table.py diff --git a/airflow/migrations/versions/0046_3_0_0_add_last_heartbeat_at_directly_to_ti.py b/airflow/migrations/versions/0045_3_0_0_add_last_heartbeat_at_directly_to_ti.py similarity index 100% rename from airflow/migrations/versions/0046_3_0_0_add_last_heartbeat_at_directly_to_ti.py rename to airflow/migrations/versions/0045_3_0_0_add_last_heartbeat_at_directly_to_ti.py diff --git a/airflow/migrations/versions/0047_3_0_0_drop_dag_pickling.py b/airflow/migrations/versions/0046_3_0_0_drop_dag_pickling.py similarity index 100% rename from airflow/migrations/versions/0047_3_0_0_drop_dag_pickling.py rename to airflow/migrations/versions/0046_3_0_0_drop_dag_pickling.py diff --git a/airflow/migrations/versions/0048_3_0_0_add_dag_versioning.py b/airflow/migrations/versions/0047_3_0_0_add_dag_versioning.py similarity index 100% rename from airflow/migrations/versions/0048_3_0_0_add_dag_versioning.py rename to airflow/migrations/versions/0047_3_0_0_add_dag_versioning.py diff --git a/airflow/migrations/versions/0049_3_0_0_add_trigger_asset_reference.py b/airflow/migrations/versions/0048_3_0_0_add_trigger_asset_reference.py similarity index 100% rename from airflow/migrations/versions/0049_3_0_0_add_trigger_asset_reference.py rename to airflow/migrations/versions/0048_3_0_0_add_trigger_asset_reference.py diff --git a/airflow/migrations/versions/0050_3_0_0_remove_pickled_data_from_xcom_table.py b/airflow/migrations/versions/0049_3_0_0_remove_pickled_data_from_xcom_table.py similarity index 100% rename from airflow/migrations/versions/0050_3_0_0_remove_pickled_data_from_xcom_table.py rename to airflow/migrations/versions/0049_3_0_0_remove_pickled_data_from_xcom_table.py diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index 7391e9c47de85..4987e63b9bbed 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1 @@ -4b6f0aec7b0cb8a2e78a5483b69d38f396102693095623d901163f0aa32e8dfd \ No newline at end of file +b42b04b6cc47650cb9e7a37258a6e8e99bdca2677253715505b8ad287192bf72 \ No newline at end of file diff --git a/docs/apache-airflow/migrations-ref.rst b/docs/apache-airflow/migrations-ref.rst index 582055210eb84..88a6079d6b6c5 100644 --- a/docs/apache-airflow/migrations-ref.rst +++ b/docs/apache-airflow/migrations-ref.rst @@ -53,9 +53,7 @@ Here's the list of all the Database Migrations that are executed via when you ru +-------------------------+------------------+-------------------+--------------------------------------------------------------+ | ``486ac7936b78`` | ``d59cbbef95eb`` | ``3.0.0`` | remove scheduler_lock column. | +-------------------------+------------------+-------------------+--------------------------------------------------------------+ -| ``d59cbbef95eb`` | ``c4a1639f0f67`` | ``3.0.0`` | Add UUID primary key to ``task_instance`` table. | -+-------------------------+------------------+-------------------+--------------------------------------------------------------+ -| ``c4a1639f0f67`` | ``05234396c6fc`` | ``3.0.0`` | create foreign key constraints for assets. | +| ``d59cbbef95eb`` | ``05234396c6fc`` | ``3.0.0`` | Add UUID primary key to ``task_instance`` table. | +-------------------------+------------------+-------------------+--------------------------------------------------------------+ | ``05234396c6fc`` | ``3a8972ecb8f9`` | ``3.0.0`` | Rename dataset as asset. | +-------------------------+------------------+-------------------+--------------------------------------------------------------+