From 4e8e72073fda29abba5128d2fcef8f9d6901c88f Mon Sep 17 00:00:00 2001 From: Ryan Rymarczyk Date: Tue, 7 Jan 2025 14:08:24 -0500 Subject: [PATCH 1/4] int to bigint for event_id columns --- .../006_36e7a7aee148_upgrade_sequence.py | 52 +++++++++++++++++++ .../rail_performance_manager_schema.py | 6 +-- 2 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 src/lamp_py/migrations/versions/performance_manager_prod/006_36e7a7aee148_upgrade_sequence.py diff --git a/src/lamp_py/migrations/versions/performance_manager_prod/006_36e7a7aee148_upgrade_sequence.py b/src/lamp_py/migrations/versions/performance_manager_prod/006_36e7a7aee148_upgrade_sequence.py new file mode 100644 index 00000000..0dd6e1b1 --- /dev/null +++ b/src/lamp_py/migrations/versions/performance_manager_prod/006_36e7a7aee148_upgrade_sequence.py @@ -0,0 +1,52 @@ +"""upgrade sequence + +Revision ID: 36e7a7aee148 +Revises: 32ba735d080c +Create Date: 2025-01-07 13:57:50.433896 + +This change upgrades the pm_event_id sequence type to bigint to avoid running out of keys + +Details +* upgrade -> drop opmi view, upgrade sequence, update sequence storage columns + +* downgrade -> not possible, can't go from bigint to int + +""" +from alembic import op +import sqlalchemy as sa + +from lamp_py.migrations.versions.performance_manager_prod.sql_strings.strings_001 import view_opmi_all_rt_fields_joined + +# revision identifiers, used by Alembic. +revision = '36e7a7aee148' +down_revision = '32ba735d080c' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # Upgrade sequence to BIGINT + op.execute("ALTER SEQUENCE vehicle_events_pm_event_id_seq as bigint MAXVALUE 9223372036854775807;") + # DROP VIEW before upgrading columns + drop_opmi_all_rt_fields_joined = "DROP VIEW IF EXISTS opmi_all_rt_fields_joined;" + op.execute(drop_opmi_all_rt_fields_joined) + # Upgrade event_id columns to BIGINT + op.alter_column('vehicle_events', 'pm_event_id', + existing_type=sa.INTEGER(), + type_=sa.BigInteger(), + existing_nullable=False, + autoincrement=True) + op.alter_column('vehicle_events', 'previous_trip_stop_pm_event_id', + existing_type=sa.INTEGER(), + type_=sa.BigInteger(), + existing_nullable=True) + op.alter_column('vehicle_events', 'next_trip_stop_pm_event_id', + existing_type=sa.INTEGER(), + type_=sa.BigInteger(), + existing_nullable=True) + op.execute(view_opmi_all_rt_fields_joined) + + +def downgrade() -> None: + # Can not migrate from INT to BIGINT without losing data. + pass diff --git a/src/lamp_py/postgres/rail_performance_manager_schema.py b/src/lamp_py/postgres/rail_performance_manager_schema.py index 26937dfe..8a407036 100644 --- a/src/lamp_py/postgres/rail_performance_manager_schema.py +++ b/src/lamp_py/postgres/rail_performance_manager_schema.py @@ -19,7 +19,7 @@ class VehicleEvents(RpmSqlBase): # pylint: disable=too-few-public-methods __tablename__ = "vehicle_events" - pm_event_id = sa.Column(sa.Integer, primary_key=True) + pm_event_id = sa.Column(sa.BigInteger, primary_key=True) # trip identifiers service_date = sa.Column(sa.Integer, nullable=False) @@ -33,8 +33,8 @@ class VehicleEvents(RpmSqlBase): # pylint: disable=too-few-public-methods parent_station = sa.Column(sa.String(60), nullable=False) # stop link fields - previous_trip_stop_pm_event_id = sa.Column(sa.Integer, nullable=True) - next_trip_stop_pm_event_id = sa.Column(sa.Integer, nullable=True) + previous_trip_stop_pm_event_id = sa.Column(sa.BigInteger, nullable=True) + next_trip_stop_pm_event_id = sa.Column(sa.BigInteger, nullable=True) # event timestamps used for metrics vp_move_timestamp = sa.Column(sa.Integer, nullable=True) From 281622627463233b3d2e1fcfff341324389a9af5 Mon Sep 17 00:00:00 2001 From: Ryan Rymarczyk Date: Tue, 7 Jan 2025 14:19:48 -0500 Subject: [PATCH 2/4] format --- .../006_36e7a7aee148_upgrade_sequence.py | 40 ++++++++++++------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/src/lamp_py/migrations/versions/performance_manager_prod/006_36e7a7aee148_upgrade_sequence.py b/src/lamp_py/migrations/versions/performance_manager_prod/006_36e7a7aee148_upgrade_sequence.py index 0dd6e1b1..c95bb113 100644 --- a/src/lamp_py/migrations/versions/performance_manager_prod/006_36e7a7aee148_upgrade_sequence.py +++ b/src/lamp_py/migrations/versions/performance_manager_prod/006_36e7a7aee148_upgrade_sequence.py @@ -12,14 +12,15 @@ * downgrade -> not possible, can't go from bigint to int """ + from alembic import op import sqlalchemy as sa from lamp_py.migrations.versions.performance_manager_prod.sql_strings.strings_001 import view_opmi_all_rt_fields_joined # revision identifiers, used by Alembic. -revision = '36e7a7aee148' -down_revision = '32ba735d080c' +revision = "36e7a7aee148" +down_revision = "32ba735d080c" branch_labels = None depends_on = None @@ -31,19 +32,28 @@ def upgrade() -> None: drop_opmi_all_rt_fields_joined = "DROP VIEW IF EXISTS opmi_all_rt_fields_joined;" op.execute(drop_opmi_all_rt_fields_joined) # Upgrade event_id columns to BIGINT - op.alter_column('vehicle_events', 'pm_event_id', - existing_type=sa.INTEGER(), - type_=sa.BigInteger(), - existing_nullable=False, - autoincrement=True) - op.alter_column('vehicle_events', 'previous_trip_stop_pm_event_id', - existing_type=sa.INTEGER(), - type_=sa.BigInteger(), - existing_nullable=True) - op.alter_column('vehicle_events', 'next_trip_stop_pm_event_id', - existing_type=sa.INTEGER(), - type_=sa.BigInteger(), - existing_nullable=True) + op.alter_column( + "vehicle_events", + "pm_event_id", + existing_type=sa.INTEGER(), + type_=sa.BigInteger(), + existing_nullable=False, + autoincrement=True, + ) + op.alter_column( + "vehicle_events", + "previous_trip_stop_pm_event_id", + existing_type=sa.INTEGER(), + type_=sa.BigInteger(), + existing_nullable=True, + ) + op.alter_column( + "vehicle_events", + "next_trip_stop_pm_event_id", + existing_type=sa.INTEGER(), + type_=sa.BigInteger(), + existing_nullable=True, + ) op.execute(view_opmi_all_rt_fields_joined) From 4bfe4ed71b87f60687fc88ddc42116113817ee2d Mon Sep 17 00:00:00 2001 From: Ryan Rymarczyk Date: Tue, 7 Jan 2025 14:37:19 -0500 Subject: [PATCH 3/4] other environments --- .../009_36e7a7aee148_upgrade_sequence.py | 62 +++++++++++++++++++ .../010_36e7a7aee148_upgrade_sequence.py | 62 +++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 src/lamp_py/migrations/versions/performance_manager_dev/009_36e7a7aee148_upgrade_sequence.py create mode 100644 src/lamp_py/migrations/versions/performance_manager_staging/010_36e7a7aee148_upgrade_sequence.py diff --git a/src/lamp_py/migrations/versions/performance_manager_dev/009_36e7a7aee148_upgrade_sequence.py b/src/lamp_py/migrations/versions/performance_manager_dev/009_36e7a7aee148_upgrade_sequence.py new file mode 100644 index 00000000..c95bb113 --- /dev/null +++ b/src/lamp_py/migrations/versions/performance_manager_dev/009_36e7a7aee148_upgrade_sequence.py @@ -0,0 +1,62 @@ +"""upgrade sequence + +Revision ID: 36e7a7aee148 +Revises: 32ba735d080c +Create Date: 2025-01-07 13:57:50.433896 + +This change upgrades the pm_event_id sequence type to bigint to avoid running out of keys + +Details +* upgrade -> drop opmi view, upgrade sequence, update sequence storage columns + +* downgrade -> not possible, can't go from bigint to int + +""" + +from alembic import op +import sqlalchemy as sa + +from lamp_py.migrations.versions.performance_manager_prod.sql_strings.strings_001 import view_opmi_all_rt_fields_joined + +# revision identifiers, used by Alembic. +revision = "36e7a7aee148" +down_revision = "32ba735d080c" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # Upgrade sequence to BIGINT + op.execute("ALTER SEQUENCE vehicle_events_pm_event_id_seq as bigint MAXVALUE 9223372036854775807;") + # DROP VIEW before upgrading columns + drop_opmi_all_rt_fields_joined = "DROP VIEW IF EXISTS opmi_all_rt_fields_joined;" + op.execute(drop_opmi_all_rt_fields_joined) + # Upgrade event_id columns to BIGINT + op.alter_column( + "vehicle_events", + "pm_event_id", + existing_type=sa.INTEGER(), + type_=sa.BigInteger(), + existing_nullable=False, + autoincrement=True, + ) + op.alter_column( + "vehicle_events", + "previous_trip_stop_pm_event_id", + existing_type=sa.INTEGER(), + type_=sa.BigInteger(), + existing_nullable=True, + ) + op.alter_column( + "vehicle_events", + "next_trip_stop_pm_event_id", + existing_type=sa.INTEGER(), + type_=sa.BigInteger(), + existing_nullable=True, + ) + op.execute(view_opmi_all_rt_fields_joined) + + +def downgrade() -> None: + # Can not migrate from INT to BIGINT without losing data. + pass diff --git a/src/lamp_py/migrations/versions/performance_manager_staging/010_36e7a7aee148_upgrade_sequence.py b/src/lamp_py/migrations/versions/performance_manager_staging/010_36e7a7aee148_upgrade_sequence.py new file mode 100644 index 00000000..6c4c9b4e --- /dev/null +++ b/src/lamp_py/migrations/versions/performance_manager_staging/010_36e7a7aee148_upgrade_sequence.py @@ -0,0 +1,62 @@ +"""upgrade sequence + +Revision ID: 36e7a7aee148 +Revises: 32ba735d080c +Create Date: 2025-01-07 13:57:50.433896 + +This change upgrades the pm_event_id sequence type to bigint to avoid running out of keys + +Details +* upgrade -> drop opmi view, upgrade sequence, update sequence storage columns + +* downgrade -> not possible, can't go from bigint to int + +""" + +from alembic import op +import sqlalchemy as sa + +from lamp_py.migrations.versions.performance_manager_staging.sql_strings.strings_001 import view_opmi_all_rt_fields_joined + +# revision identifiers, used by Alembic. +revision = "36e7a7aee148" +down_revision = "32ba735d080c" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # Upgrade sequence to BIGINT + op.execute("ALTER SEQUENCE vehicle_events_pm_event_id_seq as bigint MAXVALUE 9223372036854775807;") + # DROP VIEW before upgrading columns + drop_opmi_all_rt_fields_joined = "DROP VIEW IF EXISTS opmi_all_rt_fields_joined;" + op.execute(drop_opmi_all_rt_fields_joined) + # Upgrade event_id columns to BIGINT + op.alter_column( + "vehicle_events", + "pm_event_id", + existing_type=sa.INTEGER(), + type_=sa.BigInteger(), + existing_nullable=False, + autoincrement=True, + ) + op.alter_column( + "vehicle_events", + "previous_trip_stop_pm_event_id", + existing_type=sa.INTEGER(), + type_=sa.BigInteger(), + existing_nullable=True, + ) + op.alter_column( + "vehicle_events", + "next_trip_stop_pm_event_id", + existing_type=sa.INTEGER(), + type_=sa.BigInteger(), + existing_nullable=True, + ) + op.execute(view_opmi_all_rt_fields_joined) + + +def downgrade() -> None: + # Can not migrate from INT to BIGINT without losing data. + pass From 506ae68b08fafba27cd9c8bfe1cf03b1b2f50d8f Mon Sep 17 00:00:00 2001 From: Ryan Rymarczyk Date: Tue, 7 Jan 2025 14:39:23 -0500 Subject: [PATCH 4/4] format --- .../010_36e7a7aee148_upgrade_sequence.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lamp_py/migrations/versions/performance_manager_staging/010_36e7a7aee148_upgrade_sequence.py b/src/lamp_py/migrations/versions/performance_manager_staging/010_36e7a7aee148_upgrade_sequence.py index 6c4c9b4e..7098125c 100644 --- a/src/lamp_py/migrations/versions/performance_manager_staging/010_36e7a7aee148_upgrade_sequence.py +++ b/src/lamp_py/migrations/versions/performance_manager_staging/010_36e7a7aee148_upgrade_sequence.py @@ -16,7 +16,9 @@ from alembic import op import sqlalchemy as sa -from lamp_py.migrations.versions.performance_manager_staging.sql_strings.strings_001 import view_opmi_all_rt_fields_joined +from lamp_py.migrations.versions.performance_manager_staging.sql_strings.strings_001 import ( + view_opmi_all_rt_fields_joined, +) # revision identifiers, used by Alembic. revision = "36e7a7aee148"