Removed pseudojson class, converted all options and other json column…

…s to jsonb ones (getredash#6687) Co-authored-by: Andrew Chubatiuk <[email protected]>
eradman · Jan 11, 2024 · ec1c4d0 · ec1c4d0
1 parent 4d51039
commit ec1c4d0
Show file tree

Hide file tree

Showing 105 changed files with 549 additions and 606 deletions.
diff --git a/migrations/versions/0ec979123ba4_.py b/migrations/versions/0ec979123ba4_.py
@@ -7,7 +7,7 @@
 """
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
+from sqlalchemy.dialects.postgresql import JSON
 
 # revision identifiers, used by Alembic.
 revision = '0ec979123ba4'
@@ -18,7 +18,7 @@
 
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    op.add_column('dashboards', sa.Column('options', postgresql.JSON(astext_type=sa.Text()), server_default='{}', nullable=False))
+    op.add_column('dashboards', sa.Column('options', JSON(astext_type=sa.Text()), server_default='{}', nullable=False))
     # ### end Alembic commands ###
 
 

diff --git a/migrations/versions/640888ce445d_.py b/migrations/versions/640888ce445d_.py
@@ -10,8 +10,7 @@
 from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.sql import table
-
-from redash.models import MutableDict, PseudoJSON
+from redash.models import MutableDict
 
 
 # revision identifiers, used by Alembic.
@@ -41,7 +40,7 @@ def upgrade():
         "queries",
         sa.Column(
             "schedule",
-            MutableDict.as_mutable(PseudoJSON),
+            sa.Text(),
             nullable=False,
             server_default=json.dumps({}),
         ),
@@ -51,7 +50,7 @@ def upgrade():
     queries = table(
         "queries",
         sa.Column("id", sa.Integer, primary_key=True),
-        sa.Column("schedule", MutableDict.as_mutable(PseudoJSON)),
+        sa.Column("schedule", sa.Text()),
         sa.Column("old_schedule", sa.String(length=10)),
     )
 
@@ -85,16 +84,16 @@ def downgrade():
         "queries",
         sa.Column(
             "old_schedule",
-            MutableDict.as_mutable(PseudoJSON),
+            sa.Text(),
             nullable=False,
             server_default=json.dumps({}),
         ),
     )
 
     queries = table(
         "queries",
-        sa.Column("schedule", MutableDict.as_mutable(PseudoJSON)),
-        sa.Column("old_schedule", MutableDict.as_mutable(PseudoJSON)),
+        sa.Column("schedule", sa.Text()),
+        sa.Column("old_schedule", sa.Text()),
     )
 
     op.execute(queries.update().values({"old_schedule": queries.c.schedule}))
@@ -106,7 +105,7 @@ def downgrade():
         "queries",
         sa.Column("id", sa.Integer, primary_key=True),
         sa.Column("schedule", sa.String(length=10)),
-        sa.Column("old_schedule", MutableDict.as_mutable(PseudoJSON)),
+        sa.Column("old_schedule", sa.Text()),
     )
 
     conn = op.get_bind()

diff --git a/migrations/versions/7205816877ec_change_type_of_json_fields_from_varchar_.py b/migrations/versions/7205816877ec_change_type_of_json_fields_from_varchar_.py
@@ -0,0 +1,146 @@
+"""change type of json fields from varchar to json
+
+Revision ID: 7205816877ec
+Revises: 7ce5925f832b
+Create Date: 2024-01-03 13:55:18.885021
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects.postgresql import JSONB, JSON
+
+
+# revision identifiers, used by Alembic.
+revision = '7205816877ec'
+down_revision = '7ce5925f832b'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    connection = op.get_bind()
+    op.alter_column('queries', 'options',
+        existing_type=sa.Text(),
+        type_=JSONB(astext_type=sa.Text()),
+        nullable=True,
+        postgresql_using='options::jsonb',
+        server_default=sa.text("'{}'::jsonb"))
+    op.alter_column('queries', 'schedule',
+        existing_type=sa.Text(),
+        type_=JSONB(astext_type=sa.Text()),
+        nullable=True,
+        postgresql_using='schedule::jsonb',
+        server_default=sa.text("'{}'::jsonb"))
+    op.alter_column('events', 'additional_properties',
+        existing_type=sa.Text(),
+        type_=JSONB(astext_type=sa.Text()),
+        nullable=True,
+        postgresql_using='additional_properties::jsonb',
+        server_default=sa.text("'{}'::jsonb"))
+    op.alter_column('organizations', 'settings',
+        existing_type=sa.Text(),
+        type_=JSONB(astext_type=sa.Text()),
+        nullable=True,
+        postgresql_using='settings::jsonb',
+        server_default=sa.text("'{}'::jsonb"))
+    op.alter_column('alerts', 'options',
+        existing_type=JSON(astext_type=sa.Text()),
+        type_=JSONB(astext_type=sa.Text()),
+        nullable=True,
+        postgresql_using='options::jsonb',
+        server_default=sa.text("'{}'::jsonb"))
+    op.alter_column('dashboards', 'options',
+        existing_type=JSON(astext_type=sa.Text()),
+        type_=JSONB(astext_type=sa.Text()),
+        postgresql_using='options::jsonb',
+        server_default=sa.text("'{}'::jsonb"))
+    op.alter_column('dashboards', 'layout',
+        existing_type=sa.Text(),
+        type_=JSONB(astext_type=sa.Text()),
+        postgresql_using='layout::jsonb',
+        server_default=sa.text("'{}'::jsonb"))
+    op.alter_column('query_results', 'data',
+        existing_type=sa.Text(),
+        type_=JSONB(astext_type=sa.Text()),
+        nullable=True,
+        postgresql_using='data::text',
+        server_default=sa.text("'{}'::jsonb"))
+    op.alter_column('changes', 'change',
+        existing_type=JSON(astext_type=sa.Text()),
+        type_=JSONB(astext_type=sa.Text()),
+        postgresql_using='change::jsonb',
+        server_default=sa.text("'{}'::jsonb"))
+    op.alter_column('visualizations', 'options',
+        existing_type=sa.Text(),
+        type_=JSONB(astext_type=sa.Text()),
+        postgresql_using='options::jsonb',
+        server_default=sa.text("'{}'::jsonb"))
+    op.alter_column('widgets', 'options',
+        existing_type=sa.Text(),
+        type_=JSONB(astext_type=sa.Text()),
+        postgresql_using='options::jsonb',
+        server_default=sa.text("'{}'::jsonb"))
+
+
+def downgrade():
+    connection = op.get_bind()
+    op.alter_column('queries', 'options',
+        existing_type=JSONB(astext_type=sa.Text()),
+        type_=sa.Text(),
+        postgresql_using='options::text',
+        existing_nullable=True,
+        server_default=sa.text("'{}'::text"))
+    op.alter_column('queries', 'schedule',
+        existing_type=JSONB(astext_type=sa.Text()),
+        type_=sa.Text(),
+        postgresql_using='schedule::text',
+        existing_nullable=True,
+        server_default=sa.text("'{}'::text"))
+    op.alter_column('events', 'additional_properties',
+        existing_type=JSONB(astext_type=sa.Text()),
+        type_=sa.Text(),
+        postgresql_using='additional_properties::text',
+        existing_nullable=True,
+        server_default=sa.text("'{}'::text"))
+    op.alter_column('organizations', 'settings',
+        existing_type=JSONB(astext_type=sa.Text()),
+        type_=sa.Text(),
+        postgresql_using='settings::text',
+        existing_nullable=True,
+        server_default=sa.text("'{}'::text"))
+    op.alter_column('alerts', 'options',
+        existing_type=JSONB(astext_type=sa.Text()),
+        type_=JSON(astext_type=sa.Text()),
+        postgresql_using='options::json',
+        existing_nullable=True,
+        server_default=sa.text("'{}'::json"))
+    op.alter_column('dashboards', 'options',
+        existing_type=JSONB(astext_type=sa.Text()),
+        type_=JSON(astext_type=sa.Text()),
+        postgresql_using='options::json',
+        server_default=sa.text("'{}'::json"))
+    op.alter_column('dashboards', 'layout',
+        existing_type=JSONB(astext_type=sa.Text()),
+        type_=sa.Text(),
+        postgresql_using='layout::text',
+        server_default=sa.text("'{}'::text"))
+    op.alter_column('query_results', 'data',
+        existing_type=JSONB(astext_type=sa.Text()),
+        type_=sa.Text(),
+        postgresql_using='data::text',
+        server_default=sa.text("'{}'::text"))
+    op.alter_column('changes', 'change',
+        existing_type=JSONB(astext_type=sa.Text()),
+        type_=JSON(astext_type=sa.Text()),
+        postgresql_using='change::json',
+        server_default=sa.text("'{}'::json"))
+    op.alter_column('visualizations', 'options',
+        type_=sa.Text(),
+        existing_type=JSONB(astext_type=sa.Text()),
+        postgresql_using='options::text',
+        server_default=sa.text("'{}'::text"))
+    op.alter_column('widgets', 'options',
+        type_=sa.Text(),
+        existing_type=JSONB(astext_type=sa.Text()),
+        postgresql_using='options::text',
+        server_default=sa.text("'{}'::text"))
diff --git a/migrations/versions/73beceabb948_bring_back_null_schedule.py b/migrations/versions/73beceabb948_bring_back_null_schedule.py
@@ -7,10 +7,9 @@
 """
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
 from sqlalchemy.sql import table
 
-from redash.models import MutableDict, PseudoJSON
+from redash.models import MutableDict
 
 # revision identifiers, used by Alembic.
 revision = "73beceabb948"
@@ -43,7 +42,7 @@ def upgrade():
     queries = table(
         "queries",
         sa.Column("id", sa.Integer, primary_key=True),
-        sa.Column("schedule", MutableDict.as_mutable(PseudoJSON)),
+        sa.Column("schedule", sa.Text()),
     )
 
     conn = op.get_bind()

diff --git a/migrations/versions/98af61feea92_add_encrypted_options_to_data_sources.py b/migrations/versions/98af61feea92_add_encrypted_options_to_data_sources.py
@@ -7,7 +7,7 @@
 """
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
+from sqlalchemy.dialects.postgresql import BYTEA
 from sqlalchemy.sql import table
 from sqlalchemy_utils.types.encrypted.encrypted_type import FernetEngine
 
@@ -18,7 +18,6 @@
     Configuration,
     MutableDict,
     MutableList,
-    PseudoJSON,
 )
 
 # revision identifiers, used by Alembic.
@@ -31,7 +30,7 @@
 def upgrade():
     op.add_column(
         "data_sources",
-        sa.Column("encrypted_options", postgresql.BYTEA(), nullable=True),
+        sa.Column("encrypted_options", BYTEA(), nullable=True),
     )
 
     # copy values

diff --git a/migrations/versions/a92d92aa678e_inline_tags.py b/migrations/versions/a92d92aa678e_inline_tags.py
@@ -9,7 +9,7 @@
 from funcy import flatten, compact
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
+from sqlalchemy.dialects.postgresql import ARRAY
 from redash import models
 
 # revision identifiers, used by Alembic.
@@ -21,10 +21,10 @@
 
 def upgrade():
     op.add_column(
-        "dashboards", sa.Column("tags", postgresql.ARRAY(sa.Unicode()), nullable=True)
+        "dashboards", sa.Column("tags", ARRAY(sa.Unicode()), nullable=True)
     )
     op.add_column(
-        "queries", sa.Column("tags", postgresql.ARRAY(sa.Unicode()), nullable=True)
+        "queries", sa.Column("tags", ARRAY(sa.Unicode()), nullable=True)
     )
 
 

diff --git a/migrations/versions/d7d747033183_encrypt_alert_destinations.py b/migrations/versions/d7d747033183_encrypt_alert_destinations.py
@@ -7,7 +7,7 @@
 """
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
+from sqlalchemy.dialects.postgresql import BYTEA
 from sqlalchemy.sql import table
 from sqlalchemy_utils.types.encrypted.encrypted_type import FernetEngine
 
@@ -30,7 +30,7 @@
 def upgrade():
     op.add_column(
         "notification_destinations",
-        sa.Column("encrypted_options", postgresql.BYTEA(), nullable=True)
+        sa.Column("encrypted_options", BYTEA(), nullable=True)
     )
 
     # copy values

diff --git a/migrations/versions/e7f8a917aa8e_add_user_details_json_column.py b/migrations/versions/e7f8a917aa8e_add_user_details_json_column.py
@@ -7,7 +7,7 @@
 """
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
+from sqlalchemy.dialects.postgresql import JSON
 
 # revision identifiers, used by Alembic.
 revision = "e7f8a917aa8e"
@@ -21,7 +21,7 @@ def upgrade():
         "users",
         sa.Column(
             "details",
-            postgresql.JSON(astext_type=sa.Text()),
+            JSON(astext_type=sa.Text()),
             server_default="{}",
             nullable=True,
         ),

diff --git a/migrations/versions/fd4fc850d7ea_.py b/migrations/versions/fd4fc850d7ea_.py
@@ -7,7 +7,7 @@
 """
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
+from sqlalchemy.dialects.postgresql import JSON, JSONB
 
 from redash.models import db
 
@@ -23,8 +23,8 @@ def upgrade():
 
     ### commands auto generated by Alembic - please adjust! ###
     op.alter_column('users', 'details',
-               existing_type=postgresql.JSON(astext_type=sa.Text()),
-               type_=postgresql.JSONB(astext_type=sa.Text()),
+               existing_type=JSON(astext_type=sa.Text()),
+               type_=JSONB(astext_type=sa.Text()),
                existing_nullable=True,
                existing_server_default=sa.text("'{}'::jsonb"))
     ### end Alembic commands ###
@@ -52,8 +52,8 @@ def downgrade():
     connection.execute(update_query)
     db.session.commit()
     op.alter_column('users', 'details',
-               existing_type=postgresql.JSONB(astext_type=sa.Text()),
-               type_=postgresql.JSON(astext_type=sa.Text()),
+               existing_type=JSONB(astext_type=sa.Text()),
+               type_=JSON(astext_type=sa.Text()),
                existing_nullable=True,
                existing_server_default=sa.text("'{}'::json"))
 

diff --git a/redash/handlers/base.py b/redash/handlers/base.py
@@ -5,7 +5,7 @@
 from flask_login import current_user, login_required
 from flask_restful import Resource, abort
 from sqlalchemy import cast
-from sqlalchemy.dialects import postgresql
+from sqlalchemy.dialects.postgresql import ARRAY
 from sqlalchemy.orm.exc import NoResultFound
 from sqlalchemy_utils.functions import sort_query
 
@@ -114,7 +114,7 @@ def json_response(response):
 def filter_by_tags(result_set, column):
     if request.args.getlist("tags"):
         tags = request.args.getlist("tags")
-        result_set = result_set.filter(cast(column, postgresql.ARRAY(db.Text)).contains(tags))
+        result_set = result_set.filter(cast(column, ARRAY(db.Text)).contains(tags))
     return result_set
 
 

diff --git a/redash/handlers/dashboards.py b/redash/handlers/dashboards.py
@@ -96,7 +96,7 @@ def post(self):
             org=self.current_org,
             user=self.current_user,
             is_draft=True,
-            layout="[]",
+            layout=[],
         )
         models.db.session.add(dashboard)
         models.db.session.commit()