Replace btree with btree_gin indexes for PostgreSQL to allow larger metadata and faster nested search

tiled/catalog/adapter.py

@@ -6,7 +6,7 @@
 import shutil
 import sys
 import uuid
-from functools import partial
+from functools import partial, reduce
 from pathlib import Path
 from urllib.parse import quote_plus, urlparse
 
@@ -938,9 +938,18 @@ def _prepare_structure(structure_family, structure):
 
 def binary_op(query, tree, operation):
     dialect_name = tree.engine.url.get_dialect().name
-    attr = orm.Node.metadata_[query.key.split(".")]
+    keys = query.key.split(".")
+    attr = orm.Node.metadata_[keys]
     if dialect_name == "sqlite":
         condition = operation(_get_value(attr, type(query.value)), query.value)
+    # specific case where GIN optomized index can be used to speed up POSTGRES equals queries
+    elif dialect_name == "postgresql" and operation == operator.eq:
+        condition = orm.Node.metadata_.op("@>")(
+            type_coerce(
+                {keys[0]: reduce(lambda x, y: {y: x}, keys[1:][::-1], query.value)},
+                orm.Node.metadata_.type,
+            )
+        )
     else:
         condition = operation(attr, type_coerce(query.value, orm.Node.metadata_.type))
     return tree.new_variation(conditions=tree.conditions + [condition])

tiled/catalog/core.py

@@ -16,6 +16,9 @@ async def initialize_database(engine):
     from . import orm  # noqa: F401
 
     async with engine.connect() as connection:
+        # Install extensions
+        if engine.dialect.name == "postgresql":
+            await connection.execute(text("create extension btree_gin;"))
         # Create all tables.
         await connection.run_sync(Base.metadata.create_all)
         if engine.dialect.name == "sqlite":

tiled/catalog/orm.py

@@ -90,7 +90,7 @@ class Node(Timestamped, Base):
             "time_created",
             "id",
             "metadata",
-            postgresql_using="btree",
+            postgresql_using="gin",
         ),
         # This is used by ORDER BY with the default sorting.
         # Index("ancestors_time_created", "ancestors", "time_created"),