Add setup scripts, disable offset feature for now

.docker/setup/Dockerfile

@@ -0,0 +1,3 @@
+FROM python:3.11.9-alpine
+COPY . /app/
+RUN pip install -r /app/requirements.txt

.docker/setup/requirements.txt

@@ -0,0 +1 @@
+requests==2.32.2

.docker/setup/setup.py

@@ -0,0 +1,147 @@
+import copy
+import logging
+import os
+import pprint
+
+import requests
+
+host = os.environ.get('host') if os.environ.get('host') else 'http://localhost'
+port = os.environ.get('port') if os.environ.get('port') else '3000'
+admin_email = os.environ.get('admin_email') if os.environ.get('admin_email') else '[email protected]'
+user_email = os.environ.get('user_email') if os.environ.get('user_email') else '[email protected]'
+password = os.environ.get('password') if os.environ.get('password') else 'metabot1'
+site_name = 'ClickHouse test'
+
+endpoints = {
+    'health_check': '/api/health',
+    'properties': '/api/session/properties',
+    'setup': '/api/setup',
+    'database': '/api/database',
+    'login': '/api/session',
+    'user': '/api/user',
+}
+for k, v in endpoints.items():
+    endpoints[k] = f"{host}:{port}{v}"
+
+db_base_payload = {
+    "is_on_demand": False,
+    "is_full_sync": True,
+    "is_sample": False,
+    "cache_ttl": None,
+    "refingerprint": False,
+    "auto_run_queries": True,
+    "schedules": {},
+    "details": {
+        "host": "clickhouse",
+        "port": 8123,
+        "user": "default",
+        "password": None,
+        "dbname": "default",
+        "scan-all-databases": False,
+        "ssl": False,
+        "tunnel-enabled": False,
+        "advanced-options": False
+    },
+    "name": "Our ClickHouse",
+    "engine": "clickhouse"
+}
+
+
+def health():
+    response = requests.get(endpoints['health_check'], verify=False)
+    if response.json()['status'] == 'ok':
+        return 'healthy'
+    else:
+        health()
+
+
+def check_response(response, op):
+    if response.status_code >= 300:
+        print(f'Unexpected status {response.status_code} for {op}', response.text)
+        exit(1)
+
+
+if __name__ == '__main__':
+    print("Checking health")
+
+    if health() == 'healthy' and os.environ.get('retry') is None:
+        print("Healthy, setting up Metabase")
+
+        session = requests.Session()
+        session_token = None
+        try:
+            token = session.get(endpoints['properties'], verify=False).json()['setup-token']
+            setup_payload = {
+                'token': f'{token}',
+                'user': {
+                    'first_name': 'Admin',
+                    'last_name': 'Admin',
+                    'email': admin_email,
+                    'site_name': site_name,
+                    'password': password,
+                    'password_confirm': password
+                },
+                'database': None,
+                'invite': None,
+                'prefs': {
+                    'site_name': site_name,
+                    'site_locale': 'en',
+                    'allow_tracking': False
+                }
+            }
+            print("Getting the setup token")
+            session_token = session.post(endpoints['setup'], verify=False, json=setup_payload).json()['id']
+        except Exception as e:
+            print("The admin user was already created")
+
+        try:
+            if session_token is None:
+                session_token = session.post(endpoints['login'], verify=False,
+                                             json={"username": admin_email, "password": password})
+
+            dbs = session.get(endpoints['database'], verify=False).json()
+            print("Current databases:")
+            pprint.pprint(dbs['data'])
+
+            sample_db = next((x for x in dbs['data'] if x['id'] == 1), None)
+            if sample_db is not None:
+                print("Deleting the sample database")
+                res = session.delete(f"{endpoints['database']}/{sample_db['id']}")
+                check_response(res, 'delete sample db')
+            else:
+                print("The sample database was already deleted")
+
+            single_node_db = next((x for x in dbs['data']
+                                   if x['engine'] == 'clickhouse'
+                                   and x['details']['host'] == 'clickhouse'), None)
+            if single_node_db is None:
+                print("Creating ClickHouse single node db")
+                single_node_payload = copy.deepcopy(db_base_payload)
+                single_node_payload['name'] = 'ClickHouse (single node)'
+                res = session.post(endpoints['database'], verify=False, json=single_node_payload)
+                check_response(res, 'create single node db')
+            else:
+                print("The single node database was already created")
+
+            # cluster_db = next((x for x in dbs['data']
+            #                    if x['engine'] == 'clickhouse'
+            #                    and x['details']['host'] == 'nginx'), None)
+            # if cluster_db is None:
+            #     print("Creating ClickHouse cluster db")
+            #     cluster_db_payload = copy.deepcopy(db_base_payload)
+            #     cluster_db_payload['details']['host'] = 'nginx'
+            #     cluster_db_payload['name'] = 'ClickHouse (cluster)'
+            #     res = session.post(endpoints['database'], verify=False, json=cluster_db_payload)
+            #     check_response(res)
+            # else:
+            #     print("The cluster database was already created")
+
+            print("Creating a regular user")
+            user_payload = {"first_name": "User", "last_name": "User", "email": user_email, "password": password}
+            res = session.post(endpoints['user'], verify=False, json=user_payload)
+            check_response(res, 'create user')
+
+            print("Done!")
+        except Exception as e:
+            logging.exception("Failed to setup Metabase", e)
+            exit()

docker-compose.yml

@@ -1,7 +1,6 @@
-version: '3.8'
 services:
   clickhouse:
-    image: 'clickhouse/clickhouse-server:24.3-alpine'
+    image: 'clickhouse/clickhouse-server:24.4-alpine'
     container_name: 'metabase-driver-clickhouse-server'
     ports:
       - '8123:8123'
@@ -64,18 +63,18 @@ services:
       timeout: 5s
       retries: 10
 
-  # setup:
-  #   build: .docker/setup/.
-  #   container_name: metabase-clickhouse-setup
-  #   volumes:
-  #     - .docker/setup/setup.py:/app/setup.py
-  #   depends_on:
-  #     metabase:
-  #       condition: service_healthy
-  #   command: python /app/setup.py
-  #   environment:
-  #     host: http://metabase
-  #     port: 3000
-  #     admin_email: '[email protected]'
-  #     user_email: '[email protected]'
-  #     password: 'metabot1'
+  setup:
+    build: .docker/setup/.
+    container_name: metabase-clickhouse-setup
+    volumes:
+      - .docker/setup/setup.py:/app/setup.py
+    depends_on:
+      metabase:
+        condition: service_healthy
+    command: python /app/setup.py
+    environment:
+      host: http://metabase
+      port: 3000
+      admin_email: '[email protected]'
+      user_email: '[email protected]'
+      password: 'metabot1'

src/metabase/driver/clickhouse.clj

@@ -4,15 +4,15 @@
   (:require [clojure.core.memoize :as memoize]
             [clojure.string :as str]
             [honey.sql :as sql]
-            [metabase [config :as config]]
+            [metabase.config :as config]
             [metabase.driver :as driver]
             [metabase.driver.clickhouse-introspection]
             [metabase.driver.clickhouse-nippy]
             [metabase.driver.clickhouse-qp]
             [metabase.driver.ddl.interface :as ddl.i]
             [metabase.driver.sql :as driver.sql]
-            [metabase.driver.sql-jdbc [common :as sql-jdbc.common]
-             [connection :as sql-jdbc.conn]]
+            [metabase.driver.sql-jdbc.common :as sql-jdbc.common]
+            [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
             [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
             [metabase.driver.sql.query-processor :as sql.qp]
             [metabase.driver.sql.util :as sql.u]
@@ -38,7 +38,8 @@
                               :connection-impersonation        false
                               :schemas                         true
                               :datetime-diff                   true
-                              :upload-with-auto-pk             false}]
+                              :upload-with-auto-pk             false
+                              :window-functions/offset         false}]
 
   (defmethod driver/database-supports? [:clickhouse feature] [_driver _feature _db] supported?))
 

test/metabase/driver/clickhouse_data_types_test.clj

@@ -6,7 +6,7 @@
             [metabase.query-processor.test-util :as qp.test]
             [metabase.test :as mt]
             [metabase.test.data :as data]
-            [metabase.test.data [interface :as tx]]
+            [metabase.test.data.interface :as tx]
             [metabase.test.data.clickhouse :as ctd]))
 
 (deftest ^:parallel clickhouse-decimals

test/metabase/driver/clickhouse_substitution_test.clj

@@ -5,7 +5,7 @@
             [metabase.query-processor :as qp]
             [metabase.test :as mt]
             [metabase.test.data :as data]
-            [metabase.test.data [interface :as tx]]
+            [metabase.test.data.interface :as tx]
             [metabase.test.data.clickhouse :as ctd]
             [metabase.util :as u]
             [schema.core :as s])

test/metabase/test/data/clickhouse.clj

@@ -7,16 +7,14 @@
    [metabase.driver.ddl.interface :as ddl.i]
    [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
    [metabase.driver.sql.util :as sql.u]
-   [metabase.models [database :refer [Database]]]
+   [metabase.models.database :refer [Database]]
    [metabase.query-processor.test-util :as qp.test]
    [metabase.sync.sync-metadata :as sync-metadata]
-   [metabase.test.data
-    [interface :as tx]
-    [sql-jdbc :as sql-jdbc.tx]]
+   [metabase.test.data.interface :as tx]
    [metabase.test.data.sql :as sql.tx]
-   [metabase.test.data.sql-jdbc
-    [execute :as execute]
-    [load-data :as load-data]]
+   [metabase.test.data.sql-jdbc :as sql-jdbc.tx]
+   [metabase.test.data.sql-jdbc.execute :as execute]
+   [metabase.test.data.sql-jdbc.load-data :as load-data]
    [toucan2.tools.with-temp :as t2.with-temp]))
 
 (sql-jdbc.tx/add-test-extensions! :clickhouse)