added clickhouse support

gobitfly · Jun 14, 2024 · 1ec9b6b · 1ec9b6b
1 parent 430cc3e
commit 1ec9b6b
Show file tree

Hide file tree

Showing 7 changed files with 50 additions and 17 deletions.
diff --git a/backend/cmd/misc/main.go b/backend/cmd/misc/main.go
@@ -182,7 +182,7 @@ func main() {
 		Host:         cfg.ClickHouse.WriterDatabase.Host,
 		Port:         cfg.ClickHouse.WriterDatabase.Port,
 		MaxOpenConns: cfg.ClickHouse.WriterDatabase.MaxOpenConns,
-		SSL:          true,
+		SSL:          cfg.ClickHouse.WriterDatabase.SSL,
 		MaxIdleConns: cfg.ClickHouse.WriterDatabase.MaxIdleConns,
 	}, &types.DatabaseConfig{
 		Username:     cfg.ClickHouse.ReaderDatabase.Username,
@@ -191,7 +191,7 @@ func main() {
 		Host:         cfg.ClickHouse.ReaderDatabase.Host,
 		Port:         cfg.ClickHouse.ReaderDatabase.Port,
 		MaxOpenConns: cfg.ClickHouse.ReaderDatabase.MaxOpenConns,
-		SSL:          true,
+		SSL:          cfg.ClickHouse.ReaderDatabase.SSL,
 		MaxIdleConns: cfg.ClickHouse.ReaderDatabase.MaxIdleConns,
 	}, "clickhouse", "clickhouse")
 	defer db.ClickHouseReader.Close()

diff --git a/backend/local_deployment/README.md b/backend/local_deployment/README.md
@@ -60,7 +60,7 @@ make all
 # Start postgres, redis, little_bigtable & the eth test network
 ```
 cd ~/eth2-beaconchain-explorer/backend/local_deployment/
-kurtosis clean -a && kurtosis run --enclave my-testnet . "$(cat network-params.json)"
+kurtosis clean -a && kurtosis run --image-download always --enclave my-testnet . "$(cat network-params.json)"
 ```
 Later in your developer life (after having started Kurtosis and stopped it a few times), if you encounter an error at this step, you might need to clean up bugged cache files from previous runs that Kurtosis or Docker left behind.
 The `./stop` script [in this repository](https://github.com/thib-wien/scripts-localnetworkandexplorer) gathers cleaning commands which worked for their author (it might save you hours of browsing Stack Overflow and GitHub's issues).
@@ -75,7 +75,7 @@ This will generate a _config.yml_ to be used by the explorer and then create the
 # Start the explorer modules
 ```
 cd ~/eth2-beaconchain-explorer/local_deployment/
-docker compose up -d
+docker compose up -d --pull always
 ```
 You can start / stop the exporter submodules using `docker compose`
 

diff --git a/backend/local_deployment/main.star b/backend/local_deployment/main.star
@@ -1,14 +1,16 @@
-input_parser = import_module("github.com/kurtosis-tech/ethereum-package/src/package_io/input_parser.star")
-eth_network_module = import_module("github.com/kurtosis-tech/ethereum-package/main.star")
-transaction_spammer = import_module("github.com/kurtosis-tech/ethereum-package/src/transaction_spammer/transaction_spammer.star")
-blob_spammer = import_module("github.com/kurtosis-tech/ethereum-package/src/blob_spammer/blob_spammer.star")
-genesis_constants = import_module("github.com/kurtosis-tech/ethereum-package/src/prelaunch_data_generator/genesis_constants/genesis_constants.star")
-shared_utils = import_module("github.com/kurtosis-tech/ethereum-package/src/shared_utils/shared_utils.star")
+input_parser = import_module("github.com/ethpandaops/ethereum-package/src/package_io/input_parser.star")
+eth_network_module = import_module("github.com/ethpandaops/ethereum-package/main.star")
+transaction_spammer = import_module("github.com/ethpandaops/ethereum-package/src/transaction_spammer/transaction_spammer.star")
+blob_spammer = import_module("github.com/ethpandaops/ethereum-package/src/blob_spammer/blob_spammer.star")
+genesis_constants = import_module("github.com/ethpandaops/ethereum-package/src/prelaunch_data_generator/genesis_constants/genesis_constants.star")
+shared_utils = import_module("github.com/ethpandaops/ethereum-package/src/shared_utils/shared_utils.star")
 
 POSTGRES_PORT_ID = "postgres"
 POSTGRES_DB = "db"
 ALLOY_PORT_ID = "alloy"
 ALLOY_DB = "alloy"
+CLICKHOUSE_PORT_ID = "clickhouse"
+CLICKHOUSE_DB = "clickhouse"
 POSTGRES_USER = "postgres"
 POSTGRES_PASSWORD = "pass"
 
@@ -36,7 +38,7 @@ def run(plan, args):
 					"POSTGRES_PASSWORD": POSTGRES_PASSWORD,
 				},
 			),
-			# Add second Postgres server to simulate alloy
+			# Add second Postgres server to simulate alloy; consider switching to omni (https://cloud.google.com/alloydb/docs/omni)
 			"alloy": ServiceConfig(
 				image = "postgres:15.2-alpine",
 				ports = {
@@ -48,6 +50,18 @@ def run(plan, args):
 					"POSTGRES_PASSWORD": POSTGRES_PASSWORD,
 				},
 			),
+			# Add a Clickhouse server
+			"clickhouse": ServiceConfig(
+				image = "clickhouse/clickhouse-server:24.2",
+				ports = {
+					CLICKHOUSE_PORT_ID: PortSpec(9000, application_protocol = "clickhouse"),
+				},
+				env_vars = {
+					"CLICKHOUSE_DB": CLICKHOUSE_DB,
+					"CLICKHOUSE_USER": POSTGRES_USER,
+					"CLICKHOUSE_PASSWORD": POSTGRES_PASSWORD,
+				},
+			),
 			# Add a Redis server
 			"redis": ServiceConfig(
 				image = "redis:7",

diff --git a/backend/local_deployment/provision-explorer-config.sh b/backend/local_deployment/provision-explorer-config.sh
@@ -14,6 +14,9 @@ echo "Postgres port is $POSTGRES_PORT"
 ALLOY_PORT=$(kurtosis port print my-testnet alloy alloy --format number)
 echo "Alloy port is $ALLOY_PORT"
 
+CLICKHOUSE_PORT=$(kurtosis port print my-testnet clickhouse clickhouse --format number)
+echo "Clickhouse port is $CLICKHOUSE_PORT"
+
 LBT_PORT=$(kurtosis port print my-testnet littlebigtable littlebigtable --format number)
 echo "Little bigtable port is $LBT_PORT"
 
@@ -23,6 +26,7 @@ EL_PORT=$EL_PORT
 REDIS_PORT=$REDIS_PORT
 POSTGRES_PORT=$POSTGRES_PORT
 ALLOY_PORT=$ALLOY_PORT
+CLICKHOUSE_PORT=$CLICKHOUSE_PORT
 LBT_PORT=$LBT_PORT
 EOF
 
@@ -65,6 +69,19 @@ alloyWriter:
   port: "$ALLOY_PORT"
   user: postgres
   password: "pass"
+clickhouse:
+  readerDatabase:
+    name: clickhouse
+    host: 127.0.0.1
+    port: "$CLICKHOUSE_PORT"
+    user: postgres
+    password: "pass"
+  writerDatabase:
+    name: clickhouse
+    host: 127.0.0.1
+    port: "$CLICKHOUSE_PORT"
+    user: postgres
+    password: "pass"
 bigtable:
   project: explorer
   instance: explorer
@@ -135,7 +152,7 @@ go run ./cmd/misc/main.go -config local_deployment/config.yml -command initBigta
 echo "bigtable schema initialization completed"
 
 echo "provisioning postgres db schema"
-go run ./cmd/misc/main.go -config local_deployment/config.yml -command applyDbSchema
+go run ./cmd/misc/main.go -config local_deployment/config.yml -command applyDbSchema -target-version -2 -target-database postgres
 echo "postgres db schema initialization completed"
 
 echo "provisioning alloy db schema"

diff --git a/backend/pkg/api/data_access/data_access.go b/backend/pkg/api/data_access/data_access.go
@@ -147,7 +147,7 @@ func createDataAccessService(cfg *types.Config) *DataAccessService {
 				Port:         cfg.ClickHouse.ReaderDatabase.Port,
 				MaxOpenConns: cfg.ClickHouse.ReaderDatabase.MaxOpenConns,
 				MaxIdleConns: cfg.ClickHouse.ReaderDatabase.MaxIdleConns,
-				SSL:          true,
+				SSL:          cfg.ClickHouse.ReaderDatabase.SSL,
 			},
 			// lets just reuse reader to be extra safe
 			&types.DatabaseConfig{
@@ -158,7 +158,7 @@ func createDataAccessService(cfg *types.Config) *DataAccessService {
 				Port:         cfg.ClickHouse.ReaderDatabase.Port,
 				MaxOpenConns: cfg.ClickHouse.ReaderDatabase.MaxOpenConns,
 				MaxIdleConns: cfg.ClickHouse.ReaderDatabase.MaxIdleConns,
-				SSL:          true,
+				SSL:          cfg.ClickHouse.ReaderDatabase.SSL,
 			}, "clickhouse", "clickhouse",
 		)
 	}()

diff --git a/backend/pkg/commons/db/db.go b/backend/pkg/commons/db/db.go
@@ -132,14 +132,14 @@ func MustInitDB(writer *types.DatabaseConfig, reader *types.DatabaseConfig, driv
 
 	if driverName == "clickhouse" {
 		sslParam = "secure=false"
-		if writer.SSL {
+		if reader.SSL {
 			sslParam = "secure=true"
 		}
 		// debug
 		// sslParam += "&debug=true"
 	} else {
 		sslParam = "sslmode=disable"
-		if writer.SSL {
+		if reader.SSL {
 			sslParam = "sslmode=require"
 		}
 	}
@@ -150,7 +150,7 @@ func MustInitDB(writer *types.DatabaseConfig, reader *types.DatabaseConfig, driv
 		log.Fatal(err, "error getting Connection Reader database", 0)
 	}
 
-	dbTestConnection(dbConnReader, fmt.Sprintf("database %v:%v/%v", writer.Host, writer.Port, writer.Name))
+	dbTestConnection(dbConnReader, fmt.Sprintf("database %v:%v/%v", reader.Host, reader.Port, writer.Name))
 	dbConnReader.SetConnMaxIdleTime(time.Second * 30)
 	dbConnReader.SetConnMaxLifetime(time.Minute)
 	dbConnReader.SetMaxOpenConns(reader.MaxOpenConns)

diff --git a/backend/pkg/commons/types/config.go b/backend/pkg/commons/types/config.go
@@ -95,6 +95,7 @@ type Config struct {
 			Port         string `yaml:"port" envconfig:"CLICKHOUSE_READER_DB_PORT"`
 			MaxOpenConns int    `yaml:"maxOpenConns" envconfig:"CLICKHOUSE_READER_DB_MAX_OPEN_CONNS"`
 			MaxIdleConns int    `yaml:"maxIdleConns" envconfig:"CLICKHOUSE_READER_DB_MAX_IDLE_CONNS"`
+			SSL          bool   `yaml:"ssl" envconfig:"CLICKHOUSE_READER_DB_SSL"`
 		} `yaml:"readerDatabase"`
 		WriterDatabase struct {
 			Username     string `yaml:"user" envconfig:"CLICKHOUSE_WRITER_DB_USERNAME"`
@@ -104,6 +105,7 @@ type Config struct {
 			Port         string `yaml:"port" envconfig:"CLICKHOUSE_WRITER_DB_PORT"`
 			MaxOpenConns int    `yaml:"maxOpenConns" envconfig:"CLICKHOUSE_WRITER_DB_MAX_OPEN_CONNS"`
 			MaxIdleConns int    `yaml:"maxIdleConns" envconfig:"CLICKHOUSE_WRITER_DB_MAX_IDLE_CONNS"`
+			SSL          bool   `yaml:"ssl" envconfig:"CLICKHOUSE_WRITER_DB_SSL"`
 		} `yaml:"writerDatabase"`
 	} `yaml:"clickhouse"`
 	Indexer struct {