Merge pull request #31214 from def-/pr-gcp-terraform

GCP terraform + helm-chart test

bin/ci-builder

@@ -188,6 +188,7 @@ case "$cmd" in
             --env COMMON_ANCESTOR_OVERRIDE
             --env CONFLUENT_CLOUD_DEVEX_KAFKA_PASSWORD
             --env CONFLUENT_CLOUD_DEVEX_KAFKA_USERNAME
+            --env GCP_SERVICE_ACCOUNT_JSON
             --env GITHUB_TOKEN
             --env GPG_KEY
             --env LAUNCHDARKLY_API_TOKEN

ci/builder/Dockerfile

@@ -326,7 +326,17 @@ RUN curl -fsSL https://github.com/deb-s3/deb-s3/releases/download/0.11.3/deb-s3-
 RUN curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-$ARCH_GCC-2.17.2.zip" > awscli.zip \
     && unzip awscli.zip \
     && ./aws/install \
-    && rm -rf aws
+    && rm -rf aws awscli.zip
+
+# Install the gcloud CLI.
+
+RUN arch_gcloud=$(echo "$ARCH_GCC" | sed -e "s/aarch64/arm/" -e "s/amd64/x86_64/") \
+    && curl -fsSL "https://storage.googleapis.com/cloud-sdk-release/google-cloud-cli-507.0.0-linux-$arch_gcloud.tar.gz" > gcloud.tar.gz \
+    && tar -xzf gcloud.tar.gz -C /opt \
+    && rm gcloud.tar.gz \
+    && CLOUDSDK_CORE_DISABLE_PROMPTS=1 /opt/google-cloud-sdk/install.sh --path-update false \
+    && /opt/google-cloud-sdk/bin/gcloud config set disable_usage_reporting false \
+    && /opt/google-cloud-sdk/bin/gcloud components install gke-gcloud-auth-plugin
 
 # Install docs site dependencies. These are towards the end for the same reason
 # as the Python dependencies. These are only supported on x86_64 at the moment.
@@ -410,7 +420,7 @@ ENV TARGET_CC=$CC
 ENV TARGET_CXX=$CXX
 ENV TARGET_CXXSTDLIB=static=stdc++
 ENV TARGET_RANLIB=$RANLIB
-ENV PATH=/opt/x-tools/$ARCH_GCC-unknown-linux-gnu/bin:$PATH
+ENV PATH=/opt/google-cloud-sdk/bin:/opt/x-tools/$ARCH_GCC-unknown-linux-gnu/bin:$PATH
 ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-unknown-linux-gnu-cc
 ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-unknown-linux-gnu-cc
 ENV CARGO_TARGET_DIR=/mnt/build

ci/nightly/pipeline.template.yml

@@ -1228,15 +1228,32 @@ steps:
       depends_on: build-aarch64
       timeout_in_minutes: 1200
       concurrency: 1
-      concurrency_group: 'terraform'
+      concurrency_group: 'terraform-aws'
       agents:
         queue: linux-aarch64-small
       plugins:
         - ./ci/plugins/scratch-aws-access: ~
         - ./ci/plugins/mzcompose:
             composition: terraform
+            run: aws
       branches: "main v*.* lts-v*"
 
+    - id: terraform-gcp
+      label: "Terraform + Helm Chart E2E on GCP"
+      artifact_paths: [test/terraform/aws/terraform.tfstate]
+      depends_on: build-aarch64
+      timeout_in_minutes: 1200
+      concurrency: 1
+      concurrency_group: 'terraform-gcp'
+      agents:
+        queue: linux-aarch64-small
+      plugins:
+        - ./ci/plugins/scratch-aws-access: ~
+        - ./ci/plugins/mzcompose:
+            composition: terraform
+            run: gcp
+      #branches: "main v*.* lts-v*"
+
   - group: "Output consistency"
     key: output-consistency
     steps:

ci/plugins/mzcompose/hooks/pre-exit

@@ -85,7 +85,7 @@ timeout 300 buildkite-agent artifact upload "$artifacts_str" || true
 bin/ci-builder run stable bin/ci-annotate-errors --test-cmd="$(cat test_cmd)" --test-desc="$(cat test_desc)" "${artifacts[@]}" > ci-annotate-errors.log || CI_ANNOTATE_ERRORS_RESULT=$?
 buildkite-agent artifact upload "ci-annotate-errors.log"
 
-if [ ! -s services.log ] && [ "$BUILDKITE_LABEL" != "Maelstrom coverage of persist" ] && [ "$BUILDKITE_LABEL" != "Long single-node Maelstrom coverage of persist" ] && [ "$BUILDKITE_LABEL" != "Maelstrom coverage of txn-wal" ] && [ "$BUILDKITE_LABEL" != "Mz E2E Test" ] && [ "$BUILDKITE_LABEL" != "Output consistency (version for DFR)" ] && [ "$BUILDKITE_LABEL" != "Output consistency (version for CTF)" ] && [ "$BUILDKITE_LABEL" != "QA Canary Environment Base Load" ] && [ "$BUILDKITE_LABEL" != "Parallel Benchmark against QA Canary Environment" ] && [ "$BUILDKITE_LABEL" != "Parallel Benchmark against QA Benchmarking Staging Environment" ] && [ "$BUILDKITE_LABEL" != "Terraform + Helm Chart E2E on AWS" ]; then
+if [ ! -s services.log ] && [ "$BUILDKITE_LABEL" != "Maelstrom coverage of persist" ] && [ "$BUILDKITE_LABEL" != "Long single-node Maelstrom coverage of persist" ] && [ "$BUILDKITE_LABEL" != "Maelstrom coverage of txn-wal" ] && [ "$BUILDKITE_LABEL" != "Mz E2E Test" ] && [ "$BUILDKITE_LABEL" != "Output consistency (version for DFR)" ] && [ "$BUILDKITE_LABEL" != "Output consistency (version for CTF)" ] && [ "$BUILDKITE_LABEL" != "QA Canary Environment Base Load" ] && [ "$BUILDKITE_LABEL" != "Parallel Benchmark against QA Canary Environment" ] && [ "$BUILDKITE_LABEL" != "Parallel Benchmark against QA Benchmarking Staging Environment" ] && [ "$BUILDKITE_LABEL" != "Terraform + Helm Chart E2E on AWS" ] && [ "$BUILDKITE_LABEL" != "Terraform + Helm Chart E2E on GCP" ]; then
     echo "+++ services.log is empty, failing"
     exit 1
 fi

ci/test/lint-main/checks/check-mzcompose-files.sh

@@ -45,6 +45,7 @@ check_default_workflow_references_others() {
         -not -wholename "./test/canary-environment/mzcompose.py" `# Only run manually` \
         -not -wholename "./test/ssh-connection/mzcompose.py" `# Handled differently` \
         -not -wholename "./test/scalability/mzcompose.py" `# Other workflows are for manual usage` \
+        -not -wholename "./test/terraform/mzcompose.py" `# Handled differently` \
     )
 
     for file in "${MZCOMPOSE_TEST_FILES[@]}"; do

test/terraform/gcp/main.tf

@@ -0,0 +1,81 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+terraform {
+  required_version = ">= 1.0"
+
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = ">= 6.0"
+    }
+  }
+}
+
+provider "google" {
+  project = var.project_id
+  region  = var.region
+}
+
+module "materialize" {
+  source = "github.com/MaterializeInc/terraform-google-materialize?ref=v0.1.1"
+
+  project_id = var.project_id
+  region     = var.region
+  prefix     = "tf-gcp-test"
+
+  database_config = {
+    tier     = "db-custom-2-4096"
+    version  = "POSTGRES_15"
+    password = var.database_password
+  }
+
+  labels = {
+    environment = "simple"
+    example     = "true"
+  }
+
+  install_materialize_operator = false
+}
+
+variable "project_id" {
+  description = "GCP Project ID"
+  type        = string
+  default     = "materialize-ci"
+}
+
+variable "region" {
+  description = "GCP Region"
+  type        = string
+  default     = "us-east1"
+}
+
+variable "database_password" {
+  description = "Password for Cloud SQL database user"
+  default     = "your-strong-password"
+  type        = string
+  sensitive   = true
+}
+
+output "gke_cluster" {
+  description = "GKE cluster details"
+  value       = module.materialize.gke_cluster
+  sensitive   = true
+}
+
+output "service_accounts" {
+  description = "Service account details"
+  value       = module.materialize.service_accounts
+}
+
+output "connection_strings" {
+  description = "Connection strings for metadata and persistence backends"
+  value       = module.materialize.connection_strings
+  sensitive   = true
+}