Fixes deployment problems caused by kind changes (#2807)

opensource-observer · Jan 18, 2025 · c2157c6 · c2157c6
1 parent 8b223ff
commit c2157c6
Show file tree

Hide file tree

Showing 22 changed files with 213 additions and 85 deletions.
diff --git a/ops/clusters/local/apps.yaml b/ops/clusters/local/apps.yaml
@@ -7,6 +7,8 @@ metadata:
 spec:
   dependsOn:
     - name: infrastructure-controllers
+    - name: certificates
+    - name: operators
   interval: 1h
   retryInterval: 1m
   timeout: 5m

diff --git a/ops/clusters/local/certificates.yaml b/ops/clusters/local/certificates.yaml
@@ -0,0 +1,19 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: certificates
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: infrastructure-controllers
+  interval: 1h
+  retryInterval: 1m
+  timeout: 5m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./ops/k8s-certificates/kind
+  force: true
+  prune: true
+  wait: true
diff --git a/ops/clusters/local/operators.yaml b/ops/clusters/local/operators.yaml
@@ -0,0 +1,19 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: operators
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: certificates
+  interval: 1h
+  retryInterval: 1m
+  timeout: 5m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./ops/k8s-operators/kind
+  force: true
+  prune: true
+  wait: true
diff --git a/ops/clusters/warehouse/certificates.yaml b/ops/clusters/warehouse/certificates.yaml
@@ -0,0 +1,19 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: certificates
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: infrastructure-controllers
+  interval: 1h
+  retryInterval: 1m
+  timeout: 5m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./ops/k8s-certificates/gke
+  force: true
+  prune: true
+  wait: true
diff --git a/ops/clusters/warehouse/operators.yaml b/ops/clusters/warehouse/operators.yaml
@@ -0,0 +1,19 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: operators
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: certificates
+  interval: 1h
+  retryInterval: 1m
+  timeout: 5m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./ops/k8s-operators/gke
+  force: true
+  prune: true
+  wait: true
diff --git a/ops/clusters/warehouse/production-apps.yaml b/ops/clusters/warehouse/production-apps.yaml
@@ -7,6 +7,8 @@ metadata:
 spec:
   dependsOn:
     - name: infrastructure-controllers
+    - name: certificates
+    - name: operators
   interval: 1h
   retryInterval: 1m
   timeout: 5m

diff --git a/ops/clusters/warehouse/staging-apps.yaml b/ops/clusters/warehouse/staging-apps.yaml
@@ -7,6 +7,8 @@ metadata:
 spec:
   dependsOn:
     - name: infrastructure-controllers
+    - name: certificates
+    - name: operators
   interval: 1h
   retryInterval: 1m
   timeout: 5m

diff --git a/ops/k8s-apps/base/trino/trino.yaml b/ops/k8s-apps/base/trino/trino.yaml
@@ -34,16 +34,6 @@ spec:
     remediation:
       retries: 3
   values:
-    additionalConfigProperties:
-      - retry-policy=QUERY
-    additionalExchangeManagerProperties:
-      - "exchange.sink-buffers-per-partition=6"
-      - "exchange.sink-buffer-pool-min-size=6"
-      - "exchange.source-concurrent-readers=6"
-      - "exchange.s3.region=us"
-      - "exchange.s3.aws-access-key=${ENV:TRINO_GCS_KEY_ID}"
-      - "exchange.s3.aws-secret-key=${ENV:TRINO_GCS_SECRET}"
-      - "exchange.s3.endpoint=https://storage.googleapis.com"
     serviceAccount:
       create: true
       name: base-trino
@@ -54,8 +44,6 @@ spec:
       additionalVolumeMounts:
         - name: cache
           mountPath: /metrics-cache
-      jvm:
-        maxHeapSize: "17G"
       additionalJVMConfig:
         - "--add-opens=java.base/java.nio=ALL-UNNAMED"
 
@@ -67,71 +55,5 @@ spec:
         - name: cache
           mountPath: /metrics-cache
 
-      config:
-        query:
-          maxMemoryPerNode: 140GB
-      jvm:
-        maxHeapSize: "350G"
       additionalJVMConfig:
-        - "--add-opens=java.base/java.nio=ALL-UNNAMED"
-
-    server:
-      exchangeManager:
-        name: filesystem
-        baseDir: gs://oso-dataset-transfer-bucket/trino-exchange/
-      config:
-        query:
-          maxMemory: "1400GB"
-      workers: 1
-      autoscaling:
-        enabled: true
-        maxReplicas: 9
-        targetCPUUtilizationPercentage: 20
-        behavior:
-          scaleDown:
-            stabilizationWindowSeconds: 300
-            policies:
-            - type: Pods
-              value: 1
-              periodSeconds: 60
-          scaleUp:
-            stabilizationWindowSeconds: 0
-            policies:
-            - type: Percent
-              value: 100
-              periodSeconds: 15
-            - type: Pods
-              value: 4
-              periodSeconds: 15
-            selectPolicy: Max
-    catalogs:
-      metrics: |
-        connector.name=iceberg
-        iceberg.catalog.type=hive_metastore
-        hive.metastore.uri=thrift://10.145.192.27:9083
-        hive.metastore-cache-ttl=0s
-        hive.metastore-refresh-interval=5s
-        hive.metastore.thrift.client.connect-timeout=10s
-        hive.metastore.thrift.client.read-timeout=30s
-        iceberg.use-file-size-from-metadata=false
-        fs.native-gcs.enabled=true
-        fs.cache.enabled=true
-        fs.cache.max-sizes=300GB
-        fs.cache.directories=/metrics-cache
-        gcs.project-id=opensource-observer
-        iceberg.max-partitions-per-writer=1000
-      source: |
-        connector.name=hive
-        hive.metastore.uri=thrift://10.145.192.27:9083
-        fs.native-gcs.enabled=true
-        gcs.project-id=opensource-observer
-        hive.non-managed-table-writes-enabled=true
-      bigquery: |
-        connector.name=bigquery
-        bigquery.project-id=opensource-observer
-      clickhouse: |
-        connector.name=clickhouse
-        connection-url=${ENV:CLICKHOUSE_URL}
-        connection-user=${ENV:CLICKHOUSE_USER}
-        connection-password=${ENV:CLICKHOUSE_PASSWORD}
-        clickhouse.map-string-as-varchar=true
+        - "--add-opens=java.base/java.nio=ALL-UNNAMED"
diff --git a/ops/k8s-apps/local/kustomization.yaml b/ops/k8s-apps/local/kustomization.yaml
@@ -1,6 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-resources:
+resources: []
   - ./redis
   - ./minio
   - ./test

diff --git a/ops/k8s-apps/local/metrics-calculation-service/custom-helm-values.yaml b/ops/k8s-apps/local/metrics-calculation-service/custom-helm-values.yaml
@@ -0,0 +1,15 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: local-mcs
+spec:
+  values:
+    mcs:
+      cluster:
+        image:
+          repo: "ghcr.io/opensource-observer/oso"
+          tag: "latest"
+      trino:
+        host: local-trino-trino.local-trino.svc.cluster.local
+        port: 8080
+        user: sqlmesh
diff --git a/ops/k8s-apps/local/metrics-calculation-service/kustomization.yaml b/ops/k8s-apps/local/metrics-calculation-service/kustomization.yaml
@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../../base/metrics-calculation-service
+namespace: local-mcs
+patches:
+  - path: ./custom-helm-values.yaml
+    target:
+      kind: HelmRelease
+    options:
+      allowNameChange: true
diff --git a/ops/k8s-apps/production/trino/custom-helm-values.yaml b/ops/k8s-apps/production/trino/custom-helm-values.yaml
@@ -29,6 +29,9 @@ spec:
           effect: NoSchedule
       nodeSelector:
         pool_type: trino-coordinator
+      jvm:
+        maxHeapSize: "17G"
+
     worker:
       resources:
         requests:
@@ -40,4 +43,81 @@ spec:
           value: trino-worker
           effect: NoSchedule
       nodeSelector:
-        pool_type: trino-worker
+        pool_type: trino-worker
+			config:
+				query:
+					maxMemoryPerNode: 140GB
+			jvm:
+				maxHeapSize: "350G"
+
+    additionalConfigProperties:
+      - retry-policy=QUERY
+    additionalExchangeManagerProperties:
+      - "exchange.sink-buffers-per-partition=6"
+      - "exchange.sink-buffer-pool-min-size=6"
+      - "exchange.source-concurrent-readers=6"
+      - "exchange.s3.region=us"
+      - "exchange.s3.aws-access-key=${ENV:TRINO_GCS_KEY_ID}"
+      - "exchange.s3.aws-secret-key=${ENV:TRINO_GCS_SECRET}"
+      - "exchange.s3.endpoint=https://storage.googleapis.com"
+
+    server:
+      exchangeManager:
+        name: filesystem
+        baseDir: gs://oso-dataset-transfer-bucket/trino-exchange/
+      config:
+        query:
+          maxMemory: "1400GB"
+      workers: 1
+      autoscaling:
+        enabled: true
+        maxReplicas: 9
+        targetCPUUtilizationPercentage: 20
+        behavior:
+          scaleDown:
+            stabilizationWindowSeconds: 300
+            policies:
+            - type: Pods
+              value: 1
+              periodSeconds: 60
+          scaleUp:
+            stabilizationWindowSeconds: 0
+            policies:
+            - type: Percent
+              value: 100
+              periodSeconds: 15
+            - type: Pods
+              value: 4
+              periodSeconds: 15
+            selectPolicy: Max
+    catalogs:
+      metrics: |
+        connector.name=iceberg
+        iceberg.catalog.type=hive_metastore
+        hive.metastore.uri=thrift://10.145.192.27:9083
+        hive.metastore-cache-ttl=0s
+        hive.metastore-refresh-interval=5s
+        hive.metastore.thrift.client.connect-timeout=10s
+        hive.metastore.thrift.client.read-timeout=30s
+        iceberg.use-file-size-from-metadata=false
+        fs.native-gcs.enabled=true
+        fs.cache.enabled=true
+        fs.cache.max-sizes=300GB
+        fs.cache.directories=/metrics-cache
+        gcs.project-id=opensource-observer
+        iceberg.max-partitions-per-writer=1000
+      source: |
+        connector.name=hive
+        hive.metastore.uri=thrift://10.145.192.27:9083
+        fs.native-gcs.enabled=true
+        gcs.project-id=opensource-observer
+        hive.non-managed-table-writes-enabled=true
+      bigquery: |
+        connector.name=bigquery
+        bigquery.project-id=opensource-observer
+      clickhouse: |
+        connector.name=clickhouse
+        connection-url=${ENV:CLICKHOUSE_URL}
+        connection-user=${ENV:CLICKHOUSE_USER}
+        connection-password=${ENV:CLICKHOUSE_PASSWORD}
+        clickhouse.map-string-as-varchar=true
diff --git a/...rastructure/common/self-signed-certs.yaml → ...ertificates/common/self-signed-certs.yaml b/...rastructure/common/self-signed-certs.yaml → ...ertificates/common/self-signed-certs.yaml
diff --git a/ops/k8s-certificates/gke/kustomization.yaml b/ops/k8s-certificates/gke/kustomization.yaml
@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../common/self-signed-certs.yaml
diff --git a/ops/k8s-certificates/kind/kustomization.yaml b/ops/k8s-certificates/kind/kustomization.yaml
@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../common/self-signed-certs.yaml
diff --git a/ops/k8s-infrastructure/common/cert-manager.yaml b/ops/k8s-infrastructure/common/cert-manager.yaml
@@ -41,6 +41,8 @@ metadata:
   name: trust-manager
   namespace: cert-manager
 spec:
+  dependsOn:
+    - name: cert-manager
   interval: 30m
   chart:
     spec:

diff --git a/ops/k8s-infrastructure/gke/kustomization.yaml b/ops/k8s-infrastructure/gke/kustomization.yaml
@@ -2,7 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
 - ../common/cert-manager.yaml
-- ../common/self-signed-certs.yaml
 - kube-secrets-init.yaml
 - ingress-internal-cloudflare.yaml
 - cloudsql-proxy-operator.yaml
diff --git a/ops/k8s-infrastructure/kind/kustomization.yaml b/ops/k8s-infrastructure/kind/kustomization.yaml
@@ -2,5 +2,3 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
 - ../common/cert-manager.yaml
-- ../common/self-signed-certs.yaml
-- ./minio-operator.yaml
diff --git a/ops/k8s-operators/gke/kustomization.yaml b/ops/k8s-operators/gke/kustomization.yaml
@@ -0,0 +1,3 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources: []
diff --git a/ops/k8s-operators/kind/kustomization.yaml b/ops/k8s-operators/kind/kustomization.yaml
@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ./minio-operator.yaml
diff --git a/...s-infrastructure/kind/minio-operator.yaml → ops/k8s-operators/kind/minio-operator.yaml b/...s-infrastructure/kind/minio-operator.yaml → ops/k8s-operators/kind/minio-operator.yaml
diff --git a/ops/opsscripts/cli.py b/ops/opsscripts/cli.py
@@ -120,9 +120,13 @@ def cluster_setup(branch_name, cluster_name, repo_owner, repo_name):
                 check=True,
             )
 
+    rendered_flux_instance = flux_instance_yaml.format(
+        repo_owner=repo_owner, repo_name=repo_name, branch_name=branch_name
+    )
+
     subprocess.run(
         ["kubectl", "apply", "-f", "-"],
-        input=flux_instance_yaml.encode("utf-8"),
+        input=rendered_flux_instance.encode("utf-8"),
         check=True,
     )