diff --git a/ops/clusters/local/apps.yaml b/ops/clusters/local/apps.yaml index 4fcb50aef..f92a7ca0c 100644 --- a/ops/clusters/local/apps.yaml +++ b/ops/clusters/local/apps.yaml @@ -7,6 +7,8 @@ metadata: spec: dependsOn: - name: infrastructure-controllers + - name: certificates + - name: operators interval: 1h retryInterval: 1m timeout: 5m diff --git a/ops/clusters/local/certificates.yaml b/ops/clusters/local/certificates.yaml new file mode 100644 index 000000000..dd1595e04 --- /dev/null +++ b/ops/clusters/local/certificates.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: certificates + namespace: flux-system +spec: + dependsOn: + - name: infrastructure-controllers + interval: 1h + retryInterval: 1m + timeout: 5m + sourceRef: + kind: GitRepository + name: flux-system + path: ./ops/k8s-certificates/kind + force: true + prune: true + wait: true \ No newline at end of file diff --git a/ops/clusters/local/operators.yaml b/ops/clusters/local/operators.yaml new file mode 100644 index 000000000..038548132 --- /dev/null +++ b/ops/clusters/local/operators.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: operators + namespace: flux-system +spec: + dependsOn: + - name: certificates + interval: 1h + retryInterval: 1m + timeout: 5m + sourceRef: + kind: GitRepository + name: flux-system + path: ./ops/k8s-operators/kind + force: true + prune: true + wait: true \ No newline at end of file diff --git a/ops/clusters/warehouse/certificates.yaml b/ops/clusters/warehouse/certificates.yaml new file mode 100644 index 000000000..93a419b9c --- /dev/null +++ b/ops/clusters/warehouse/certificates.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: certificates + namespace: flux-system +spec: + dependsOn: + - name: infrastructure-controllers + interval: 1h + retryInterval: 1m + timeout: 5m + sourceRef: + kind: GitRepository + name: flux-system + path: ./ops/k8s-certificates/gke + force: true + prune: true + wait: true \ No newline at end of file diff --git a/ops/clusters/warehouse/operators.yaml b/ops/clusters/warehouse/operators.yaml new file mode 100644 index 000000000..b900cd4cb --- /dev/null +++ b/ops/clusters/warehouse/operators.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: operators + namespace: flux-system +spec: + dependsOn: + - name: certificates + interval: 1h + retryInterval: 1m + timeout: 5m + sourceRef: + kind: GitRepository + name: flux-system + path: ./ops/k8s-operators/gke + force: true + prune: true + wait: true \ No newline at end of file diff --git a/ops/clusters/warehouse/production-apps.yaml b/ops/clusters/warehouse/production-apps.yaml index f99c3afb0..c80c1b0d0 100644 --- a/ops/clusters/warehouse/production-apps.yaml +++ b/ops/clusters/warehouse/production-apps.yaml @@ -7,6 +7,8 @@ metadata: spec: dependsOn: - name: infrastructure-controllers + - name: certificates + - name: operators interval: 1h retryInterval: 1m timeout: 5m diff --git a/ops/clusters/warehouse/staging-apps.yaml b/ops/clusters/warehouse/staging-apps.yaml index e13fe7b8e..66de3a872 100644 --- a/ops/clusters/warehouse/staging-apps.yaml +++ b/ops/clusters/warehouse/staging-apps.yaml @@ -7,6 +7,8 @@ metadata: spec: dependsOn: - name: infrastructure-controllers + - name: certificates + - name: operators interval: 1h retryInterval: 1m timeout: 5m diff --git a/ops/k8s-apps/base/trino/trino.yaml b/ops/k8s-apps/base/trino/trino.yaml index 9e65fe7b6..20b1ce5c6 100644 --- a/ops/k8s-apps/base/trino/trino.yaml +++ b/ops/k8s-apps/base/trino/trino.yaml @@ -34,16 +34,6 @@ spec: remediation: retries: 3 values: - additionalConfigProperties: - - retry-policy=QUERY - additionalExchangeManagerProperties: - - "exchange.sink-buffers-per-partition=6" - - "exchange.sink-buffer-pool-min-size=6" - - "exchange.source-concurrent-readers=6" - - "exchange.s3.region=us" - - "exchange.s3.aws-access-key=${ENV:TRINO_GCS_KEY_ID}" - - "exchange.s3.aws-secret-key=${ENV:TRINO_GCS_SECRET}" - - "exchange.s3.endpoint=https://storage.googleapis.com" serviceAccount: create: true name: base-trino @@ -54,8 +44,6 @@ spec: additionalVolumeMounts: - name: cache mountPath: /metrics-cache - jvm: - maxHeapSize: "17G" additionalJVMConfig: - "--add-opens=java.base/java.nio=ALL-UNNAMED" @@ -67,71 +55,5 @@ spec: - name: cache mountPath: /metrics-cache - config: - query: - maxMemoryPerNode: 140GB - jvm: - maxHeapSize: "350G" additionalJVMConfig: - - "--add-opens=java.base/java.nio=ALL-UNNAMED" - - server: - exchangeManager: - name: filesystem - baseDir: gs://oso-dataset-transfer-bucket/trino-exchange/ - config: - query: - maxMemory: "1400GB" - workers: 1 - autoscaling: - enabled: true - maxReplicas: 9 - targetCPUUtilizationPercentage: 20 - behavior: - scaleDown: - stabilizationWindowSeconds: 300 - policies: - - type: Pods - value: 1 - periodSeconds: 60 - scaleUp: - stabilizationWindowSeconds: 0 - policies: - - type: Percent - value: 100 - periodSeconds: 15 - - type: Pods - value: 4 - periodSeconds: 15 - selectPolicy: Max - catalogs: - metrics: | - connector.name=iceberg - iceberg.catalog.type=hive_metastore - hive.metastore.uri=thrift://10.145.192.27:9083 - hive.metastore-cache-ttl=0s - hive.metastore-refresh-interval=5s - hive.metastore.thrift.client.connect-timeout=10s - hive.metastore.thrift.client.read-timeout=30s - iceberg.use-file-size-from-metadata=false - fs.native-gcs.enabled=true - fs.cache.enabled=true - fs.cache.max-sizes=300GB - fs.cache.directories=/metrics-cache - gcs.project-id=opensource-observer - iceberg.max-partitions-per-writer=1000 - source: | - connector.name=hive - hive.metastore.uri=thrift://10.145.192.27:9083 - fs.native-gcs.enabled=true - gcs.project-id=opensource-observer - hive.non-managed-table-writes-enabled=true - bigquery: | - connector.name=bigquery - bigquery.project-id=opensource-observer - clickhouse: | - connector.name=clickhouse - connection-url=${ENV:CLICKHOUSE_URL} - connection-user=${ENV:CLICKHOUSE_USER} - connection-password=${ENV:CLICKHOUSE_PASSWORD} - clickhouse.map-string-as-varchar=true + - "--add-opens=java.base/java.nio=ALL-UNNAMED" \ No newline at end of file diff --git a/ops/k8s-apps/local/kustomization.yaml b/ops/k8s-apps/local/kustomization.yaml index 57a71ae43..24b313023 100644 --- a/ops/k8s-apps/local/kustomization.yaml +++ b/ops/k8s-apps/local/kustomization.yaml @@ -1,6 +1,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -resources: +resources: [] - ./redis - ./minio - ./test diff --git a/ops/k8s-apps/local/metrics-calculation-service/custom-helm-values.yaml b/ops/k8s-apps/local/metrics-calculation-service/custom-helm-values.yaml new file mode 100644 index 000000000..9aa3731c9 --- /dev/null +++ b/ops/k8s-apps/local/metrics-calculation-service/custom-helm-values.yaml @@ -0,0 +1,15 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: local-mcs +spec: + values: + mcs: + cluster: + image: + repo: "ghcr.io/opensource-observer/oso" + tag: "latest" + trino: + host: local-trino-trino.local-trino.svc.cluster.local + port: 8080 + user: sqlmesh \ No newline at end of file diff --git a/ops/k8s-apps/local/metrics-calculation-service/kustomization.yaml b/ops/k8s-apps/local/metrics-calculation-service/kustomization.yaml new file mode 100644 index 000000000..bfd93e372 --- /dev/null +++ b/ops/k8s-apps/local/metrics-calculation-service/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../../base/metrics-calculation-service +namespace: local-mcs +patches: + - path: ./custom-helm-values.yaml + target: + kind: HelmRelease + options: + allowNameChange: true diff --git a/ops/k8s-apps/production/trino/custom-helm-values.yaml b/ops/k8s-apps/production/trino/custom-helm-values.yaml index f1e5d69f9..30be3817a 100644 --- a/ops/k8s-apps/production/trino/custom-helm-values.yaml +++ b/ops/k8s-apps/production/trino/custom-helm-values.yaml @@ -29,6 +29,9 @@ spec: effect: NoSchedule nodeSelector: pool_type: trino-coordinator + jvm: + maxHeapSize: "17G" + worker: resources: requests: @@ -40,4 +43,81 @@ spec: value: trino-worker effect: NoSchedule nodeSelector: - pool_type: trino-worker \ No newline at end of file + pool_type: trino-worker + config: + query: + maxMemoryPerNode: 140GB + jvm: + maxHeapSize: "350G" + + additionalConfigProperties: + - retry-policy=QUERY + additionalExchangeManagerProperties: + - "exchange.sink-buffers-per-partition=6" + - "exchange.sink-buffer-pool-min-size=6" + - "exchange.source-concurrent-readers=6" + - "exchange.s3.region=us" + - "exchange.s3.aws-access-key=${ENV:TRINO_GCS_KEY_ID}" + - "exchange.s3.aws-secret-key=${ENV:TRINO_GCS_SECRET}" + - "exchange.s3.endpoint=https://storage.googleapis.com" + + server: + exchangeManager: + name: filesystem + baseDir: gs://oso-dataset-transfer-bucket/trino-exchange/ + config: + query: + maxMemory: "1400GB" + workers: 1 + autoscaling: + enabled: true + maxReplicas: 9 + targetCPUUtilizationPercentage: 20 + behavior: + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: Pods + value: 1 + periodSeconds: 60 + scaleUp: + stabilizationWindowSeconds: 0 + policies: + - type: Percent + value: 100 + periodSeconds: 15 + - type: Pods + value: 4 + periodSeconds: 15 + selectPolicy: Max + catalogs: + metrics: | + connector.name=iceberg + iceberg.catalog.type=hive_metastore + hive.metastore.uri=thrift://10.145.192.27:9083 + hive.metastore-cache-ttl=0s + hive.metastore-refresh-interval=5s + hive.metastore.thrift.client.connect-timeout=10s + hive.metastore.thrift.client.read-timeout=30s + iceberg.use-file-size-from-metadata=false + fs.native-gcs.enabled=true + fs.cache.enabled=true + fs.cache.max-sizes=300GB + fs.cache.directories=/metrics-cache + gcs.project-id=opensource-observer + iceberg.max-partitions-per-writer=1000 + source: | + connector.name=hive + hive.metastore.uri=thrift://10.145.192.27:9083 + fs.native-gcs.enabled=true + gcs.project-id=opensource-observer + hive.non-managed-table-writes-enabled=true + bigquery: | + connector.name=bigquery + bigquery.project-id=opensource-observer + clickhouse: | + connector.name=clickhouse + connection-url=${ENV:CLICKHOUSE_URL} + connection-user=${ENV:CLICKHOUSE_USER} + connection-password=${ENV:CLICKHOUSE_PASSWORD} + clickhouse.map-string-as-varchar=true \ No newline at end of file diff --git a/ops/k8s-infrastructure/common/self-signed-certs.yaml b/ops/k8s-certificates/common/self-signed-certs.yaml similarity index 100% rename from ops/k8s-infrastructure/common/self-signed-certs.yaml rename to ops/k8s-certificates/common/self-signed-certs.yaml diff --git a/ops/k8s-certificates/gke/kustomization.yaml b/ops/k8s-certificates/gke/kustomization.yaml new file mode 100644 index 000000000..67609b1f1 --- /dev/null +++ b/ops/k8s-certificates/gke/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../common/self-signed-certs.yaml diff --git a/ops/k8s-certificates/kind/kustomization.yaml b/ops/k8s-certificates/kind/kustomization.yaml new file mode 100644 index 000000000..67609b1f1 --- /dev/null +++ b/ops/k8s-certificates/kind/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../common/self-signed-certs.yaml diff --git a/ops/k8s-infrastructure/common/cert-manager.yaml b/ops/k8s-infrastructure/common/cert-manager.yaml index 964aa2387..747598873 100644 --- a/ops/k8s-infrastructure/common/cert-manager.yaml +++ b/ops/k8s-infrastructure/common/cert-manager.yaml @@ -41,6 +41,8 @@ metadata: name: trust-manager namespace: cert-manager spec: + dependsOn: + - name: cert-manager interval: 30m chart: spec: diff --git a/ops/k8s-infrastructure/gke/kustomization.yaml b/ops/k8s-infrastructure/gke/kustomization.yaml index ff7d0854a..4db51a9f1 100644 --- a/ops/k8s-infrastructure/gke/kustomization.yaml +++ b/ops/k8s-infrastructure/gke/kustomization.yaml @@ -2,7 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ../common/cert-manager.yaml -- ../common/self-signed-certs.yaml - kube-secrets-init.yaml - ingress-internal-cloudflare.yaml - cloudsql-proxy-operator.yaml diff --git a/ops/k8s-infrastructure/kind/kustomization.yaml b/ops/k8s-infrastructure/kind/kustomization.yaml index 1c63d5f6e..803b11324 100644 --- a/ops/k8s-infrastructure/kind/kustomization.yaml +++ b/ops/k8s-infrastructure/kind/kustomization.yaml @@ -2,5 +2,3 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ../common/cert-manager.yaml -- ../common/self-signed-certs.yaml -- ./minio-operator.yaml diff --git a/ops/k8s-operators/gke/kustomization.yaml b/ops/k8s-operators/gke/kustomization.yaml new file mode 100644 index 000000000..b83b23e57 --- /dev/null +++ b/ops/k8s-operators/gke/kustomization.yaml @@ -0,0 +1,3 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: [] diff --git a/ops/k8s-operators/kind/kustomization.yaml b/ops/k8s-operators/kind/kustomization.yaml new file mode 100644 index 000000000..f32bced69 --- /dev/null +++ b/ops/k8s-operators/kind/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ./minio-operator.yaml diff --git a/ops/k8s-infrastructure/kind/minio-operator.yaml b/ops/k8s-operators/kind/minio-operator.yaml similarity index 100% rename from ops/k8s-infrastructure/kind/minio-operator.yaml rename to ops/k8s-operators/kind/minio-operator.yaml diff --git a/ops/opsscripts/cli.py b/ops/opsscripts/cli.py index 593a6dcf4..e04ef958b 100644 --- a/ops/opsscripts/cli.py +++ b/ops/opsscripts/cli.py @@ -120,9 +120,13 @@ def cluster_setup(branch_name, cluster_name, repo_owner, repo_name): check=True, ) + rendered_flux_instance = flux_instance_yaml.format( + repo_owner=repo_owner, repo_name=repo_name, branch_name=branch_name + ) + subprocess.run( ["kubectl", "apply", "-f", "-"], - input=flux_instance_yaml.encode("utf-8"), + input=rendered_flux_instance.encode("utf-8"), check=True, )