Local Kind (#2801)

* Add Flux v2.4.0 component manifests

* Add Flux v2.4.0 component manifests

* Add Flux sync manifests

* remove need for flux-system secret

* use public repo

* remove

* Setup kind infrastructure base

* move kind cluster config out of flux area

* Test redis deployment

* connect apps

* add minio

* connect minio

* connect minio in apps

* fix minio

* use updated docs for minio

* fix cluster issuance

* install trust manager

* make into a deployment

* remove unnecessary files

* centralize log handler setup

---------

Co-authored-by: Flux <>

ops/clusters/local/README.md

@@ -0,0 +1,12 @@
+# Local testing cluster
+
+This is a local testing cluster that we can use to test trino and dagster
+deployment.
+
+## Setup
+
+From the root of the repository run this command:
+
+```bash
+$ oso kind setup
+```

ops/clusters/local/apps.yaml

@@ -0,0 +1,19 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: local-apps
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: infrastructure-controllers
+  interval: 1h
+  retryInterval: 1m
+  timeout: 5m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./ops/k8s-apps/local
+  force: true
+  prune: true
+  wait: true

ops/clusters/local/infrastructure.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: infrastructure-controllers
+  namespace: flux-system
+spec:
+  interval: 1h
+  retryInterval: 1m
+  timeout: 5m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./ops/k8s-infrastructure/kind
+  prune: true
+  wait: true

ops/clusters/warehouse/infrastructure.yaml

@@ -11,6 +11,6 @@ spec:
   sourceRef:
     kind: GitRepository
     name: flux-system
-  path: ./ops/k8s-infrastructure/controllers
+  path: ./ops/k8s-infrastructure/gke
   prune: true
   wait: true

ops/k8s-apps/local/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./redis
+  - ./minio
+  - ./test
+  #- ./trino

ops/k8s-apps/local/minio/kustomization.yaml

@@ -0,0 +1,6 @@
+# We use minio to emulate an s3 storage for trino and iceberg
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: local-minio
+resources:
+  - minio.yaml

ops/k8s-apps/local/minio/minio.yaml

@@ -0,0 +1,90 @@
+# Intentionally the password is admin as this is for local testing.
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: local-minio
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: minio-configuration
+  namespace: local-minio
+type: Opaque
+stringData:
+  config.env: |-
+    export MINIO_ROOT_USER="admin"
+    export MINIO_ROOT_PASSWORD="password"
+    export MINIO_STORAGE_CLASS_STANDARD="EC:0"
+    export MINIO_BROWSER="on"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: storage-user
+  namespace: local-minio
+type: Opaque
+data:
+  CONSOLE_ACCESS_KEY: c3RvcmFnZS11c2Vy # storage-user
+  CONSOLE_SECRET_KEY: cGFzc3dvcmQ=     # password
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: tenant-certmanager-cert
+  namespace: local-minio
+spec:
+  # If you see "Empty issuer DN not allowed in X509Certificates"
+  # You will need to add spec.subject.organizations or spec.commonName
+  # As documented in https://cert-manager.io/docs/usage/certificate/
+  # This specially if your other apps can't trust MinIO cert issued by cert-manager
+  # Example
+  #
+  # subject:
+  #   organizations:
+  #     - jetstack
+  # commonName: minio.local-minio.svc.cluster.local
+  dnsNames:
+    - "minio.local-minio"
+    - "minio.local-minio.svc"
+    - "minio.local-minio.svc.cluster.local"
+    - "*.local-minio.svc.cluster.local"
+    - "*.minio.local-minio.svc.cluster.local"
+  secretName: local-minio-tls
+  issuerRef:
+    name: cluster-ca
+    kind: ClusterIssuer
+    group: cert-manager.io
+---
+apiVersion: minio.min.io/v2
+kind: Tenant
+metadata:
+  name: minio-configuration
+  namespace: local-minio
+spec:
+  image: quay.io/minio/minio:RELEASE.2024-11-07T00-52-20Z
+  mountPath: /export
+  imagePullPolicy: IfNotPresent
+  pools:
+  - name: pool-0
+    servers: 1
+    volumeClaimTemplate:
+      metadata:
+        name: data
+      spec:
+        accessModes:
+        - ReadWriteOnce
+        resources:
+          requests:
+            storage: "1Gi"
+        storageClassName: standard
+    volumesPerServer: 1
+  requestAutoCert: false
+  ## Secret with credentials and configurations to be used by MinIO Tenant.
+  configuration:
+    name: minio-configuration
+  users:
+  - name: storage-user
+  externalCertSecret:
+    - name: local-minio-tls
+      type: cert-manager.io/v1

ops/k8s-apps/local/redis/kustomization.yaml

@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../../base/redis
+namespace: local-redis
+patches:
+  - path: redis-deployment.yaml
+    target:
+      kind: Deployment

ops/k8s-apps/local/redis/redis-deployment.yaml

@@ -0,0 +1,24 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: redis
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: redis
+  template:
+    metadata:
+      labels:
+        app: redis
+    spec:
+      containers:
+      - name: redis
+        image: redis:latest
+        ports:
+        - containerPort: 6379
+        env:
+        - name: REDIS_MAXMEMORY
+          value: "20mb"
+        - name: REDIS_MAXMEMORY_POLICY
+          value: "allkeys-lru"

ops/k8s-apps/local/test/README.md

@@ -0,0 +1 @@
+# A testing pod that runs forever

ops/k8s-apps/local/test/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: test
+resources:
+  - pod.yaml

ops/k8s-apps/local/test/pod.yaml

@@ -0,0 +1,39 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: test
+  labels:
+    opensource.observer/cert-inject: "enabled"
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test-deployment
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: test-pod
+  template:
+    metadata:
+      labels:
+        app: test-pod
+    spec:
+      containers:
+      - name: test-pod
+        image: ubuntu
+        command: ["bash", "-c", "while true; do sleep 30; done;" ]
+        volumeMounts:
+          - mountPath: /etc/ssl/certs/
+            name: cluster-self-signed-bundle
+            readOnly: true
+      volumes:
+        - name: cluster-self-signed-bundle
+          configMap:
+            name: cluster-self-signed-bundle
+            defaultMode: 0644
+            optional: false
+            items:
+            - key: ca-certificates.crt
+              path: ca-certificates.crt

ops/k8s-apps/local/trino/custom-helm-values.yaml

@@ -0,0 +1,8 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: production-trino
+spec:
+  values:
+    serviceAccount:
+      name: production-trino

ops/k8s-apps/local/trino/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../../base/trino
+namespace: local-trino
+patches:
+  - path: ./custom-helm-values.yaml
+    target:
+      kind: HelmRelease
+    options:
+      allowNameChange: true

ops/k8s-infrastructure/controllers/cert-manager.yaml → ops/k8s-infrastructure/common/cert-manager.yaml

@@ -10,7 +10,7 @@ metadata:
 apiVersion: source.toolkit.fluxcd.io/v1
 kind: HelmRepository
 metadata:
-  name: cert-manager
+  name: jetstack 
   namespace: cert-manager
 spec:
   interval: 24h
@@ -29,8 +29,25 @@ spec:
       version: "1.x"
       sourceRef:
         kind: HelmRepository
-        name: cert-manager
+        name: jetstack 
         namespace: cert-manager
       interval: 12h
   values:
-    installCRDs: true
+    installCRDs: true
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: trust-manager
+  namespace: cert-manager
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: trust-manager
+      version: "*"
+      sourceRef:
+        kind: HelmRepository
+        name: jetstack 
+        namespace: cert-manager
+      interval: 12h

ops/k8s-infrastructure/common/self-signed-certs.yaml

@@ -0,0 +1,65 @@
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: self-signed-root
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: self-signed-ca
+  namespace: cert-manager
+spec:
+  isCA: true
+  commonName: self-signed-ca
+  secretName: self-signed-root-secret
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    name: self-signed-root
+    kind: ClusterIssuer
+    group: cert-manager.io
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: cluster-ca 
+spec:
+  ca:
+    secretName: self-signed-root-secret
+---
+apiVersion: trust.cert-manager.io/v1alpha1
+kind: Bundle
+metadata:
+  name: cluster-self-signed-bundle  # The bundle name will also be used for the target
+spec:
+  sources:
+  # Include a bundle of publicly trusted certificates which can be
+  # used to validate most TLS certificates on the internet, such as
+  # those issued by Let's Encrypt, Google, Amazon and others.
+  - useDefaultCAs: true
+
+  # A Secret in the "trust" namespace; see "Trust Namespace" below for further details
+  - secret:
+      name: self-signed-root-secret
+      key: "ca.crt"
+
+  target:
+    # Sync the bundle to a ConfigMap called `my-org.com` in every namespace which
+    # has the label "linkerd.io/inject=enabled"
+    # All ConfigMaps will include a PEM-formatted bundle, here named "root-certs.pem"
+    # and in this case we also request binary formatted bundles in JKS and PKCS#12 formats,
+    # here named "bundle.jks" and "bundle.p12".
+    configMap:
+      key: "ca-certificates.crt"
+    additionalFormats:
+      jks:
+        key: "ca-certificates.jks"
+      pkcs12:
+        key: "ca-certificates.p12"
+    namespaceSelector:
+      matchLabels:
+        opensource.observer/cert-inject: "enabled"

ops/k8s-infrastructure/controllers/kustomization.yaml → ops/k8s-infrastructure/gke/kustomization.yaml

@@ -1,7 +1,8 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- cert-manager.yaml
+- ../common/cert-manager.yaml
+- ../common/self-signed-certs.yaml
 - kube-secrets-init.yaml
 - ingress-internal-cloudflare.yaml
 - cloudsql-proxy-operator.yaml