feat: add cloudnative-pg

prehor · May 19, 2024 · 9d0a312 · 9d0a312
1 parent 000a07a
commit 9d0a312
Show file tree

Hide file tree

Showing 17 changed files with 442 additions and 0 deletions.
diff --git a/.github/renovate/versioning.json5 b/.github/renovate/versioning.json5
@@ -6,6 +6,13 @@
       "matchDatasources": ["github-releases"],
       "versioning": "regex:^v(?<major>\\d+)\\.(?<minor>\\d+)\\.(?<patch>\\d+)(?<compatibility>\\+k.s)\\.?(?<build>\\d+)$",
       "matchPackagePatterns": ["k3s"]
+    },
+    {
+      "description": ["Use custom versioning for TimescaleDB"],
+      "matchDatasources": ["docker"],
+      // https://docs.renovatebot.com/modules/versioning/#regular-expression-versioning
+      "versioning": "regex:^(?<major>\\d+)(-(?<minor>\\d+)\\.(?<patch>\\d+)(-(?<build>\\d+))?)?$",
+      "matchPackagePatterns": ["timescaledb"]
     }
   ]
 }
diff --git a/kubernetes/main/apps/database/cloudnative-pg/app/helmrelease.yaml b/kubernetes/main/apps/database/cloudnative-pg/app/helmrelease.yaml
@@ -0,0 +1,35 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2beta2.json
+apiVersion: helm.toolkit.fluxcd.io/v2beta2
+kind: HelmRelease
+metadata:
+  name: &app postgres-operator
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: cloudnative-pg
+      version: 0.21.2
+      sourceRef:
+        kind: HelmRepository
+        name: cloudnative-pg
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      strategy: rollback
+      retries: 3
+  dependsOn:
+    - name: openebs
+      namespace: storage
+  values:
+    crds:
+      create: true
+    fullnameOverride: *app
+    monitoring:
+      podMonitorEnabled: false
+      # grafanaDashboard:
+      #   create: true
diff --git a/kubernetes/main/apps/database/cloudnative-pg/app/kustomization.yaml b/kubernetes/main/apps/database/cloudnative-pg/app/kustomization.yaml
@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./helmrelease.yaml
+
diff --git a/kubernetes/main/apps/database/cloudnative-pg/cluster/cluster16.yaml b/kubernetes/main/apps/database/cloudnative-pg/cluster/cluster16.yaml
@@ -0,0 +1,91 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/postgresql.cnpg.io/cluster_v1.json
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: postgres16
+spec:
+  instances: 3
+  imageCatalogRef:
+    apiGroup: postgresql.cnpg.io
+    kind: ImageCatalog
+    name: timescaledb
+    major: 16
+  primaryUpdateStrategy: unsupervised
+  storage:
+    size: 15Gi
+    storageClass: local-database
+  superuserSecret:
+    name: postgres-cluster-secret
+  enableSuperuserAccess: true
+  postgresql:
+    parameters:
+      max_wal_size: "4096"  # See walSegmentSize: 1024 in bootstrap.initdb
+      shared_buffers: "256MB"
+      # ZFS tuning
+      # https://openzfs.github.io/openzfs-docs/Performance%20and%20Tuning/Workload%20Tuning.html#postgresql
+      # https://vadosware.io/post/everything-ive-seen-on-optimizing-postgres-on-zfs-on-linux/
+      # https://kubeblocks.io/blog/A-testing-report-for-optimizing-PG-performance-on-Kubernetes
+      # full_page_writes: "off"   # https://github.com/cloudnative-pg/cloudnative-pg/discussions/3058
+      recovery_prefetch: "try"
+      wal_init_zero: "off"
+      wal_recycle: "off"
+    shared_preload_libraries:
+      - timescaledb
+  nodeMaintenanceWindow:
+    inProgress: false
+    reusePVC: true
+  resources:
+    requests:
+      cpu: 100m
+    limits:
+      memory: 4Gi
+  monitoring:
+    enablePodMonitor: true
+    # https://github.com/cloudnative-pg/cloudnative-pg/issues/2501
+    podMonitorMetricRelabelings:
+      - { sourceLabels: ["cluster"], targetLabel: cnpg_cluster, action: replace }
+      - { regex: cluster, action: labeldrop }
+  backup:
+    retentionPolicy: 30d
+    barmanObjectStore: &barmanObjectStore
+      data:
+        compression: bzip2
+        encryption: AES256
+      wal:
+        compression: bzip2
+        encryption: AES256
+      destinationPath: s3://home-ops-postgresql
+      endpointURL: https://s3.${STORAGE_DOMAIN}
+      # Note: serverName version needs to be inclemented
+      # when recovering from an existing cnpg cluster
+      serverName: &currentCluster postgres16-v0
+      s3Credentials:
+        accessKeyId:
+          name: postgres-cluster-secret
+          key: MINIO_ACCESS_KEY
+        secretAccessKey:
+          name: postgres-cluster-secret
+          key: MINIO_SECRET_KEY
+  # Note: previousCluster needs to be set to the name of the previous
+  # cluster when recovering from an existing cnpg cluster
+  bootstrap:
+    initdb:
+      # The postgres-db-manager user will be used to create databases using onedr0p/postgres-init
+      database: postgres-db-manager
+      owner: postgres-db-manager
+      secret:
+        name: postgres-secret
+      postInitApplicationSQL:
+        - ALTER USER "postgres-db-manager" WITH CREATEDB;
+        - ALTER USER "postgres-db-manager" WITH CREATEROLE;
+      # ZFS tuning
+      walSegmentSize: 1024
+  #   recovery:
+  #     source: &previousCluster postgres16-v0
+  # # Note: externalClusters is needed when recovering from an existing cnpg cluster
+  # externalClusters:
+  #   - name: *previousCluster
+  #     barmanObjectStore:
+  #       <<: *barmanObjectStore
+  #       serverName: *previousCluster
diff --git a/kubernetes/main/apps/database/cloudnative-pg/cluster/imagecatalog.yaml b/kubernetes/main/apps/database/cloudnative-pg/cluster/imagecatalog.yaml
@@ -0,0 +1,10 @@
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: ImageCatalog
+metadata:
+  name: timescaledb
+spec:
+  images:
+    - major: 16
+      # renovate: datasource=docker
+      image: ghcr.io/imusmanmalik/timescaledb-postgis:16-3.4-42
diff --git a/kubernetes/main/apps/database/cloudnative-pg/cluster/kustomization.yaml b/kubernetes/main/apps/database/cloudnative-pg/cluster/kustomization.yaml
@@ -0,0 +1,10 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./cluster16.yaml
+  - ./imagecatalog.yaml
+  - ./prometheusrule.yaml
+  - ./secret.sops.yaml
+  - ./scheduledbackup.yaml
diff --git a/kubernetes/main/apps/database/cloudnative-pg/cluster/prometheusrule.yaml b/kubernetes/main/apps/database/cloudnative-pg/cluster/prometheusrule.yaml
@@ -0,0 +1,67 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/monitoring.coreos.com/prometheusrule_v1.json
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: cloudnative-pg-rules
+  labels:
+    prometheus: k8s
+    role: alert-rules
+spec:
+  groups:
+    - name: cloudnative-pg.rules
+      rules:
+        - alert: LongRunningTransaction
+          annotations:
+            description: Pod {{ $labels.pod }} is taking more than 5 minutes (300 seconds) for a query.
+            summary: A query is taking longer than 5 minutes.
+          expr: |-
+            cnpg_backends_max_tx_duration_seconds > 300
+          for: 1m
+          labels:
+            severity: warning
+        - alert: BackendsWaiting
+          annotations:
+            description: Pod {{ $labels.pod }} has been waiting for longer than 5 minutes
+            summary: If a backend is waiting for longer than 5 minutes
+          expr: |-
+            cnpg_backends_waiting_total > 300
+          for: 1m
+          labels:
+            severity: warning
+        - alert: PGDatabase
+          annotations:
+            description: Over 150,000,000 transactions from frozen xid on pod {{ $labels.pod }}
+            summary: Number of transactions from the frozen XID to the current one
+          expr: |-
+            cnpg_pg_database_xid_age > 150000000
+          for: 1m
+          labels:
+            severity: warning
+        - alert: PGReplication
+          annotations:
+            description: Standby is lagging behind by over 300 seconds (5 minutes)
+            summary: The standby is lagging behind the primary
+          expr: |-
+            cnpg_pg_replication_lag > 300
+          for: 1m
+          labels:
+            severity: warning
+        - alert: LastFailedArchiveTime
+          annotations:
+            description: Archiving failed for {{ $labels.pod }}
+            summary: Checks the last time archiving failed. Will be < 0 when it has not failed.
+          expr: |-
+            (cnpg_pg_stat_archiver_last_failed_time - cnpg_pg_stat_archiver_last_archived_time) > 1
+          for: 1m
+          labels:
+            severity: warning
+        - alert: DatabaseDeadlockConflicts
+          annotations:
+            description: There are over 10 deadlock conflicts in {{ $labels.pod }}
+            summary: Checks the number of database conflicts
+          expr: |-
+            cnpg_pg_stat_database_deadlocks > 10
+          for: 1m
+          labels:
+            severity: warning
diff --git a/kubernetes/main/apps/database/cloudnative-pg/cluster/scheduledbackup.yaml b/kubernetes/main/apps/database/cloudnative-pg/cluster/scheduledbackup.yaml
@@ -0,0 +1,12 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/postgresql.cnpg.io/scheduledbackup_v1.json
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: postgres
+spec:
+  schedule: "@daily"
+  immediate: true
+  backupOwnerReference: self
+  cluster:
+    name: postgres16
diff --git a/kubernetes/main/apps/database/cloudnative-pg/cluster/secret.sops.yaml b/kubernetes/main/apps/database/cloudnative-pg/cluster/secret.sops.yaml
@@ -0,0 +1,30 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: postgres-cluster-secret
+type: kubernetes.io/basic-auth
+stringData:
+    username: ENC[AES256_GCM,data:3Ku+Mxga5Lk=,iv:dapK4E/J0e49TniC4CLwy4U7+EATxtqawrcv7h9dQv0=,tag:nAw2jeyoWc4hzLMBJ6vt2Q==,type:str]
+    password: ENC[AES256_GCM,data:yReM/1DM14MLyaE9oHU/U3U=,iv:ugpgMXRWKo+sAcTuKqPr+obY3aqlm9JQ+9hW1ViEitE=,tag:8vBsK00a9D0sKJq397zr2Q==,type:str]
+    MINIO_ACCESS_KEY: ENC[AES256_GCM,data:sc6ivOv+5xzafapxy00mzJVsQOk=,iv:OwKP+5ivwq2aG0xC2PKTsKef/LTzVdCYwBhaqeBRsUc=,tag:kFHdLrw8uhLlVCURU2by6Q==,type:str]
+    MINIO_SECRET_KEY: ENC[AES256_GCM,data:gXRBjrngjTIBNgDoVxZuE1NcH4aD8aBQXzU05oS49eYC+ZvU9otoiA==,iv:mM2rvNR5FGQ8M4GrkxZzzv1LibM3BXoSjUz/UC0obws=,tag:fTpwfImq7kNfzfG/0pbIKw==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1ve9kzacrwq7l9l0emvs326uk6t576d75r596e083r2tq6xu28qcsacy3s7
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2eEpNd2dxOWhwSjBqYmxa
+            VER4Z0g1RDRndC9ack1PQkhZYTVyRDk2c0dRCi9HZlJ3bjNmZEhiZmkvdTRyQjhZ
+            SzMyVUp1NFpvbUw4Y2NVYTQ2UTJaVncKLS0tIEZxVDRaOVNERUdGdUdrZ05Jcm16
+            MnpTa1UxUjFNT1YvN08rdXFVanN3dkEKiNbt9G2CadgtArpBIBbgd9KQG5Oe48e7
+            PGY9Pd/MI1O0tlbeGfPnei+k8klOzzt6HnbxX09eVeSonG4czng6gQ==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-05-19T14:39:41Z"
+    mac: ENC[AES256_GCM,data:SsIDOFVpAgvtKjUn4UxUJZVii+LZ2ggRfgIIxeL4lyZ5KE+CIRaegBx8DF0v91m9WtVlo+JpurK4fWvhAKB6jF3IFrSoTTo3Q1cuieth3xXth1cJP/OqyEvH1VosCZXcmnRExoo/KDHplXpt6GenWQPc57D9rmwb3YhTqEEtqBc=,iv:oKGZHL0ho9aVjz0rg6HOtrWEAHO244FVQES7PpBdkXc=,tag:IPY5DMp+xiFPSYoNqi87Kw==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.8.1
diff --git a/kubernetes/main/apps/database/cloudnative-pg/ks.yaml b/kubernetes/main/apps/database/cloudnative-pg/ks.yaml
@@ -0,0 +1,66 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app cloudnative-pg-operator
+  namespace: flux-system
+spec:
+  targetNamespace: database
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  path: ./kubernetes/main/apps/database/cloudnative-pg/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-ops
+  wait: true
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app cloudnative-pg-secret-store
+  namespace: flux-system
+spec:
+  targetNamespace: database
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  path: ./kubernetes/main/apps/database/cloudnative-pg/secret-store
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-ops
+  wait: true
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app cloudnative-pg-cluster
+  namespace: flux-system
+spec:
+  targetNamespace: database
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  dependsOn:
+    - name: cloudnative-pg-operator
+    - name: cloudnative-pg-secret-store
+  path: ./kubernetes/main/apps/database/cloudnative-pg/cluster
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-ops
+  wait: true
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
diff --git a/kubernetes/main/apps/database/cloudnative-pg/secret-store/clustersecretstore.yaml b/kubernetes/main/apps/database/cloudnative-pg/secret-store/clustersecretstore.yaml
@@ -0,0 +1,19 @@
+---
+apiVersion: external-secrets.io/v1beta1
+kind: ClusterSecretStore
+metadata:
+  name: postgres-secret-store
+spec:
+  provider:
+    kubernetes:
+      remoteNamespace: database
+      auth:
+        serviceAccount:
+          name: postgres-secret-manager
+          namespace: database
+      server:
+        caProvider:
+          type: ConfigMap
+          name: kube-root-ca.crt
+          namespace: database
+          key: ca.crt
diff --git a/kubernetes/main/apps/database/cloudnative-pg/secret-store/kustomization.yaml b/kubernetes/main/apps/database/cloudnative-pg/secret-store/kustomization.yaml
@@ -0,0 +1,8 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./clustersecretstore.yaml
+  - ./rbac.yaml
+  - ./secret.sops.yaml