feat(staging): Rebuilding cluster continues

* prometheus-crds, coredns, cilium, kubelet-csr-approver, spegel installed
szinn · Jan 10, 2025 · cced568 · cced568
1 parent a12fd69
commit cced568
Show file tree

Hide file tree

Showing 37 changed files with 821 additions and 96 deletions.
diff --git a/.envrc b/.envrc
@@ -8,7 +8,7 @@ use_sops() {
 
 dotenv $HOME/.local/syscfg/volsync-credentials.txt
 dotenv $HOME/.local/syscfg/s3-credentials.txt
-dotenv $HOME/.local/syscfg/op-connect-credentials.txt
+# dotenv $HOME/.local/syscfg/op-connect-credentials.txt
 dotenv $HOME/.local/syscfg/postgres-main.txt
 
 export DBBACKUP=$HOME/Ragnar/k8s/main/backup/dbms

diff --git a/.taskfiles/Bootstrap/Taskfile.yaml b/.taskfiles/Bootstrap/Taskfile.yaml
@@ -4,7 +4,7 @@ version: "3"
 
 tasks:
   main:
-    desc: "Bootstrap main cluster"
+    desc: Bootstrap main cluster
     vars:
       cluster: main
     cmds:
@@ -14,25 +14,63 @@ tasks:
           nodes: k8s-1 k8s-2 k8s-3 k8s-4 k8s-5 k8s-6
 
   staging:
-    desc: "Bootstrap staging cluster"
+    desc: Bootstrap staging cluster
     cmds:
       - task: :proxmox:reset-staging
       - task: :talos:bootstrap
         vars:
           cluster: staging
-          nodes: stage-1 stage-2 stage-3
       - task: :proxmox:unmount-staging-cdrom
 
+  merge:
+    desc: Merge kubeconfig and talosconfig configurations
+    requires:
+      vars:
+        - cluster
+    vars:
+      CLUSTER_DIR: "{{.KUBERNETES_DIR}}/{{.cluster}}"
+    cmds:
+      - cp $HOME/.kube/config /tmp/kubectl-config-backup
+      - cp $HOME/.talos/config /tmp/talosconfig-config-backup
+      - cmd: kubectl --kubeconfig $HOME/.kube/config config delete-context {{.cluster}}
+        ignore_error: true
+      - cmd: talosctl --talosconfig $HOME/.talos/config config remove -y {{.cluster}}
+        ignore_error: true
+      - KUBECONFIG="$HOME/.kube/config:{{.CLUSTER_DIR}}/kubeconfig" kubectl config view --flatten > $HOME/.kube/config-new
+      - mv $HOME/.kube/config-new $HOME/.kube/config
+      - talosctl  --talosconfig $HOME/.talos/config config merge {{.CLUSTER_DIR}}/talosconfig
+
   config:
     desc: Rebuild all cluster configuration
     cmd: bash .taskfiles/Bootstrap/_scripts/build-config.sh
 
   deploy:
-    desc: Deploy a cluster
-    cmd: bash .taskfiles/Bootstrap/_scripts/deploy-cluster.sh {{.cluster}}
+    desc: Bootstrap Apps [K8S_CLUSTER={{.K8S_CLUSTER}}]
+    preconditions:
+      - which helmfile kubectl
+      - test -f "${TALOSCONFIG}"
+      - test -f {{.CLUSTER_DIR}}/bootstrap/helmfile.yaml
+      - test -f {{.CLUSTER_DIR}}/bootstrap/templates/resources.yaml.j2
+      - talosctl --context {{.cluster}} config info
     requires:
       vars:
         - cluster
+    env:
+      TALOSCONFIG: "{{.CLUSTER_DIR}}/talosconfig"
+    vars:
+      CLUSTER_DIR: "{{.KUBERNETES_DIR}}/{{.cluster}}"
+    cmds:
+      - until kubectl --context {{.cluster}} wait nodes --for=condition=Ready=False --all --timeout=10m; do sleep 5; done
+      - op run --env-file {{.CLUSTER_DIR}}/bootstrap/bootstrap.env --no-masking -- minijinja-cli "{{.CLUSTER_DIR}}/bootstrap/templates/resources.yaml.j2" | kubectl --context {{.cluster}} apply --server-side --filename -
+      - helmfile --kube-context {{.cluster}} --quiet --file {{.CLUSTER_DIR}}/bootstrap/helmfile.yaml apply --skip-diff-on-install --suppress-diff
+      - until kubectl --context {{.cluster}} wait nodes --for=condition=Ready --all --timeout=10m; do sleep 5; done
+
+  # deploy:
+  #   desc: Deploy a cluster
+  #   cmd: bash .taskfiles/Bootstrap/_scripts/deploy-cluster.sh {{.cluster}}
+  #   requires:
+  #     vars:
+  #       - cluster
 
   get-certs:
     desc: Fetch certificates from cluster

diff --git a/.taskfiles/talos/Taskfile.yaml b/.taskfiles/talos/Taskfile.yaml
@@ -7,19 +7,31 @@ vars:
 
 tasks:
   gen-secrets:
-    desc: "Generate cluster secrets"
-    cmds:
-      - talhelper gensecret > kubernetes/{{.cluster}}/bootstrap/talos/talsecret.sops.yaml
-      - sops -e -i kubernetes/{{.cluster}}/bootstrap/talos/talsecret.sops.yaml
+    desc: Generate cluster secrets
+    summary: |
+      Args:
+        cluster: Cluster to run command against (required)
+    preconditions:
+      - which test talhelper
+      - test -f {{.CLUSTER_DIR}}/bootstrap/talos/talsecret.sops.yaml
+      - test -f {{.CLUSTER_DIR}}/bootstrap/talos/talconfig.yaml
     requires:
       vars:
         - cluster
+    vars:
+      CLUSTER_DIR: "{{.KUBERNETES_DIR}}/{{.cluster}}"
+    cmds:
+      - talhelper gensecret > {{.CLUSTER_DIR}}/bootstrap/talos/talsecret.sops.yaml
+      - sops -e -i {{.CLUSTER_DIR}}/bootstrap/talos/talsecret.sops.yaml
 
   bootstrap:
     desc: Bootstrap the cluster
     summary: |
       Args:
         cluster: Cluster to run command against (required)
+    requires:
+      vars:
+        - cluster
     prompt: Bootstrap Talos on the '{{.cluster}}' cluster ... continue?
     cmds:
       - task: generate-config
@@ -34,106 +46,127 @@ tasks:
       - task: fetch-kubeconfig
         vars:
           cluster: "{{.cluster}}"
-      - until kubectl --context staging --kubeconfig="{{.KUBERNETES_DIR}}/{{.cluster}}/kubeconfig" wait nodes --for=condition=Ready=False --all --timeout=10m; do sleep 5; done
+      - until kubectl --context {{.cluster}} --kubeconfig="{{.KUBERNETES_DIR}}/{{.cluster}}/kubeconfig" wait nodes --for=condition=Ready=False --all --timeout=10m; do sleep 5; done
+
   #     - task: bootstrap-core
   #       vars:
   #         cluster: "{{.cluster}}"
-  #   requires:
-  #     vars:
-  #       - cluster
-  #       - nodes
 
   generate-config:
     desc: Generate Talos configurations
     summary: |
       Args:
         cluster: Cluster to run command against (required)
-    cmds:
-      - |
-        talhelper genconfig \
-          --secret-file {{.CONFIG_ROOT}}/talsecret.sops.yaml \
-          --config-file {{.CONFIG_ROOT}}/talconfig.yaml \
-          --out-dir {{.CONFIG_ROOT}}/clusterconfig
-      - cp {{.CONFIG_ROOT}}/clusterconfig/talosconfig {{.KUBERNETES_DIR}}/{{.cluster}}
-    vars:
-      CONFIG_ROOT: "{{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/talos"
+    preconditions:
+      - which test talhelper cp
+      - test -f {{.CLUSTER_DIR}}/bootstrap/talos/talsecret.sops.yaml
+      - test -f {{.CLUSTER_DIR}}/bootstrap/talos/talconfig.yaml
     requires:
       vars:
         - cluster
+    vars:
+      CLUSTER_DIR: "{{.KUBERNETES_DIR}}/{{.cluster}}"
+      TALOS_CONFIG_ROOT: "{{.CLUSTER_DIR}}/bootstrap/talos"
+    cmds:
+      - |
+        talhelper genconfig \
+          --secret-file {{.TALOS_CONFIG_ROOT}}/talsecret.sops.yaml \
+          --config-file {{.TALOS_CONFIG_ROOT}}/talconfig.yaml \
+          --out-dir {{.TALOS_CONFIG_ROOT}}/clusterconfig
+      - cp {{.TALOS_CONFIG_ROOT}}/clusterconfig/talosconfig {{.CLUSTER_DIR}}
 
   apply-config:
     desc: Apply Talos configurations
     summary: |
       Args:
         cluster: Cluster to run command against (required)
-        nodes: List of hostnames in the cluster to configure
-    cmds:
-      - for: {var: CONFIG_FILES}
-        cmd: |
-          talosctl apply-config -i -n {{ trimPrefix (printf "%s-" .cluster) (base .ITEM) | trimSuffix ".yaml" }} \
-            --talosconfig {{.KUBERNETES_DIR}}/{{.cluster}}/talosconfig \
-            --context {{.cluster}} \
-            -f {{.ITEM}}
+    preconditions:
+      - which ls
+      - test -f {{.KUBERNETES_DIR}}/{{.cluster}}/talosconfig
+    requires:
+      vars:
+        - cluster
     vars:
-      CONFIG_ROOT: "{{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/talos/clusterconfig"
+      CLUSTER_DIR: "{{.KUBERNETES_DIR}}/{{.cluster}}"
+      TALOS_CONFIG_ROOT: "{{.CLUSTER_DIR}}/bootstrap/talos"
+      CLUSTER_CONFIG_ROOT: "{{.TALOS_CONFIG_ROOT}}/clusterconfig"
       CONFIG_FILES:
-        sh: ls {{.CONFIG_ROOT}}/*.yaml
+        sh: ls {{.CLUSTER_CONFIG_ROOT}}/*.yaml
+      TALOSCONFIG: "{{.CLUSTER_DIR}}/talosconfig"
+    cmds:
+      - for: {var: CONFIG_FILES}
+        task: _apply-machineconfig
+        vars:
+          cluster: "{{.cluster}}"
+          FILENAME: "{{.ITEM}}"
+          HOSTNAME: |-
+            {{ trimPrefix (printf "%s-" .cluster) (base .ITEM) | trimSuffix ".yaml" }}
+
+  _apply-machineconfig:
+    internal: true
+    desc: Apply a single Talos machineConfig to a Talos node
+    preconditions:
+      - which talosctl
+      - test -f "{{.FILENAME}}"
     requires:
       vars:
         - cluster
-    preconditions:
-      - test -f {{.KUBERNETES_DIR}}/{{.cluster}}/talosconfig
+        - HOSTNAME
+        - FILENAME
+    cmds:
+      - talosctl apply-config
+        --context "{{.cluster}}"
+        --nodes "{{.HOSTNAME}}"
+        --file "{{.FILENAME}}"
+        --insecure
 
   bootstrap-etcd:
     desc: Bootstrap Etcd
     summary: |
       Args:
         cluster: Cluster to run command against (required)
-    cmd: until talosctl --talosconfig {{.KUBERNETES_DIR}}/{{.cluster}}/talosconfig --context {{.cluster}} --nodes {{.controller}} bootstrap; do sleep 10; done
-    env:
-      TALOSCONFIG: "{{.KUBERNETES_DIR}}/{{.cluster}}/talosconfig"
-    vars:
-      controller:
-        sh: talosctl --talosconfig {{.KUBERNETES_DIR}}/{{.cluster}}/talosconfig --context {{.cluster}} config info --output json | jq --raw-output '.endpoints[0]'
+    preconditions:
+      - test -f {{.TALOSCONFIG}}
+      - talosctl --context {{.cluster}} config info >/dev/null 2>&1
     requires:
       vars:
         - cluster
-    preconditions:
-      - test -f {{.KUBERNETES_DIR}}/{{.cluster}}/talosconfig
-      - talosctl --talosconfig {{.KUBERNETES_DIR}}/{{.cluster}}/talosconfig --context {{.cluster}} config info >/dev/null 2>&1
+    env:
+      TALOSCONFIG: "{{.CLUSTER_DIR}}/talosconfig"
+    vars:
+      CLUSTER_DIR: "{{.KUBERNETES_DIR}}/{{.cluster}}"
+      TALOS_CONTROLLER:
+        sh: talosctl --talosconfig "{{.CLUSTER_DIR}}/talosconfig" --context {{.cluster}} config info --output json | jq --raw-output '.endpoints[0]'
+    cmds:
+      - until talosctl --context {{.cluster}} --nodes {{.TALOS_CONTROLLER}} bootstrap; do sleep 10; done
 
   fetch-kubeconfig:
     desc: Fetch kubeconfig from Talos controllers
     summary: |
       Args:
         cluster: Cluster to run command against (required)
-    cmds:
-      - |
-        talosctl kubeconfig \
-          --talosconfig {{.KUBERNETES_DIR}}/{{.cluster}}/talosconfig \
-          --context {{.cluster}} \
-          --nodes {{.controller}} \
-          --force \
-          --force-context-name {{.cluster}} \
-          {{.KUBERNETES_DIR}}/{{.cluster}}
-      # Need to do it twice since the context name force doesn't happen the first time
-      - |
-        talosctl kubeconfig \
-          --talosconfig {{.KUBERNETES_DIR}}/{{.cluster}}/talosconfig \
-          --context {{.cluster}} \
-          --nodes {{.controller}} \
-          --force \
-          --force-context-name {{.cluster}} \
-          {{.KUBERNETES_DIR}}/{{.cluster}}
-    vars:
-      controller:
-        sh: talosctl --talosconfig {{.KUBERNETES_DIR}}/{{.cluster}}/talosconfig --context {{.cluster}} config info --output json | jq --raw-output '.endpoints[0]'
+    preconditions:
+      - test -f {{.CLUSTER_DIR}}/talosconfig
+      - talosctl --context {{.cluster}} config info >/dev/null 2>&1
     requires:
       vars:
         - cluster
-    preconditions:
-      - test -f {{.KUBERNETES_DIR}}/{{.cluster}}/talosconfig
-      - talosctl --talosconfig {{.KUBERNETES_DIR}}/{{.cluster}}/talosconfig --context {{.cluster}} config info >/dev/null 2>&1
+    env:
+      TALOSCONFIG: "{{.CLUSTER_DIR}}/talosconfig"
+    vars:
+      CLUSTER_DIR: "{{.KUBERNETES_DIR}}/{{.cluster}}"
+      KUBECONFIG: "{{.CLUSTER_DIR}}/kubeconfig"
+      TALOS_CONTROLLER:
+        sh: talosctl --talosconfig "{{.CLUSTER_DIR}}/talosconfig" --context {{.cluster}} config info --output json | jq --raw-output '.endpoints[0]'
+    cmds:
+      - talosctl kubeconfig
+          --context {{.cluster}}
+          --nodes {{.TALOS_CONTROLLER}}
+          --force
+          --force-context-name {{.cluster}}
+          "{{.KUBECONFIG}}"
+      - cmd: kubectl config delete-context "admin@{{.cluster}}"
+        ignore_error: true
 
   # bootstrap-core:
   #   desc: Bootstrap core apps needed for Talos

diff --git a/Taskfile.yaml b/Taskfile.yaml
@@ -1,13 +1,18 @@
 ---
 version: "3"
 
+set:
+  - pipefail
+shopt:
+  - globstar
+
 vars:
   ANSIBLE_DIR: "{{.ROOT_DIR}}/ansible"
   KUBERNETES_DIR: "{{.ROOT_DIR}}/kubernetes"
 
 env:
-  KUBECONFIG: "{{.KUBERNETES_DIR}}/main/kubeconfig:{{.KUBERNETES_DIR}}/staging/kubeconfig"
   MINIJINJA_CONFIG_FILE: "{{.ROOT_DIR}}/.minijinja.toml"
+  KUBECONFIG: "{{.KUBERNETES_DIR}}/main/kubeconfig:{{.KUBERNETES_DIR}}/staging/kubeconfig"
 
 includes:
   ansible: .taskfiles/Ansible

diff --git a/kubernetes/repositories/helm-charts/coredns-charts.yaml b/kubernetes/repositories/helm-charts/coredns-charts.yaml
@@ -0,0 +1,10 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/refs/heads/main/helmrepository-source-v1.json
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: coredns-charts
+  namespace: flux-system
+spec:
+  interval: 2h
+  url: https://coredns.github.io/helm
diff --git a/kubernetes/repositories/helm-charts/kustomization.yaml b/kubernetes/repositories/helm-charts/kustomization.yaml
@@ -9,6 +9,7 @@ resources:
   - bjw-s-charts.yaml
   - cilium-charts.yaml
   - cloudnative-pg-charts.yaml
+  - coredns-charts.yaml
   - deliveryhero-charts.yaml
   - descheduler-charts.yaml
   - dragonfly-charts.yaml