Merge pull request #472 from TomerFi/add-clusterdeployments

feat: use hive clusterdeployment for creating spoke clusters
validatedpatterns · May 30, 2024 · 7ba9db5 · 7ba9db5
2 parents e6441ce + 6cd4e85
commit 7ba9db5
Show file tree

Hide file tree

Showing 11 changed files with 860 additions and 95 deletions.
diff --git a/acm/templates/provision/clusterdeployment.yaml b/acm/templates/provision/clusterdeployment.yaml
@@ -0,0 +1,83 @@
+{{- range .Values.clusterGroup.managedClusterGroups }}
+{{- $group := . }}
+
+{{- range $group.clusterDeployments}}
+{{ $cluster := . }}
+
+{{- if (eq $cluster.name nil) }}
+{{- fail (printf "managedClusterGroup clusterDeployment cluster name is empty: %s" $cluster) }}
+{{- end }}
+{{- if (eq $group.name nil) }}
+{{- fail (printf "managedClusterGroup clusterDeployment group name is empty: %s" $cluster) }}
+{{- end }}
+
+{{- $deploymentName := print $cluster.name "-" $group.name }}
+
+{{- $cloud := "None" }}
+{{- $region := "None" }}
+
+{{- if $cluster.platform.aws }}
+{{- $cloud = "aws" }}
+{{- $region = $cluster.platform.aws.region }}
+{{- else if $cluster.platform.azure }}
+{{- $cloud = "azure" }}
+{{- $region = $cluster.platform.azure.region }}
+{{- end }}
+
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: {{ $deploymentName }}
+
+---
+apiVersion: hive.openshift.io/v1
+kind: ClusterDeployment
+metadata:
+  name: {{ $deploymentName }}
+  namespace: {{ $deploymentName }}
+  labels:
+    vendor: OpenShift
+  annotations:
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+spec:
+  baseDomain: {{ $cluster.baseDomain }}
+  clusterName: {{ $deploymentName }}
+  installAttemptsLimit: 1
+  platform:
+    {{ $cloud }}:
+      credentialsSecretRef:
+        name: {{ $deploymentName }}-creds
+      region: {{ $region }}
+  provisioning:
+    installConfigSecretRef:
+      name: {{ $deploymentName }}-install-config
+    sshPrivateKeySecretRef:
+      name: {{ $deploymentName }}-ssh-private-key
+    imageSetRef:
+      name: img{{ $cluster.openshiftVersion }}-multi-appsub
+  pullSecretRef:
+    name: {{ $deploymentName }}-pull-secret
+
+---
+apiVersion: cluster.open-cluster-management.io/v1
+kind: ManagedCluster
+metadata:
+  labels:
+    cluster.open-cluster-management.io/clusterset: {{ $group.name }}
+  {{- if (not $group.acmlabels) }}
+    clusterGroup: {{ $group.name }}
+  {{- else if eq (len $group.acmlabels) 0 }}
+    clusterGroup: {{ $group.name }}
+  {{- else }}
+    {{- range $group.acmlabels }}
+    {{ .name }}: {{ .value }}
+    {{- end }}
+  {{- end }}
+  name: {{ $deploymentName }}
+  annotations:
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+spec:
+  hubAcceptsClient: true
+{{- end }}{{- /* range $group.clusterDeployments */}}
+{{- end }}{{- /* range .Values.clusterGroup.managedClusterGroups */}}
diff --git a/acm/templates/provision/clusterpool.yaml b/acm/templates/provision/clusterpool.yaml
@@ -1,17 +1,5 @@
 {{- range .Values.clusterGroup.managedClusterGroups }}
 {{- $group := . }}
-{{- if .clusterPools }}{{- /* We only create ManagedClusterSets if there are clusterPools defined */}}
-apiVersion: cluster.open-cluster-management.io/v1beta1
-kind: ManagedClusterSet
-metadata:
-  annotations:
-    cluster.open-cluster-management.io/submariner-broker-ns: {{ .name }}-broker
-    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
-  name: {{ .name }}
-spec:
-  clusterSelector:
-    selectorType: LegacyClusterSetLabel
----
 {{- range .clusterPools }}
 
 {{- $pool := . }}
@@ -54,7 +42,7 @@ spec:
   runningCount: {{ $numClusters }}
   baseDomain: {{ .baseDomain }}
   installConfigSecretTemplateRef:
-    name: {{ $poolName }}-install-config 
+    name: {{ $poolName }}-install-config
   imageSetRef:
     name: img{{ .openshiftVersion }}-multi-appsub
   pullSecretRef:
@@ -91,5 +79,4 @@ spec:
 ---
 {{- end }}{{- /* range .range clusters */}}
 {{- end }}{{- /* range .clusterPools */}}
-{{- end }}{{- /* if .clusterPools) */}}
 {{- end }}{{- /* range .Values.clusterGroup.managedClusterGroups */}}
diff --git a/acm/templates/provision/managedclusterset.yaml b/acm/templates/provision/managedclusterset.yaml
@@ -0,0 +1,13 @@
+{{- range .Values.clusterGroup.managedClusterGroups }}
+{{- if or .clusterPools .clusterDeployments }}{{- /* We only create ManagedClusterSets if there are clusterPools or clusterDeployments defined */}}
+---
+apiVersion: cluster.open-cluster-management.io/v1beta2
+kind: ManagedClusterSet
+metadata:
+  annotations:
+    cluster.open-cluster-management.io/submariner-broker-ns: {{ .name }}-broker
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+  name: {{ .name }}
+
+{{- end }}{{- /* if .clusterPools) */}}
+{{- end }}{{- /* range .Values.clusterGroup.managedClusterGroups */}}
diff --git a/acm/templates/provision/secrets-aws.yaml b/acm/templates/provision/secrets-aws.yaml
@@ -3,58 +3,88 @@
 {{- range .clusterPools }}
 {{- $poolName := print .name "-" $group.name }}
 {{- if .platform.aws }}
+---
+{{- template "externalsecret.aws.creds" (dict "name" $poolName "context" . "secretStore" $.Values.secretStore) }}
+---
+{{- template "externalsecret.aws.infra-creds" (dict "name" $poolName "context" . "secretStore" $.Values.secretStore) }}
+
+{{- end }}{{- /* if .platform.aws */}}
+{{- end }}{{- /* range .clusterPools  */}}
+
+{{- range .clusterDeployments }}
+{{- $deploymentName := print .name "-" $group.name }}
+{{- if .platform.aws }}
+---
+{{- template "externalsecret.aws.creds" (dict "name" $deploymentName "context" . "secretStore" $.Values.secretStore "namespaced" true) }}
+---
+{{- template "externalsecret.aws.infra-creds" (dict "name" $deploymentName "context" . "secretStore" $.Values.secretStore "namespaced" true) }}
+
+{{- end }}{{- /* if .platform.aws */}}
+{{- end }}{{- /* range .clusterDeployments  */}}
+
+{{- end }}{{- /* range .Values.clusterGroup.managedClusterGroups */}}
+
+{{- define "externalsecret.aws.creds" }}
 apiVersion: external-secrets.io/v1beta1
 kind: ExternalSecret
 metadata:
-  name: {{ $poolName }}-creds
+  name: {{ .name }}-creds
+  {{- if .namespaced }}
+  namespace: {{ .name }}
+  {{- end }}
 spec:
   dataFrom:
   - extract:
       # Expects entries called: aws_access_key_id and aws_secret_access_key
-      key: {{ default "secret/data/hub/aws" .awsKeyPath }}
+      key: {{ default "secret/data/hub/aws" .context.awsKeyPath }}
   refreshInterval: 24h0m0s
   secretStoreRef:
-    name: {{ $.Values.secretStore.name }}
-    kind: {{ $.Values.secretStore.kind }}
+    name: {{ .secretStore.name }}
+    kind: {{ .secretStore.kind }}
   target:
-    name: {{ $poolName }}-creds
+    name: {{ .name }}-creds
     creationPolicy: Owner
     template:
       type: Opaque
----
+{{- end}}
+
+{{- define "externalsecret.aws.infra-creds"}}
 # For use when manually creating clusters with ACM
 apiVersion: external-secrets.io/v1beta1
 kind: ExternalSecret
 metadata:
-  name: {{ $poolName }}-infra-creds
-spec: 
+  name: {{ .name }}-infra-creds
+  {{- if .namespaced }}
+  namespace: {{ .name }}
+  {{- end }}
+spec:
   data:
   - secretKey: openshiftPullSecret
     remoteRef:
-      key: {{ default "secret/data/hub/openshiftPullSecret" .pullSecretKeyPath }}
+      key: {{ default "secret/data/hub/openshiftPullSecret" .context.pullSecretKeyPath }}
       property: content
   - secretKey: awsKeyId
     remoteRef:
-      key: {{ default "secret/data/hub/aws" .awsKeyPath }}
+      key: {{ default "secret/data/hub/aws" .context.awsKeyPath }}
       property: aws_access_key_id
   - secretKey: awsAccessKey
     remoteRef:
-      key: {{ default "secret/data/hub/aws" .awsKeyPath }}
+      key: {{ default "secret/data/hub/aws" .context.awsKeyPath }}
       property: aws_secret_access_key
   - secretKey: sshPublicKey
     remoteRef:
-      key: {{ default "secret/data/hub/publickey" .sshPublicKeyPath }}
+      key: {{ default "secret/data/hub/publickey" .context.sshPublicKeyPath }}
       property: content
   - secretKey: sshPrivateKey
     remoteRef:
-      key: {{ default "secret/data/hub/privatekey" .sshPrivateKeyPath }}
+      key: {{ default "secret/data/hub/privatekey" .context.sshPrivateKeyPath }}
       property: content
   refreshInterval: 24h0m0s
-  secretStoreRef: 
-    name: {{ $.Values.secretStore.name }}
-    kind: {{ $.Values.secretStore.kind }}
+  secretStoreRef:
+    name: {{ .secretStore.name }}
+    kind: {{ .secretStore.kind }}
   target:
-    name: {{ $poolName }}-infra-creds
+    name: {{ .name }}-infra-creds
     creationPolicy: Owner
     template:
       type: Opaque
@@ -63,7 +93,7 @@ spec:
           cluster.open-cluster-management.io/credentials: ""
           cluster.open-cluster-management.io/type: aws
       data:
-        baseDomain: "{{ .baseDomain }}"
+        baseDomain: "{{ .context.baseDomain }}"
         pullSecret: |-
           {{ "{{ .openshiftPullSecret | toString }}" }}
         aws_access_key_id: |-
@@ -78,7 +108,4 @@ spec:
         httpsProxy: ""
         noProxy: ""
         additionalTrustBundle: ""
----
-{{- end }}
-{{- end }}
-{{- end }}
+{{- end}}
diff --git a/acm/templates/provision/secrets-azure.yaml b/acm/templates/provision/secrets-azure.yaml
@@ -3,58 +3,90 @@
 {{- range .clusterPools }}
 {{- $poolName := print .name "-" $group.name }}
 {{- if .platform.azure }}
+---
+{{- template "externalsecret.azure.creds" (dict "name" $poolName "context" . "secretStore" $.Values.secretStore) }}
+---
+{{- template "externalsecret.azure.infra-creds" (dict "name" $poolName "context" . "secretStore" $.Values.secretStore) }}
+
+---
+{{- end }}{{- /* if .platform.azure */}}
+{{- end }}{{- /* range .clusterPools  */}}
+
+{{- range .clusterDeployments }}
+{{- $deploymentName := print .name "-" $group.name }}
+{{- if .platform.azure }}
+---
+{{- template "externalsecret.azure.creds" (dict "name" $deploymentName "context" . "secretStore" $.Values.secretStore "namespaced" true) }}
+---
+{{- template "externalsecret.azure.infra-creds" (dict "name" $deploymentName "context" . "secretStore" $.Values.secretStore "namespaced" true) }}
+
+
+{{- end }}{{- /* if .platform.azure */}}
+{{- end }}{{- /* range .clusterPools  */}}
+
+{{- end }}{{- /* range .Values.clusterGroup.managedClusterGroups  */}}
+
+{{- define "externalsecret.azure.creds" }}
 apiVersion: external-secrets.io/v1beta1
 kind: ExternalSecret
 metadata:
-  name: {{ $poolName }}-creds
+  name: {{ .name }}-creds
+  {{- if .namespaced }}
+  namespace: {{ .name }}
+  {{- end }}
 spec:
   data:
   - secretKey: azureOsServicePrincipal
     remoteRef:
-      key: {{ default "secret/data/hub/azureOsServicePrincipal" .azureKeyPath }}
+      key: {{ default "secret/data/hub/azureOsServicePrincipal" .context.azureKeyPath }}
       property: content
   refreshInterval: 24h0m0s
   secretStoreRef:
-    name: {{ $.Values.secretStore.name }}
-    kind: {{ $.Values.secretStore.kind }}
+    name: {{ .secretStore.name }}
+    kind: {{ .secretStore.kind }}
   target:
-    name: {{ $poolName }}-creds
+    name: {{ .name }}-creds
     creationPolicy: Owner
     template:
       type: Opaque
       data:
         osServicePrincipal.json: |-
           {{ "{{ .azureOsServicePrincipal | toString }}" }}
----
+{{- end }}
+
+{{- define "externalsecret.azure.infra-creds"}}
 # For use when manually creating clusters with ACM
 apiVersion: external-secrets.io/v1beta1
 kind: ExternalSecret
 metadata:
-  name: {{ $poolName }}-infra-creds
-spec: 
+  name: {{ .name }}-infra-creds
+  {{- if .namespaced }}
+  namespace: {{ .name }}
+  {{- end }}
+spec:
   data:
   - secretKey: openshiftPullSecret
     remoteRef:
-      key: {{ default "secret/data/hub/openshiftPullSecret" .pullSecretKeyPath }}
+      key: {{ default "secret/data/hub/openshiftPullSecret" .context.pullSecretKeyPath }}
       property: content
   - secretKey: sshPublicKey
     remoteRef:
-      key: {{ default "secret/data/hub/publickey" .sshPublicKeyPath }}
+      key: {{ default "secret/data/hub/publickey" .context.sshPublicKeyPath }}
       property: content
   - secretKey: sshPrivateKey
     remoteRef:
-      key: {{ default "secret/data/hub/privatekey" .sshPrivateKeyPath }}
+      key: {{ default "secret/data/hub/privatekey" .context.sshPrivateKeyPath }}
       property: content
   - secretKey: azureOsServicePrincipal
     remoteRef:
-      key: {{ default "secret/data/hub/azureOsServicePrincipal" .azureKeyPath }}
+      key: {{ default "secret/data/hub/azureOsServicePrincipal" .context.azureKeyPath }}
       property: content
   refreshInterval: 24h0m0s
-  secretStoreRef: 
-    name: {{ $.Values.secretStore.name }}
-    kind: {{ $.Values.secretStore.kind }}
+  secretStoreRef:
+    name: {{ .secretStore.name }}
+    kind: {{ .secretStore.kind }}
   target:
-    name: {{ $poolName }}-infra-creds
+    name: {{ .name }}-infra-creds
     creationPolicy: Owner
     template:
       type: Opaque
@@ -66,8 +98,8 @@ spec:
         cloudName: AzurePublicCloud
         osServicePrincipal.json: |-
           {{ "{{ .azureOsServicePrincipal | toString }}" }}
-        baseDomain: "{{ .baseDomain }}"
-        baseDomainResourceGroupName: "{{ .platform.azure.baseDomainResourceGroupName | toString }}"
+        baseDomain: "{{ .context.baseDomain }}"
+        baseDomainResourceGroupName: "{{ .context.platform.azure.baseDomainResourceGroupName | toString }}"
         pullSecret: |-
           {{ "{{ .openshiftPullSecret | toString }}" }}
         ssh-privatekey: |-
@@ -78,7 +110,4 @@ spec:
         httpsProxy: ""
         noProxy: ""
         additionalTrustBundle: ""
----
-{{- end }}
-{{- end }}
 {{- end }}