cs namespace and service account parameterization (#1102)

Signed-off-by: Gerd Oberlechner <[email protected]>

cluster-service/Makefile

@@ -6,9 +6,9 @@ ZONE_NAME ?= "${REGIONAL_DNS_SUBDOMAIN}.${CX_PARENT_DNS_ZONE_NAME}"
 
 
 deploy:
-	@kubectl create namespace cluster-service --dry-run=client -o json | kubectl apply -f - && \
-	kubectl label namespace cluster-service "istio.io/rev=${ISTO_TAG}" --overwrite=true && \
-	AZURE_CS_MI_CLIENT_ID=$(shell az identity show -g ${RESOURCEGROUP} -n clusters-service --query clientId -o tsv) && \
+	@kubectl create namespace ${NAMESPACE} --dry-run=client -o json | kubectl apply -f - && \
+	kubectl label namespace ${NAMESPACE} "istio.io/rev=${ISTO_TAG}" --overwrite=true && \
+	AZURE_CS_MI_CLIENT_ID=$(shell az identity show -g ${RESOURCEGROUP} -n ${MI_NAME} --query clientId -o tsv) && \
 	CS_SERVICE_PRINCIPAL_CREDS_BASE64='$(shell az keyvault secret show --vault-name "${SERVICE_KV}" --name "aro-hcp-dev-sp-cs" | jq .value -r | base64 | tr -d '\n')' && \
 	TENANT_ID=$(shell az account show --query tenantId --output tsv) && \
 	OIDC_BLOB_SERVICE_ENDPOINT=$(shell az storage account show -n ${OIDC_STORAGE_ACCOUNT} -g ${RESOURCEGROUP} --query primaryEndpoints.blob -o tsv) && \
@@ -27,8 +27,9 @@ deploy:
 	CX_SECRETS_KV_URL=$(shell az keyvault show -n ${CX_SECRETS_KV_NAME} -g ${MGMT_RESOURCEGROUP} --query properties.vaultUri -o tsv) && \
 	CX_MI_KV_URL=$(shell az keyvault show -n ${CX_MI_KV_NAME} -g ${MGMT_RESOURCEGROUP} --query properties.vaultUri -o tsv) && \
 	${HELM_CMD} cluster-service deploy/helm \
-	  --namespace cluster-service \
+	  --namespace ${NAMESPACE} \
 	  -f deploy/helm/$${OVERRIDES} \
+	  --set serviceAccountName=${SERVICE_ACCOUNT_NAME} \
 	  --set azureCsMiClientId=$${AZURE_CS_MI_CLIENT_ID} \
 	  --set oidcIssuerBlobServiceUrl=$${OIDC_BLOB_SERVICE_ENDPOINT} \
 	  --set oidcIssuerBaseUrl=$${OIDC_ISSUER_BASE_ENDPOINT} \

cluster-service/deploy/helm/templates/deployment.yaml

@@ -25,8 +25,8 @@ spec:
         checksum/cs: {{ include (print $.Template.BasePath "/clusters-service.secret.yaml") . | sha256sum }}
         checksum/runtime: {{ include (print $.Template.BasePath "/azure-runtime-config.configmap.yaml") . | sha256sum }}
     spec:
-      serviceAccount: clusters-service
-      serviceAccountName: clusters-service
+      serviceAccount: {{ .Values.serviceAccountName }}
+      serviceAccountName: {{ .Values.serviceAccountName }}
       volumes:
       - name: service
         secret:

cluster-service/deploy/helm/templates/istio.yml

@@ -2,7 +2,7 @@ apiVersion: security.istio.io/v1beta1
 kind: PeerAuthentication
 metadata:
   name: default
-  namespace: cluster-service
+  namespace: {{ .Release.Namespace }}
 spec:
   selector:
     matchLabels:
@@ -15,7 +15,7 @@ apiVersion: security.istio.io/v1
 kind: AuthorizationPolicy
 metadata:
   name: allow-metrics
-  namespace: cluster-service
+  namespace: {{ .Release.Namespace }}
 spec:
   action: "ALLOW"
   rules:
@@ -32,14 +32,14 @@ apiVersion: security.istio.io/v1
 kind: AuthorizationPolicy
 metadata:
   name: allow-nothing
-  namespace: cluster-service
+  namespace: {{ .Release.Namespace }}
 spec: {}
 ---
 apiVersion: security.istio.io/v1
 kind: AuthorizationPolicy
 metadata:
   name: allow-frontend
-  namespace: cluster-service
+  namespace: {{ .Release.Namespace }}
 spec:
   action: "ALLOW"
   rules:
@@ -61,13 +61,13 @@ apiVersion: security.istio.io/v1
 kind: AuthorizationPolicy
 metadata:
   name: allow-intra-namespace
-  namespace: cluster-service
+  namespace: {{ .Release.Namespace }}
 spec:
   action: "ALLOW"
   rules:
     - from:
       - source:
-          principals: ["cluster.local/ns/cluster-service/sa/clusters-service"]
+          principals: ["cluster.local/ns/{{ .Release.Namespace }}/sa/{{ .Values.serviceAccountName }}"]
       to:
       - operation:
           ports:

cluster-service/deploy/helm/templates/serviceaccount.yaml

@@ -2,7 +2,7 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: clusters-service
+  name: {{ .Values.serviceAccountName }}
   namespace: {{ .Release.Namespace }}
   labels:
     app: clusters-service

cluster-service/deploy/helm/values.yaml

@@ -1,3 +1,7 @@
+# service account name
+serviceAccountName: clusters-service
+
+
 # TODO: This parameter isn't currently used, but kept to avoid failures in the
 # execution of saasherder. It will be removed once the version of the service
 # that doesn't use it is deployed to all environments.

cluster-service/pipeline.yaml

@@ -72,6 +72,12 @@ resourceGroups:
       configRef: clusterService.azureOperatorsManagedIdentities.cloudNetworkConfig.roleName
     - name: ISTO_TAG
       configRef: svc.istio.tag
+    - name: MI_NAME
+      configRef: clusterService.managedIdentityName
+    - name: NAMESPACE
+      configRef: clusterService.k8s.namespace
+    - name: SERVICE_ACCOUNT_NAME
+      configRef: clusterService.k8s.serviceAccountName
 
 
     # this is maestro consumer registration stuff

config/config.msft.yaml

@@ -86,6 +86,10 @@ defaults:
       deploy: true
       private: false
       minTLSVersion: 'TLSV1.2'
+    managedIdentityName: clusters-service
+    k8s:
+      namespace: cluster-service
+      serviceAccountName: clusters-service
 
   # Image Sync
   imageSync:

config/config.schema.json

@@ -63,6 +63,26 @@
           "imageTag": {
             "type": "string"
           },
+          "managedIdentityName": {
+            "type": "string",
+            "description": "The name of the MSI that will be used by CS to interact with Azure"
+          },
+          "k8s": {
+            "type": "object",
+            "properties": {
+              "namespace": {
+                "type": "string"
+              },
+              "serviceAccountName": {
+                "type": "string"
+              }
+            },
+            "additionalProperties": false,
+            "required": [
+              "namespace",
+              "serviceAccountName"
+            ]
+          },
           "postgres": {
             "type": "object",
             "properties": {
@@ -125,6 +145,9 @@
           "acrRG",
           "imageRepo",
           "imageTag",
+          "managedIdentityName",
+          "k8s",
+          "azureOperatorsManagedIdentities",
           "postgres"
         ]
       },

config/config.yaml

@@ -86,6 +86,10 @@ defaults:
       deploy: true
       private: false
       minTLSVersion: 'TLSV1.2'
+    managedIdentityName: clusters-service
+    k8s:
+      namespace: cluster-service
+      serviceAccountName: clusters-service
 
   # Image Sync
   imageSync:

config/public-cloud-cs-pr.json

@@ -31,6 +31,11 @@
     },
     "imageRepo": "app-sre/uhc-clusters-service",
     "imageTag": "a51079c",
+    "k8s": {
+      "namespace": "cluster-service",
+      "serviceAccountName": "clusters-service"
+    },
+    "managedIdentityName": "clusters-service",
     "postgres": {
       "deploy": true,
       "minTLSVersion": "TLSV1.2",

config/public-cloud-dev.json

@@ -31,6 +31,11 @@
     },
     "imageRepo": "app-sre/uhc-clusters-service",
     "imageTag": "a51079c",
+    "k8s": {
+      "namespace": "cluster-service",
+      "serviceAccountName": "clusters-service"
+    },
+    "managedIdentityName": "clusters-service",
     "postgres": {
       "deploy": true,
       "minTLSVersion": "TLSV1.2",

config/public-cloud-msft-int.json

@@ -31,6 +31,11 @@
     },
     "imageRepo": "app-sre/uhc-clusters-service",
     "imageTag": "ecd15ad",
+    "k8s": {
+      "namespace": "cluster-service",
+      "serviceAccountName": "clusters-service"
+    },
+    "managedIdentityName": "clusters-service",
     "postgres": {
       "deploy": true,
       "minTLSVersion": "TLSV1.2",

config/public-cloud-personal-dev.json

@@ -31,6 +31,11 @@
     },
     "imageRepo": "app-sre/uhc-clusters-service",
     "imageTag": "a51079c",
+    "k8s": {
+      "namespace": "cluster-service",
+      "serviceAccountName": "clusters-service"
+    },
+    "managedIdentityName": "clusters-service",
     "postgres": {
       "deploy": false,
       "minTLSVersion": "TLSV1.2",

dev-infrastructure/Makefile

@@ -475,7 +475,7 @@ cs-current-user-pg-connect:
 .PHONY: cs-current-user-pg-connect
 
 cs-miwi-pg-connect:
-	@scripts/cs-miwi-pg-connect.sh $(SVC_RESOURCEGROUP) $(CS_PG_NAME) "clusters-service" "cluster-service" "clusters-service"
+	@scripts/cs-miwi-pg-connect.sh $(SVC_RESOURCEGROUP) $(CS_PG_NAME) $(CS_MI_NAME) $(CS_NS_NAME) $(CS_SA_NAME)
 .PHONY: cs-miwi-pg-connect
 
 maestro-current-user-pg-connect:

dev-infrastructure/config.tmpl.mk

@@ -11,6 +11,9 @@ IMAGE_SYNC_ENVIRONMENT ?= {{ .imageSync.environmentName }}
 ARO_HCP_IMAGE_ACR ?= {{ .svcAcrName }}
 AKS_NAME ?= {{ .aksName }}
 CS_PG_NAME ?= {{ .clusterService.postgres.name }}
+CS_MI_NAME ?= {{ .clusterService.managedIdentityName }}
+CS_NS_NAME ?= {{ .clusterService.k8s.namespace }}
+CS_SA_NAME ?= {{ .clusterService.k8s.serviceAccountName }}
 MAESTRO_PG_NAME ?= {{ .maestro.postgres.name }}
 OIDC_STORAGE_ACCOUNT ?= {{ .oidcStorageAccountName }}
 CX_KV_NAME ?= {{ .cxKeyVault.name }}

dev-infrastructure/configurations/cs-integ-msi.tmpl.bicepparam

@@ -1,5 +1,6 @@
 using '../templates/cs-integration-msi.bicep'
 
 param namespaceFormatString = 'sandbox-jenkins-{0}-aro-hcp'
-param clusterServiceManagedIdentityName = 'clusters-service'
+param clusterServiceManagedIdentityName = '{{ .clusterService.managedIdentityName }}'
 param clusterName = '{{ .aksName }}'
+param clusterServiceServiceAccountName = '{{ .clusterService.k8s.serviceAccountName }}'

dev-infrastructure/configurations/output-region.tmpl.bicepparam

@@ -1 +1,3 @@
 using '../templates/output-region.bicep'
+
+param csMIName = '{{ .clusterService.managedIdentityName }}'

dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam

@@ -38,6 +38,9 @@ param csPostgresDeploy = {{ .clusterService.postgres.deploy }}
 param csPostgresServerName = '{{ .clusterService.postgres.name }}'
 param csPostgresServerMinTLSVersion = '{{ .clusterService.postgres.minTLSVersion }}'
 param clusterServicePostgresPrivate = {{ .clusterService.postgres.private }}
+param csMIName = '{{ .clusterService.managedIdentityName }}'
+param csNamespace = '{{ .clusterService.k8s.namespace }}'
+param csServiceAccountName = '{{ .clusterService.k8s.serviceAccountName }}'
 
 param serviceKeyVaultName = '{{ .serviceKeyVault.name }}'
 param serviceKeyVaultResourceGroup = '{{ .serviceKeyVault.rg }}'

dev-infrastructure/templates/cs-integration-msi.bicep

@@ -7,6 +7,9 @@ param clusterServiceManagedIdentityName string
 @description('The name of the cluster to integrate with')
 param clusterName string
 
+@description('The name of the CS service account')
+param clusterServiceServiceAccountName string
+
 resource uami 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = {
   name: clusterServiceManagedIdentityName
 }
@@ -25,7 +28,7 @@ resource uami_fedcred 'Microsoft.ManagedIdentity/userAssignedIdentities/federate
         'api://AzureADTokenExchange'
       ]
       issuer: aksCluster.properties.oidcIssuerProfile.issuerURL
-      subject: 'system:serviceaccount:${format(namespaceFormatString, i)}:clusters-service'
+      subject: 'system:serviceaccount:${format(namespaceFormatString, i)}:${clusterServiceServiceAccountName}'
     }
   }
 ]

dev-infrastructure/templates/output-region.bicep

@@ -1,5 +1,8 @@
+@description('The name of the CS managed identity')
+param csMIName string
+
 resource csMSI 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = {
-  name: 'clusters-service'
+  name: csMIName
   location: resourceGroup().location
 }
 

dev-infrastructure/templates/svc-cluster.bicep

@@ -149,7 +149,14 @@ param frontendIngressCertName string
 @description('The name of the Azure Monitor Workspace (stores prometheus metrics)')
 param azureMonitorWorkspaceName string
 
-var clusterServiceMIName = 'clusters-service'
+@description('The name of the CS managed identity')
+param csMIName string
+
+@description('The namespace of the CS managed identity')
+param csNamespace string
+
+@description('The service account name of the CS managed identity')
+param csServiceAccountName string
 
 resource serviceKeyVault 'Microsoft.KeyVault/vaults@2024-04-01-preview' existing = {
   name: serviceKeyVaultName
@@ -195,9 +202,9 @@ module svcCluster '../modules/aks-cluster-base.bicep' = {
         serviceAccountName: maestroServiceAccountName
       }
       cs_wi: {
-        uamiName: clusterServiceMIName
-        namespace: 'cluster-service'
-        serviceAccountName: 'clusters-service'
+        uamiName: csMIName
+        namespace: csNamespace
+        serviceAccountName: csServiceAccountName
       }
       image_sync_wi: {
         uamiName: 'image-sync'
@@ -303,10 +310,7 @@ module serviceKeyVaultPrivateEndpoint '../modules/private-endpoint.bicep' = {
 //   C L U S T E R   S E R V I C E
 //
 
-var csManagedIdentityPrincipalId = filter(
-  svcCluster.outputs.userAssignedIdentities,
-  id => id.uamiName == clusterServiceMIName
-)[0].uamiPrincipalID
+var csManagedIdentityPrincipalId = filter(svcCluster.outputs.userAssignedIdentities, id => id.uamiName == csMIName)[0].uamiPrincipalID
 
 module cs '../modules/cluster-service.bicep' = {
   name: 'cluster-service'
@@ -318,7 +322,7 @@ module cs '../modules/cluster-service.bicep' = {
     deployPostgres: csPostgresDeploy
     postgresServerPrivate: clusterServicePostgresPrivate
     clusterServiceManagedIdentityPrincipalId: csManagedIdentityPrincipalId
-    clusterServiceManagedIdentityName: clusterServiceMIName
+    clusterServiceManagedIdentityName: csMIName
     serviceKeyVaultName: serviceKeyVault.name
     serviceKeyVaultResourceGroup: serviceKeyVaultResourceGroup
     regionalCXDNSZoneName: regionalCXDNSZoneName
@@ -338,7 +342,7 @@ module oidc '../modules/oidc/main.bicep' = {
   params: {
     location: location
     storageAccountName: oidcStorageAccountName
-    rpMsiName: clusterServiceMIName
+    rpMsiName: csMIName
     skuName: oidcStorageAccountSku
     msiId: aroDevopsMsiId
     deploymentScriptLocation: location

frontend/Makefile

@@ -84,6 +84,8 @@ deploy:
 		--set deployment.imageName=${ARO_HCP_FRONTEND_IMAGE} \
 		--set pullBinding.registry=${ARO_HCP_IMAGE_REGISTRY} \
 		--set pullBinding.scope=repository:${ARO_HCP_IMAGE_REPOSITORY}:pull \
+		--set clusterService.namespace=${CS_NAMESPACE} \
+		--set clusterService.serviceAccount=${CS_SERVICE_ACCOUNT_NAME} \
 		--namespace aro-hcp
 .PHONY: deploy
 

frontend/deploy/helm/frontend/templates/frontend.deployment.yaml

@@ -27,7 +27,7 @@ spec:
         - name: aro-hcp-frontend
           image: '{{ .Values.deployment.imageName }}'
           imagePullPolicy: Always
-          args: ["--clusters-service-url", "http://clusters-service.cluster-service.svc.cluster.local:8000"]
+          args: ["--clusters-service-url", "http://clusters-service.{{ .Values.clusterService.namespace }}.svc.cluster.local:8000"]
           env:
           - name: DB_NAME
             valueFrom:

frontend/deploy/helm/frontend/values.yaml

@@ -20,3 +20,6 @@ pullBinding:
   scope: ""
   workloadIdentityClientId: ""
   workloadIdentityTenantId: ""
+clusterService:
+  namespace: cluster-service
+  serviceAccount: clusters-service

frontend/pipeline.yaml

@@ -32,3 +32,7 @@ resourceGroups:
       configRef: frontend.cert.name
     - name: ISTO_TAG
       configRef: svc.istio.tag
+    - name: CS_NAMESPACE
+      configRef: clusterService.k8s.namespace
+    - name: CS_SERVICE_ACCOUNT_NAME
+      configRef: clusterService.k8s.serviceAccountName

maestro/server/Makefile

@@ -23,5 +23,7 @@ deploy:
 		--set image.base=${IMAGE_BASE} \
 		--set image.tag=${IMAGE_TAG} \
 		--set database.host=$${DATABASE_HOST} \
-		--set database.name=${DATABASE_NAME}
+		--set database.name=${DATABASE_NAME} \
+		--set clusterService.namespace=${CS_NAMESPACE} \
+		--set clusterService.serviceAccount=${CS_SERVICE_ACCOUNT_NAME}
 .PHONY: deploy