From 4ecd4bce6cc69c97b7548e2a06c8c17b342652da Mon Sep 17 00:00:00 2001 From: Saurabh Date: Fri, 25 Aug 2023 11:22:37 +0530 Subject: [PATCH] Feature/us 561435 - Init container to have cpu and memory limits defined. (#622) * US-561435 - Changes to assign cpu and memory to init containers * US-561435 - Indenting the code * US-561435 - Addressing code review comments. Moving definiton to supplements.tpl US-561435 - Addressing code review comments. Moving definiton to supplements.tpl Co-authored-by: saurabh.kumar2@in.pega.com --- .../hazelcast/templates/_supplemental.tpl | 11 +++++ .../charts/installer/templates/_helpers.tpl | 8 +++- .../installer/templates/_supplemental.tpl | 11 +++++ .../pegasearch/templates/_supplemental.tpl | 11 +++++ .../templates/pega-search-deployment.yaml | 2 + charts/pega/templates/_helpers.tpl | 2 + charts/pega/templates/_supplemental.tpl | 11 +++++ terratest/src/test/pega/utilities.go | 43 ++++++++++++------- 8 files changed, 82 insertions(+), 17 deletions(-) diff --git a/charts/pega/charts/hazelcast/templates/_supplemental.tpl b/charts/pega/charts/hazelcast/templates/_supplemental.tpl index 5cb39e6cb..e8eb70789 100644 --- a/charts/pega/charts/hazelcast/templates/_supplemental.tpl +++ b/charts/pega/charts/hazelcast/templates/_supplemental.tpl @@ -22,6 +22,17 @@ charts to render standalone. See: https://github.com/helm/helm/issues/11260 for {{- $depName -}}-credentials-secret {{- end }} +{{- define "initContainerResources" }} + resources: + # Resources requests/limits for initContainers + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 50m + memory: 64Mi +{{- end }} + {{- define "pegaCredentialVolumeTemplate" }} - name: {{ template "pegaVolumeCredentials" }} projected: diff --git a/charts/pega/charts/installer/templates/_helpers.tpl b/charts/pega/charts/installer/templates/_helpers.tpl index 03b2c4327..46940d56e 100644 --- a/charts/pega/charts/installer/templates/_helpers.tpl +++ b/charts/pega/charts/installer/templates/_helpers.tpl @@ -90,11 +90,12 @@ image: {{ .Values.global.utilityImages.k8s_wait_for.image }} imagePullPolicy: {{ .Values.global.utilityImages.k8s_wait_for.imagePullPolicy }} args: [ 'job', '{{ template "pegaDBInstall" }}'] - env: + env: - name: WAIT_TIME value: "{{ template "k8sWaitForWaitTime" $ }}" - name: MAX_RETRIES value: "{{ template "k8sWaitForMaxRetries" $ }}" +{{- include "initContainerResources" $ }} {{- end }} {{- define "waitForPegaDBZDTUpgrade" -}} @@ -108,6 +109,7 @@ value: "{{ template "k8sWaitForWaitTime" $ }}" - name: MAX_RETRIES value: "{{ template "k8sWaitForMaxRetries" $ }}" +{{- include "initContainerResources" $ }} {{- end }} {{- define "waitForPreDBUpgrade" -}} @@ -115,11 +117,12 @@ image: {{ .Values.global.utilityImages.k8s_wait_for.image }} imagePullPolicy: {{ .Values.global.utilityImages.k8s_wait_for.imagePullPolicy }} args: [ 'job', '{{ template "pegaPreDBUpgrade" }}'] - env: + env: - name: WAIT_TIME value: "{{ template "k8sWaitForWaitTime" $ }}" - name: MAX_RETRIES value: "{{ template "k8sWaitForMaxRetries" $ }}" +{{- include "initContainerResources" $ }} {{- end }} {{- define "waitForRollingUpdates" -}} @@ -154,6 +157,7 @@ value: "{{ template "k8sWaitForWaitTime" $ }}" - name: MAX_RETRIES value: "{{ template "k8sWaitForMaxRetries" $ }}" +{{- include "initContainerResources" $ }} {{- end }} {{- define "initContainerEnvs" -}} diff --git a/charts/pega/charts/installer/templates/_supplemental.tpl b/charts/pega/charts/installer/templates/_supplemental.tpl index 5cb39e6cb..e8eb70789 100644 --- a/charts/pega/charts/installer/templates/_supplemental.tpl +++ b/charts/pega/charts/installer/templates/_supplemental.tpl @@ -22,6 +22,17 @@ charts to render standalone. See: https://github.com/helm/helm/issues/11260 for {{- $depName -}}-credentials-secret {{- end }} +{{- define "initContainerResources" }} + resources: + # Resources requests/limits for initContainers + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 50m + memory: 64Mi +{{- end }} + {{- define "pegaCredentialVolumeTemplate" }} - name: {{ template "pegaVolumeCredentials" }} projected: diff --git a/charts/pega/charts/pegasearch/templates/_supplemental.tpl b/charts/pega/charts/pegasearch/templates/_supplemental.tpl index 5cb39e6cb..e8eb70789 100644 --- a/charts/pega/charts/pegasearch/templates/_supplemental.tpl +++ b/charts/pega/charts/pegasearch/templates/_supplemental.tpl @@ -22,6 +22,17 @@ charts to render standalone. See: https://github.com/helm/helm/issues/11260 for {{- $depName -}}-credentials-secret {{- end }} +{{- define "initContainerResources" }} + resources: + # Resources requests/limits for initContainers + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 50m + memory: 64Mi +{{- end }} + {{- define "pegaCredentialVolumeTemplate" }} - name: {{ template "pegaVolumeCredentials" }} projected: diff --git a/charts/pega/charts/pegasearch/templates/pega-search-deployment.yaml b/charts/pega/charts/pegasearch/templates/pega-search-deployment.yaml index cba0741cd..7522ad89d 100644 --- a/charts/pega/charts/pegasearch/templates/pega-search-deployment.yaml +++ b/charts/pega/charts/pegasearch/templates/pega-search-deployment.yaml @@ -40,6 +40,7 @@ spec: image: {{ .Values.global.utilityImages.busybox.image }} imagePullPolicy: {{ .Values.global.utilityImages.busybox.imagePullPolicy }} command: ['sh', '-c', 'chown -R {{ .Values.podSecurityContext.runAsUser | default 1000 }}:{{ .Values.podSecurityContext.runAsUser | default 1000 }} /usr/share/elasticsearch/data'] + {{- include "initContainerResources" . | indent 6 }} volumeMounts: - name: esstorage mountPath: /usr/share/elasticsearch/data @@ -49,6 +50,7 @@ spec: image: {{ .Values.global.utilityImages.busybox.image }} imagePullPolicy: {{ .Values.global.utilityImages.busybox.imagePullPolicy }} command: ['sysctl', '-w', 'vm.max_map_count=262144'] + {{- include "initContainerResources" . | indent 6 }} securityContext: privileged: true {{ end }} diff --git a/charts/pega/templates/_helpers.tpl b/charts/pega/templates/_helpers.tpl index 0f261be71..3a050eb44 100644 --- a/charts/pega/templates/_helpers.tpl +++ b/charts/pega/templates/_helpers.tpl @@ -194,6 +194,7 @@ imagePullPolicy: {{ .Values.global.utilityImages.busybox.imagePullPolicy }} # Init container for waiting for Elastic Search to initialize. The URL should point at your Elastic Search instance. command: ['sh', '-c', 'until $(wget -q -S --spider --timeout=2 -O /dev/null {{ include "pegaSearchURL" $ }}); do echo Waiting for search to become live...; sleep 10; done;'] +{{- include "initContainerResources" $ }} {{- end }} {{- define "waitForCassandra" -}} @@ -205,6 +206,7 @@ # -p is password # final 2 args for cqlsh are cassandra host and port respectively command: ['sh', '-c', '{{- template "waitForCassandraScript" dict "nodes" (include "getCassandraSubchartService" .) "node" .Values.dds -}}'] +{{- include "initContainerResources" $ }} {{- end -}} {{- end }} diff --git a/charts/pega/templates/_supplemental.tpl b/charts/pega/templates/_supplemental.tpl index 5cb39e6cb..e8eb70789 100644 --- a/charts/pega/templates/_supplemental.tpl +++ b/charts/pega/templates/_supplemental.tpl @@ -22,6 +22,17 @@ charts to render standalone. See: https://github.com/helm/helm/issues/11260 for {{- $depName -}}-credentials-secret {{- end }} +{{- define "initContainerResources" }} + resources: + # Resources requests/limits for initContainers + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 50m + memory: 64Mi +{{- end }} + {{- define "pegaCredentialVolumeTemplate" }} - name: {{ template "pegaVolumeCredentials" }} projected: diff --git a/terratest/src/test/pega/utilities.go b/terratest/src/test/pega/utilities.go index 1c04b0f0f..4b66c0975 100644 --- a/terratest/src/test/pega/utilities.go +++ b/terratest/src/test/pega/utilities.go @@ -24,7 +24,7 @@ func compareConfigMapData(t *testing.T, actualFileData string, expectedFileName require.Equal(t, expectedFileData, actualFileData) } -//aksSpecificUpgraderDeployEnvs - Test aks specific upgrade-deploy environmnet variables in case of upgrade-deploy +// aksSpecificUpgraderDeployEnvs - Test aks specific upgrade-deploy environmnet variables in case of upgrade-deploy func aksSpecificUpgraderDeployEnvs(t *testing.T, options *helm.Options, container k8score.Container) { if options.SetValues["global.provider"] == "aks" && options.SetValues["global.actions.execute"] == "upgrade-deploy" { require.Equal(t, container.Env[0].Name, "KUBERNETES_SERVICE_HOST") @@ -36,6 +36,13 @@ func aksSpecificUpgraderDeployEnvs(t *testing.T, options *helm.Options, containe } } +func VerifyInitContainerResources(t *testing.T, container k8score.Container) { + require.Equal(t, "50m", container.Resources.Limits.Cpu().String()) + require.Equal(t, "64Mi", container.Resources.Limits.Memory().String()) + require.Equal(t, "50m", container.Resources.Requests.Cpu().String()) + require.Equal(t, "64Mi", container.Resources.Requests.Memory().String()) +} + // VerifyInitContainerData - Verifies any possible initContainer that can occur in pega helm chart deployments func VerifyInitContainerData(t *testing.T, containers []k8score.Container, options *helm.Options) { var depName = getDeploymentName(options) @@ -51,26 +58,32 @@ func VerifyInitContainerData(t *testing.T, containers []k8score.Container, optio require.Equal(t, "pegasystems/k8s-wait-for", container.Image) VerifyK8sWaitForArgs(t, container) require.Equal(t, []string{"job", "pega-db-install"}, container.Args) + VerifyInitContainerResources(t, container) } else if name == "wait-for-pegasearch" { require.Equal(t, "busybox:1.31.0", container.Image) require.Equal(t, []string{"sh", "-c", "until $(wget -q -S --spider --timeout=2 -O /dev/null http://" + depName + "-search); do echo Waiting for search to become live...; sleep 10; done;"}, container.Command) + VerifyInitContainerResources(t, container) } else if name == "wait-for-cassandra" { require.Equal(t, "cassandra:3.11.3", container.Image) //The cassandra svc name below is derived from helm release name and not .Values.global.deploymentName like search svc require.Equal(t, []string{"sh", "-c", "until cqlsh -u \"dnode_ext\" -p \"dnode_ext\" -e \"describe cluster\" pega-cassandra 9042 ; do echo Waiting for cassandra to become live...; sleep 10; done;"}, container.Command) + VerifyInitContainerResources(t, container) } else if name == "wait-for-pegaupgrade" { require.Equal(t, "pegasystems/k8s-wait-for", container.Image) VerifyK8sWaitForArgs(t, container) require.Equal(t, []string{"job", "pega-zdt-upgrade"}, container.Args) aksSpecificUpgraderDeployEnvs(t, options, container) + VerifyInitContainerResources(t, container) } else if name == "wait-for-pre-dbupgrade" { require.Equal(t, "pegasystems/k8s-wait-for", container.Image) VerifyK8sWaitForArgs(t, container) require.Equal(t, []string{"job", "pega-pre-upgrade"}, container.Args) + VerifyInitContainerResources(t, container) } else if name == "wait-for-rolling-updates" { require.Equal(t, "pegasystems/k8s-wait-for", container.Image) VerifyK8sWaitForArgs(t, container) require.Equal(t, []string{"sh", "-c", " kubectl rollout status deployment/" + depName + "-web --namespace default && kubectl rollout status deployment/" + depName + "-batch --namespace default && kubectl rollout status statefulset/" + depName + "-stream --namespace default"}, container.Command) + VerifyInitContainerResources(t, container) } else { fmt.Println("invalid init containers found.. please check the list", name) t.Fail() @@ -79,21 +92,21 @@ func VerifyInitContainerData(t *testing.T, containers []k8score.Container, optio } func VerifyK8sWaitForArgs(t *testing.T, container k8score.Container) { - BOTH_PARAMS_HAPPEN_ONCE := 6 - require.Equal(t, "pegasystems/k8s-wait-for", container.Image) + BOTH_PARAMS_HAPPEN_ONCE := 6 + require.Equal(t, "pegasystems/k8s-wait-for", container.Image) - envvars := container.Env - k8sWaitForParamsProduct := 1 - for i := 0; i < len(envvars); i++ { - envvar := envvars[i] - if envvar.Name == "WAIT_TIME" { - k8sWaitForParamsProduct *= 2 - } - if envvar.Name == "MAX_RETRIES" { - k8sWaitForParamsProduct *= 3 - } - } - require.Equal(t, BOTH_PARAMS_HAPPEN_ONCE, k8sWaitForParamsProduct) + envvars := container.Env + k8sWaitForParamsProduct := 1 + for i := 0; i < len(envvars); i++ { + envvar := envvars[i] + if envvar.Name == "WAIT_TIME" { + k8sWaitForParamsProduct *= 2 + } + if envvar.Name == "MAX_RETRIES" { + k8sWaitForParamsProduct *= 3 + } + } + require.Equal(t, BOTH_PARAMS_HAPPEN_ONCE, k8sWaitForParamsProduct) } func getDeploymentName(options *helm.Options) string {