Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[NET-11297] Purge on disable #4378

Merged
merged 11 commits into from
Oct 21, 2024
3 changes: 3 additions & 0 deletions .changelog/4378.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
```release-note:enhancement
catalog-sync: Added field to helm chart to purge all services registered with catalog-sync from consul on disabling of catalog-sync.
```
174 changes: 174 additions & 0 deletions charts/consul/templates/sync-catalog-cleanup-on-uninstall-job.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
{{- if .Values.syncCatalog.cleanupNodeOnRemoval }}
apiVersion: batch/v1
kind: Job
metadata:
name: {{ template "consul.fullname" . }}-sync-catalog-cleanup-on-uninstall
namespace: {{ .Release.Namespace }}
labels:
app: {{ template "consul.name" . }}
chart: {{ template "consul.chart" . }}
heritage: {{ .Release.Service }}
release: {{ .Release.Name }}
component: sync-catalog-cleanup
{{- if .Values.global.extraLabels }}
{{- toYaml .Values.global.extraLabels | nindent 4 }}
{{- end }}
annotations:
"helm.sh/hook": pre-delete
"helm.sh/hook-weight": "0"
"helm.sh/hook-delete-policy": hook-succeeded,hook-failed
spec:
template:
metadata:
name: {{ template "consul.fullname" . }}-sync-catalog-cleanup-on-uninstall
labels:
app: {{ template "consul.name" . }}
chart: {{ template "consul.chart" . }}
release: {{ .Release.Name }}
component: sync-catalog-cleanup
{{- if .Values.syncCatalog.extraLabels }}
{{- toYaml .Values.syncCatalog.extraLabels | nindent 8 }}
{{- end }}
{{- if .Values.global.extraLabels }}
{{- toYaml .Values.global.extraLabels | nindent 8 }}
{{- end }}
annotations:
"consul.hashicorp.com/connect-inject": "false"
"consul.hashicorp.com/mesh-inject": "false"
{{- if .Values.syncCatalog.annotations }}
{{- tpl .Values.syncCatalog.annotations . | nindent 8 }}
{{- end }}
{{- if (and .Values.global.secretsBackend.vault.enabled .Values.global.tls.enabled) }}
"vault.hashicorp.com/agent-init-first": "true"
"vault.hashicorp.com/agent-inject": "true"
"vault.hashicorp.com/role": {{ .Values.global.secretsBackend.vault.consulCARole }}
"vault.hashicorp.com/agent-inject-secret-serverca.crt": {{ .Values.global.tls.caCert.secretName }}
"vault.hashicorp.com/agent-inject-template-serverca.crt": {{ template "consul.serverTLSCATemplate" . }}
{{- if and .Values.global.secretsBackend.vault.ca.secretName .Values.global.secretsBackend.vault.ca.secretKey }}
"vault.hashicorp.com/agent-extra-secret": "{{ .Values.global.secretsBackend.vault.ca.secretName }}"
"vault.hashicorp.com/ca-cert": "/vault/custom/{{ .Values.global.secretsBackend.vault.ca.secretKey }}"
{{- end }}
{{- if .Values.global.secretsBackend.vault.agentAnnotations }}
{{ tpl .Values.global.secretsBackend.vault.agentAnnotations . | nindent 8 | trim }}
{{- end }}
{{- if (and (.Values.global.secretsBackend.vault.vaultNamespace) (not (hasKey (default "" .Values.global.secretsBackend.vault.agentAnnotations | fromYaml) "vault.hashicorp.com/namespace")))}}
"vault.hashicorp.com/namespace": "{{ .Values.global.secretsBackend.vault.vaultNamespace }}"
{{- end }}
{{- end }}
{{- if .Values.syncCatalog.metrics.enabled | default .Values.global.metrics.enabled }}
"prometheus.io/scrape": "true"
{{- if not (hasKey (default "" .Values.syncCatalog.annotations | fromYaml) "prometheus.io/path")}}
"prometheus.io/path": {{ default "/metrics" .Values.syncCatalog.metrics.path }}
{{- end }}
"prometheus.io/port": {{ .Values.syncCatalog.metrics.port | default "20300" | quote }}
{{- end }}
spec:
restartPolicy: Never
serviceAccountName: {{ template "consul.fullname" . }}-sync-catalog-cleanup
containers:
- name: sync-catalog-cleanup-job
image: "{{ default .Values.global.imageK8S .Values.syncCatalog.image }}"
{{ template "consul.imagePullPolicy" . }}
{{- include "consul.restrictedSecurityContext" . | nindent 8 }}
env:
{{- include "consul.consulK8sConsulServerEnvVars" . | nindent 8 }}
{{- if .Values.global.acls.manageSystemACLs }}
- name: CONSUL_LOGIN_AUTH_METHOD
{{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter .Values.global.enableConsulNamespaces }}
value: {{ template "consul.fullname" . }}-k8s-component-auth-method-{{ .Values.global.datacenter }}
{{- else }}
value: {{ template "consul.fullname" . }}-k8s-component-auth-method
{{- end }}
- name: CONSUL_LOGIN_DATACENTER
{{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter .Values.global.enableConsulNamespaces }}
value: {{ .Values.global.federation.primaryDatacenter }}
{{- else }}
value: {{ .Values.global.datacenter }}
{{- end }}
- name: CONSUL_LOGIN_META
value: "component=sync-catalog,pod=$(NAMESPACE)/$(POD_NAME)"
{{- end }}
- name: NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
{{- if (and .Values.syncCatalog.aclSyncToken.secretName .Values.syncCatalog.aclSyncToken.secretKey) }}
- name: CONSUL_ACL_TOKEN
valueFrom:
secretKeyRef:
name: {{ .Values.syncCatalog.aclSyncToken.secretName }}
key: {{ .Values.syncCatalog.aclSyncToken.secretKey }}
{{- end }}
volumeMounts:
{{- if .Values.global.tls.enabled }}
{{- if not (or (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots) .Values.global.secretsBackend.vault.enabled) }}
- name: consul-ca-cert
mountPath: /consul/tls/ca
readOnly: true
{{- end }}
{{- end }}
command:
- "/bin/sh"
- "-ec"
- |
exec consul-k8s-control-plane sync-catalog \
-log-level={{ default .Values.global.logLevel .Values.syncCatalog.logLevel }} \
-log-json={{ .Values.global.logJSON }} \
-k8s-default-sync={{ .Values.syncCatalog.default }} \
{{- if .Values.global.adminPartitions.enabled }}
-partition={{ .Values.global.adminPartitions.name }} \
{{- end }}
{{- if .Values.syncCatalog.consulNodeName }}
-consul-node-name={{ .Values.syncCatalog.consulNodeName }} \
{{- end }}
{{- if .Values.syncCatalog.metrics.enabled | default .Values.global.metrics.enabled }}
-enable-metrics \
{{- end }}
{{- if .Values.syncCatalog.metrics.path }}
-metrics-path={{ .Values.syncCatalog.metrics.path }} \
{{- end }}
{{- if .Values.syncCatalog.metrics.port }}
-metrics-port={{ .Values.syncCatalog.metrics.port }} \
{{- end }}
-prometheus-retention-time={{ .Values.global.metrics.agentMetricsRetentionTime }} \
-purge-k8s-services-from-node
{{- with .Values.syncCatalog.resources }}
resources:
{{- toYaml . | nindent 10 }}
{{- end }}
{{- if or (eq (.Values.syncCatalog.metrics.enabled | toString) "-") .Values.syncCatalog.metrics.enabled .Values.global.metrics.enabled }}
ports:
- name: prometheus
containerPort: {{ .Values.syncCatalog.metrics.port | default "20300" | int }}
{{- end }}
{{- if .Values.syncCatalog.priorityClassName }}
priorityClassName: {{ .Values.syncCatalog.priorityClassName | quote }}
{{- end }}
{{- if .Values.syncCatalog.nodeSelector }}
nodeSelector:
{{ tpl .Values.syncCatalog.nodeSelector . | indent 8 | trim }}
{{- end }}
{{- if .Values.syncCatalog.affinity }}
affinity:
{{ tpl .Values.syncCatalog.affinity . | indent 8 | trim }}
{{- end }}
{{- if .Values.syncCatalog.tolerations }}
tolerations:
{{ tpl .Values.syncCatalog.tolerations . | indent 8 | trim }}
{{- end }}
volumes:
{{- if .Values.global.tls.enabled }}
{{- if not (or (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots) .Values.global.secretsBackend.vault.enabled) }}
- name: consul-ca-cert
secret:
{{- if .Values.global.tls.caCert.secretName }}
secretName: {{ .Values.global.tls.caCert.secretName }}
{{- else }}
secretName: {{ template "consul.fullname" . }}-ca-cert
{{- end }}
items:
- key: {{ default "tls.crt" .Values.global.tls.caCert.secretKey }}
path: tls.crt
{{- end }}
{{- end }}
{{- end }}
175 changes: 175 additions & 0 deletions charts/consul/templates/sync-catalog-cleanup-on-upgrade-job.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,175 @@
{{- $syncCatalogEnabled := (or (and (ne (.Values.syncCatalog.enabled | toString) "-") .Values.syncCatalog.enabled) (and (eq (.Values.syncCatalog.enabled | toString) "-") .Values.global.enabled)) }}
{{- if and (not $syncCatalogEnabled) .Values.syncCatalog.cleanupNodeOnRemoval .Release.IsUpgrade }}
apiVersion: batch/v1
kind: Job
metadata:
name: {{ template "consul.fullname" . }}-sync-catalog-cleanup-on-upgrade
namespace: {{ .Release.Namespace }}
labels:
app: {{ template "consul.name" . }}
chart: {{ template "consul.chart" . }}
heritage: {{ .Release.Service }}
release: {{ .Release.Name }}
component: sync-catalog-cleanup
{{- if .Values.global.extraLabels }}
{{- toYaml .Values.global.extraLabels | nindent 4 }}
{{- end }}
annotations:
"helm.sh/hook": pre-upgrade
"helm.sh/hook-weight": "0"
"helm.sh/hook-delete-policy": hook-succeeded,hook-failed
spec:
template:
metadata:
name: {{ template "consul.fullname" . }}-sync-catalog-cleanup-on-upgrade
labels:
app: {{ template "consul.name" . }}
chart: {{ template "consul.chart" . }}
release: {{ .Release.Name }}
component: sync-catalog-cleanup
{{- if .Values.syncCatalog.extraLabels }}
{{- toYaml .Values.syncCatalog.extraLabels | nindent 8 }}
{{- end }}
{{- if .Values.global.extraLabels }}
{{- toYaml .Values.global.extraLabels | nindent 8 }}
{{- end }}
annotations:
"consul.hashicorp.com/connect-inject": "false"
"consul.hashicorp.com/mesh-inject": "false"
{{- if .Values.syncCatalog.annotations }}
{{- tpl .Values.syncCatalog.annotations . | nindent 8 }}
{{- end }}
{{- if (and .Values.global.secretsBackend.vault.enabled .Values.global.tls.enabled) }}
"vault.hashicorp.com/agent-init-first": "true"
"vault.hashicorp.com/agent-inject": "true"
"vault.hashicorp.com/role": {{ .Values.global.secretsBackend.vault.consulCARole }}
"vault.hashicorp.com/agent-inject-secret-serverca.crt": {{ .Values.global.tls.caCert.secretName }}
"vault.hashicorp.com/agent-inject-template-serverca.crt": {{ template "consul.serverTLSCATemplate" . }}
{{- if and .Values.global.secretsBackend.vault.ca.secretName .Values.global.secretsBackend.vault.ca.secretKey }}
"vault.hashicorp.com/agent-extra-secret": "{{ .Values.global.secretsBackend.vault.ca.secretName }}"
"vault.hashicorp.com/ca-cert": "/vault/custom/{{ .Values.global.secretsBackend.vault.ca.secretKey }}"
{{- end }}
{{- if .Values.global.secretsBackend.vault.agentAnnotations }}
{{ tpl .Values.global.secretsBackend.vault.agentAnnotations . | nindent 8 | trim }}
{{- end }}
{{- if (and (.Values.global.secretsBackend.vault.vaultNamespace) (not (hasKey (default "" .Values.global.secretsBackend.vault.agentAnnotations | fromYaml) "vault.hashicorp.com/namespace")))}}
"vault.hashicorp.com/namespace": "{{ .Values.global.secretsBackend.vault.vaultNamespace }}"
{{- end }}
{{- end }}
{{- if .Values.syncCatalog.metrics.enabled | default .Values.global.metrics.enabled }}
"prometheus.io/scrape": "true"
{{- if not (hasKey (default "" .Values.syncCatalog.annotations | fromYaml) "prometheus.io/path")}}
"prometheus.io/path": {{ default "/metrics" .Values.syncCatalog.metrics.path }}
{{- end }}
"prometheus.io/port": {{ .Values.syncCatalog.metrics.port | default "20300" | quote }}
{{- end }}
spec:
restartPolicy: Never
serviceAccountName: {{ template "consul.fullname" . }}-sync-catalog-cleanup
containers:
- name: sync-catalog-cleanup-job
image: "{{ default .Values.global.imageK8S .Values.syncCatalog.image }}"
{{ template "consul.imagePullPolicy" . }}
{{- include "consul.restrictedSecurityContext" . | nindent 8 }}
env:
{{- include "consul.consulK8sConsulServerEnvVars" . | nindent 8 }}
{{- if .Values.global.acls.manageSystemACLs }}
- name: CONSUL_LOGIN_AUTH_METHOD
{{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter .Values.global.enableConsulNamespaces }}
value: {{ template "consul.fullname" . }}-k8s-component-auth-method-{{ .Values.global.datacenter }}
{{- else }}
value: {{ template "consul.fullname" . }}-k8s-component-auth-method
{{- end }}
- name: CONSUL_LOGIN_DATACENTER
{{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter .Values.global.enableConsulNamespaces }}
value: {{ .Values.global.federation.primaryDatacenter }}
{{- else }}
value: {{ .Values.global.datacenter }}
{{- end }}
- name: CONSUL_LOGIN_META
value: "component=sync-catalog,pod=$(NAMESPACE)/$(POD_NAME)"
{{- end }}
- name: NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
{{- if (and .Values.syncCatalog.aclSyncToken.secretName .Values.syncCatalog.aclSyncToken.secretKey) }}
- name: CONSUL_ACL_TOKEN
valueFrom:
secretKeyRef:
name: {{ .Values.syncCatalog.aclSyncToken.secretName }}
key: {{ .Values.syncCatalog.aclSyncToken.secretKey }}
{{- end }}
volumeMounts:
{{- if .Values.global.tls.enabled }}
{{- if not (or (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots) .Values.global.secretsBackend.vault.enabled) }}
- name: consul-ca-cert
mountPath: /consul/tls/ca
readOnly: true
{{- end }}
{{- end }}
command:
- "/bin/sh"
- "-ec"
- |
exec consul-k8s-control-plane sync-catalog \
-log-level={{ default .Values.global.logLevel .Values.syncCatalog.logLevel }} \
-log-json={{ .Values.global.logJSON }} \
-k8s-default-sync={{ .Values.syncCatalog.default }} \
{{- if .Values.global.adminPartitions.enabled }}
-partition={{ .Values.global.adminPartitions.name }} \
{{- end }}
{{- if .Values.syncCatalog.consulNodeName }}
-consul-node-name={{ .Values.syncCatalog.consulNodeName }} \
{{- end }}
{{- if .Values.syncCatalog.metrics.enabled | default .Values.global.metrics.enabled }}
-enable-metrics \
{{- end }}
{{- if .Values.syncCatalog.metrics.path }}
-metrics-path={{ .Values.syncCatalog.metrics.path }} \
{{- end }}
{{- if .Values.syncCatalog.metrics.port }}
-metrics-port={{ .Values.syncCatalog.metrics.port }} \
{{- end }}
-prometheus-retention-time={{ .Values.global.metrics.agentMetricsRetentionTime }} \
-purge-k8s-services-from-node
{{- with .Values.syncCatalog.resources }}
resources:
{{- toYaml . | nindent 10 }}
{{- end }}
{{- if or (eq (.Values.syncCatalog.metrics.enabled | toString) "-") .Values.syncCatalog.metrics.enabled .Values.global.metrics.enabled }}
ports:
- name: prometheus
containerPort: {{ .Values.syncCatalog.metrics.port | default "20300" | int }}
{{- end }}
{{- if .Values.syncCatalog.priorityClassName }}
priorityClassName: {{ .Values.syncCatalog.priorityClassName | quote }}
{{- end }}
{{- if .Values.syncCatalog.nodeSelector }}
nodeSelector:
{{ tpl .Values.syncCatalog.nodeSelector . | indent 8 | trim }}
{{- end }}
{{- if .Values.syncCatalog.affinity }}
affinity:
{{ tpl .Values.syncCatalog.affinity . | indent 8 | trim }}
{{- end }}
{{- if .Values.syncCatalog.tolerations }}
tolerations:
{{ tpl .Values.syncCatalog.tolerations . | indent 8 | trim }}
{{- end }}
volumes:
{{- if .Values.global.tls.enabled }}
{{- if not (or (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots) .Values.global.secretsBackend.vault.enabled) }}
- name: consul-ca-cert
secret:
{{- if .Values.global.tls.caCert.secretName }}
secretName: {{ .Values.global.tls.caCert.secretName }}
{{- else }}
secretName: {{ template "consul.fullname" . }}-ca-cert
{{- end }}
items:
- key: {{ default "tls.crt" .Values.global.tls.caCert.secretKey }}
path: tls.crt
{{- end }}
{{- end }}
{{- end }}
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{{- if .Values.syncCatalog.cleanupNodeOnRemoval }}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ template "consul.fullname" . }}-sync-catalog-cleanup
namespace: {{ .Release.Namespace }}
labels:
app: {{ template "consul.name" . }}
chart: {{ template "consul.chart" . }}
heritage: {{ .Release.Service }}
release: {{ .Release.Name }}
component: sync-catalog-cleanup
{{- if .Values.syncCatalog.serviceAccount.annotations }}
annotations:
{{ tpl .Values.syncCatalog.serviceAccount.annotations . | nindent 4 | trim }}
{{- end }}
{{- with .Values.global.imagePullSecrets }}
imagePullSecrets:
{{- range . }}
- name: {{ .name }}
{{- end }}
{{- end }}
{{- end }}
3 changes: 2 additions & 1 deletion charts/consul/templates/sync-catalog-deployment.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
{{- if (or (and (ne (.Values.syncCatalog.enabled | toString) "-") .Values.syncCatalog.enabled) (and (eq (.Values.syncCatalog.enabled | toString) "-") .Values.global.enabled)) }}
{{- $syncCatalogEnabled := (or (and (ne (.Values.syncCatalog.enabled | toString) "-") .Values.syncCatalog.enabled) (and (eq (.Values.syncCatalog.enabled | toString) "-") .Values.global.enabled)) }}
{{- if $syncCatalogEnabled }}
{{- template "consul.reservedNamesFailer" (list .Values.syncCatalog.consulNamespaces.consulDestinationNamespace "syncCatalog.consulNamespaces.consulDestinationNamespace") }}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

❤️

{{ template "consul.validateRequiredCloudSecretsExist" . }}
{{ template "consul.validateCloudSecretKeys" . }}
Expand Down
4 changes: 4 additions & 0 deletions charts/consul/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2090,6 +2090,10 @@ syncCatalog:
# global.enabled.
enabled: false

# True if you want to deregister all services in this cluster from consul that have been registered by the catalog sync when the `enabled` flag is set to false.
# @type: boolean
cleanupNodeOnRemoval: false

# The name of the Docker image (including any tag) for consul-k8s-control-plane
# to run the sync program.
# @type: string
Expand Down
Loading
Loading