feat(metrics-server): use cert-manager certs

charts/spin-operator/templates/deployment.yaml

@@ -71,21 +71,9 @@ spec:
         - mountPath: /tmp/k8s-webhook-server/serving-certs
           name: cert
           readOnly: true
-      - args: {{- toYaml .Values.controllerManager.kubeRbacProxy.args | nindent 8 }}
-        env:
-        - name: KUBERNETES_CLUSTER_DOMAIN
-          value: {{ quote .Values.kubernetesClusterDomain }}
-        image: {{ .Values.controllerManager.kubeRbacProxy.image.repository }}:{{ .Values.controllerManager.kubeRbacProxy.image.tag
-          | default .Chart.AppVersion }}
-        name: kube-rbac-proxy
-        ports:
-        - containerPort: 8443
-          name: https
-          protocol: TCP
-        resources: {{- toYaml .Values.controllerManager.kubeRbacProxy.resources | nindent
-          10 }}
-        securityContext: {{- toYaml .Values.controllerManager.kubeRbacProxy.containerSecurityContext
-          | nindent 10 }}
+        - mountPath: /tmp/k8s-metrics-server/metrics-certs
+          name: metrics-certs
+          readOnly: true
       securityContext:
         runAsNonRoot: true
       serviceAccountName: {{ include "spin-operator.fullname" . }}-controller-manager
@@ -94,4 +82,15 @@ spec:
       - name: cert
         secret:
           defaultMode: 420
-          secretName: webhook-server-cert
+          secretName: webhook-server-cert
+      - name: metrics-certs
+        secret:
+          items:
+          - key: ca.crt
+            path: ca.crt
+          - key: tls.crt
+            path: tls.crt
+          - key: tls.key
+            path: tls.key
+          optional: false
+          secretName: metrics-server-cert

charts/spin-operator/templates/manager-rbac.yaml

@@ -8,18 +8,14 @@ rules:
 - apiGroups:
   - ""
   resources:
-  - secrets
+  - events
   verbs:
   - create
-  - delete
-  - get
-  - list
   - patch
-  - update
-  - watch
 - apiGroups:
   - ""
   resources:
+  - secrets
   - services
   verbs:
   - create
@@ -47,17 +43,11 @@ rules:
   - deployments/status
   verbs:
   - get
-- apiGroups:
-  - ""
-  resources:
-  - events
-  verbs:
-  - create
-  - patch
 - apiGroups:
   - core.spinkube.dev
   resources:
   - spinappexecutors
+  - spinapps
   verbs:
   - create
   - delete
@@ -76,25 +66,6 @@ rules:
   - core.spinkube.dev
   resources:
   - spinappexecutors/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - core.spinkube.dev
-  resources:
-  - spinapps
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - core.spinkube.dev
-  resources:
   - spinapps/status
   verbs:
   - get

charts/spin-operator/templates/metrics-auth-rbac.yaml

@@ -0,0 +1,34 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "spin-operator.fullname" . }}-metrics-auth-role
+  labels:
+  {{- include "spin-operator.labels" . | nindent 4 }}
+rules:
+- apiGroups:
+  - authentication.k8s.io
+  resources:
+  - tokenreviews
+  verbs:
+  - create
+- apiGroups:
+  - authorization.k8s.io
+  resources:
+  - subjectaccessreviews
+  verbs:
+  - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "spin-operator.fullname" . }}-metrics-auth-rolebinding
+  labels:
+  {{- include "spin-operator.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: '{{ include "spin-operator.fullname" . }}-metrics-auth-role'
+subjects:
+- kind: ServiceAccount
+  name: '{{ include "spin-operator.fullname" . }}-controller-manager'
+  namespace: '{{ .Release.Namespace }}'

charts/spin-operator/templates/metrics-certs.yaml

@@ -0,0 +1,16 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ include "spin-operator.fullname" . }}-metrics-certs
+  labels:
+  {{- include "spin-operator.labels" . | nindent 4 }}
+spec:
+  dnsNames:
+  - '{{ include "{{ .Release.Namespace }}.fullname" . }}-metrics-service.{{ .Release.Namespace
+    }}.svc'
+  - '{{ include "{{ .Release.Namespace }}.fullname" . }}-metrics-service.{{ .Release.Namespace
+    }}.svc.{{ .Values.kubernetesClusterDomain }}'
+  issuerRef:
+    kind: Issuer
+    name: '{{ include "spin-operator.fullname" . }}-selfsigned-issuer'
+  secretName: metrics-server-cert

charts/spin-operator/templates/metrics-reader-rbac.yaml

@@ -3,9 +3,6 @@ kind: ClusterRole
 metadata:
   name: {{ include "spin-operator.fullname" . }}-metrics-reader
   labels:
-    app.kubernetes.io/component: kube-rbac-proxy
-    app.kubernetes.io/created-by: spin-operator
-    app.kubernetes.io/part-of: spin-operator
   {{- include "spin-operator.labels" . | nindent 4 }}
 rules:
 - nonResourceURLs:

charts/spin-operator/templates/proxy-rbac.yaml → charts/spin-operator/templates/metrics-role-and-binding.yaml

@@ -1,9 +1,9 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: {{ include "spin-operator.fullname" . }}-proxy-role
+  name: {{ include "spin-operator.fullname" . }}-metrics-role
   labels:
-    app.kubernetes.io/component: kube-rbac-proxy
+    app.kubernetes.io/component: metrics-server
     app.kubernetes.io/created-by: spin-operator
     app.kubernetes.io/part-of: spin-operator
   {{- include "spin-operator.labels" . | nindent 4 }}
@@ -24,16 +24,16 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: {{ include "spin-operator.fullname" . }}-proxy-rolebinding
+  name: {{ include "spin-operator.fullname" . }}-metrics-rolebinding
   labels:
-    app.kubernetes.io/component: kube-rbac-proxy
+    app.kubernetes.io/component: metrics-server
     app.kubernetes.io/created-by: spin-operator
     app.kubernetes.io/part-of: spin-operator
   {{- include "spin-operator.labels" . | nindent 4 }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: '{{ include "spin-operator.fullname" . }}-proxy-role'
+  name: '{{ include "spin-operator.fullname" . }}-metrics-role'
 subjects:
 - kind: ServiceAccount
   name: '{{ include "spin-operator.fullname" . }}-controller-manager'

charts/spin-operator/templates/metrics-service.yaml

@@ -1,11 +1,8 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ include "spin-operator.fullname" . }}-controller-manager-metrics-service
+  name: {{ include "spin-operator.fullname" . }}-metrics-service
   labels:
-    app.kubernetes.io/component: kube-rbac-proxy
-    app.kubernetes.io/created-by: spin-operator
-    app.kubernetes.io/part-of: spin-operator
     control-plane: controller-manager
   {{- include "spin-operator.labels" . | nindent 4 }}
 spec:

charts/spin-operator/values.yaml

@@ -8,7 +8,8 @@ controllerManager:
     ## In general, these should be left as-is.
     args:
     - --health-probe-bind-address=:8082
-    - --metrics-bind-address=127.0.0.1:8080
+    - --metrics-bind-address=:8443
+    - --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs
     - --leader-elect
     - --enable-webhooks
     ## containerSecurityContext defines privilege and access control for the
@@ -38,38 +39,6 @@ controllerManager:
         cpu: 10m
         memory: 64Mi
 
-  ## kubeRbacProxy handles RBAC authorization with the Kubernetes API server.
-  kubeRbacProxy:
-    ## args are the default arguments to supply to the RBAC proxy.
-    ## In general, these should be left as-is.
-    args:
-    - --secure-listen-address=0.0.0.0:8443
-    - --upstream=http://127.0.0.1:8080/
-    - --logtostderr=true
-    - --v=0
-    ## containerSecurityContext defines privilege and access control for the
-    ## container.
-    ## See https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
-    containerSecurityContext:
-      allowPrivilegeEscalation: false
-      capabilities:
-        drop:
-        - ALL
-    ## image indicates which repository and tag combination will be used for
-    ## pulling the RBAC proxy image.
-    image:
-      repository: gcr.io/kubebuilder/kube-rbac-proxy
-      tag: v0.15.0
-    ## resources represent default cpu/mem limits for the RBAC proxy container.
-    resources:
-      # TODO: update these per https://github.com/spinkube/spin-operator/issues/21
-      limits:
-        cpu: 500m
-        memory: 128Mi
-      requests:
-        cpu: 5m
-        memory: 64Mi
-
   # replicas represent how many pod replicas of the controllerManager to run.
   replicas: 1
 
@@ -83,13 +52,13 @@ kubernetesClusterDomain: cluster.local
 
 ## metricsService configuration.
 ## This configuration should only be updated in tandem with corresponding
-## controller and RBAC proxy configuration.
+## controller configuration.
 metricsService:
   ports:
   - name: https
     port: 8443
     protocol: TCP
-    targetPort: https
+    targetPort: 8443
   type: ClusterIP
 
 ## webhookService configuration.