Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ref(*): remove kube-rbac-proxy #363

Merged
merged 1 commit into from
Jan 10, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 0 additions & 15 deletions charts/spin-operator/templates/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -71,21 +71,6 @@ spec:
- mountPath: /tmp/k8s-webhook-server/serving-certs
name: cert
readOnly: true
- args: {{- toYaml .Values.controllerManager.kubeRbacProxy.args | nindent 8 }}
env:
- name: KUBERNETES_CLUSTER_DOMAIN
value: {{ quote .Values.kubernetesClusterDomain }}
image: {{ .Values.controllerManager.kubeRbacProxy.image.repository }}:{{ .Values.controllerManager.kubeRbacProxy.image.tag
| default .Chart.AppVersion }}
name: kube-rbac-proxy
ports:
- containerPort: 8443
name: https
protocol: TCP
resources: {{- toYaml .Values.controllerManager.kubeRbacProxy.resources | nindent
10 }}
securityContext: {{- toYaml .Values.controllerManager.kubeRbacProxy.containerSecurityContext
| nindent 10 }}
securityContext:
runAsNonRoot: true
serviceAccountName: {{ include "spin-operator.fullname" . }}-controller-manager
Expand Down
35 changes: 3 additions & 32 deletions charts/spin-operator/templates/manager-rbac.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,14 @@ rules:
- apiGroups:
- ""
resources:
- secrets
- events
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
- secrets
- services
verbs:
- create
Expand Down Expand Up @@ -47,17 +43,11 @@ rules:
- deployments/status
verbs:
- get
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- apiGroups:
- core.spinkube.dev
resources:
- spinappexecutors
- spinapps
verbs:
- create
- delete
Expand All @@ -76,25 +66,6 @@ rules:
- core.spinkube.dev
resources:
- spinappexecutors/status
verbs:
- get
- patch
- update
- apiGroups:
- core.spinkube.dev
resources:
- spinapps
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- core.spinkube.dev
resources:
- spinapps/status
verbs:
- get
Expand Down
34 changes: 34 additions & 0 deletions charts/spin-operator/templates/metrics-auth-rbac.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: {{ include "spin-operator.fullname" . }}-metrics-auth-role
labels:
{{- include "spin-operator.labels" . | nindent 4 }}
rules:
- apiGroups:
- authentication.k8s.io
resources:
- tokenreviews
verbs:
- create
- apiGroups:
- authorization.k8s.io
resources:
- subjectaccessreviews
verbs:
- create
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: {{ include "spin-operator.fullname" . }}-metrics-auth-rolebinding
labels:
{{- include "spin-operator.labels" . | nindent 4 }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: '{{ include "spin-operator.fullname" . }}-metrics-auth-role'
subjects:
- kind: ServiceAccount
name: '{{ include "spin-operator.fullname" . }}-controller-manager'
namespace: '{{ .Release.Namespace }}'
3 changes: 0 additions & 3 deletions charts/spin-operator/templates/metrics-reader-rbac.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,6 @@ kind: ClusterRole
metadata:
name: {{ include "spin-operator.fullname" . }}-metrics-reader
labels:
app.kubernetes.io/component: kube-rbac-proxy
app.kubernetes.io/created-by: spin-operator
app.kubernetes.io/part-of: spin-operator
{{- include "spin-operator.labels" . | nindent 4 }}
rules:
- nonResourceURLs:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: {{ include "spin-operator.fullname" . }}-proxy-role
name: {{ include "spin-operator.fullname" . }}-metrics-role
labels:
app.kubernetes.io/component: kube-rbac-proxy
app.kubernetes.io/component: metrics-server
app.kubernetes.io/created-by: spin-operator
app.kubernetes.io/part-of: spin-operator
{{- include "spin-operator.labels" . | nindent 4 }}
Expand All @@ -24,16 +24,16 @@ rules:
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: {{ include "spin-operator.fullname" . }}-proxy-rolebinding
name: {{ include "spin-operator.fullname" . }}-metrics-rolebinding
labels:
app.kubernetes.io/component: kube-rbac-proxy
app.kubernetes.io/component: metrics-server
app.kubernetes.io/created-by: spin-operator
app.kubernetes.io/part-of: spin-operator
{{- include "spin-operator.labels" . | nindent 4 }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: '{{ include "spin-operator.fullname" . }}-proxy-role'
name: '{{ include "spin-operator.fullname" . }}-metrics-role'
subjects:
- kind: ServiceAccount
name: '{{ include "spin-operator.fullname" . }}-controller-manager'
Expand Down
5 changes: 1 addition & 4 deletions charts/spin-operator/templates/metrics-service.yaml
Original file line number Diff line number Diff line change
@@ -1,11 +1,8 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "spin-operator.fullname" . }}-controller-manager-metrics-service
name: {{ include "spin-operator.fullname" . }}-metrics-service
labels:
app.kubernetes.io/component: kube-rbac-proxy
app.kubernetes.io/created-by: spin-operator
app.kubernetes.io/part-of: spin-operator
control-plane: controller-manager
{{- include "spin-operator.labels" . | nindent 4 }}
spec:
Expand Down
38 changes: 3 additions & 35 deletions charts/spin-operator/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ controllerManager:
## In general, these should be left as-is.
args:
- --health-probe-bind-address=:8082
- --metrics-bind-address=127.0.0.1:8080
- --metrics-bind-address=:8443
- --leader-elect
- --enable-webhooks
## containerSecurityContext defines privilege and access control for the
Expand Down Expand Up @@ -38,38 +38,6 @@ controllerManager:
cpu: 10m
memory: 64Mi

## kubeRbacProxy handles RBAC authorization with the Kubernetes API server.
kubeRbacProxy:
## args are the default arguments to supply to the RBAC proxy.
## In general, these should be left as-is.
args:
- --secure-listen-address=0.0.0.0:8443
- --upstream=http://127.0.0.1:8080/
- --logtostderr=true
- --v=0
## containerSecurityContext defines privilege and access control for the
## container.
## See https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
containerSecurityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
## image indicates which repository and tag combination will be used for
## pulling the RBAC proxy image.
image:
repository: gcr.io/kubebuilder/kube-rbac-proxy
tag: v0.15.0
## resources represent default cpu/mem limits for the RBAC proxy container.
resources:
# TODO: update these per https://github.com/spinkube/spin-operator/issues/21
limits:
cpu: 500m
memory: 128Mi
requests:
cpu: 5m
memory: 64Mi

# replicas represent how many pod replicas of the controllerManager to run.
replicas: 1

Expand All @@ -83,13 +51,13 @@ kubernetesClusterDomain: cluster.local

## metricsService configuration.
## This configuration should only be updated in tandem with corresponding
## controller and RBAC proxy configuration.
## controller configuration.
metricsService:
ports:
- name: https
port: 8443
protocol: TCP
targetPort: https
targetPort: 8443
type: ClusterIP

## webhookService configuration.
Expand Down
48 changes: 47 additions & 1 deletion cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package main

import (
"crypto/tls"
"flag"
"os"

Expand All @@ -32,6 +33,7 @@ import (
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"

spinv1alpha1 "github.com/spinkube/spin-operator/api/v1alpha1"
Expand All @@ -58,12 +60,19 @@ func main() {
var enableLeaderElection bool
var probeAddr string
var enableWebhooks bool
var secureMetrics bool
var enableHTTP2 bool
var tlsOpts []func(*tls.Config)
flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
flag.StringVar(&probeAddr, "health-probe-bind-address", ":8082", "The address the probe endpoint binds to.")
flag.BoolVar(&enableLeaderElection, "leader-elect", false,
"Enable leader election for controller manager. "+
"Enabling this will ensure there is only one active controller manager.")
flag.BoolVar(&enableWebhooks, "enable-webhooks", false, "Enable admission webhooks")
flag.BoolVar(&secureMetrics, "metrics-secure", true,
"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
flag.BoolVar(&enableHTTP2, "enable-http2", false,
"If set, HTTP/2 will be enabled for the metrics server")
opts := zap.Options{
Development: true,
}
Expand All @@ -78,9 +87,46 @@ func main() {
"enableLeaderElection", enableLeaderElection,
"enableWebhooks", enableWebhooks)

// if the enable-http2 flag is false (the default), http/2 should be disabled
// due to its vulnerabilities. More specifically, disabling http/2 will
// prevent from being vulnerable to the HTTP/2 Stream Cancellation and
// Rapid Reset CVEs. For more information see:
// - https://github.com/advisories/GHSA-qppj-fm5r-hxr3
// - https://github.com/advisories/GHSA-4374-p667-p6c8
disableHTTP2 := func(c *tls.Config) {
setupLog.Info("disabling http/2")
c.NextProtos = []string{"http/1.1"}
}

if !enableHTTP2 {
tlsOpts = append(tlsOpts, disableHTTP2)
}

// Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server.
// More info:
// - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/server
// - https://book.kubebuilder.io/reference/metrics.html
metricsServerOptions := metricsserver.Options{
BindAddress: metricsAddr,
SecureServing: secureMetrics,
TLSOpts: tlsOpts,
}

if secureMetrics {
// FilterProvider is used to protect the metrics endpoint with authn/authz.
// These configurations ensure that only authorized users and service accounts
// can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info:
// https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/filters#WithAuthenticationAndAuthorization
metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization

// TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was lifted from the upstream example project code, but I also see instructions in kubernetes-sigs/kubebuilder#3907 to have cert-manager cover certs for the metrics-server, presumably similar to the webhook-server. Shall we do this?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah we probably should

Copy link
Contributor Author

@vdice vdice Jan 9, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, created a follow-up: #364

// generate self-signed certificates for the metrics server. While convenient for development and testing,
// this setup is not recommended for production.
}

mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
Scheme: scheme,
Metrics: metricsserver.Options{BindAddress: metricsAddr},
Metrics: metricsServerOptions,
HealthProbeBindAddress: probeAddr,
LeaderElection: enableLeaderElection,
LeaderElectionID: "90ba2d18.spinkube.dev",
Expand Down
38 changes: 3 additions & 35 deletions config/chart/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ controllerManager:
## In general, these should be left as-is.
args:
- --health-probe-bind-address=:8082
- --metrics-bind-address=127.0.0.1:8080
- --metrics-bind-address=:8443
- --leader-elect
- --enable-webhooks
## containerSecurityContext defines privilege and access control for the
Expand Down Expand Up @@ -38,38 +38,6 @@ controllerManager:
cpu: 10m
memory: 64Mi

## kubeRbacProxy handles RBAC authorization with the Kubernetes API server.
kubeRbacProxy:
## args are the default arguments to supply to the RBAC proxy.
## In general, these should be left as-is.
args:
- --secure-listen-address=0.0.0.0:8443
- --upstream=http://127.0.0.1:8080/
- --logtostderr=true
- --v=0
## containerSecurityContext defines privilege and access control for the
## container.
## See https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
containerSecurityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
## image indicates which repository and tag combination will be used for
## pulling the RBAC proxy image.
image:
repository: gcr.io/kubebuilder/kube-rbac-proxy
tag: v0.15.0
## resources represent default cpu/mem limits for the RBAC proxy container.
resources:
# TODO: update these per https://github.com/spinkube/spin-operator/issues/21
limits:
cpu: 500m
memory: 128Mi
requests:
cpu: 5m
memory: 64Mi

# replicas represent how many pod replicas of the controllerManager to run.
replicas: 1

Expand All @@ -83,13 +51,13 @@ kubernetesClusterDomain: cluster.local

## metricsService configuration.
## This configuration should only be updated in tandem with corresponding
## controller and RBAC proxy configuration.
## controller configuration.
metricsService:
ports:
- name: https
port: 8443
protocol: TCP
targetPort: https
targetPort: 8443
type: ClusterIP

## webhookService configuration.
Expand Down
Loading
Loading