fixed ClusterRole for node proxy

charts/k3k/templates/rbac.yaml

@@ -11,4 +11,26 @@ roleRef:
 subjects:
   - kind: ServiceAccount
     name: {{ include "k3k.serviceAccountName" . }}
-    namespace: {{ .Values.namespace }}
+    namespace: {{ .Values.namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "k3k.fullname" . }}-node-proxy
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - "nodes"
+  - "nodes/proxy"
+  verbs:
+  - "*"
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ include "k3k.fullname" . }}-node-proxy
+roleRef:
+  kind: ClusterRole
+  name: {{ include "k3k.fullname" . }}-node-proxy
+  apiGroup: rbac.authorization.k8s.io

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/go-logr/zapr v1.3.0
 	github.com/onsi/ginkgo/v2 v2.20.1
 	github.com/onsi/gomega v1.36.0
+	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_model v0.6.1
 	github.com/rancher/dynamiclistener v1.27.5
 	github.com/sirupsen/logrus v1.9.3
@@ -29,6 +30,7 @@ require (
 	k8s.io/apimachinery v0.29.11
 	k8s.io/apiserver v0.29.11
 	k8s.io/client-go v0.29.11
+	k8s.io/component-base v0.29.11
 	k8s.io/metrics v0.29.11
 	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
 	sigs.k8s.io/controller-runtime v0.17.5
@@ -80,7 +82,6 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_golang v1.19.1 // indirect
 	github.com/prometheus/common v0.55.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
@@ -119,7 +120,6 @@ require (
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.29.2 // indirect
-	k8s.io/component-base v0.29.11 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
 	k8s.io/kms v0.31.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect

pkg/controller/cluster/agent/shared.go

@@ -16,7 +16,7 @@ import (
 
 const (
 	sharedKubeletConfigPath = "/opt/rancher/k3k/config.yaml"
-	sharedNodeAgentName     = "kubelet"
+	SharedNodeAgentName     = "kubelet"
 	SharedNodeMode          = "shared"
 )
 
@@ -69,8 +69,6 @@ func (s *SharedAgent) Resources() []ctrlruntimeclient.Object {
 		s.serviceAccount(),
 		s.role(),
 		s.roleBinding(),
-		s.clusterRole(),
-		s.clusterRoleBinding(),
 		s.service(),
 		s.deployment(),
 		s.dnsService(),
@@ -293,55 +291,8 @@ func (s *SharedAgent) roleBinding() *rbacv1.RoleBinding {
 	}
 }
 
-func (s *SharedAgent) clusterRole() *rbacv1.ClusterRole {
-	return &rbacv1.ClusterRole{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "ClusterRole",
-			APIVersion: "rbac.authorization.k8s.io/v1",
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      s.Name(),
-			Namespace: s.cluster.Namespace,
-		},
-		Rules: []rbacv1.PolicyRule{
-			{
-				APIGroups: []string{""},
-				// nodes and nodes/proxy are needed to gather the stats and metrics from the host from
-				// '/api/v1/nodes/<node>/proxy/stats/summary' and '/api/v1/nodes/<node>/proxy/metrics/resource' endpoints
-				// in the GetStatsSummary and GetMetricsResource provider implementation of the Virtual Kubelet
-				Resources: []string{"nodes", "nodes/proxy"},
-				Verbs:     []string{"*"},
-			},
-		},
-	}
-}
-
-func (s *SharedAgent) clusterRoleBinding() *rbacv1.ClusterRoleBinding {
-	return &rbacv1.ClusterRoleBinding{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "ClusterRoleBinding",
-			APIVersion: "rbac.authorization.k8s.io/v1",
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name: s.Name(),
-		},
-		RoleRef: rbacv1.RoleRef{
-			APIGroup: "rbac.authorization.k8s.io",
-			Kind:     "ClusterRole",
-			Name:     s.Name(),
-		},
-		Subjects: []rbacv1.Subject{
-			{
-				Kind:      "ServiceAccount",
-				Name:      s.Name(),
-				Namespace: s.cluster.Namespace,
-			},
-		},
-	}
-}
-
 func (s *SharedAgent) Name() string {
-	return controller.SafeConcatNameWithPrefix(s.cluster.Name, sharedNodeAgentName)
+	return controller.SafeConcatNameWithPrefix(s.cluster.Name, SharedNodeAgentName)
 }
 
 func (s *SharedAgent) DNSName() string {

pkg/controller/cluster/cluster.go

@@ -8,19 +8,21 @@ import (
 	"time"
 
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	"github.com/rancher/k3k/pkg/controller"
 	"github.com/rancher/k3k/pkg/controller/cluster/agent"
 	"github.com/rancher/k3k/pkg/controller/cluster/server"
 	"github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap"
 	"github.com/rancher/k3k/pkg/log"
 	"go.uber.org/zap"
 	v1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/controller"
+	ctrlruntimecontroller "sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -59,7 +61,7 @@ func Add(ctx context.Context, mgr manager.Manager, sharedAgentImage string, logg
 	}
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&v1alpha1.Cluster{}).
-		WithOptions(controller.Options{
+		WithOptions(ctrlruntimecontroller.Options{
 			MaxConcurrentReconciles: maxConcurrentReconciles,
 		}).
 		Complete(&reconciler)
@@ -100,6 +102,11 @@ func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request
 			}
 		}
 	}
+
+	if err := c.unbindNodeProxyClusterRole(ctx, &cluster); err != nil {
+		return reconcile.Result{}, err
+	}
+
 	if controllerutil.ContainsFinalizer(&cluster, clusterFinalizerName) {
 		// remove finalizer from the cluster and update it.
 		controllerutil.RemoveFinalizer(&cluster, clusterFinalizerName)
@@ -130,9 +137,6 @@ func (c *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1
 			cluster.Status.Persistence.StorageRequestSize = defaultStoragePersistentSize
 		}
 	}
-	if err := c.Client.Update(ctx, cluster); err != nil {
-		return err
-	}
 
 	cluster.Status.ClusterCIDR = cluster.Spec.ClusterCIDR
 	if cluster.Status.ClusterCIDR == "" {
@@ -189,6 +193,10 @@ func (c *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1
 		}
 	}
 
+	if err := c.bindNodeProxyClusterRole(ctx, cluster); err != nil {
+		return err
+	}
+
 	return c.Client.Update(ctx, cluster)
 }
 
@@ -279,6 +287,56 @@ func (c *ClusterReconciler) server(ctx context.Context, cluster *v1alpha1.Cluste
 	return nil
 }
 
+func (c *ClusterReconciler) bindNodeProxyClusterRole(ctx context.Context, cluster *v1alpha1.Cluster) error {
+	clusterRoleBinding := &rbacv1.ClusterRoleBinding{}
+	if err := c.Client.Get(ctx, types.NamespacedName{Name: "k3k-node-proxy"}, clusterRoleBinding); err != nil {
+		return fmt.Errorf("failed to get or find k3k-node-proxy ClusterRoleBinding: %w", err)
+	}
+
+	subjectName := controller.SafeConcatNameWithPrefix(cluster.Name, agent.SharedNodeAgentName)
+
+	found := false
+	for _, subject := range clusterRoleBinding.Subjects {
+		if subject.Name == subjectName && subject.Namespace == cluster.Namespace {
+			found = true
+		}
+	}
+
+	if !found {
+		clusterRoleBinding.Subjects = append(clusterRoleBinding.Subjects, rbacv1.Subject{
+			Kind:      "ServiceAccount",
+			Name:      subjectName,
+			Namespace: cluster.Namespace,
+		})
+	}
+
+	return c.Client.Update(ctx, clusterRoleBinding)
+}
+
+func (c *ClusterReconciler) unbindNodeProxyClusterRole(ctx context.Context, cluster *v1alpha1.Cluster) error {
+	clusterRoleBinding := &rbacv1.ClusterRoleBinding{}
+	if err := c.Client.Get(ctx, types.NamespacedName{Name: "k3k-node-proxy"}, clusterRoleBinding); err != nil {
+		return fmt.Errorf("failed to get or find k3k-node-proxy ClusterRoleBinding: %w", err)
+	}
+
+	subjectName := controller.SafeConcatNameWithPrefix(cluster.Name, agent.SharedNodeAgentName)
+
+	var cleanedSubjects []rbacv1.Subject
+	for _, subject := range clusterRoleBinding.Subjects {
+		if subject.Name != subjectName || subject.Namespace != cluster.Namespace {
+			cleanedSubjects = append(cleanedSubjects, subject)
+		}
+	}
+
+	// if no subject was removed, all good
+	if reflect.DeepEqual(clusterRoleBinding.Subjects, cleanedSubjects) {
+		return nil
+	}
+
+	clusterRoleBinding.Subjects = cleanedSubjects
+	return c.Client.Update(ctx, clusterRoleBinding)
+}
+
 func (c *ClusterReconciler) agent(ctx context.Context, cluster *v1alpha1.Cluster, serviceIP, token string) error {
 	agent := agent.New(cluster, serviceIP, c.SharedAgentImage, token)
 	agentsConfig, err := agent.Config()