disable traefik and svclb and fix tls certs for kubelet

Signed-off-by: galal-hussein <[email protected]>

k3k-kubelet/kubelet.go

@@ -5,6 +5,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
+	"net"
 	"net/http"
 	"time"
 
@@ -157,7 +158,7 @@ func clusterIP(ctx context.Context, serviceName, clusterNamespace string, hostCl
 
 func (k *kubelet) registerNode(ctx context.Context, agentIP, srvPort, namespace, name, hostname, serverIP, dnsIP string) error {
 	providerFunc := k.newProviderFunc(namespace, name, hostname, agentIP, serverIP, dnsIP)
-	nodeOpts := k.nodeOpts(ctx, srvPort, namespace, name, hostname)
+	nodeOpts := k.nodeOpts(ctx, srvPort, namespace, name, hostname, agentIP)
 
 	var err error
 	k.node, err = nodeutil.NewNode(k.name, providerFunc, nodeutil.WithClient(k.virtClient), nodeOpts)
@@ -216,7 +217,7 @@ func (k *kubelet) newProviderFunc(namespace, name, hostname, agentIP, serverIP,
 	}
 }
 
-func (k *kubelet) nodeOpts(ctx context.Context, srvPort, namespace, name, hostname string) nodeutil.NodeOpt {
+func (k *kubelet) nodeOpts(ctx context.Context, srvPort, namespace, name, hostname, agentIP string) nodeutil.NodeOpt {
 	return func(c *nodeutil.NodeConfig) error {
 		c.HTTPListenAddr = fmt.Sprintf(":%s", srvPort)
 		// set up the routes
@@ -226,7 +227,7 @@ func (k *kubelet) nodeOpts(ctx context.Context, srvPort, namespace, name, hostna
 		}
 		c.Handler = mux
 
-		tlsConfig, err := loadTLSConfig(ctx, k.hostClient, name, namespace, k.name, hostname, k.token)
+		tlsConfig, err := loadTLSConfig(ctx, k.hostClient, name, namespace, k.name, hostname, k.token, agentIP)
 		if err != nil {
 			return fmt.Errorf("unable to get tls config: %w", err)
 		}
@@ -296,7 +297,7 @@ func kubeconfigBytes(url string, serverCA, clientCert, clientKey []byte) ([]byte
 	return clientcmd.Write(*config)
 }
 
-func loadTLSConfig(ctx context.Context, hostClient ctrlruntimeclient.Client, clusterName, clusterNamespace, nodeName, hostname, token string) (*tls.Config, error) {
+func loadTLSConfig(ctx context.Context, hostClient ctrlruntimeclient.Client, clusterName, clusterNamespace, nodeName, hostname, token, agentIP string) (*tls.Config, error) {
 	var (
 		cluster v1alpha1.Cluster
 		b       *bootstrap.ControlRuntimeBootstrap
@@ -314,8 +315,10 @@ func loadTLSConfig(ctx context.Context, hostClient ctrlruntimeclient.Client, clu
 	}); err != nil {
 		return nil, fmt.Errorf("unable to decode bootstrap: %w", err)
 	}
+	ip := net.ParseIP(agentIP)
 	altNames := certutil.AltNames{
 		DNSNames: []string{hostname},
+		IPs:      []net.IP{ip},
 	}
 	cert, key, err := kubeconfig.CreateClientCertKey(nodeName, nil, &altNames, []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, 0, b.ServerCA.Content, b.ServerCAKey.Content)
 	if err != nil {

pkg/controller/cluster/cluster.go

@@ -121,7 +121,7 @@ func (c *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1
 		return err
 	}
 
-	s := server.New(cluster, c.Client, token)
+	s := server.New(cluster, c.Client, token, string(cluster.Spec.Mode))
 
 	if cluster.Spec.Persistence != nil {
 		cluster.Status.Persistence = cluster.Spec.Persistence

pkg/controller/cluster/server/config.go

@@ -68,7 +68,7 @@ func serverOptions(cluster *v1alpha1.Cluster, token string) string {
 		}
 	}
 	if cluster.Spec.Mode != agent.VirtualNodeMode {
-		opts = opts + "disable-agent: true\negress-selector-mode: disabled\n"
+		opts = opts + "disable-agent: true\negress-selector-mode: disabled\ndisable:\n- servicelb\n- traefik"
 	}
 	// TODO: Add extra args to the options
 

pkg/controller/cluster/server/server.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
 	"github.com/rancher/k3k/pkg/controller"
+	"github.com/rancher/k3k/pkg/controller/cluster/agent"
 	apps "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -31,14 +32,16 @@ const (
 type Server struct {
 	cluster *v1alpha1.Cluster
 	client  client.Client
+	mode    string
 	token   string
 }
 
-func New(cluster *v1alpha1.Cluster, client client.Client, token string) *Server {
+func New(cluster *v1alpha1.Cluster, client client.Client, token, mode string) *Server {
 	return &Server{
 		cluster: cluster,
 		client:  client,
 		token:   token,
+		mode:    mode,
 	}
 }
 
@@ -130,9 +133,6 @@ func (s *Server) podSpec(image, name string, persistent bool, affinitySelector *
 						},
 					},
 				},
-				SecurityContext: &v1.SecurityContext{
-					Privileged: ptr.To(true),
-				},
 				Command: []string{
 					"/bin/sh",
 					"-c",
@@ -219,6 +219,12 @@ func (s *Server) podSpec(image, name string, persistent bool, affinitySelector *
 		},
 	}
 
+	// start the pod unprivileged in shared mode
+	if s.mode == agent.VirtualNodeMode {
+		podSpec.Containers[0].SecurityContext = &v1.SecurityContext{
+			Privileged: ptr.To(true),
+		}
+	}
 	return podSpec
 }