virtual-kubelet controller integration

.github/workflows/release.yml

@@ -40,14 +40,23 @@ jobs:
         username: ${{ env.DOCKER_USERNAME }}
         password: ${{ env.DOCKER_PASSWORD }}
 
-    - name: Build container image
+    - name: Build controller image
       uses: docker/build-push-action@v5
       with:
         context: .
         push: true
         tags: rancher/k3k:${{ github.ref_name }}
         file: package/Dockerfile
         platforms: linux/amd64
+
+    - name: Build Virtual Kubelet image
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        push: true
+        tags: rancher/k3k:k3k-kubelet-dev
+        file: package/Dockerfile.kubelet
+        platforms: linux/amd64
 
 
 

charts/k3k/crds/k3k.io_clusters.yaml

@@ -135,6 +135,15 @@ spec:
                 description: NodeSelector is the node selector that will be applied
                   to all server/agent pods
                 type: object
+              mode:
+                description: Mode is the cluster provisioning mode which can be either
+                  "virtual" or "shared". Defaults to "shared"
+                type: string
+                x-kubernetes-validations:
+                - message: mode is immutable
+                  rule: self == oldSelf
+                - message: invalid value for mode
+                  rule: self == "virtual" || self == "shared"
               persistence:
                 description: |-
                   Persistence contains options controlling how the etcd data of the virtual cluster is persisted. By default, no data
@@ -191,6 +200,7 @@ spec:
                 type: string
             required:
             - agents
+            - mode
             - servers
             - token
             - version

charts/k3k/templates/deployment.yaml

@@ -19,9 +19,11 @@ spec:
         - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           name: {{ .Chart.Name }}
-          environment:
+          env:
           - name: CLUSTER_CIDR
             value: {{ .Values.host.clusterCIDR }}
+          - name: SHARED_AGENT_IMAGE
+            value: "{{ .Values.sharedAgent.image.repository }}:{{ .Values.sharedAgent.image.tag }}"
           ports:
           - containerPort: 8080
             name: https

charts/k3k/values.yaml

@@ -22,3 +22,9 @@ serviceAccount:
   # The name of the service account to use.
   # If not set and create is true, a name is generated using the fullname template
   name: ""
+
+# configuration related to the shared agent mode in k3k
+sharedAgent:
+  image:
+    repository: "rancher/k3k"
+    tag: "k3k-kubelet-dev"

cli/cmds/cluster/create.go

@@ -7,36 +7,26 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/rancher/k3k/cli/cmds"
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	"github.com/rancher/k3k/pkg/controller"
 	"github.com/rancher/k3k/pkg/controller/cluster"
 	"github.com/rancher/k3k/pkg/controller/cluster/server"
 	"github.com/rancher/k3k/pkg/controller/kubeconfig"
-	"github.com/rancher/k3k/pkg/controller/util"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/authentication/user"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/util/retry"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-var (
-	Scheme  = runtime.NewScheme()
-	backoff = wait.Backoff{
-		Steps:    5,
-		Duration: 20 * time.Second,
-		Factor:   2,
-		Jitter:   0.1,
-	}
-)
+var Scheme = runtime.NewScheme()
 
 func init() {
 	_ = clientgoscheme.AddToScheme(Scheme)
@@ -120,7 +110,7 @@ var (
 
 func create(clx *cli.Context) error {
 	ctx := context.Background()
-	if err := validateCreateFlags(clx); err != nil {
+	if err := validateCreateFlags(); err != nil {
 		return err
 	}
 
@@ -173,13 +163,13 @@ func create(clx *cli.Context) error {
 
 	logrus.Infof("Extracting Kubeconfig for [%s] cluster", name)
 	cfg := &kubeconfig.KubeConfig{
-		CN:         util.AdminCommonName,
+		CN:         controller.AdminCommonName,
 		ORG:        []string{user.SystemPrivilegedGroup},
 		ExpiryDate: 0,
 	}
 	logrus.Infof("waiting for cluster to be available..")
 	var kubeconfig []byte
-	if err := retry.OnError(backoff, apierrors.IsNotFound, func() error {
+	if err := retry.OnError(controller.Backoff, apierrors.IsNotFound, func() error {
 		kubeconfig, err = cfg.Extract(ctx, ctrlClient, cluster, host[0])
 		if err != nil {
 			return err
@@ -203,7 +193,7 @@ func create(clx *cli.Context) error {
 	return os.WriteFile(cluster.Name+"-kubeconfig.yaml", kubeconfig, 0644)
 }
 
-func validateCreateFlags(clx *cli.Context) error {
+func validateCreateFlags() error {
 	if persistenceType != server.EphermalNodesType &&
 		persistenceType != server.DynamicNodesType {
 		return errors.New("invalid persistence type")

cli/cmds/kubeconfig/kubeconfig.go

@@ -10,14 +10,13 @@ import (
 
 	"github.com/rancher/k3k/cli/cmds"
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	"github.com/rancher/k3k/pkg/controller"
 	"github.com/rancher/k3k/pkg/controller/kubeconfig"
-	"github.com/rancher/k3k/pkg/controller/util"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/authentication/user"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/clientcmd"
@@ -31,19 +30,13 @@ func init() {
 }
 
 var (
-	Scheme         = runtime.NewScheme()
-	name           string
-	cn             string
-	org            cli.StringSlice
-	altNames       cli.StringSlice
-	expirationDays int64
-	configName     string
-	backoff        = wait.Backoff{
-		Steps:    5,
-		Duration: 20 * time.Second,
-		Factor:   2,
-		Jitter:   0.1,
-	}
+	Scheme                  = runtime.NewScheme()
+	name                    string
+	cn                      string
+	org                     cli.StringSlice
+	altNames                cli.StringSlice
+	expirationDays          int64
+	configName              string
 	generateKubeconfigFlags = []cli.Flag{
 		cli.StringFlag{
 			Name:        "name",
@@ -59,7 +52,7 @@ var (
 			Name:        "cn",
 			Usage:       "Common name (CN) of the generated certificates for the kubeconfig",
 			Destination: &cn,
-			Value:       util.AdminCommonName,
+			Value:       controller.AdminCommonName,
 		},
 		cli.StringSliceFlag{
 			Name:  "org",
@@ -141,7 +134,7 @@ func generate(clx *cli.Context) error {
 	}
 	logrus.Infof("waiting for cluster to be available..")
 	var kubeconfig []byte
-	if err := retry.OnError(backoff, apierrors.IsNotFound, func() error {
+	if err := retry.OnError(controller.Backoff, apierrors.IsNotFound, func() error {
 		kubeconfig, err = cfg.Extract(ctx, ctrlClient, &cluster, host[0])
 		if err != nil {
 			return err

go.mod

@@ -25,6 +25,7 @@ require (
 	go.etcd.io/etcd/api/v3 v3.5.14
 	go.etcd.io/etcd/client/v3 v3.5.14
 	go.uber.org/zap v1.26.0
+	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.31.1
 	k8s.io/apimachinery v0.31.1
 	k8s.io/apiserver v0.31.0
@@ -119,7 +120,6 @@ require (
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.29.2 // indirect
 	k8s.io/component-base v0.29.2 // indirect

k3k-kubelet/config.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	"errors"
+	"os"
+
+	"gopkg.in/yaml.v2"
+)
+
+// config has all virtual-kubelet startup options
+type config struct {
+	ClusterName       string `yaml:"clusterName,omitempty"`
+	ClusterNamespace  string `yaml:"clusterNamespace,omitempty"`
+	NodeName          string `yaml:"nodeName,omitempty"`
+	Token             string `yaml:"token,omitempty"`
+	AgentHostname     string `yaml:"agentHostname,omitempty"`
+	HostConfigPath    string `yaml:"hostConfigPath,omitempty"`
+	VirtualConfigPath string `yaml:"virtualConfigPath,omitempty"`
+	KubeletPort       string `yaml:"kubeletPort,omitempty"`
+}
+
+func (c *config) unmarshalYAML(data []byte) error {
+	var conf config
+
+	if err := yaml.Unmarshal(data, &conf); err != nil {
+		return err
+	}
+
+	if c.ClusterName == "" {
+		c.ClusterName = conf.ClusterName
+	}
+	if c.ClusterNamespace == "" {
+		c.ClusterNamespace = conf.ClusterNamespace
+	}
+	if c.HostConfigPath == "" {
+		c.HostConfigPath = conf.HostConfigPath
+	}
+	if c.VirtualConfigPath == "" {
+		c.VirtualConfigPath = conf.VirtualConfigPath
+	}
+	if c.KubeletPort == "" {
+		c.KubeletPort = conf.KubeletPort
+	}
+	if c.AgentHostname == "" {
+		c.AgentHostname = conf.AgentHostname
+	}
+	if c.NodeName == "" {
+		c.NodeName = conf.NodeName
+	}
+	return nil
+}
+
+func (c *config) validate() error {
+	if c.ClusterName == "" {
+		return errors.New("cluster name is not provided")
+	}
+	if c.ClusterNamespace == "" {
+		return errors.New("cluster namespace is not provided")
+	}
+	if c.AgentHostname == "" {
+		return errors.New("agent Hostname is not provided")
+	}
+	return nil
+}
+
+func (c *config) parse(path string) error {
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return nil
+	}
+
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return err
+	}
+	return c.unmarshalYAML(b)
+}