Skip to content

Commit

Permalink
virtual-kubelet controller integration (rancher#130)
Browse files Browse the repository at this point in the history
* Virtual kubelet controller integration

Signed-off-by: galal-hussein <[email protected]>

* Add k3k-kubelet image to the release workflow

Signed-off-by: galal-hussein <[email protected]>

* Add k3k-kubelet image to the release workflow

Signed-off-by: galal-hussein <[email protected]>

* Fix build/release workflow

Signed-off-by: galal-hussein <[email protected]>

* Remove pkg directory in k3k-kubelet

Signed-off-by: galal-hussein <[email protected]>

* rename Type to Config

Signed-off-by: galal-hussein <[email protected]>

* Move the kubelet and config outside of pkg

Signed-off-by: galal-hussein <[email protected]>

* fix comments

Signed-off-by: galal-hussein <[email protected]>

* Fix naming throughout the package

Signed-off-by: galal-hussein <[email protected]>

* Fix comments

Signed-off-by: galal-hussein <[email protected]>

* more fixes to naming

Signed-off-by: galal-hussein <[email protected]>

* fixes

Signed-off-by: galal-hussein <[email protected]>

* fixes

Signed-off-by: galal-hussein <[email protected]>

* fixes

Signed-off-by: galal-hussein <[email protected]>

* fixes

Signed-off-by: galal-hussein <[email protected]>

---------

Signed-off-by: galal-hussein <[email protected]>
  • Loading branch information
galal-hussein authored and briandowns committed Dec 3, 2024
1 parent 593455e commit 823ed32
Show file tree
Hide file tree
Showing 36 changed files with 1,252 additions and 837 deletions.
11 changes: 10 additions & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,14 +40,23 @@ jobs:
username: ${{ env.DOCKER_USERNAME }}
password: ${{ env.DOCKER_PASSWORD }}

- name: Build container image
- name: Build controller image
uses: docker/build-push-action@v5
with:
context: .
push: true
tags: rancher/k3k:${{ github.ref_name }}
file: package/Dockerfile
platforms: linux/amd64

- name: Build Virtual Kubelet image
uses: docker/build-push-action@v5
with:
context: .
push: true
tags: rancher/k3k:k3k-kubelet-dev
file: package/Dockerfile.kubelet
platforms: linux/amd64



10 changes: 10 additions & 0 deletions charts/k3k/crds/k3k.io_clusters.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,15 @@ spec:
description: NodeSelector is the node selector that will be applied
to all server/agent pods
type: object
mode:
description: Mode is the cluster provisioning mode which can be either
"virtual" or "shared". Defaults to "shared"
type: string
x-kubernetes-validations:
- message: mode is immutable
rule: self == oldSelf
- message: invalid value for mode
rule: self == "virtual" || self == "shared"
persistence:
description: |-
Persistence contains options controlling how the etcd data of the virtual cluster is persisted. By default, no data
Expand Down Expand Up @@ -191,6 +200,7 @@ spec:
type: string
required:
- agents
- mode
- servers
- token
- version
Expand Down
4 changes: 3 additions & 1 deletion charts/k3k/templates/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,11 @@ spec:
- image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
name: {{ .Chart.Name }}
environment:
env:
- name: CLUSTER_CIDR
value: {{ .Values.host.clusterCIDR }}
- name: SHARED_AGENT_IMAGE
value: "{{ .Values.sharedAgent.image.repository }}:{{ .Values.sharedAgent.image.tag }}"
ports:
- containerPort: 8080
name: https
Expand Down
6 changes: 6 additions & 0 deletions charts/k3k/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,3 +22,9 @@ serviceAccount:
# The name of the service account to use.
# If not set and create is true, a name is generated using the fullname template
name: ""

# configuration related to the shared agent mode in k3k
sharedAgent:
image:
repository: "rancher/k3k"
tag: "k3k-kubelet-dev"
22 changes: 6 additions & 16 deletions cli/cmds/cluster/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,36 +7,26 @@ import (
"os"
"path/filepath"
"strings"
"time"

"github.com/rancher/k3k/cli/cmds"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
"github.com/rancher/k3k/pkg/controller"
"github.com/rancher/k3k/pkg/controller/cluster"
"github.com/rancher/k3k/pkg/controller/cluster/server"
"github.com/rancher/k3k/pkg/controller/kubeconfig"
"github.com/rancher/k3k/pkg/controller/util"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/authentication/user"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/util/retry"
"sigs.k8s.io/controller-runtime/pkg/client"
)

var (
Scheme = runtime.NewScheme()
backoff = wait.Backoff{
Steps: 5,
Duration: 20 * time.Second,
Factor: 2,
Jitter: 0.1,
}
)
var Scheme = runtime.NewScheme()

func init() {
_ = clientgoscheme.AddToScheme(Scheme)
Expand Down Expand Up @@ -120,7 +110,7 @@ var (

func create(clx *cli.Context) error {
ctx := context.Background()
if err := validateCreateFlags(clx); err != nil {
if err := validateCreateFlags(); err != nil {
return err
}

Expand Down Expand Up @@ -173,13 +163,13 @@ func create(clx *cli.Context) error {

logrus.Infof("Extracting Kubeconfig for [%s] cluster", name)
cfg := &kubeconfig.KubeConfig{
CN: util.AdminCommonName,
CN: controller.AdminCommonName,
ORG: []string{user.SystemPrivilegedGroup},
ExpiryDate: 0,
}
logrus.Infof("waiting for cluster to be available..")
var kubeconfig []byte
if err := retry.OnError(backoff, apierrors.IsNotFound, func() error {
if err := retry.OnError(controller.Backoff, apierrors.IsNotFound, func() error {
kubeconfig, err = cfg.Extract(ctx, ctrlClient, cluster, host[0])
if err != nil {
return err
Expand All @@ -203,7 +193,7 @@ func create(clx *cli.Context) error {
return os.WriteFile(cluster.Name+"-kubeconfig.yaml", kubeconfig, 0644)
}

func validateCreateFlags(clx *cli.Context) error {
func validateCreateFlags() error {
if persistenceType != server.EphermalNodesType &&
persistenceType != server.DynamicNodesType {
return errors.New("invalid persistence type")
Expand Down
27 changes: 10 additions & 17 deletions cli/cmds/kubeconfig/kubeconfig.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,13 @@ import (

"github.com/rancher/k3k/cli/cmds"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
"github.com/rancher/k3k/pkg/controller"
"github.com/rancher/k3k/pkg/controller/kubeconfig"
"github.com/rancher/k3k/pkg/controller/util"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/authentication/user"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/tools/clientcmd"
Expand All @@ -31,19 +30,13 @@ func init() {
}

var (
Scheme = runtime.NewScheme()
name string
cn string
org cli.StringSlice
altNames cli.StringSlice
expirationDays int64
configName string
backoff = wait.Backoff{
Steps: 5,
Duration: 20 * time.Second,
Factor: 2,
Jitter: 0.1,
}
Scheme = runtime.NewScheme()
name string
cn string
org cli.StringSlice
altNames cli.StringSlice
expirationDays int64
configName string
generateKubeconfigFlags = []cli.Flag{
cli.StringFlag{
Name: "name",
Expand All @@ -59,7 +52,7 @@ var (
Name: "cn",
Usage: "Common name (CN) of the generated certificates for the kubeconfig",
Destination: &cn,
Value: util.AdminCommonName,
Value: controller.AdminCommonName,
},
cli.StringSliceFlag{
Name: "org",
Expand Down Expand Up @@ -141,7 +134,7 @@ func generate(clx *cli.Context) error {
}
logrus.Infof("waiting for cluster to be available..")
var kubeconfig []byte
if err := retry.OnError(backoff, apierrors.IsNotFound, func() error {
if err := retry.OnError(controller.Backoff, apierrors.IsNotFound, func() error {
kubeconfig, err = cfg.Extract(ctx, ctrlClient, &cluster, host[0])
if err != nil {
return err
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ require (
go.etcd.io/etcd/api/v3 v3.5.14
go.etcd.io/etcd/client/v3 v3.5.14
go.uber.org/zap v1.26.0
gopkg.in/yaml.v2 v2.4.0
k8s.io/api v0.31.1
k8s.io/apimachinery v0.31.1
k8s.io/apiserver v0.31.0
Expand Down Expand Up @@ -119,7 +120,6 @@ require (
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/apiextensions-apiserver v0.29.2 // indirect
k8s.io/component-base v0.29.2 // indirect
Expand Down
File renamed without changes.
76 changes: 76 additions & 0 deletions k3k-kubelet/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
package main

import (
"errors"
"os"

"gopkg.in/yaml.v2"
)

// config has all virtual-kubelet startup options
type config struct {
ClusterName string `yaml:"clusterName,omitempty"`
ClusterNamespace string `yaml:"clusterNamespace,omitempty"`
NodeName string `yaml:"nodeName,omitempty"`
Token string `yaml:"token,omitempty"`
AgentHostname string `yaml:"agentHostname,omitempty"`
HostConfigPath string `yaml:"hostConfigPath,omitempty"`
VirtualConfigPath string `yaml:"virtualConfigPath,omitempty"`
KubeletPort string `yaml:"kubeletPort,omitempty"`
}

func (c *config) unmarshalYAML(data []byte) error {
var conf config

if err := yaml.Unmarshal(data, &conf); err != nil {
return err
}

if c.ClusterName == "" {
c.ClusterName = conf.ClusterName
}
if c.ClusterNamespace == "" {
c.ClusterNamespace = conf.ClusterNamespace
}
if c.HostConfigPath == "" {
c.HostConfigPath = conf.HostConfigPath
}
if c.VirtualConfigPath == "" {
c.VirtualConfigPath = conf.VirtualConfigPath
}
if c.KubeletPort == "" {
c.KubeletPort = conf.KubeletPort
}
if c.AgentHostname == "" {
c.AgentHostname = conf.AgentHostname
}
if c.NodeName == "" {
c.NodeName = conf.NodeName
}
return nil
}

func (c *config) validate() error {
if c.ClusterName == "" {
return errors.New("cluster name is not provided")
}
if c.ClusterNamespace == "" {
return errors.New("cluster namespace is not provided")
}
if c.AgentHostname == "" {
return errors.New("agent Hostname is not provided")
}
return nil
}

func (c *config) parse(path string) error {
if _, err := os.Stat(path); os.IsNotExist(err) {
return nil
}

b, err := os.ReadFile(path)
if err != nil {
return err
}
return c.unmarshalYAML(b)
}
Loading

0 comments on commit 823ed32

Please sign in to comment.