Adding Networkpolicy to ClusterSets

.gitignore

@@ -6,3 +6,4 @@
 .idea
 .vscode/
 __debug*
+*-kubeconfig.yaml

charts/k3k/crds/k3k.io_clusters.yaml

@@ -77,6 +77,31 @@ spec:
                 x-kubernetes-validations:
                 - message: clusterDNS is immutable
                   rule: self == oldSelf
+              clusterLimit:
+                description: Limit is the limits that apply for the server/worker
+                  nodes.
+                properties:
+                  serverLimit:
+                    additionalProperties:
+                      anyOf:
+                      - type: integer
+                      - type: string
+                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                      x-kubernetes-int-or-string: true
+                    description: ServerLimit is the limits (cpu/mem) that apply to
+                      the server nodes
+                    type: object
+                  workerLimit:
+                    additionalProperties:
+                      anyOf:
+                      - type: integer
+                      - type: string
+                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                      x-kubernetes-int-or-string: true
+                    description: WorkerLimit is the limits (cpu/mem) that apply to
+                      the agent nodes
+                    type: object
+                type: object
               expose:
                 description: |-
                   Expose contains options for exposing the apiserver inside/outside of the cluster. By default, this is only exposed as a
@@ -88,9 +113,6 @@ spec:
                         type: boolean
                       ingressClassName:
                         type: string
-                    required:
-                    - enabled
-                    - ingressClassName
                     type: object
                   loadbalancer:
                     properties:
@@ -106,10 +128,12 @@ spec:
                     required:
                     - enabled
                     type: object
-                required:
-                - ingress
-                - loadbalancer
-                - nodePort
+                type: object
+              nodeSelector:
+                additionalProperties:
+                  type: string
+                description: NodeSelector is the node selector that will be applied
+                  to all server/agent pods
                 type: object
               persistence:
                 description: |-

charts/k3k/crds/k3k.io_clustersets.yaml

@@ -0,0 +1,176 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.14.0
+  name: clustersets.k3k.io
+spec:
+  group: k3k.io
+  names:
+    kind: ClusterSet
+    listKind: ClusterSetList
+    plural: clustersets
+    singular: clusterset
+  scope: Namespaced
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: Spec is the spec of the ClusterSet
+            properties:
+              defaultLimits:
+                description: DefaultLimits are the limits used for servers/agents
+                  when a cluster in the set doesn't provide any
+                properties:
+                  serverLimit:
+                    additionalProperties:
+                      anyOf:
+                      - type: integer
+                      - type: string
+                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                      x-kubernetes-int-or-string: true
+                    description: ServerLimit is the limits (cpu/mem) that apply to
+                      the server nodes
+                    type: object
+                  workerLimit:
+                    additionalProperties:
+                      anyOf:
+                      - type: integer
+                      - type: string
+                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                      x-kubernetes-int-or-string: true
+                    description: WorkerLimit is the limits (cpu/mem) that apply to
+                      the agent nodes
+                    type: object
+                type: object
+              defaultNodeSelector:
+                additionalProperties:
+                  type: string
+                description: DefaultNodeSelector is the node selector that applies
+                  to all clusters (server + agent) in the set
+                type: object
+              maxLimits:
+                additionalProperties:
+                  anyOf:
+                  - type: integer
+                  - type: string
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+                description: MaxLimits are the limits that apply to all clusters (server
+                  + agent) in the set
+                type: object
+            type: object
+          status:
+            description: Status is the status of the ClusterSet
+            properties:
+              conditions:
+                description: Conditions are the invidual conditions for the cluster
+                  set
+                items:
+                  description: "Condition contains details for one aspect of the current
+                    state of this API Resource.\n---\nThis struct is intended for
+                    direct use as an array at the field path .status.conditions.  For
+                    example,\n\n\n\ttype FooStatus struct{\n\t    // Represents the
+                    observations of a foo's current state.\n\t    // Known .status.conditions.type
+                    are: \"Available\", \"Progressing\", and \"Degraded\"\n\t    //
+                    +patchMergeKey=type\n\t    // +patchStrategy=merge\n\t    // +listType=map\n\t
+                    \   // +listMapKey=type\n\t    Conditions []metav1.Condition `json:\"conditions,omitempty\"
+                    patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t
+                    \   // other fields\n\t}"
+                  properties:
+                    lastTransitionTime:
+                      description: |-
+                        lastTransitionTime is the last time the condition transitioned from one status to another.
+                        This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                      format: date-time
+                      type: string
+                    message:
+                      description: |-
+                        message is a human readable message indicating details about the transition.
+                        This may be an empty string.
+                      maxLength: 32768
+                      type: string
+                    observedGeneration:
+                      description: |-
+                        observedGeneration represents the .metadata.generation that the condition was set based upon.
+                        For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+                        with respect to the current state of the instance.
+                      format: int64
+                      minimum: 0
+                      type: integer
+                    reason:
+                      description: |-
+                        reason contains a programmatic identifier indicating the reason for the condition's last transition.
+                        Producers of specific condition types may define expected values and meanings for this field,
+                        and whether the values are considered a guaranteed API.
+                        The value should be a CamelCase string.
+                        This field may not be empty.
+                      maxLength: 1024
+                      minLength: 1
+                      pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                      type: string
+                    status:
+                      description: status of the condition, one of True, False, Unknown.
+                      enum:
+                      - "True"
+                      - "False"
+                      - Unknown
+                      type: string
+                    type:
+                      description: |-
+                        type of condition in CamelCase or in foo.example.com/CamelCase.
+                        ---
+                        Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be
+                        useful (see .node.status.conditions), the ability to deconflict is important.
+                        The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)
+                      maxLength: 316
+                      pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                      type: string
+                  required:
+                  - lastTransitionTime
+                  - message
+                  - reason
+                  - status
+                  - type
+                  type: object
+                type: array
+              lastUpdateTime:
+                description: LastUpdate is the timestamp when the status was last
+                  updated
+                type: string
+              observedGeneration:
+                description: ObservedGeneration was the generation at the time the
+                  status was updated.
+                format: int64
+                type: integer
+              summary:
+                description: Sumamry is a summary of the status (error, ready)
+                type: string
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

charts/k3k/templates/deployment.yaml

@@ -19,8 +19,14 @@ spec:
         - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           name: {{ .Chart.Name }}
+          environment:
+          - name: CLUSTER_CIDR
+            value: {{ .Values.host.clusterCIDR }}
           ports:
           - containerPort: 8080
             name: https
             protocol: TCP
-      serviceAccountName: {{ include "k3k.serviceAccountName" . }}
+          - containerPort: 9443
+            name: https-webhook
+            protocol: TCP
+      serviceAccountName: {{ include "k3k.serviceAccountName" . }}

charts/k3k/templates/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata: 
+  name: k3k-webhook
+  labels:
+    {{- include "k3k.labels" . | nindent 4 }}
+  namespace: {{ .Values.namespace }}
+spec:
+  ports:
+    - port: 443
+      protocol: TCP
+      name: https-webhook
+      targetPort: 9443
+  selector:
+    {{- include "k3k.selectorLabels" . | nindent 6 }}

charts/k3k/values.yaml

@@ -2,7 +2,7 @@ replicaCount: 1
 namespace: k3k-system
 
 image:
-  repository: rancher/k3k
+  repository: rancher/k3k 
   pullPolicy: Always
   # Overrides the image tag whose default is the chart appVersion.
   tag: "v0.2.1"
@@ -11,6 +11,10 @@ imagePullSecrets: []
 nameOverride: ""
 fullnameOverride: ""
 
+host:
+# clusterCIDR specifies the clusterCIDR that will be added to the default networkpolicy for clustersets
+  clusterCIDR: ""
+
 serviceAccount:
   # Specifies whether a service account should be created
   create: true

main.go

@@ -4,32 +4,45 @@ package main
 import (
 	"context"
 	"flag"
+	"os"
 
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
 	"github.com/rancher/k3k/pkg/controller/cluster"
+	"github.com/rancher/k3k/pkg/controller/clusterset"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
-	ctrlconfig "sigs.k8s.io/controller-runtime/pkg/client/config"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 )
 
-var Scheme = runtime.NewScheme()
+const (
+	clusterCIDRFlagName = "cluster-cidr"
+	clusterCIDREnvVar   = "CLUSTER_CIDR"
+	KubeconfigFlagName  = "kubeconfig"
+)
+
+var (
+	Scheme      = runtime.NewScheme()
+	clusterCIDR string
+	kubeconfig  string
+)
 
 func init() {
 	_ = clientgoscheme.AddToScheme(Scheme)
 	_ = v1alpha1.AddToScheme(Scheme)
 }
 
 func main() {
-	ctrlconfig.RegisterFlags(nil)
-	flag.Parse()
-
+	fs := addFlags()
+	fs.Parse(os.Args[1:])
 	ctx := context.Background()
 
-	kubeconfig := flag.Lookup("kubeconfig").Value.String()
+	if clusterCIDR == "" {
+		clusterCIDR = os.Getenv(clusterCIDREnvVar)
+	}
+
 	restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
 	if err != nil {
 		klog.Fatalf("Failed to create config from kubeconfig file: %v", err)
@@ -38,12 +51,26 @@ func main() {
 	mgr, err := ctrl.NewManager(restConfig, manager.Options{
 		Scheme: Scheme,
 	})
+
 	if err != nil {
 		klog.Fatalf("Failed to create new controller runtime manager: %v", err)
 	}
 
 	if err := cluster.Add(ctx, mgr); err != nil {
-		klog.Fatalf("Failed to add the new controller: %v", err)
+		klog.Fatalf("Failed to add the new cluster controller: %v", err)
+	}
+
+	if err := cluster.AddPodController(ctx, mgr); err != nil {
+		klog.Fatalf("Failed to add the new cluster controller: %v", err)
+	}
+	klog.Info("adding clusterset controller")
+	if err := clusterset.Add(ctx, mgr, clusterCIDR); err != nil {
+		klog.Fatalf("Failed to add the clusterset controller: %v", err)
+	}
+
+	klog.Info("adding networkpolicy node controller")
+	if err := clusterset.AddNodeController(ctx, mgr, clusterCIDR); err != nil {
+		klog.Fatalf("Failed to add the clusterset controller: %v", err)
 	}
 
 	if err := cluster.AddPodController(ctx, mgr); err != nil {
@@ -54,3 +81,10 @@ func main() {
 		klog.Fatalf("Failed to start the manager: %v", err)
 	}
 }
+
+func addFlags() *flag.FlagSet {
+	fs := flag.NewFlagSet("k3k", flag.ExitOnError)
+	fs.StringVar(&clusterCIDR, clusterCIDRFlagName, "", "The host's cluster CIDR")
+	fs.StringVar(&kubeconfig, KubeconfigFlagName, "", "Paths to a kubeconfig. Only required if out-of-cluster.")
+	return fs
+}

pkg/apis/k3k.io/v1alpha1/register.go

@@ -21,7 +21,10 @@ func Resource(resource string) schema.GroupResource {
 func addKnownTypes(s *runtime.Scheme) error {
 	s.AddKnownTypes(SchemeGroupVersion,
 		&Cluster{},
-		&ClusterList{})
+		&ClusterList{},
+		&ClusterSet{},
+		&ClusterSetList{},
+	)
 	metav1.AddToGroupVersion(s, SchemeGroupVersion)
 	return nil
 }