diff --git a/PROJECT b/PROJECT index 2c8b7d3..484534b 100644 --- a/PROJECT +++ b/PROJECT @@ -38,4 +38,7 @@ resources: kind: CustomizedUserRemediationTemplate path: github.com/medik8s/customized-user-remediation/api/v1alpha1 version: v1alpha1 + webhooks: + defaulting: true + webhookVersion: v1 version: "3" diff --git a/api/v1alpha1/customizeduserremediationtemplate_webhook.go b/api/v1alpha1/customizeduserremediationtemplate_webhook.go new file mode 100644 index 0000000..9c31822 --- /dev/null +++ b/api/v1alpha1/customizeduserremediationtemplate_webhook.go @@ -0,0 +1,51 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + commonAnnotations "github.com/medik8s/common/pkg/annotations" + + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +// log is for logging in this package. +var customizeduserremediationtemplatelog = logf.Log.WithName("customizeduserremediationtemplate-resource") + +func (r *CustomizedUserRemediationTemplate) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(r). + Complete() +} + +// TODO(user): EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! + +//+kubebuilder:webhook:path=/mutate-customized-user-remediation-medik8s-io-v1alpha1-customizeduserremediationtemplate,mutating=true,failurePolicy=fail,sideEffects=None,groups=customized-user-remediation.medik8s.io,resources=customizeduserremediationtemplates,verbs=create;update,versions=v1alpha1,name=mcustomizeduserremediationtemplate.kb.io,admissionReviewVersions=v1 + +var _ webhook.Defaulter = &CustomizedUserRemediationTemplate{} + +// Default implements webhook.Defaulter so a webhook will be registered for the type +func (r *CustomizedUserRemediationTemplate) Default() { + customizeduserremediationtemplatelog.Info("default", "name", r.Name) + if r.GetAnnotations() == nil { + r.Annotations = make(map[string]string) + } + if _, isSameKindAnnotationSet := r.GetAnnotations()[commonAnnotations.MultipleTemplatesSupportedAnnotation]; !isSameKindAnnotationSet { + r.Annotations[commonAnnotations.MultipleTemplatesSupportedAnnotation] = "true" + } +} diff --git a/api/v1alpha1/webhook_suite_test.go b/api/v1alpha1/webhook_suite_test.go new file mode 100644 index 0000000..fa14442 --- /dev/null +++ b/api/v1alpha1/webhook_suite_test.go @@ -0,0 +1,133 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "path/filepath" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + admissionv1beta1 "k8s.io/api/admission/v1beta1" + + //+kubebuilder:scaffold:imports + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var cfg *rest.Config +var k8sClient client.Client +var testEnv *envtest.Environment +var ctx context.Context +var cancel context.CancelFunc + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Webhook Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: false, + WebhookInstallOptions: envtest.WebhookInstallOptions{ + Paths: []string{filepath.Join("..", "..", "config", "webhook")}, + }, + } + + var err error + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + scheme := runtime.NewScheme() + err = AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + + err = admissionv1beta1.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + // start webhook server using Manager + webhookInstallOptions := &testEnv.WebhookInstallOptions + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme, + Host: webhookInstallOptions.LocalServingHost, + Port: webhookInstallOptions.LocalServingPort, + CertDir: webhookInstallOptions.LocalServingCertDir, + LeaderElection: false, + MetricsBindAddress: "0", + }) + Expect(err).NotTo(HaveOccurred()) + + err = (&CustomizedUserRemediationTemplate{}).SetupWebhookWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:webhook + + go func() { + defer GinkgoRecover() + err = mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred()) + }() + + // wait for the webhook server to get ready + dialer := &net.Dialer{Timeout: time.Second} + addrPort := fmt.Sprintf("%s:%d", webhookInstallOptions.LocalServingHost, webhookInstallOptions.LocalServingPort) + Eventually(func() error { + conn, err := tls.DialWithDialer(dialer, "tcp", addrPort, &tls.Config{InsecureSkipVerify: true}) + if err != nil { + return err + } + conn.Close() + return nil + }).Should(Succeed()) + +}) + +var _ = AfterSuite(func() { + cancel() + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/bundle/manifests/customized-user-remediation-webhook-service_v1_service.yaml b/bundle/manifests/customized-user-remediation-webhook-service_v1_service.yaml new file mode 100644 index 0000000..e28846c --- /dev/null +++ b/bundle/manifests/customized-user-remediation-webhook-service_v1_service.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: webhook + app.kubernetes.io/created-by: customized-user-remediation + app.kubernetes.io/instance: webhook-service + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: service + app.kubernetes.io/part-of: customized-user-remediation + name: customized-user-remediation-webhook-service +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + selector: + control-plane: controller-manager +status: + loadBalancer: {} diff --git a/bundle/manifests/customized-user-remediation.clusterserviceversion.yaml b/bundle/manifests/customized-user-remediation.clusterserviceversion.yaml index c28dcd7..5eabd56 100644 --- a/bundle/manifests/customized-user-remediation.clusterserviceversion.yaml +++ b/bundle/manifests/customized-user-remediation.clusterserviceversion.yaml @@ -351,3 +351,24 @@ spec: name: Medik8s url: https://www.medik8s.io/ version: 0.0.1 + webhookdefinitions: + - admissionReviewVersions: + - v1 + containerPort: 443 + deploymentName: customized-user-remediation-controller-manager + failurePolicy: Fail + generateName: mcustomizeduserremediationtemplate.kb.io + rules: + - apiGroups: + - customized-user-remediation.medik8s.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - customizeduserremediationtemplates + sideEffects: None + targetPort: 9443 + type: MutatingAdmissionWebhook + webhookPath: /mutate-customized-user-remediation-medik8s-io-v1alpha1-customizeduserremediationtemplate diff --git a/config/certmanager/certificate.yaml b/config/certmanager/certificate.yaml new file mode 100644 index 0000000..ae3225d --- /dev/null +++ b/config/certmanager/certificate.yaml @@ -0,0 +1,39 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: issuer + app.kubernetes.io/instance: selfsigned-issuer + app.kubernetes.io/component: certificate + app.kubernetes.io/created-by: customized-user-remediation + app.kubernetes.io/part-of: customized-user-remediation + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: certificate + app.kubernetes.io/instance: serving-cert + app.kubernetes.io/component: certificate + app.kubernetes.io/created-by: customized-user-remediation + app.kubernetes.io/part-of: customized-user-remediation + app.kubernetes.io/managed-by: kustomize + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize + dnsNames: + - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc + - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/config/certmanager/kustomization.yaml b/config/certmanager/kustomization.yaml new file mode 100644 index 0000000..bebea5a --- /dev/null +++ b/config/certmanager/kustomization.yaml @@ -0,0 +1,5 @@ +resources: +- certificate.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/config/certmanager/kustomizeconfig.yaml b/config/certmanager/kustomizeconfig.yaml new file mode 100644 index 0000000..e631f77 --- /dev/null +++ b/config/certmanager/kustomizeconfig.yaml @@ -0,0 +1,16 @@ +# This configuration is for teaching kustomize how to update name ref and var substitution +nameReference: +- kind: Issuer + group: cert-manager.io + fieldSpecs: + - kind: Certificate + group: cert-manager.io + path: spec/issuerRef/name + +varReference: +- kind: Certificate + group: cert-manager.io + path: spec/commonName +- kind: Certificate + group: cert-manager.io + path: spec/dnsNames diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 917c41e..b0cdf19 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -18,7 +18,7 @@ bases: - ../manager # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml -#- ../webhook +- ../webhook # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. #- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. diff --git a/config/default/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml new file mode 100644 index 0000000..738de35 --- /dev/null +++ b/config/default/manager_webhook_patch.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: webhook-server-cert diff --git a/config/default/webhookcainjection_patch.yaml b/config/default/webhookcainjection_patch.yaml new file mode 100644 index 0000000..fa13ad4 --- /dev/null +++ b/config/default/webhookcainjection_patch.yaml @@ -0,0 +1,29 @@ +# This patch add annotation to admission webhook config and +# the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize. +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/name: mutatingwebhookconfiguration + app.kubernetes.io/instance: mutating-webhook-configuration + app.kubernetes.io/component: webhook + app.kubernetes.io/created-by: customized-user-remediation + app.kubernetes.io/part-of: customized-user-remediation + app.kubernetes.io/managed-by: kustomize + name: mutating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/name: validatingwebhookconfiguration + app.kubernetes.io/instance: validating-webhook-configuration + app.kubernetes.io/component: webhook + app.kubernetes.io/created-by: customized-user-remediation + app.kubernetes.io/part-of: customized-user-remediation + app.kubernetes.io/managed-by: kustomize + name: validating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml new file mode 100644 index 0000000..9cf2613 --- /dev/null +++ b/config/webhook/kustomization.yaml @@ -0,0 +1,6 @@ +resources: +- manifests.yaml +- service.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/config/webhook/kustomizeconfig.yaml b/config/webhook/kustomizeconfig.yaml new file mode 100644 index 0000000..25e21e3 --- /dev/null +++ b/config/webhook/kustomizeconfig.yaml @@ -0,0 +1,25 @@ +# the following config is for teaching kustomize where to look at when substituting vars. +# It requires kustomize v2.1.0 or newer to work properly. +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + - kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + +namespace: +- kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true +- kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true + +varReference: +- path: metadata/annotations diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml new file mode 100644 index 0000000..909ad59 --- /dev/null +++ b/config/webhook/manifests.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-customized-user-remediation-medik8s-io-v1alpha1-customizeduserremediationtemplate + failurePolicy: Fail + name: mcustomizeduserremediationtemplate.kb.io + rules: + - apiGroups: + - customized-user-remediation.medik8s.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - customizeduserremediationtemplates + sideEffects: None diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml new file mode 100644 index 0000000..93ed86c --- /dev/null +++ b/config/webhook/service.yaml @@ -0,0 +1,20 @@ + +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: service + app.kubernetes.io/instance: webhook-service + app.kubernetes.io/component: webhook + app.kubernetes.io/created-by: customized-user-remediation + app.kubernetes.io/part-of: customized-user-remediation + app.kubernetes.io/managed-by: kustomize + name: webhook-service + namespace: system +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + selector: + control-plane: controller-manager diff --git a/controllers/customizeduserremediation_controller_test.go b/controllers/customizeduserremediation_controller_test.go index b396a49..8a3fc72 100644 --- a/controllers/customizeduserremediation_controller_test.go +++ b/controllers/customizeduserremediation_controller_test.go @@ -2,6 +2,7 @@ package controllers import ( "context" + "fmt" "strings" "time" @@ -48,7 +49,7 @@ var _ = Describe("CUR Controller", func() { Expect(node.CreationTimestamp).ToNot(BeZero()) }) - It("check the job is created correctly on the node", func() { + testJob := func() { jobs := &batchv1.JobList{} Expect(k8sClient.List(context.Background(), jobs)).To(Not(HaveOccurred())) @@ -65,7 +66,19 @@ var _ = Describe("CUR Controller", func() { //verify that the container on the Job is running the correct script defined by th user verifyContainer(scriptJob.Spec.Template.Spec.Containers[0]) + } + When("node name is stored in remediation name", func() { + It("check the job is created correctly on the node", testJob) + }) + //remediation is created from escalation remediation supporting same kind template + When("node name is stored in remediation's annotation", func() { + BeforeEach(func() { + cur.Name = fmt.Sprintf("%s-%s", unhealthyNodeName, "pseudo-random-test-sufix") + cur.Annotations = map[string]string{"remediation.medik8s.io/node-name": unhealthyNodeName} + }) + It("check the job is created correctly on the node", testJob) }) + }) func deleteRemediations() { diff --git a/main.go b/main.go index 6c249f0..a86a156 100644 --- a/main.go +++ b/main.go @@ -121,6 +121,10 @@ func main() { os.Exit(1) } + if err = (&customizeduserremediationv1alpha1.CustomizedUserRemediationTemplate{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "CustomizedUserRemediationTemplate") + os.Exit(1) + } //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/pkg/script/manager.go b/pkg/script/manager.go index 8ff5685..63467a7 100644 --- a/pkg/script/manager.go +++ b/pkg/script/manager.go @@ -8,6 +8,7 @@ import ( "time" "github.com/go-logr/logr" + commonAnnotations "github.com/medik8s/common/pkg/annotations" batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" @@ -40,6 +41,7 @@ type manager struct { } func (m *manager) RunScriptAsJob(ctx context.Context, cur *customizeduserremediationv1alpha1.CustomizedUserRemediation) error { + nodeName := getNodeName(cur) randomLabelValue, err := GenerateRandomLabelValue(6) if err != nil { @@ -77,7 +79,7 @@ func (m *manager) RunScriptAsJob(ctx context.Context, cur *customizeduserremedia }, //TODO mshitrit consider whether v1.RestartPolicyOnFailure is a better choice RestartPolicy: v1.RestartPolicyNever, - NodeName: cur.Name, + NodeName: nodeName, ServiceAccountName: "customized-user-remediation-controller-manager", Containers: []v1.Container{ { @@ -165,3 +167,15 @@ func GenerateRandomLabelValue(length int) (string, error) { return string(bytes), nil } + +// getNodeName checks for the node name in cur's commonAnnotations.NodeNameAnnotation if it does not exist it assumes the node name equals to far CR's name and return it. +func getNodeName(cur *customizeduserremediationv1alpha1.CustomizedUserRemediation) string { + ann := cur.GetAnnotations() + if ann == nil { + return cur.GetName() + } + if nodeName, isNodeNameAnnotationExist := ann[commonAnnotations.NodeNameAnnotation]; isNodeNameAnnotationExist { + return nodeName + } + return cur.GetName() +}