From 21f617f85ec87ab0bffdd9566dbfeed44d8ff7d4 Mon Sep 17 00:00:00 2001 From: Ved Ratan <82467006+VedRatan@users.noreply.github.com> Date: Fri, 5 Jul 2024 11:34:12 +0530 Subject: [PATCH] (feat): added status based policy creation logic (#212) * added status based policy creation logic * included fine grained condition based trigger * added policy equality check and check status logic --------- Signed-off-by: Ved Ratan --- pkg/adapter/nimbus-kyverno/manager/manager.go | 78 +++++++++++-------- .../nimbus-kyverno/processor/kpbuilder.go | 11 ++- pkg/adapter/nimbus-kyverno/utils/utils.go | 72 +++++++++++++++++ .../nimbus-kyverno/watcher/kpwatcher.go | 73 +++++++++++------ 4 files changed, 174 insertions(+), 60 deletions(-) diff --git a/pkg/adapter/nimbus-kyverno/manager/manager.go b/pkg/adapter/nimbus-kyverno/manager/manager.go index c28b6792..23e0167f 100644 --- a/pkg/adapter/nimbus-kyverno/manager/manager.go +++ b/pkg/adapter/nimbus-kyverno/manager/manager.go @@ -5,7 +5,6 @@ package manager import ( "context" - "fmt" "strings" "github.com/go-logr/logr" @@ -25,6 +24,7 @@ import ( "github.com/5GSEC/nimbus/pkg/adapter/common" "github.com/5GSEC/nimbus/pkg/adapter/k8s" "github.com/5GSEC/nimbus/pkg/adapter/nimbus-kyverno/processor" + "github.com/5GSEC/nimbus/pkg/adapter/nimbus-kyverno/utils" "github.com/5GSEC/nimbus/pkg/adapter/nimbus-kyverno/watcher" adapterutil "github.com/5GSEC/nimbus/pkg/adapter/util" globalwatcher "github.com/5GSEC/nimbus/pkg/adapter/watcher" @@ -55,10 +55,9 @@ func Run(ctx context.Context) { deletedKcpCh := make(chan string) go watcher.WatchKcps(ctx, updatedKcpCh, deletedKcpCh) - addKpCh := make(chan common.Request) updatedKpCh := make(chan common.Request) deletedKpCh := make(chan common.Request) - go watcher.WatchKps(ctx, addKpCh, updatedKpCh, deletedKpCh) + go watcher.WatchKps(ctx, updatedKpCh, deletedKpCh) for { select { @@ -72,7 +71,6 @@ func Run(ctx context.Context) { close(updatedKcpCh) close(deletedKcpCh) - close(addKpCh) close(updatedKpCh) close(deletedKpCh) return @@ -84,10 +82,9 @@ func Run(ctx context.Context) { logKpToDelete(ctx, deletedNp) case deletedCnp := <-deletedClusterNpChan: logKcpToDelete(ctx, deletedCnp) - case addedKp := <-addKpCh: - createTriggerForKp(ctx, addedKp) case updatedKp := <-updatedKpCh: - reconcileKp(ctx, updatedKp.Name, updatedKp.Namespace, true) + createTriggerForKp(ctx, updatedKp) + reconcileKp(ctx, updatedKp.Name, updatedKp.Namespace, false) case updatedKcp := <-updatedKcpCh: reconcileKcp(ctx, updatedKcp, false) case deletedKcp := <-deletedKcpCh: @@ -112,7 +109,7 @@ func reconcileKp(ctx context.Context, kpName, namespace string, deleted bool) { if deleted { logger.V(2).Info("Reconciling deleted KyvernoPolicy", "KyvernoPolicy.Name", kpName, "KyvernoPolicy.Namespace", namespace) } else { - logger.V(2).Info("Reconciling modified KyvernoPolicy", "KyvernoPolicy.Name", kpName, "KyvernoPolicy.Namespace", namespace) + logger.V(2).Info("Reconciling modified or added KyvernoPolicy", "KyvernoPolicy.Name", kpName, "KyvernoPolicy.Namespace", namespace) } createOrUpdateKp(ctx, npName, namespace) } @@ -168,26 +165,35 @@ func createOrUpdateKp(ctx context.Context, npName, npNamespace string) { logger.Error(err, "failed to get existing KyvernoPolicy", "KyvernoPolicy.Name", kp.Name, "KyvernoPolicy.Namespace", kp.Namespace) return } + if err != nil { if errors.IsNotFound(err) { if err = k8sClient.Create(ctx, &kp); err != nil { logger.Error(err, "failed to create KyvernoPolicy", "KyvernoPolicy.Name", kp.Name, "KyvernoPolicy.Namespace", kp.Namespace) return } + if err = adapterutil.UpdateNpStatus(ctx, k8sClient, "KyvernoPolicy/"+kp.Name, np.Name, np.Namespace, false); err != nil { + logger.Error(err, "failed to update KyvernoPolicies status in NimbusPolicy") + } logger.Info("KyvernoPolicy created", "KyvernoPolicy.Name", kp.Name, "KyvernoPolicy.Namespace", kp.Namespace) } } else { - kp.ObjectMeta.ResourceVersion = existingKp.ObjectMeta.ResourceVersion - if err = k8sClient.Update(ctx, &kp); err != nil { - logger.Error(err, "failed to configure existing KyvernoPolicy", "KyvernoPolicy.Name", existingKp.Name, "KyvernoPolicy.Namespace", existingKp.Namespace) - return + reason, isEqual := utils.PolEqual(existingKp, kp) + if !isEqual { + kp.ObjectMeta.ResourceVersion = existingKp.ObjectMeta.ResourceVersion + if err = k8sClient.Update(ctx, &kp); err != nil { + logger.Error(err, "failed to configure existing KyvernoPolicy", "KyvernoPolicy.Name", existingKp.Name, "KyvernoPolicy.Namespace", existingKp.Namespace) + return + } + if err = adapterutil.UpdateNpStatus(ctx, k8sClient, "KyvernoPolicy/"+kp.Name, np.Name, np.Namespace, false); err != nil { + logger.Error(err, "failed to update KyvernoPolicies status in NimbusPolicy") + } + logger.Info("KyvernoPolicy configured", "KyvernoPolicy.Name", existingKp.Name, "KyvernoPolicy.Namespace", existingKp.Namespace, "Reason", reason) + } else { + continue } - logger.Info("KyvernoPolicy configured", "KyvernoPolicy.Name", existingKp.Name, "KyvernoPolicy.Namespace", existingKp.Namespace) } - if err = adapterutil.UpdateNpStatus(ctx, k8sClient, "KyvernoPolicy/"+kp.Name, np.Name, np.Namespace, false); err != nil { - logger.Error(err, "failed to update KyvernoPolicies status in NimbusPolicy") - } } } @@ -314,8 +320,8 @@ func deleteDanglingkps(ctx context.Context, np v1alpha1.NimbusPolicy, logger log kpsToDelete[kpOwnedByNp.Name] = kpOwnedByNp } - for _, nimbusRule := range np.Spec.NimbusRules { - kpName := np.Name + "-" + strings.ToLower(nimbusRule.ID) + for _, pol := range np.Status.Policies { + kpName := strings.ToLower(pol)[14:] delete(kpsToDelete, kpName) } @@ -410,6 +416,17 @@ func deleteDanglingkcps(ctx context.Context, cnp v1alpha1.ClusterNimbusPolicy, l func createTriggerForKp(ctx context.Context, nameNamespace common.Request) { logger := log.FromContext(ctx) + var existingKp kyvernov1.Policy + var existingConfigMap corev1.ConfigMap + err := k8sClient.Get(ctx, types.NamespacedName{Name: nameNamespace.Name, Namespace: nameNamespace.Namespace}, &existingKp) + if err != nil && !errors.IsNotFound(err) { + logger.Error(err, "failed to get existing KyvernoPolicy", "KyvernoPolicy.Name", existingKp.Name, "KyvernoPolicy.Namespace", nameNamespace.Namespace) + return + } + if !strings.Contains(existingKp.GetName(), "mutateexisting") || !utils.CheckIfReady(existingKp.Status.Conditions) { // check if the policy is ready and the policy is the mutateexisting one + return + } + configMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: nameNamespace.Name + "-trigger-configmap", @@ -420,26 +437,21 @@ func createTriggerForKp(ctx context.Context, nameNamespace common.Request) { }, } - var existingKp kyvernov1.Policy - err := k8sClient.Get(ctx, types.NamespacedName{Name: nameNamespace.Name, Namespace: nameNamespace.Namespace}, &existingKp) - if err != nil && !errors.IsNotFound(err) { - logger.Error(err, "failed to get existing KyvernoPolicy", "KyvernoPolicy.Name", existingKp.Name, "KyvernoPolicy.Namespace", nameNamespace.Namespace) - return - } - - // Set MutateExistingKyvernoPolicy as the owner of the zConfigMap + // Set MutateExistingKyvernoPolicy as the owner of the ConfigMap if err := ctrl.SetControllerReference(&existingKp, &configMap.ObjectMeta, scheme); err != nil { logger.Error(err, "failed to set OwnerReference on ConfigMap", "Name", configMap.Name) return } - // Create the ConfigMap - err = k8sClient.Create(context.TODO(), configMap) + err = k8sClient.Get(ctx, types.NamespacedName{Name: nameNamespace.Name + "-trigger-configmap", Namespace: nameNamespace.Namespace}, &existingConfigMap) + if err != nil && errors.IsNotFound(err) { + // Create the ConfigMap + err = k8sClient.Create(context.TODO(), configMap) - if err != nil { - logger.Error(err, "failed to create trigger ConfigMap in namespace", "Namespace", configMap.Namespace) - } else { - fmt.Println(nameNamespace) - logger.Info("Created trigger ConfigMap in namespace ", "Namespace" ,configMap.Namespace) + if err != nil { + logger.Error(err, "Failed to create trigger ConfigMap", "Namespace", configMap.Namespace) + } else { + logger.Info("Created trigger ConfigMap", "Namespace", configMap.Namespace) + } } } diff --git a/pkg/adapter/nimbus-kyverno/processor/kpbuilder.go b/pkg/adapter/nimbus-kyverno/processor/kpbuilder.go index ebc11090..af85ee0f 100644 --- a/pkg/adapter/nimbus-kyverno/processor/kpbuilder.go +++ b/pkg/adapter/nimbus-kyverno/processor/kpbuilder.go @@ -32,6 +32,9 @@ func init() { func BuildKpsFrom(logger logr.Logger, np *v1alpha1.NimbusPolicy) []kyvernov1.Policy { // Build KPs based on given IDs var allkps []kyvernov1.Policy + admission := true + background := true + skipBackgroundAdmissionReq := true for _, nimbusRule := range np.Spec.NimbusRules { id := nimbusRule.ID if idpool.IsIdSupportedBy(id, "kyverno") { @@ -46,6 +49,10 @@ func BuildKpsFrom(logger logr.Logger, np *v1alpha1.NimbusPolicy) []kyvernov1.Pol kp.Namespace = np.Namespace kp.Annotations = make(map[string]string) kp.Annotations["policies.kyverno.io/description"] = nimbusRule.Description + kp.Spec.Admission = &admission + kp.Spec.Background = &background + kp.Spec.Rules[0].SkipBackgroundRequests = skipBackgroundAdmissionReq + if nimbusRule.Rule.RuleAction == "Block" { kp.Spec.ValidationFailureAction = kyvernov1.ValidationFailureAction("Enforce") } else { @@ -103,10 +110,6 @@ func cocoRuntimeAddition(np *v1alpha1.NimbusPolicy) ([]kyvernov1.Policy, error) } deploymentsGVR := schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"} - // labelSelector := metav1.LabelSelector{MatchLabels: labels} - // listOptions := metav1.ListOptions{ - // LabelSelector: apiLabels.Set(labelSelector.MatchLabels).String(), - // } deployments, err := client.Resource(deploymentsGVR).Namespace(np.Namespace).List(context.TODO(), metav1.ListOptions{}) if err != nil { errs = append(errs, err) diff --git a/pkg/adapter/nimbus-kyverno/utils/utils.go b/pkg/adapter/nimbus-kyverno/utils/utils.go index 22c33c2f..73ac5bb6 100644 --- a/pkg/adapter/nimbus-kyverno/utils/utils.go +++ b/pkg/adapter/nimbus-kyverno/utils/utils.go @@ -5,10 +5,13 @@ package utils import ( "fmt" + "reflect" "strings" + kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1" "golang.org/x/text/cases" "golang.org/x/text/language" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func GetGVK(kind string) string { @@ -45,6 +48,75 @@ func GetGVK(kind string) string { return fmt.Sprintf("%s/%s", apiVersion, Title(kind)) } +// sort.Slice(planets, func(i, j int) bool { +// return planets[i].Axis < planets[j].Axis +// }) + +func PolEqual(a, b kyvernov1.Policy) (string, bool) { + if len(a.Spec.Rules[0].MatchResources.Any) != len(b.Spec.Rules[0].MatchResources.Any) { + return "diff: labels not equal", false + } + if a.ObjectMeta.Name != b.ObjectMeta.Name { + return "diff: name", false + } + if a.ObjectMeta.Namespace != b.ObjectMeta.Namespace { + return "diff: Namespace", false + } + + if !reflect.DeepEqual(a.ObjectMeta.Labels, b.ObjectMeta.Labels) { + return "diff: Labels", false + } + + if !reflect.DeepEqual(a.ObjectMeta.OwnerReferences, b.ObjectMeta.OwnerReferences) { + return "diff: OwnerReferences", false + } + + if !checkLabels(a, b) { + return "diff: labels", false + } + + if !reflect.DeepEqual(a.Spec, b.Spec) { + return "diff: Spec", false + } + return "", true +} + +func CheckIfReady(conditions []metav1.Condition) bool { + for _, condition := range conditions { + if condition.Type == "Ready" && condition.Reason == "Succeeded" { + return true + } + } + return false +} +func checkLabels(a, b kyvernov1.Policy) bool { + resourceFiltersA := a.Spec.Rules[0].MatchResources.Any + resourceFiltersB := b.Spec.Rules[0].MatchResources.Any + if len(resourceFiltersA) != len(resourceFiltersB) { + return false + } + mp := make(map[string]bool) + for _, filter := range resourceFiltersA { + if filter.Selector != nil { + for k,v := range filter.Selector.MatchLabels { + key := k+v + mp[key] = true + } + } + } + + for _, filter := range resourceFiltersB { + if filter.Selector != nil { + for k,v := range filter.Selector.MatchLabels { + key := k+v + if !mp[key] { + return false + } + } + } + } + return true +} func Title(input string) string { toTitle := cases.Title(language.Und) diff --git a/pkg/adapter/nimbus-kyverno/watcher/kpwatcher.go b/pkg/adapter/nimbus-kyverno/watcher/kpwatcher.go index f11c2f91..9ddc0522 100644 --- a/pkg/adapter/nimbus-kyverno/watcher/kpwatcher.go +++ b/pkg/adapter/nimbus-kyverno/watcher/kpwatcher.go @@ -10,13 +10,15 @@ import ( "github.com/5GSEC/nimbus/pkg/adapter/common" "github.com/5GSEC/nimbus/pkg/adapter/k8s" + "github.com/5GSEC/nimbus/pkg/adapter/nimbus-kyverno/utils" + adapterutil "github.com/5GSEC/nimbus/pkg/adapter/util" + kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic/dynamicinformer" "k8s.io/client-go/tools/cache" "sigs.k8s.io/controller-runtime/pkg/log" - - adapterutil "github.com/5GSEC/nimbus/pkg/adapter/util" ) var ( @@ -39,42 +41,65 @@ func kpInformer() cache.SharedIndexInformer { // WatchKps watches update and delete event for KyvernoPolicies owned by // NimbusPolicy or ClusterNimbusPolicy and put their info on respective channels. -func WatchKps(ctx context.Context, addKpch, updatedKpCh, deletedKpCh chan common.Request) { +func WatchKps(ctx context.Context, updatedKpCh, deletedKpCh chan common.Request) { logger := log.FromContext(ctx) informer := kpInformer() handlers := cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - u := obj.(*unstructured.Unstructured) - if adapterutil.IsOrphan(u.GetOwnerReferences(), "NimbusPolicy") { - logger.V(4).Info("Ignoring orphan KyvernoPolicy", "KyvernoPolicy.Name", u.GetName(), "KyvernoPolicy.Namespace", u.GetNamespace(), "Operation", "Delete") - return - } - kpNamespacedName := common.Request{ - Name: u.GetName(), - Namespace: u.GetNamespace(), - } - if strings.Contains(kpNamespacedName.Name, "mutateexisting") { - addKpch <- kpNamespacedName - } - }, UpdateFunc: func(oldObj, newObj interface{}) { oldU := oldObj.(*unstructured.Unstructured) newU := newObj.(*unstructured.Unstructured) - if adapterutil.IsOrphan(newU.GetOwnerReferences(), "NimbusPolicy") { logger.V(4).Info("Ignoring orphan KyvernoPolicy", "KyvernoPolicy.Name", oldU.GetName(), "KyvernoPolicy.Namespace", oldU.GetNamespace(), "Operation", "Update") return } - if oldU.GetGeneration() == newU.GetGeneration() { + var oldKp kyvernov1.Policy + var newKp kyvernov1.Policy + err := runtime.DefaultUnstructuredConverter.FromUnstructured(oldU.Object, &oldKp) + if err != nil { + logger.Error(err, "failed to convert to kyverno policy") return } - kpNamespacedName := common.Request{ - Name: newU.GetName(), - Namespace: newU.GetNamespace(), + err = runtime.DefaultUnstructuredConverter.FromUnstructured(newU.Object, &newKp) + if err != nil { + logger.Error(err, "failed to convert to kyverno policy") + return + } + + oldConditions := oldKp.Status.Conditions + newConditions := newKp.Status.Conditions + + if !strings.Contains(newKp.GetName(), "mutateexisting") { + if oldU.GetGeneration() == newU.GetGeneration() { + return + } + kpNamespacedName := common.Request{ + Name: newU.GetName(), + Namespace: newU.GetNamespace(), + } + updatedKpCh <- kpNamespacedName + return + } + + // for mutate existing policy + if oldU.GetGeneration() == newU.GetGeneration() { + if utils.CheckIfReady(newConditions) && !utils.CheckIfReady(oldConditions) { + kpNamespacedName := common.Request{ + Name: newU.GetName(), + Namespace: newU.GetNamespace(), + } + updatedKpCh <- kpNamespacedName + return + } + return + } else { + kpNamespacedName := common.Request{ + Name: newU.GetName(), + Namespace: newU.GetNamespace(), + } + updatedKpCh <- kpNamespacedName } - updatedKpCh <- kpNamespacedName }, DeleteFunc: func(obj interface{}) { u := obj.(*unstructured.Unstructured) @@ -97,3 +122,5 @@ func WatchKps(ctx context.Context, addKpch, updatedKpCh, deletedKpCh chan common logger.Info("KyvernoPolicy watcher started") informer.Run(ctx.Done()) } + +