e2e: uninstall: check for lack of dangling objects

WIP TBD

Signed-off-by: Francesco Romani <[email protected]>

test/e2e/uninstall/uninstall_test.go

@@ -18,20 +18,18 @@ package uninstall
 
 import (
 	"context"
-
-	"k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/klog/v2"
-	"sigs.k8s.io/controller-runtime/pkg/client"
+	"fmt"
+	"strings"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
-	machineconfigv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/k8stopologyawareschedwg/deployer/pkg/deployer/platform"
 
-	nropmcp "github.com/openshift-kni/numaresources-operator/internal/machineconfigpools"
-	"github.com/openshift-kni/numaresources-operator/pkg/objectnames"
+	nropv1 "github.com/openshift-kni/numaresources-operator/api/numaresourcesoperator/v1"
+	"github.com/openshift-kni/numaresources-operator/internal/dangling"
 
 	e2eclient "github.com/openshift-kni/numaresources-operator/test/utils/clients"
 	"github.com/openshift-kni/numaresources-operator/test/utils/configuration"
@@ -53,70 +51,69 @@ var _ = Describe("[Uninstall] clusterCleanup", Serial, func() {
 	})
 
 	Context("with a running cluster with all the components", func() {
-		It("should delete all components after NRO deletion", func() {
+		It("should delete all components after NRO deletion", func(ctx context.Context) {
 			By("deleting the NRO object")
-			// since we are getting an existing object, we don't need the real labels here
-			nroObj := objects.TestNRO(objects.EmptyMatchLabels())
+			var nroObj nropv1.NUMAResourcesOperator
+			Expect(e2eclient.Client.Get(ctx, objects.NROObjectKey(), &nroObj)).To(Succeed(), "failed to get the NUMA resource operator %q", nroObj.Name)
+
 			By("deleting the KC object")
 			kcObj, err := objects.TestKC(objects.EmptyMatchLabels())
 			Expect(err).To(Not(HaveOccurred()))
 
-			// failed to get the NRO object, nothing else we can do
-			if err := e2eclient.Client.Get(context.TODO(), client.ObjectKeyFromObject(nroObj), nroObj); err != nil {
-				if !errors.IsNotFound(err) {
-					klog.Warningf("failed to get the NUMA resource operator %q: %v", nroObj.Name, err)
-				}
-
-				return
-			}
-
 			unpause, err := e2epause.MachineConfigPoolsByNodeGroups(nroObj.Spec.NodeGroups)
 			Expect(err).NotTo(HaveOccurred())
 
-			if err := e2eclient.Client.Delete(context.TODO(), nroObj); err != nil {
-				klog.Warningf("failed to delete the numaresourcesoperators %q", nroObj.Name)
-				return
-			}
-
-			if err := e2eclient.Client.Delete(context.TODO(), kcObj); err != nil && !errors.IsNotFound(err) {
-				klog.Warningf("failed to delete the kubeletconfigs %q", kcObj.Name)
-			}
+			Expect(e2eclient.Client.Delete(ctx, &nroObj)).To(Succeed(), "failed to delete the numaresourcesoperators %q", nroObj.Name)
+			Expect(e2eclient.Client.Delete(ctx, kcObj)).To(Succeed(), "failed to delete the kubeletconfigs %q", kcObj.Name)
 
 			err = unpause()
 			Expect(err).NotTo(HaveOccurred())
 
 			if configuration.Plat == platform.Kubernetes {
 				mcpObj := objects.TestMCP()
-				if err := e2eclient.Client.Delete(context.TODO(), mcpObj); err != nil {
-					klog.Warningf("failed to delete the machine config pool %q", mcpObj.Name)
-				}
+				Expect(e2eclient.Client.Delete(ctx, mcpObj)).To(Succeed(), "failed to delete the machine config pool %q", mcpObj.Name)
 			}
 
-			if configuration.Plat == platform.OpenShift {
-				Eventually(func() bool {
-					mcps, err := nropmcp.GetListByNodeGroupsV1(context.TODO(), e2eclient.Client, nroObj.Spec.NodeGroups)
+			By("checking that owned objects are deleted")
+			// TODO: check expected bits are all deleted
+
+			By("checking there are not dangling objects")
+			Eventually(func() error {
+				// sort by less likely to break (or expected so)
+				dsKeys, err := dangling.DaemonSets(e2eclient.Client, ctx, &nroObj)
+				if err != nil {
+					return err
+				}
+				if len(dsKeys) > 0 {
+					return fmt.Errorf("found dangling DaemonSets: %s", stringifyObjectKeys(dsKeys))
+				}
+
+				if configuration.Plat == platform.OpenShift {
+					mcKeys, err := dangling.MachineConfigs(e2eclient.Client, ctx, &nroObj)
 					if err != nil {
-						klog.Warningf("failed to get machine config pools: %v", err)
-						return false
+						return err
 					}
-
-					for _, mcp := range mcps {
-						mcName := objectnames.GetMachineConfigName(nroObj.Name, mcp.Name)
-						for _, s := range mcp.Status.Configuration.Source {
-							// the config is still existing under the machine config pool
-							if s.Name == mcName {
-								return false
-							}
-						}
-
-						if machineconfigv1.IsMachineConfigPoolConditionFalse(mcp.Status.Conditions, machineconfigv1.MachineConfigPoolUpdated) {
-							return false
-						}
+					if len(mcKeys) > 0 {
+						return fmt.Errorf("found dangling MachineConfigs: %s", stringifyObjectKeys(mcKeys))
 					}
-
-					return true
-				}).WithTimeout(configuration.MachineConfigPoolUpdateTimeout).WithPolling(configuration.MachineConfigPoolUpdateInterval).Should(BeTrue())
-			}
+				}
+				return nil
+			}).WithTimeout(configuration.MachineConfigPoolUpdateTimeout).WithPolling(configuration.MachineConfigPoolUpdateInterval).Should(Succeed())
 		})
 	})
 })
+
+func stringifyObjectKeys(keys []client.ObjectKey) string {
+	if len(keys) == 0 {
+		return ""
+	}
+	if len(keys) == 1 {
+		return keys[0].String()
+	}
+	var sb strings.Builder
+	fmt.Fprintf(&sb, "%s", keys[0].String())
+	for _, key := range keys[1:] {
+		fmt.Fprintf(&sb, ",%s", key.String())
+	}
+	return sb.String()
+}