[YUNIKORN-2289] Isolate e2e tests in separate namespaces (#913)

Closes: #913 Signed-off-by: Craig Condit <[email protected]>
apache · Sep 10, 2024 · c959d23 · c959d23
1 parent 8add17d
commit c959d23
Show file tree

Hide file tree

Showing 11 changed files with 71 additions and 95 deletions.
diff --git a/test/e2e/basic_scheduling/basic_scheduling_suite_test.go b/test/e2e/basic_scheduling/basic_scheduling_suite_test.go
@@ -54,5 +54,6 @@ var By = ginkgo.By
 
 var Ω = gomega.Ω
 var BeNil = gomega.BeNil
+var dev string
 var HaveOccurred = gomega.HaveOccurred
 var BeEquivalentTo = gomega.BeEquivalentTo
diff --git a/test/e2e/basic_scheduling/basic_scheduling_test.go b/test/e2e/basic_scheduling/basic_scheduling_test.go
@@ -37,7 +37,6 @@ var suiteName string
 var kClient k8s.KubeCtl
 var restClient yunikorn.RClient
 var sleepRespPod *v1.Pod
-var dev = "dev" + common.RandSeq(5)
 var appsInfo *dao.ApplicationDAOInfo
 var oldConfigMap = new(v1.ConfigMap)
 
@@ -52,20 +51,18 @@ var _ = ginkgo.BeforeSuite(func() {
 	gomega.Ω(kClient.SetClient()).To(gomega.BeNil())
 	// Initializing rest client
 	restClient = yunikorn.RClient{}
-
 	yunikorn.EnsureYuniKornConfigsPresent()
-
 	By("Port-forward the scheduler pod")
 	err := kClient.PortForwardYkSchedulerPod()
 	Ω(err).NotTo(HaveOccurred())
-
 	yunikorn.UpdateConfigMapWrapper(oldConfigMap, "fifo")
-
+})
+var _ = ginkgo.BeforeEach(func() {
+	dev = "dev" + common.RandSeq(5)
 	ginkgo.By("create development namespace")
 	ns1, err := kClient.CreateNamespace(dev, nil)
 	gomega.Ω(err).NotTo(gomega.HaveOccurred())
 	gomega.Ω(ns1.Status.Phase).To(gomega.Equal(v1.NamespaceActive))
-
 	ginkgo.By("Deploy the sleep pod to the development namespace")
 	initPod, podErr := k8s.InitSleepPod(sleepPodConfigs)
 	gomega.Ω(podErr).NotTo(gomega.HaveOccurred())
@@ -74,17 +71,17 @@ var _ = ginkgo.BeforeSuite(func() {
 	// Wait for pod to move to running state
 	err = kClient.WaitForPodRunning(dev, sleepPodConfigs.Name, 30*time.Second)
 	gomega.Ω(err).NotTo(gomega.HaveOccurred())
-
 	appsInfo, err = restClient.GetAppInfo("default", "root."+dev, sleepRespPod.ObjectMeta.Labels["applicationId"])
 	gomega.Ω(err).NotTo(gomega.HaveOccurred())
 	gomega.Ω(appsInfo).NotTo(gomega.BeNil())
 })
-
-var _ = ginkgo.AfterSuite(func() {
+var _ = ginkgo.AfterEach(func() {
 	ginkgo.By("Tear down namespace: " + dev)
 	err := kClient.TearDownNamespace(dev)
 	Ω(err).NotTo(HaveOccurred())
+})
 
+var _ = ginkgo.AfterSuite(func() {
 	yunikorn.RestoreConfigMapWrapper(oldConfigMap)
 })
 

diff --git a/test/e2e/persistent_volume/persistent_volume_suite_test.go b/test/e2e/persistent_volume/persistent_volume_suite_test.go
@@ -48,3 +48,4 @@ func TestPersistentVolume(t *testing.T) {
 
 var Ω = gomega.Ω
 var HaveOccurred = gomega.HaveOccurred
+var dev string
diff --git a/test/e2e/persistent_volume/persistent_volume_test.go b/test/e2e/persistent_volume/persistent_volume_test.go
@@ -40,7 +40,6 @@ import (
 var suiteName string
 var kClient k8s.KubeCtl
 var restClient yunikorn.RClient
-var dev = "dev-" + common.RandSeq(5)
 
 const (
 	LocalTypePv    = "Local"
@@ -57,27 +56,30 @@ var _ = ginkgo.BeforeSuite(func() {
 	// Initializing rest client
 	restClient = yunikorn.RClient{}
 	Ω(restClient).NotTo(gomega.BeNil())
-
 	yunikorn.EnsureYuniKornConfigsPresent()
-
+})
+var _ = ginkgo.BeforeEach(func() {
 	// Create namespace
+	dev = "dev-" + common.RandSeq(5)
 	ginkgo.By("Create namespace " + dev)
 	ns, err := kClient.CreateNamespace(dev, nil)
 	Ω(err).NotTo(HaveOccurred())
 	Ω(ns.Status.Phase).To(gomega.Equal(v1.NamespaceActive))
 })
 
+var _ = ginkgo.AfterEach(func() {
+	ginkgo.By("Tearing down namespace: " + dev)
+	err := kClient.TearDownNamespace(dev)
+	Ω(err).NotTo(HaveOccurred())
+})
+
 var _ = ginkgo.AfterSuite(func() {
 	// Clean up
 	ginkgo.By("Deleting PVCs and PVs")
 	err := kClient.DeletePVCs(dev)
 	err2 := kClient.DeletePVs(dev)
-	ginkgo.By("Tearing down namespace: " + dev)
-	err3 := kClient.TearDownNamespace(dev)
-
 	Ω(err).NotTo(HaveOccurred())
 	Ω(err2).NotTo(HaveOccurred())
-	Ω(err3).NotTo(HaveOccurred())
 })
 
 var _ = ginkgo.Describe("PersistentVolume", func() {

diff --git a/test/e2e/recovery_and_restart/recovery_and_restart_suite_test.go b/test/e2e/recovery_and_restart/recovery_and_restart_suite_test.go
@@ -50,3 +50,4 @@ func TestRecoveryAndRestart(t *testing.T) {
 }
 
 var Ω = gomega.Ω
+var dev string
diff --git a/test/e2e/recovery_and_restart/recovery_and_restart_test.go b/test/e2e/recovery_and_restart/recovery_and_restart_test.go
@@ -52,7 +52,6 @@ var kClient k8s.KubeCtl
 var restClient yunikorn.RClient
 var oldConfigMap = new(v1.ConfigMap)
 var sleepRespPod *v1.Pod
-var dev = "dev" + common.RandSeq(5)
 
 // Define sleepPod
 var sleepPodConfigs = k8s.SleepPodConfig{Name: "sleepjob", NS: dev}
@@ -70,6 +69,15 @@ var _ = ginkgo.BeforeSuite(func() {
 	yunikorn.EnsureYuniKornConfigsPresent()
 	yunikorn.UpdateConfigMapWrapper(oldConfigMap, "")
 
+	ginkgo.By("Restart the scheduler pod")
+	yunikorn.RestartYunikorn(&kClient)
+
+	ginkgo.By("Port-forward scheduler pod after restart")
+	yunikorn.RestorePortForwarding(&kClient)
+})
+
+var _ = ginkgo.BeforeEach(func() {
+	dev = "dev" + common.RandSeq(5)
 	ginkgo.By("create development namespace")
 	ns1, err := kClient.CreateNamespace(dev, nil)
 	gomega.Ω(err).NotTo(gomega.HaveOccurred())
@@ -85,13 +93,6 @@ var _ = ginkgo.BeforeSuite(func() {
 		fmt.Sprintf("applicationId=%s", sleepRespPod.ObjectMeta.Labels["applicationId"]),
 		60)
 	gomega.Ω(err).NotTo(gomega.HaveOccurred())
-
-	ginkgo.By("Restart the scheduler pod")
-	yunikorn.RestartYunikorn(&kClient)
-
-	ginkgo.By("Port-forward scheduler pod after restart")
-	yunikorn.RestorePortForwarding(&kClient)
-
 	ginkgo.By("Deploy 2nd sleep pod to the development namespace")
 	sleepObj2, podErr := k8s.InitSleepPod(sleepPod2Configs)
 	Ω(podErr).NotTo(gomega.HaveOccurred())
@@ -104,10 +105,13 @@ var _ = ginkgo.BeforeSuite(func() {
 	gomega.Ω(err).NotTo(gomega.HaveOccurred())
 })
 
-var _ = ginkgo.AfterSuite(func() {
+var _ = ginkgo.AfterEach(func() {
 	ginkgo.By("Tear down namespace: " + dev)
 	err := kClient.TearDownNamespace(dev)
 	Ω(err).NotTo(gomega.HaveOccurred())
+})
+
+var _ = ginkgo.AfterSuite(func() {
 
 	// call the healthCheck api to check scheduler health
 	ginkgo.By("Check Yunikorn's health")

diff --git a/test/e2e/restart_changed_config/restart_changed_config_test.go b/test/e2e/restart_changed_config/restart_changed_config_test.go
@@ -62,10 +62,11 @@ var _ = ginkgo.BeforeSuite(func() {
 	ginkgo.By("Port-forward the scheduler pod")
 	var err = kClient.PortForwardYkSchedulerPod()
 	Ω(err).NotTo(gomega.HaveOccurred())
+})
 
+var _ = ginkgo.BeforeEach(func() {
 	ginkgo.By("create development namespace")
-	var ns *v1.Namespace
-	ns, err = kClient.CreateNamespace(dev, nil)
+	ns, err := kClient.CreateNamespace(dev, nil)
 	gomega.Ω(err).NotTo(gomega.HaveOccurred())
 	gomega.Ω(ns.Status.Phase).To(gomega.Equal(v1.NamespaceActive))
 	ns, err = kClient.CreateNamespace(test, nil)
@@ -74,14 +75,6 @@ var _ = ginkgo.BeforeSuite(func() {
 })
 
 var _ = ginkgo.AfterSuite(func() {
-	ginkgo.By("Tear down namespace: " + dev)
-	err := kClient.TearDownNamespace(dev)
-	Ω(err).NotTo(gomega.HaveOccurred())
-
-	ginkgo.By("Tear down namespace: " + test)
-	err = kClient.TearDownNamespace(test)
-	Ω(err).NotTo(gomega.HaveOccurred())
-
 	// call the healthCheck api to check scheduler health
 	ginkgo.By("Check YuniKorn's health")
 	checks, err2 := yunikorn.GetFailedHealthChecks()
@@ -172,5 +165,12 @@ var _ = ginkgo.Describe("PodInRecoveryQueue", func() {
 
 	ginkgo.AfterEach(func() {
 		tests.DumpClusterInfoIfSpecFailed(suiteName, []string{dev, test})
+		ginkgo.By("Tear down namespace: " + dev)
+		err := kClient.TearDownNamespace(dev)
+		Ω(err).NotTo(gomega.HaveOccurred())
+
+		ginkgo.By("Tear down namespace: " + test)
+		err = kClient.TearDownNamespace(test)
+		Ω(err).NotTo(gomega.HaveOccurred())
 	})
 })
diff --git a/test/e2e/simple_preemptor/simple_preemptor_suite_test.go b/test/e2e/simple_preemptor/simple_preemptor_suite_test.go
@@ -48,3 +48,4 @@ func TestSimplePreemptor(t *testing.T) {
 
 var Ω = gomega.Ω
 var HaveOccurred = gomega.HaveOccurred
+var dev string
diff --git a/test/e2e/simple_preemptor/simple_preemptor_test.go b/test/e2e/simple_preemptor/simple_preemptor_test.go
@@ -24,24 +24,22 @@ import (
 	"strings"
 	"time"
 
-	"github.com/onsi/ginkgo/v2"
-	"github.com/onsi/gomega"
-	v1 "k8s.io/api/core/v1"
-	k8serrors "k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/api/resource"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
 	tests "github.com/apache/yunikorn-k8shim/test/e2e"
+
 	"github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common"
 	"github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s"
 	"github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 )
 
 var suiteName string
 var kClient k8s.KubeCtl
 var restClient yunikorn.RClient
-var ns *v1.Namespace
-var dev = "dev" + common.RandSeq(5)
 var oldConfigMap = new(v1.ConfigMap)
 
 // Nodes
@@ -71,11 +69,6 @@ var _ = ginkgo.BeforeSuite(func() {
 	var err = kClient.PortForwardYkSchedulerPod()
 	Ω(err).NotTo(gomega.HaveOccurred())
 
-	ginkgo.By("create development namespace")
-	ns, err = kClient.CreateNamespace(dev, nil)
-	gomega.Ω(err).NotTo(gomega.HaveOccurred())
-	gomega.Ω(ns.Status.Phase).To(gomega.Equal(v1.NamespaceActive))
-
 	var nodes *v1.NodeList
 	nodes, err = kClient.GetNodes()
 	Ω(err).NotTo(gomega.HaveOccurred())
@@ -126,6 +119,14 @@ var _ = ginkgo.BeforeSuite(func() {
 	sleepPodMemLimit2 = int64(float64(Worker2Res.Value())/3.5) / (1000 * 1000)
 })
 
+var _ = ginkgo.BeforeEach(func() {
+	dev = "dev" + common.RandSeq(5)
+	ginkgo.By("create development namespace")
+	ns, err := kClient.CreateNamespace(dev, nil)
+	gomega.Ω(err).NotTo(gomega.HaveOccurred())
+	gomega.Ω(ns.Status.Phase).To(gomega.Equal(v1.NamespaceActive))
+})
+
 var _ = ginkgo.AfterSuite(func() {
 
 	ginkgo.By("Untainting some nodes")
@@ -136,10 +137,6 @@ var _ = ginkgo.AfterSuite(func() {
 	checks, err := yunikorn.GetFailedHealthChecks()
 	Ω(err).NotTo(gomega.HaveOccurred())
 	Ω(checks).To(gomega.Equal(""), checks)
-	ginkgo.By("Tearing down namespace: " + ns.Name)
-	err = kClient.TearDownNamespace(ns.Name)
-	Ω(err).NotTo(gomega.HaveOccurred())
-
 	yunikorn.RestoreConfigMapWrapper(oldConfigMap)
 })
 
@@ -205,9 +202,7 @@ var _ = ginkgo.Describe("SimplePreemptor", func() {
 		gomega.Ω(err).NotTo(gomega.HaveOccurred())
 
 		// Wait for pod to move to running state
-		err = kClient.WaitForPodBySelectorRunning(dev,
-			fmt.Sprintf("app=%s", sleepRespPod.ObjectMeta.Labels["app"]),
-			600)
+		err = kClient.WaitForPodRunning(dev, sleepRespPod.Name, 1*time.Minute)
 		gomega.Ω(err).NotTo(gomega.HaveOccurred())
 
 		// assert sleeppod4 is killed
@@ -217,29 +212,9 @@ var _ = ginkgo.Describe("SimplePreemptor", func() {
 	})
 
 	ginkgo.AfterEach(func() {
-		tests.DumpClusterInfoIfSpecFailed(suiteName, []string{ns.Name})
-
-		// Delete all sleep pods
-		ginkgo.By("Delete all sleep pods")
-		pods, err := kClient.GetPodNamesFromNS(ns.Name)
-		if err == nil {
-			for _, each := range pods {
-				if strings.Contains(each, "sleep") {
-					ginkgo.By("Deleting sleep pod: " + each)
-					err = kClient.DeletePod(each, ns.Name)
-					if err != nil {
-						if statusErr, ok := err.(*k8serrors.StatusError); ok {
-							if statusErr.ErrStatus.Reason == metav1.StatusReasonNotFound {
-								fmt.Fprintf(ginkgo.GinkgoWriter, "Failed to delete pod %s - reason is %s, it "+
-									"has been deleted in the meantime\n", each, statusErr.ErrStatus.Reason)
-								continue
-							}
-						}
-					}
-				}
-			}
-		} else {
-			fmt.Fprintf(ginkgo.GinkgoWriter, "Failed to get pods from namespace %s - reason is %s\n", ns.Name, err.Error())
-		}
+		tests.DumpClusterInfoIfSpecFailed(suiteName, []string{dev})
+		ginkgo.By("Tearing down namespace: " + dev)
+		err := kClient.TearDownNamespace(dev)
+		Ω(err).NotTo(gomega.HaveOccurred())
 	})
 })
diff --git a/test/e2e/user_group_limit/user_group_limit_suite_test.go b/test/e2e/user_group_limit/user_group_limit_suite_test.go
@@ -48,3 +48,4 @@ func TestUserGroupLimit(t *testing.T) {
 
 var Ω = gomega.Ω
 var HaveOccurred = gomega.HaveOccurred
+var dev string
diff --git a/test/e2e/user_group_limit/user_group_limit_test.go b/test/e2e/user_group_limit/user_group_limit_test.go
@@ -68,8 +68,6 @@ var (
 	suiteName             string
 	kClient               k8s.KubeCtl
 	restClient            yunikorn.RClient
-	ns                    *v1.Namespace
-	dev                   = "dev" + common.RandSeq(5)
 	oldConfigMap          = new(v1.ConfigMap)
 	admissionCustomConfig = map[string]string{
 		"log.core.scheduler.ugm.level":   "debug",
@@ -92,9 +90,12 @@ var _ = ginkgo.BeforeSuite(func() {
 	ginkgo.By("Port-forward the scheduler pod")
 	var err = kClient.PortForwardYkSchedulerPod()
 	Ω(err).NotTo(gomega.HaveOccurred())
+})
 
+var _ = ginkgo.BeforeEach(func() {
+	dev = "dev" + common.RandSeq(5)
 	ginkgo.By("create development namespace")
-	ns, err = kClient.CreateNamespace(dev, nil)
+	ns, err := kClient.CreateNamespace(dev, nil)
 	gomega.Ω(err).NotTo(gomega.HaveOccurred())
 	gomega.Ω(ns.Status.Phase).To(gomega.Equal(v1.NamespaceActive))
 })
@@ -104,9 +105,6 @@ var _ = ginkgo.AfterSuite(func() {
 	checks, err := yunikorn.GetFailedHealthChecks()
 	Ω(err).NotTo(gomega.HaveOccurred())
 	Ω(checks).To(gomega.Equal(""), checks)
-	ginkgo.By("Tearing down namespace: " + ns.Name)
-	err = kClient.TearDownNamespace(ns.Name)
-	Ω(err).NotTo(gomega.HaveOccurred())
 })
 
 var _ = ginkgo.Describe("UserGroupLimit", func() {
@@ -912,15 +910,10 @@ var _ = ginkgo.Describe("UserGroupLimit", func() {
 	})
 
 	ginkgo.AfterEach(func() {
-		tests.DumpClusterInfoIfSpecFailed(suiteName, []string{ns.Name})
-
-		// Delete all sleep pods
-		ginkgo.By("Delete all sleep pods")
-		err := kClient.DeletePods(ns.Name)
-		if err != nil {
-			fmt.Fprintf(ginkgo.GinkgoWriter, "Failed to delete pods in namespace %s - reason is %s\n", ns.Name, err.Error())
-		}
-
+		tests.DumpClusterInfoIfSpecFailed(suiteName, []string{dev})
+		ginkgo.By("Tearing down namespace: " + dev)
+		err := kClient.TearDownNamespace(dev)
+		Ω(err).NotTo(gomega.HaveOccurred())
 		// reset config
 		ginkgo.By("Restoring YuniKorn configuration")
 		yunikorn.RestoreConfigMapWrapper(oldConfigMap)
Original file line number	Diff line number	Diff line change
Expand Up		@@ -48,3 +48,4 @@ func TestPersistentVolume(t *testing.T) {

		var Ω = gomega.Ω
		var HaveOccurred = gomega.HaveOccurred
		var dev string
Original file line number	Diff line number	Diff line change
Expand Up		@@ -50,3 +50,4 @@ func TestRecoveryAndRestart(t *testing.T) {
		}

		var Ω = gomega.Ω
		var dev string
Original file line number	Diff line number	Diff line change
Expand Up		@@ -48,3 +48,4 @@ func TestSimplePreemptor(t *testing.T) {

		var Ω = gomega.Ω
		var HaveOccurred = gomega.HaveOccurred
		var dev string
Original file line number	Diff line number	Diff line change
Expand Up		@@ -48,3 +48,4 @@ func TestUserGroupLimit(t *testing.T) {

		var Ω = gomega.Ω
		var HaveOccurred = gomega.HaveOccurred
		var dev string