Remove completed pods in e2e

EKS cluster suffers from limited number of private IPs, which are exausted by test suites running in parallel. It affects GitRepo processing, where a pod created to update content of the repo in order to apply manifests to the cluster. Since the pod is not able to be scheduled, e2e tests are failing. Signed-off-by: Danil-Grigorev <[email protected]>
rancher · Jan 22, 2025 · d01728d · d01728d
1 parent 3d5dd5d
commit d01728d
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 7 deletions.
diff --git a/test/e2e/specs/import_gitops_mgmtv3.go b/test/e2e/specs/import_gitops_mgmtv3.go
@@ -93,7 +93,7 @@ type CreateMgmtV3UsingGitOpsSpecInput struct {
 // automatically imports into Rancher Manager.
 func CreateMgmtV3UsingGitOpsSpec(ctx context.Context, inputGetter func() CreateMgmtV3UsingGitOpsSpecInput) {
 	var (
-		specName              = "creategitops-v3"
+		specName              = "creategitops"
 		input                 CreateMgmtV3UsingGitOpsSpecInput
 		namespace             *corev1.Namespace
 		repoName              string
@@ -172,6 +172,7 @@ func CreateMgmtV3UsingGitOpsSpec(ctx context.Context, inputGetter func() CreateM
 		input = inputGetter()
 		Expect(turtlesframework.Parse(&input)).To(Succeed(), "Failed to parse environment variables")
 
+		Expect(input.GitAddr).ToNot(BeEmpty(), "Invalid argument. input.GitAddr can't be empty when calling %s spec", specName)
 		Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName)
 		Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName)
 		Expect(os.MkdirAll(input.ArtifactFolder, 0750)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName)

diff --git a/test/e2e/suites/import-gitops-v3/import_gitops_v3_test.go b/test/e2e/suites/import-gitops-v3/import_gitops_v3_test.go
@@ -98,7 +98,7 @@ var _ = Describe("[Azure] [AKS] - [management.cattle.io/v3] Create and delete CA
 			E2EConfig:                      e2e.LoadE2EConfig(),
 			BootstrapClusterProxy:          bootstrapClusterProxy,
 			ClusterTemplate:                e2e.CAPIAzureAKSTopology,
-			ClusterName:                    "highlander-e2e-topology",
+			ClusterName:                    "cluster-aks",
 			ControlPlaneMachineCount:       ptr.To[int](1),
 			WorkerMachineCount:             ptr.To[int](1),
 			GitAddr:                        gitAddress,
@@ -125,7 +125,7 @@ var _ = Describe("[AWS] [EKS] - [management.cattle.io/v3] Create and delete CAPI
 			E2EConfig:                      e2e.LoadE2EConfig(),
 			BootstrapClusterProxy:          bootstrapClusterProxy,
 			ClusterTemplate:                e2e.CAPIAwsEKSMMP,
-			ClusterName:                    "clusterv3-eks",
+			ClusterName:                    "cluster-eks",
 			ControlPlaneMachineCount:       ptr.To[int](1),
 			WorkerMachineCount:             ptr.To[int](1),
 			GitAddr:                        gitAddress,
@@ -152,7 +152,7 @@ var _ = Describe("[GCP] [GKE] - [management.cattle.io/v3] Create and delete CAPI
 			E2EConfig:                      e2e.LoadE2EConfig(),
 			BootstrapClusterProxy:          bootstrapClusterProxy,
 			ClusterTemplate:                e2e.CAPIGCPGKE,
-			ClusterName:                    "clusterv3-gke",
+			ClusterName:                    "cluster-gke",
 			ControlPlaneMachineCount:       ptr.To[int](1),
 			WorkerMachineCount:             ptr.To[int](1),
 			GitAddr:                        gitAddress,

diff --git a/test/e2e/suites/import-gitops-v3/suite_test.go b/test/e2e/suites/import-gitops-v3/suite_test.go
@@ -21,13 +21,13 @@ package import_gitops_v3
 
 import (
 	"context"
-	"encoding/json"
 	"testing"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/rancher/turtles/test/e2e"
 	"github.com/rancher/turtles/test/testenv"
+	"k8s.io/apimachinery/pkg/util/json"
 	capiframework "sigs.k8s.io/cluster-api/test/framework"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -132,6 +132,8 @@ var _ = SynchronizedBeforeSuite(
 			CustomIngressConfig:   e2e.GiteaIngress,
 		})
 
+		Expect(testenv.CleanupClusterPodPool(ctx, setupClusterResult.BootstrapClusterProxy.GetClient())).To(Succeed())
+
 		data, err := json.Marshal(e2e.Setup{
 			ClusterName:     setupClusterResult.ClusterName,
 			KubeconfigPath:  setupClusterResult.KubeconfigPath,
@@ -158,11 +160,11 @@ var _ = SynchronizedAfterSuite(
 	},
 	func() {
 		testenv.UninstallGitea(ctx, testenv.UninstallGiteaInput{
-			BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
+			BootstrapClusterProxy: bootstrapClusterProxy,
 		})
 
 		testenv.UninstallRancherTurtles(ctx, testenv.UninstallRancherTurtlesInput{
-			BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
+			BootstrapClusterProxy: bootstrapClusterProxy,
 		})
 
 		testenv.CleanupTestCluster(ctx, testenv.CleanupTestClusterInput{

diff --git a/test/testenv/cleanup.go b/test/testenv/cleanup.go
@@ -29,6 +29,10 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	turtlesframework "github.com/rancher/turtles/test/framework"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/selection"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 )
 
@@ -139,3 +143,16 @@ func dumpBootstrapCluster(ctx context.Context) {
 		return
 	}
 }
+
+// CleanupClusterPodPool removes some of the finished pods after rancher rollout to cleanup pod pool to
+// allow scheduling new pods.
+func CleanupClusterPodPool(ctx context.Context, cl client.Client) error {
+	req, err := labels.NewRequirement("pod-impersonation.cattle.io/token", selection.Exists, []string{})
+	if err != nil {
+		return err
+	}
+
+	return cl.DeleteAllOf(ctx, &corev1.Pod{}, client.InNamespace(""), client.MatchingLabelsSelector{
+		Selector: labels.NewSelector().Add(*req),
+	})
+}