test(kubernetes): improve the upgrade test

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/onsi/gomega v1.36.2
 	github.com/spf13/cobra v1.8.1
 	k8s.io/api v0.32.0 // indirect
-	k8s.io/apimachinery v0.32.0 // indirect
+	k8s.io/apimachinery v0.32.0
 	k8s.io/client-go v0.32.0
 	k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect
 )

mk/run.mk

@@ -1,3 +1,16 @@
+# Extract major, minor, and patch versions
+MAJOR := $(word 1,$(subst ., ,$(SMOKE_PRODUCT_VERSION)))
+MINOR := $(word 2,$(subst ., ,$(SMOKE_PRODUCT_VERSION)))
+PATCH := $(word 3,$(subst ., ,$(SMOKE_PRODUCT_VERSION)))
+
+PREV_MINOR := $(if $(filter-out 1,$(MAJOR)),$(shell echo $$(($(MINOR)-1))),$(MINOR))
+SMOKE_PRODUCT_VERSION_PREV_MINOR := $(MAJOR).$(PREV_MINOR).0
+KUMACTLBIN_PREV_MINOR = $(TOP)/build/$(SMOKE_PRODUCT_NAME)-$(SMOKE_PRODUCT_VERSION_PREV_MINOR)/bin/kumactl
+
+PREV_PATCH := $(if $(filter-out 0,$(PATCH)),$(shell echo $$(($(PATCH)-1))),0)
+SMOKE_PRODUCT_VERSION_PREV_PATCH := $(MAJOR).$(MINOR).$(PREV_PATCH)
+KUMACTLBIN_PREV_PATCH = $(TOP)/build/$(SMOKE_PRODUCT_NAME)-$(SMOKE_PRODUCT_VERSION_PREV_PATCH)/bin/kumactl
+
 KUMACTLBIN = $(TOP)/build/$(SMOKE_PRODUCT_NAME)-$(SMOKE_PRODUCT_VERSION)/bin/kumactl
 
 E2E_ENV_VARS += KUMA_K8S_TYPE=kind
@@ -6,6 +19,10 @@ E2E_ENV_VARS += E2E_CONFIG_FILE="$(TOP)/test/cfg-$(SMOKE_PRODUCT_NAME).yaml"
 E2E_ENV_VARS += KUMA_DEBUG_DIR="$(TOP)/build/debug-output"
 E2E_ENV_VARS += KUMACTLBIN="$(KUMACTLBIN)"
 E2E_ENV_VARS += KUMA_GLOBAL_IMAGE_TAG="$(SMOKE_PRODUCT_VERSION)"
+E2E_ENV_VARS += KUMACTLBIN_PREV_MINOR="$(KUMACTLBIN_PREV_MINOR)"
+E2E_ENV_VARS += SMOKE_PRODUCT_VERSION_PREV_MINOR="$(SMOKE_PRODUCT_VERSION_PREV_MINOR)"
+E2E_ENV_VARS += KUMACTLBIN_PREV_PATCH="$(KUMACTLBIN_PREV_PATCH)"
+E2E_ENV_VARS += SMOKE_PRODUCT_VERSION_PREV_PATCH="$(SMOKE_PRODUCT_VERSION_PREV_PATCH)"
 
 INSTALLER_URL=https://kuma.io/installer.sh
 ifeq ($(SMOKE_PRODUCT_NAME),kong-mesh)
@@ -16,6 +33,8 @@ endif
 fetch-product:
 	@mkdir -p build
 	@[ -f $(KUMACTLBIN) ] || (cd build && echo "Downloading installer of $(SMOKE_PRODUCT_NAME) (version $(SMOKE_PRODUCT_VERSION))" && curl -L $(INSTALLER_URL) | VERSION=$(SMOKE_PRODUCT_VERSION) sh -)
+	@[ -f $(KUMACTLBIN_PREV_MINOR) ] || (cd build && echo "Downloading installer of $(SMOKE_PRODUCT_NAME) (version $(SMOKE_PRODUCT_VERSION_PREV_MINOR))" && curl -L $(INSTALLER_URL) | VERSION=$(SMOKE_PRODUCT_VERSION_PREV_MINOR) sh -)
+	@[ -f $(KUMACTLBIN_PREV_PATCH) ] || (cd build && echo "Downloading installer of $(SMOKE_PRODUCT_NAME) (version $(SMOKE_PRODUCT_VERSION_PREV_PATCH))" && curl -L $(INSTALLER_URL) | VERSION=$(SMOKE_PRODUCT_VERSION_PREV_PATCH) sh -)
 
 .PHONY: deploy-kubernetes
 deploy-kubernetes:

test/kubernetes/install_test.go

@@ -92,7 +92,9 @@ spec:
 			Expect(cluster.Install(YamlK8s(demoAppYAML))).To(Succeed())
 
 			for _, fn := range []InstallFunc{
+				WaitNumPods(TestNamespace, 1, demoApp),
 				WaitPodsAvailable(TestNamespace, demoApp),
+				WaitNumPods(TestNamespace, 1, demoGateway),
 				WaitPodsAvailable(TestNamespace, demoGateway)} {
 				Expect(fn(cluster)).To(Succeed())
 			}

test/kubernetes/suite_test.go

@@ -2,10 +2,12 @@ package kubernetes_test
 
 import (
 	"fmt"
+	"github.com/blang/semver/v4"
 	"github.com/kumahq/kuma/pkg/test"
 	. "github.com/kumahq/kuma/test/framework"
 	. "github.com/onsi/ginkgo/v2"
 	"os"
+	"strings"
 	"testing"
 )
 
@@ -14,6 +16,7 @@ func TestE2E(t *testing.T) {
 }
 
 var cluster *K8sCluster
+var currentVersion, prevMinorVersion, prevPatchVersion semver.Version
 
 func createKumaDeployOptions(installMode InstallationMode, cni cniMode, version string) []KumaDeploymentOption {
 	opts := []KumaDeploymentOption{
@@ -31,7 +34,6 @@ func createKumaDeployOptions(installMode InstallationMode, cni cniMode, version
 			WithHelmReleaseName(fmt.Sprintf("smoke-%s-%s", installMode, cni)),
 		)
 	} else {
-		// todo: support version?
 		opts = append(opts,
 			WithCtlOpts(map[string]string{
 				"--set": "" +
@@ -48,6 +50,23 @@ func createKumaDeployOptions(installMode InstallationMode, cni cniMode, version
 	return opts
 }
 
+func init() {
+	var err error
+	currentVersion, err = semver.Parse(strings.TrimPrefix(Config.KumaImageTag, "v"))
+	if err != nil {
+		panic(fmt.Sprintf("Failed to parse current version: %s", Config.KumaImageTag))
+	}
+
+	prevMinorVersion, err = semver.Parse(strings.TrimPrefix(os.Getenv("SMOKE_PRODUCT_VERSION_PREV_MINOR"), "v"))
+	if err != nil {
+		panic(fmt.Sprintf("Failed to parse previous minor version: %s", os.Getenv("SMOKE_PRODUCT_VERSION_PREV_MINOR")))
+	}
+	prevPatchVersion, err = semver.Parse(strings.TrimPrefix(os.Getenv("SMOKE_PRODUCT_VERSION_PREV_PATCH"), "v"))
+	if err != nil {
+		panic(fmt.Sprintf("Failed to parse previous patch version: %s", os.Getenv("SMOKE_PRODUCT_VERSION_PREV_PATCH")))
+	}
+}
+
 var _ = BeforeSuite(func() {
 	kubeConfigPath := os.Getenv("KUBECONFIG")
 	if kubeConfigPath == "" {

test/kubernetes/upgrade_test.go

@@ -10,7 +10,6 @@ import (
 	"github.com/pkg/errors"
 	"os"
 	"path/filepath"
-	"strings"
 	"time"
 
 	"github.com/kumahq/kuma/pkg/config/core"
@@ -27,48 +26,30 @@ func Upgrade() {
 	kicName := "kic"
 	stabilizationDuration := 30 * time.Second
 
-	currentVersion, err := semver.Parse(strings.TrimPrefix(Config.KumaImageTag, "v"))
-	Expect(err).ToNot(HaveOccurred())
-	prevMinorVersion := currentVersion
-	if prevMinorVersion.Major > 1 {
-		prevMinorVersion.Minor--
-		prevMinorVersion.Patch = 0
-	}
-	prevPatchVersion := currentVersion
-	if currentVersion.Patch > 0 {
-		prevPatchVersion.Patch--
-	}
-
 	DescribeTableSubtree("upgrade Kuma with a running workload", func(prevVersion semver.Version, installMode InstallationMode, cni cniMode) {
 		if prevVersion.String() == currentVersion.String() {
-			Skip(fmt.Sprintf("Skipping because the previous version is the same as the current version %s", currentVersion))
+			Logf("Skipping because the previous version is the same as the current version %s", currentVersion)
 			return
 		}
+		originalKumactl := Config.KumactlBin
+		originalImageTag := Config.KumaImageTag
 
 		BeforeAll(func() {
-			if installMode == HelmInstallationMode {
-				setupHelmRepo(cluster.GetTesting())
+			Logf("Testing upgrading from %s to %s", prevVersion, currentVersion)
+			versionEnvName := "KUMACTLBIN_PREV_MINOR"
+			if prevVersion.String() == prevPatchVersion.String() {
+				versionEnvName = "KUMACTLBIN_PREV_PATCH"
 			}
-		})
-
-		originalKumactl := ""
-		if cluster.GetKuma() != nil {
-			originalKumactl = cluster.GetKumactlOptions().Kumactl
-		}
-
-		BeforeAll(func() {
-			versionEnvName := "KUMACTLBIN_" + prevVersion.String()
-			versionEnvName = strings.Replace(versionEnvName, ".", "_", -1)
-			versionEnvName = strings.Replace(versionEnvName, "-", "_", -1)
-			versionEnvName = strings.ToUpper(versionEnvName)
 			prevKumactl := os.Getenv(versionEnvName)
 			if installMode == KumactlInstallationMode {
 				if prevKumactl == "" {
 					Fail(fmt.Sprintf("Please set path to version %s kumactl using envirionment variable %s", prevVersion, versionEnvName))
 					return
 				}
-
-				cluster.GetKumactlOptions().Kumactl = prevKumactl
+				Config.KumactlBin = prevKumactl
+				Config.KumaImageTag = prevVersion.String()
+			} else {
+				setupHelmRepo(cluster.GetTesting())
 			}
 
 			err := NewClusterSetup().
@@ -82,9 +63,9 @@ func Upgrade() {
 			Expect(cluster.DeleteNamespace(TestNamespace)).To(Succeed())
 			Expect(cluster.DeleteNamespace(kicName)).To(Succeed())
 			Expect(cluster.DeleteKuma()).To(Succeed())
-			if originalKumactl != "" {
-				cluster.GetKumactlOptions().Kumactl = originalKumactl
-			}
+			cluster.SetCP(nil)
+			Config.KumactlBin = originalKumactl
+			Config.KumaImageTag = originalImageTag
 		})
 
 		It("should run the demo app with mTLS and gateways", func() {
@@ -97,7 +78,9 @@ func Upgrade() {
 			Expect(cluster.Install(YamlK8s(demoAppYAML))).To(Succeed())
 
 			for _, fn := range []InstallFunc{
+				WaitNumPods(TestNamespace, 1, demoApp),
 				WaitPodsAvailable(TestNamespace, demoApp),
+				WaitNumPods(TestNamespace, 1, demoGateway),
 				WaitPodsAvailable(TestNamespace, demoGateway)} {
 				Expect(fn(cluster)).To(Succeed())
 			}