test(kubernetes): add the upgrade tests on Kubernetes

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/onsi/gomega v1.36.2
 	github.com/spf13/cobra v1.8.1
 	k8s.io/api v0.32.0 // indirect
-	k8s.io/apimachinery v0.32.0 // indirect
+	k8s.io/apimachinery v0.32.0
 	k8s.io/client-go v0.32.0
 	k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect
 )

mk/build.mk

@@ -3,3 +3,8 @@
 build:
 	@mkdir -p build
 	@go build -o ./build/kuma-smoke ./cmd/.
+
+.PHONY: clean
+clean: cleanup-kubernetes
+	@rm -rf build
+	@rm -f raw-report.json

mk/run.mk

@@ -1,3 +1,16 @@
+# Extract major, minor, and patch versions
+MAJOR := $(word 1,$(subst ., ,$(SMOKE_PRODUCT_VERSION)))
+MINOR := $(word 2,$(subst ., ,$(SMOKE_PRODUCT_VERSION)))
+PATCH := $(word 3,$(subst ., ,$(SMOKE_PRODUCT_VERSION)))
+
+PREV_MINOR := $(if $(filter-out 1,$(MAJOR)),$(shell echo $$(($(MINOR)-1))),$(MINOR))
+SMOKE_PRODUCT_VERSION_PREV_MINOR := $(MAJOR).$(PREV_MINOR).0
+KUMACTLBIN_PREV_MINOR = $(TOP)/build/$(SMOKE_PRODUCT_NAME)-$(SMOKE_PRODUCT_VERSION_PREV_MINOR)/bin/kumactl
+
+PREV_PATCH := $(if $(filter-out 0,$(PATCH)),$(shell echo $$(($(PATCH)-1))),0)
+SMOKE_PRODUCT_VERSION_PREV_PATCH := $(MAJOR).$(MINOR).$(PREV_PATCH)
+KUMACTLBIN_PREV_PATCH = $(TOP)/build/$(SMOKE_PRODUCT_NAME)-$(SMOKE_PRODUCT_VERSION_PREV_PATCH)/bin/kumactl
+
 KUMACTLBIN = $(TOP)/build/$(SMOKE_PRODUCT_NAME)-$(SMOKE_PRODUCT_VERSION)/bin/kumactl
 
 E2E_ENV_VARS += KUMA_K8S_TYPE=kind
@@ -6,6 +19,10 @@ E2E_ENV_VARS += E2E_CONFIG_FILE="$(TOP)/test/cfg-$(SMOKE_PRODUCT_NAME).yaml"
 E2E_ENV_VARS += KUMA_DEBUG_DIR="$(TOP)/build/debug-output"
 E2E_ENV_VARS += KUMACTLBIN="$(KUMACTLBIN)"
 E2E_ENV_VARS += KUMA_GLOBAL_IMAGE_TAG="$(SMOKE_PRODUCT_VERSION)"
+E2E_ENV_VARS += KUMACTLBIN_PREV_MINOR="$(KUMACTLBIN_PREV_MINOR)"
+E2E_ENV_VARS += SMOKE_PRODUCT_VERSION_PREV_MINOR="$(SMOKE_PRODUCT_VERSION_PREV_MINOR)"
+E2E_ENV_VARS += KUMACTLBIN_PREV_PATCH="$(KUMACTLBIN_PREV_PATCH)"
+E2E_ENV_VARS += SMOKE_PRODUCT_VERSION_PREV_PATCH="$(SMOKE_PRODUCT_VERSION_PREV_PATCH)"
 
 INSTALLER_URL=https://kuma.io/installer.sh
 ifeq ($(SMOKE_PRODUCT_NAME),kong-mesh)
@@ -16,6 +33,8 @@ endif
 fetch-product:
 	@mkdir -p build
 	@[ -f $(KUMACTLBIN) ] || (cd build && echo "Downloading installer of $(SMOKE_PRODUCT_NAME) (version $(SMOKE_PRODUCT_VERSION))" && curl -L $(INSTALLER_URL) | VERSION=$(SMOKE_PRODUCT_VERSION) sh -)
+	@[ -f $(KUMACTLBIN_PREV_MINOR) ] || (cd build && echo "Downloading installer of $(SMOKE_PRODUCT_NAME) (version $(SMOKE_PRODUCT_VERSION_PREV_MINOR))" && curl -L $(INSTALLER_URL) | VERSION=$(SMOKE_PRODUCT_VERSION_PREV_MINOR) sh -)
+	@[ -f $(KUMACTLBIN_PREV_PATCH) ] || (cd build && echo "Downloading installer of $(SMOKE_PRODUCT_NAME) (version $(SMOKE_PRODUCT_VERSION_PREV_PATCH))" && curl -L $(INSTALLER_URL) | VERSION=$(SMOKE_PRODUCT_VERSION_PREV_PATCH) sh -)
 
 .PHONY: deploy-kubernetes
 deploy-kubernetes:

test/kubernetes/install_test.go

@@ -42,7 +42,7 @@ func Install() {
 
 		BeforeAll(func() {
 			err := NewClusterSetup().
-				Install(Kuma(core.Zone, createKumaDeployOptions(installMode, cni)...)).
+				Install(Kuma(core.Zone, createKumaDeployOptions(installMode, cni, Config.KumaImageTag)...)).
 				Install(NamespaceWithSidecarInjection(TestNamespace)).
 				Setup(cluster)
 			Expect(err).ToNot(HaveOccurred())
@@ -92,7 +92,9 @@ spec:
 			Expect(cluster.Install(YamlK8s(demoAppYAML))).To(Succeed())
 
 			for _, fn := range []InstallFunc{
+				WaitNumPods(TestNamespace, 1, demoApp),
 				WaitPodsAvailable(TestNamespace, demoApp),
+				WaitNumPods(TestNamespace, 1, demoGateway),
 				WaitPodsAvailable(TestNamespace, demoGateway)} {
 				Expect(fn(cluster)).To(Succeed())
 			}
@@ -108,7 +110,7 @@ spec:
 		})
 
 		It("should distribute certs when mTLS is enabled", func() {
-			By("enable mTLS on the cluster")
+			By("enable mTLS on the mesh")
 			Expect(cluster.Install(MTLSMeshKubernetes(defaultMesh))).To(Succeed())
 
 			Eventually(func(g Gomega) {
@@ -130,21 +132,7 @@ spec:
 			})
 
 			By("the demo-app should be requested successfully with a MeshTrafficPermission applied")
-			mtp := `
-apiVersion: kuma.io/v1alpha1
-kind: MeshTrafficPermission
-metadata:
-  name: allow-any
-  namespace: %s
-spec:
-  targetRef:
-    kind: Mesh
-  from:
-    - targetRef:
-        kind: Mesh
-      default:
-        action: Allow`
-			Expect(cluster.Install(YamlK8s(fmt.Sprintf(mtp, Config.KumaNamespace)))).To(Succeed())
+			Expect(cluster.Install(YamlK8s(meshTrafficPermission(Config.KumaNamespace)))).To(Succeed())
 
 			By("request the demo app")
 			requestFromGateway(demoGateway, "", "/", func(g Gomega, out string) {
@@ -171,7 +159,52 @@ spec:
 			Expect(err).ToNot(HaveOccurred())
 
 			By("install the GatewayAPI resources using Kong Gateway")
-			ingress := `
+			Expect(cluster.Install(YamlK8s(demoAppGatewayResources(kicName, TestNamespace)))).To(Succeed())
+
+			By("request the demo app via the delegated gateway")
+			requestFromGateway(demoGateway, kicIP, "/", func(g Gomega, out string) {
+				g.Expect(out).To(ContainSubstring("200 OK"))
+			})
+		})
+
+		It("should maintain a stable control plane", func() {
+			time.Sleep(10 * time.Second)
+
+			Expect(CpRestarted(cluster)).To(BeFalse(), cluster.Name()+" restarted in this suite, this should not happen.")
+
+			logOutputFile := filepath.Join(Config.DebugDir, fmt.Sprintf("%s-install-logs-%s-%s.log",
+				Config.KumaServiceName, installMode, cni))
+			logs, err := cluster.GetKumaCPLogs()
+
+			Expect(err).To(Not(HaveOccurred()))
+			Expect(os.WriteFile(logOutputFile, []byte(logs), 0o600)).To(Succeed())
+		})
+	},
+		Entry("kumactl, kuma-init (CNI disabled)", KumactlInstallationMode, cniDisabled),
+		Entry("helm, kuma-cni (CNI enabled)", HelmInstallationMode, cniEnabled),
+	)
+}
+
+func meshTrafficPermission(namespace string) string {
+	mtp := `
+apiVersion: kuma.io/v1alpha1
+kind: MeshTrafficPermission
+metadata:
+  name: allow-any
+  namespace: %s
+spec:
+  targetRef:
+    kind: Mesh
+  from:
+    - targetRef:
+        kind: Mesh
+      default:
+        action: Allow`
+	return fmt.Sprintf(mtp, namespace)
+}
+
+func demoAppGatewayResources(kic, appNamespace string) string {
+	ingress := `
 ---
 apiVersion: gateway.networking.k8s.io/v1
 kind: GatewayClass
@@ -219,32 +252,9 @@ spec:
         type: PathPrefix
         value: /
 `
-			ingress = strings.Replace(ingress, "KIC_NAME", kicName, -1)
-			ingress = strings.Replace(ingress, "APP_NAMESPACE", TestNamespace, -1)
-			Expect(cluster.Install(YamlK8s(ingress))).To(Succeed())
-
-			By("request the demo app via the delegated gateway")
-			requestFromGateway(demoGateway, kicIP, "/", func(g Gomega, out string) {
-				g.Expect(out).To(ContainSubstring("200 OK"))
-			})
-		})
-
-		It("should maintain a stable control plane", func() {
-			time.Sleep(10 * time.Second)
-
-			Expect(CpRestarted(cluster)).To(BeFalse(), cluster.Name()+" restarted in this suite, this should not happen.")
-
-			logOutputFile := filepath.Join(Config.DebugDir, fmt.Sprintf("%s-logs-%s-%s.log",
-				Config.KumaServiceName, installMode, cni))
-			logs, err := cluster.GetKumaCPLogs()
-
-			Expect(err).To(Not(HaveOccurred()))
-			Expect(os.WriteFile(logOutputFile, []byte(logs), 0o600)).To(Succeed())
-		})
-	},
-		Entry("kumactl, kuma-init (CNI disabled)", KumactlInstallationMode, cniDisabled),
-		Entry("helm, kuma-cni (CNI enabled)", HelmInstallationMode, cniEnabled),
-	)
+	ingress = strings.Replace(ingress, "KIC_NAME", kic, -1)
+	ingress = strings.Replace(ingress, "APP_NAMESPACE", appNamespace, -1)
+	return ingress
 }
 
 // requestFromGateway uses the builtin gateway pod as the client and requests an endpoint within the cluster

test/kubernetes/suite_test.go

@@ -2,10 +2,12 @@ package kubernetes_test
 
 import (
 	"fmt"
+	"github.com/blang/semver/v4"
 	"github.com/kumahq/kuma/pkg/test"
 	. "github.com/kumahq/kuma/test/framework"
 	. "github.com/onsi/ginkgo/v2"
 	"os"
+	"strings"
 	"testing"
 )
 
@@ -14,8 +16,9 @@ func TestE2E(t *testing.T) {
 }
 
 var cluster *K8sCluster
+var targetVersion, prevMinorVersion, prevPatchVersion semver.Version
 
-func createKumaDeployOptions(installMode InstallationMode, cni cniMode) []KumaDeploymentOption {
+func createKumaDeployOptions(installMode InstallationMode, cni cniMode, version string) []KumaDeploymentOption {
 	opts := []KumaDeploymentOption{
 		WithInstallationMode(installMode),
 	}
@@ -27,7 +30,7 @@ func createKumaDeployOptions(installMode InstallationMode, cni cniMode) []KumaDe
 			WithHelmOpt("controlPlane.resources.limits.memory", "4Gi"),
 			WithHelmChartPath(Config.HelmChartName),
 			WithoutHelmOpt("global.image.tag"),
-			WithHelmChartVersion(Config.KumaImageTag),
+			WithHelmChartVersion(version),
 			WithHelmReleaseName(fmt.Sprintf("smoke-%s-%s", installMode, cni)),
 		)
 	} else {
@@ -47,6 +50,23 @@ func createKumaDeployOptions(installMode InstallationMode, cni cniMode) []KumaDe
 	return opts
 }
 
+func init() {
+	var err error
+	targetVersion, err = semver.Parse(strings.TrimPrefix(Config.KumaImageTag, "v"))
+	if err != nil {
+		panic(fmt.Sprintf("Failed to parse test target version: %s", Config.KumaImageTag))
+	}
+
+	prevMinorVersion, err = semver.Parse(strings.TrimPrefix(os.Getenv("SMOKE_PRODUCT_VERSION_PREV_MINOR"), "v"))
+	if err != nil {
+		panic(fmt.Sprintf("Failed to parse previous minor version: %s", os.Getenv("SMOKE_PRODUCT_VERSION_PREV_MINOR")))
+	}
+	prevPatchVersion, err = semver.Parse(strings.TrimPrefix(os.Getenv("SMOKE_PRODUCT_VERSION_PREV_PATCH"), "v"))
+	if err != nil {
+		panic(fmt.Sprintf("Failed to parse previous patch version: %s", os.Getenv("SMOKE_PRODUCT_VERSION_PREV_PATCH")))
+	}
+}
+
 var _ = BeforeSuite(func() {
 	kubeConfigPath := os.Getenv("KUBECONFIG")
 	if kubeConfigPath == "" {
@@ -58,5 +78,6 @@ var _ = BeforeSuite(func() {
 })
 
 var (
-	_ = Describe("Single Zone on Kubernetes", Install, Ordered)
+	_ = Describe("Single Zone on Kubernetes - Install", Install, Ordered)
+	_ = Describe("Single Zone on Kubernetes - Upgrade", Upgrade, Ordered)
 )