e2e: introduce upgrade suite

Makefile

@@ -145,6 +145,9 @@ test-install-e2e: build-e2e-all
 test-uninstall-e2e: build-e2e-all
 	hack/run-test-uninstall-e2e.sh
 
+test-upgrade-e2e: build-e2e-all
+	hack/run-test-upgrade-e2e.sh
+
 test-must-gather-e2e: build-must-gather-e2e
 	hack/run-test-must-gather-e2e.sh
 
@@ -188,6 +191,9 @@ binary-e2e-rte: binary-e2e-rte-local generate-source
 binary-e2e-install: generate-source
 	go test -v -c -o bin/e2e-nrop-install.test ./test/e2e/install && go test -v -c -o bin/e2e-nrop-sched-install.test ./test/e2e/sched/install
 
+binary-e2e-upgrade: generate-source
+	go test -v -c -o bin/e2e-nrop-upgrade.test ./test/e2e/upgrade
+
 binary-e2e-uninstall: generate-source
 	go test -v -c -o bin/e2e-nrop-uninstall.test ./test/e2e/uninstall && go test -v -c -o bin/e2e-nrop-sched-uninstall.test ./test/e2e/sched/uninstall
 
@@ -208,6 +214,7 @@ binary-must-gather-e2e: binary-e2e-must-gather
 
 binary-e2e-all: goversion \
 	binary-e2e-install \
+	binary-e2e-upgrade \
 	binary-e2e-rte \
 	binary-e2e-sched \
 	binary-e2e-uninstall \
@@ -244,6 +251,8 @@ build-e2e-rte: generate-source fmt vet binary-e2e-rte
 
 build-e2e-install: fmt vet binary-e2e-install
 
+build-e2e-upgrade: fmt vet binary-e2e-upgrade
+
 build-e2e-uninstall: fmt vet binary-e2e-uninstall
 
 build-e2e-all: generate-source fmt vet binary-e2e-all

hack/run-test-upgrade-e2e.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+source hack/common.sh
+
+NO_COLOR=""
+if ! which tput &> /dev/null 2>&1 || [[ $(tput -T$TERM colors) -lt 8 ]]; then
+  echo "Terminal does not seem to support colored output, disabling it"
+  NO_COLOR="-ginkgo.no-color"
+fi
+
+setupreport
+
+# Run upgrade test suite
+echo "Running NRO upgrade test suite"
+if ! "${BIN_DIR}"/e2e-nrop-upgrade.test ${NO_COLOR} --ginkgo.v --ginkgo.timeout=1h --ginkgo.fail-fast --ginkgo.junit-report=${REPORT_DIR}/install.xml --ginkgo.label-filter="upgrade"; then
+  echo "Failed to run NRO upgrade test suite"
+  exit 1
+fi

test/e2e/upgrade/test_suite_upgrade_test.go

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2024 Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package upgrade
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	e2eclient "github.com/openshift-kni/numaresources-operator/test/utils/clients"
+)
+
+func TestInstall(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Upgrade")
+}
+
+var _ = BeforeSuite(func() {
+	Expect(e2eclient.ClientsEnabled).To(BeTrue(), "failed to create runtime-controller client")
+})

test/e2e/upgrade/upgrade_test.go

@@ -0,0 +1,101 @@
+/*
+ * Copyright 2024 Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package upgrade
+
+import (
+	"context"
+	"encoding/json"
+	"path/filepath"
+	"strings"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/util/version"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	nropv1 "github.com/openshift-kni/numaresources-operator/api/numaresourcesoperator/v1"
+	"github.com/openshift-kni/numaresources-operator/internal/api/annotations"
+	"github.com/openshift-kni/numaresources-operator/internal/api/buildinfo"
+	nropmcp "github.com/openshift-kni/numaresources-operator/internal/machineconfigpools"
+	"github.com/openshift-kni/numaresources-operator/internal/remoteexec"
+	"github.com/openshift-kni/numaresources-operator/pkg/objectnames"
+	e2eclient "github.com/openshift-kni/numaresources-operator/test/utils/clients"
+	"github.com/openshift-kni/numaresources-operator/test/utils/deploy"
+	"github.com/openshift-kni/numaresources-operator/test/utils/objects"
+
+	machineconfigv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
+)
+
+var _ = Describe("Upgrade", Label("upgrade"), func() {
+	var err error
+	var initialized bool
+	BeforeEach(func() {
+		if !initialized {
+			Expect(e2eclient.ClientsEnabled).To(BeTrue(), "failed to create runtime-controller client")
+		}
+
+		nropObj := objects.TestNRO()
+		nname := client.ObjectKeyFromObject(nropObj)
+
+		err = e2eclient.Client.Get(context.TODO(), nname, nropObj)
+		Expect(err).NotTo(HaveOccurred(), "failed to get the NRO resource: %v", err)
+
+		pod, err := deploy.FindNUMAResourcesOperatorPod(context.TODO(), e2eclient.Client, nropObj)
+		Expect(err).ToNot(HaveOccurred())
+		stdout, _, err := remoteexec.CommandOnPod(context.TODO(), e2eclient.K8sClient, pod, "/bin/cat", filepath.Join("/usr/local/share/", "buildinfo.json"))
+		Expect(err).ToNot(HaveOccurred())
+
+		bi := &buildinfo.BuildInfo{}
+		err = json.Unmarshal(stdout, &bi)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(bi.Branch).ToNot(BeEmpty())
+		operatorVersion := version.MustParse(strings.TrimPrefix(bi.Branch, "release-"))
+		minVersion := version.MustParse("4.18")
+		if operatorVersion.LessThan(minVersion) {
+			Skip("Upgrade suite is only supported on operator versions 4.18 or newer")
+		}
+		initialized = true
+	})
+
+	Context("after operator upgrade", func() {
+		It("should remove machineconfigs when no SElinux policy annotation is present", func() {
+			updatedNROObj := &nropv1.NUMAResourcesOperator{}
+
+			err := e2eclient.Client.Get(context.TODO(), objects.NROObjectKey(), updatedNROObj)
+			Expect(err).NotTo(HaveOccurred())
+
+			if annotations.IsCustomPolicyEnabled(updatedNROObj.Annotations) {
+				Skip("SElinux policy annotation is present")
+			}
+			mcps, err := nropmcp.GetListByNodeGroupsV1(context.TODO(), e2eclient.Client, updatedNROObj.Spec.NodeGroups)
+			Expect(err).NotTo(HaveOccurred())
+
+			for _, mcp := range mcps {
+				mc := &machineconfigv1.MachineConfig{}
+				// Check mc not created
+				mcKey := client.ObjectKey{
+					Name: objectnames.GetMachineConfigName(updatedNROObj.Name, mcp.Name),
+				}
+
+				err := e2eclient.Client.Get(context.TODO(), mcKey, mc)
+				Expect(err).ToNot(BeNil(), "MachineConfig %s is not expected to to be present", mcKey.String())
+				Expect(errors.IsNotFound(err)).To(BeTrue(), "Unexpected error occurred while getting MachineConfig %s: %v", mcKey.String(), err)
+			}
+		})
+	})
+})

test/utils/deploy/find.go

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2024 Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package deploy
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/klog/v2"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	nropv1 "github.com/openshift-kni/numaresources-operator/api/numaresourcesoperator/v1"
+)
+
+func FindNUMAResourcesOperatorPod(ctx context.Context, cli client.Client, nrop *nropv1.NUMAResourcesOperator) (*corev1.Pod, error) {
+	if len(nrop.Status.NodeGroups) < 1 {
+		return nil, errors.New("node groups not reported, nothing to do")
+	}
+	// nrop places all daemonsets in the same namespace on which it resides, so any group is fine
+	namespace := nrop.Status.NodeGroups[0].DaemonSet.Namespace // shortcut
+	klog.InfoS("NROP pod", "namespace", namespace)
+
+	sel, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{
+		MatchLabels: map[string]string{
+			"app": "numaresources-operator",
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+	klog.InfoS("NROP pod", "selector", sel.String())
+
+	podList := corev1.PodList{}
+	err = cli.List(ctx, &podList, &client.ListOptions{Namespace: namespace, LabelSelector: sel})
+	if err != nil {
+		return nil, err
+	}
+
+	if len(podList.Items) != 1 {
+		return nil, fmt.Errorf("unexpected number of pods found: %d", len(podList.Items))
+	}
+
+	return &podList.Items[0], nil
+}