e2e: verify NROP machineconfigs are removed

Starting in version 4.18, NROP MachineConfigs containing the custom SELinux policy are expected to be removed unless a specific annotation is set in the NUMAResourcesOperator CR to enforce the use of the custom (legacy) SELinux policy. To ensure this behavior, we added a test that verifies MachineConfigs are removed when the annotation is absent in the CR.

Signed-off-by: Ronny Baturov <[email protected]>

Makefile

@@ -132,6 +132,9 @@ test-e2e: build-e2e-all
 test-install-e2e: build-e2e-all
 	hack/run-test-install-e2e.sh
 
+test-upgrade-e2e: build-e2e-all
+	hack/run-test-upgrade-e2e.sh
+
 test-must-gather-e2e: build-must-gather-e2e
 	hack/run-test-must-gather-e2e.sh
 
@@ -171,6 +174,9 @@ binary-e2e-rte: binary-e2e-rte-local
 binary-e2e-install:
 	go test -v -c -o bin/e2e-nrop-install.test ./test/e2e/install && go test -v -c -o bin/e2e-nrop-sched-install.test ./test/e2e/sched/install
 
+binary-e2e-upgrade:
+	go test -v -c -o bin/e2e-nrop-upgrade.test ./test/e2e/upgrade
+
 binary-e2e-uninstall:
 	go test -v -c -o bin/e2e-nrop-uninstall.test ./test/e2e/uninstall && go test -v -c -o bin/e2e-nrop-sched-uninstall.test ./test/e2e/sched/uninstall
 
@@ -193,6 +199,7 @@ binary-must-gather-e2e: binary-e2e-must-gather
 
 binary-e2e-all: goversion \
 	binary-e2e-install \
+	binary-e2e-upgrade \
 	binary-e2e-rte \
 	binary-e2e-sched \
 	binary-e2e-uninstall \
@@ -223,6 +230,8 @@ build-e2e-rte: fmt vet binary-e2e-rte
 
 build-e2e-install: fmt vet binary-e2e-install
 
+build-e2e-upgrade: fmt vet binary-e2e-upgrade
+
 build-e2e-uninstall: fmt vet binary-e2e-uninstall
 
 build-e2e-all: fmt vet binary-e2e-all

hack/run-test-upgrade-e2e.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+source hack/common.sh
+
+NO_COLOR=""
+if ! which tput &> /dev/null 2>&1 || [[ $(tput -T$TERM colors) -lt 8 ]]; then
+  echo "Terminal does not seem to support colored output, disabling it"
+  NO_COLOR="-ginkgo.no-color"
+fi
+
+setupreport
+
+# Run upgrade test suite
+echo "Running NRO upgrade test suite"
+if ! "${BIN_DIR}"/e2e-nrop-upgrade.test ${NO_COLOR} --ginkgo.v --ginkgo.timeout=1h --ginkgo.fail-fast --ginkgo.junit-report=${REPORT_DIR}/install.xml --ginkgo.focus='\[Upgrade\]'; then
+  echo "Failed to run NRO upgrade test suite"
+  exit 1
+fi

test/e2e/upgrade/test_suite_upgrade_test.go

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2024 Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package upgrade
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	e2eclient "github.com/openshift-kni/numaresources-operator/test/utils/clients"
+)
+
+func TestInstall(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Upgrade")
+}
+
+var _ = BeforeSuite(func() {
+	By("Creating all test resources")
+	Expect(e2eclient.ClientsEnabled).To(BeTrue(), "failed to create runtime-controller client")
+})

test/e2e/upgrade/upgrade_test.go

@@ -0,0 +1,55 @@
+package upgrade
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	nropv1 "github.com/openshift-kni/numaresources-operator/api/numaresourcesoperator/v1"
+	"github.com/openshift-kni/numaresources-operator/internal/api/annotations"
+	nropmcp "github.com/openshift-kni/numaresources-operator/internal/machineconfigpools"
+	"github.com/openshift-kni/numaresources-operator/pkg/objectnames"
+	e2eclient "github.com/openshift-kni/numaresources-operator/test/utils/clients"
+	"github.com/openshift-kni/numaresources-operator/test/utils/objects"
+	machineconfigv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
+)
+
+var _ = Describe("[Upgrade]", func() {
+	var initialized bool
+
+	BeforeEach(func() {
+		if !initialized {
+			Expect(e2eclient.ClientsEnabled).To(BeTrue(), "failed to create runtime-controller client")
+		}
+		initialized = true
+	})
+
+	Context("after operator upgrade", func() {
+		It("should remove machineconfigs when no SElinux policy annotation is present", func() {
+			updatedNROObj := &nropv1.NUMAResourcesOperator{}
+
+			err := e2eclient.Client.Get(context.TODO(), objects.NROObjectKey(), updatedNROObj)
+			Expect(err).NotTo(HaveOccurred())
+
+			if !annotations.IsCustomPolicyEnabled(updatedNROObj.Annotations) {
+				mcps, err := nropmcp.GetListByNodeGroupsV1(context.TODO(), e2eclient.Client, updatedNROObj.Spec.NodeGroups)
+				Expect(err).NotTo(HaveOccurred())
+
+				for _, mcp := range mcps {
+					mc := &machineconfigv1.MachineConfig{}
+					// Check mc not created
+					mcKey := client.ObjectKey{
+						Name: objectnames.GetMachineConfigName(updatedNROObj.Name, mcp.Name),
+					}
+
+					err := e2eclient.Client.Get(context.TODO(), mcKey, mc)
+					Expect(err).ToNot(BeNil(), "MachineConfig %s is not expected to to be present", mcKey.String())
+					Expect(errors.IsNotFound(err)).To(BeTrue(), "Unexpected error occurred while getting MachineConfig %s: %v", mcKey.String(), err)
+				}
+			}
+		})
+	})
+})