diff --git a/Makefile b/Makefile index 2f7006475..e94d3b05d 100644 --- a/Makefile +++ b/Makefile @@ -145,6 +145,9 @@ test-install-e2e: build-e2e-all test-uninstall-e2e: build-e2e-all hack/run-test-uninstall-e2e.sh +test-upgrade-e2e: build-e2e-all + hack/run-test-upgrade-e2e.sh + test-must-gather-e2e: build-must-gather-e2e hack/run-test-must-gather-e2e.sh @@ -188,6 +191,9 @@ binary-e2e-rte: binary-e2e-rte-local generate-source binary-e2e-install: generate-source go test -v -c -o bin/e2e-nrop-install.test ./test/e2e/install && go test -v -c -o bin/e2e-nrop-sched-install.test ./test/e2e/sched/install +binary-e2e-upgrade: generate-source + go test -v -c -o bin/e2e-nrop-upgrade.test ./test/e2e/upgrade + binary-e2e-uninstall: generate-source go test -v -c -o bin/e2e-nrop-uninstall.test ./test/e2e/uninstall && go test -v -c -o bin/e2e-nrop-sched-uninstall.test ./test/e2e/sched/uninstall @@ -208,6 +214,7 @@ binary-must-gather-e2e: binary-e2e-must-gather binary-e2e-all: goversion \ binary-e2e-install \ + binary-e2e-upgrade \ binary-e2e-rte \ binary-e2e-sched \ binary-e2e-uninstall \ @@ -244,6 +251,8 @@ build-e2e-rte: generate-source fmt vet binary-e2e-rte build-e2e-install: fmt vet binary-e2e-install +build-e2e-upgrade: fmt vet binary-e2e-upgrade + build-e2e-uninstall: fmt vet binary-e2e-uninstall build-e2e-all: generate-source fmt vet binary-e2e-all diff --git a/hack/run-test-upgrade-e2e.sh b/hack/run-test-upgrade-e2e.sh new file mode 100755 index 000000000..5420fc096 --- /dev/null +++ b/hack/run-test-upgrade-e2e.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +source hack/common.sh + +NO_COLOR="" +if ! which tput &> /dev/null 2>&1 || [[ $(tput -T$TERM colors) -lt 8 ]]; then + echo "Terminal does not seem to support colored output, disabling it" + NO_COLOR="-ginkgo.no-color" +fi + +setupreport + +# Run upgrade test suite +echo "Running NRO upgrade test suite" +if ! "${BIN_DIR}"/e2e-nrop-upgrade.test ${NO_COLOR} --ginkgo.v --ginkgo.timeout=1h --ginkgo.fail-fast --ginkgo.junit-report=${REPORT_DIR}/install.xml --ginkgo.label-filter="upgrade"; then + echo "Failed to run NRO upgrade test suite" + exit 1 +fi diff --git a/test/e2e/upgrade/test_suite_upgrade_test.go b/test/e2e/upgrade/test_suite_upgrade_test.go new file mode 100644 index 000000000..ae93fb0c5 --- /dev/null +++ b/test/e2e/upgrade/test_suite_upgrade_test.go @@ -0,0 +1,35 @@ +/* + * Copyright 2024 Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package upgrade + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + e2eclient "github.com/openshift-kni/numaresources-operator/test/utils/clients" +) + +func TestInstall(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Upgrade") +} + +var _ = BeforeSuite(func() { + Expect(e2eclient.ClientsEnabled).To(BeTrue(), "failed to create runtime-controller client") +}) diff --git a/test/e2e/upgrade/upgrade_test.go b/test/e2e/upgrade/upgrade_test.go new file mode 100644 index 000000000..33fc60c6f --- /dev/null +++ b/test/e2e/upgrade/upgrade_test.go @@ -0,0 +1,101 @@ +/* + * Copyright 2024 Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package upgrade + +import ( + "context" + "encoding/json" + "path/filepath" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/util/version" + "sigs.k8s.io/controller-runtime/pkg/client" + + nropv1 "github.com/openshift-kni/numaresources-operator/api/numaresourcesoperator/v1" + "github.com/openshift-kni/numaresources-operator/internal/api/annotations" + "github.com/openshift-kni/numaresources-operator/internal/api/buildinfo" + nropmcp "github.com/openshift-kni/numaresources-operator/internal/machineconfigpools" + "github.com/openshift-kni/numaresources-operator/internal/remoteexec" + "github.com/openshift-kni/numaresources-operator/pkg/objectnames" + e2eclient "github.com/openshift-kni/numaresources-operator/test/utils/clients" + "github.com/openshift-kni/numaresources-operator/test/utils/deploy" + "github.com/openshift-kni/numaresources-operator/test/utils/objects" + + machineconfigv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" +) + +var _ = Describe("Upgrade", Label("upgrade"), func() { + var err error + var initialized bool + BeforeEach(func() { + if !initialized { + Expect(e2eclient.ClientsEnabled).To(BeTrue(), "failed to create runtime-controller client") + } + + nropObj := objects.TestNRO() + nname := client.ObjectKeyFromObject(nropObj) + + err = e2eclient.Client.Get(context.TODO(), nname, nropObj) + Expect(err).NotTo(HaveOccurred(), "failed to get the NRO resource: %v", err) + + pod, err := deploy.FindNUMAResourcesOperatorPod(context.TODO(), e2eclient.Client, nropObj) + Expect(err).ToNot(HaveOccurred()) + stdout, _, err := remoteexec.CommandOnPod(context.TODO(), e2eclient.K8sClient, pod, "/bin/cat", filepath.Join("/usr/local/share/", "buildinfo.json")) + Expect(err).ToNot(HaveOccurred()) + + bi := &buildinfo.BuildInfo{} + err = json.Unmarshal(stdout, &bi) + Expect(err).ToNot(HaveOccurred()) + Expect(bi.Branch).ToNot(BeEmpty()) + operatorVersion := version.MustParse(strings.TrimPrefix(bi.Branch, "release-")) + minVersion := version.MustParse("4.18") + if operatorVersion.LessThan(minVersion) { + Skip("Upgrade suite is only supported on operator versions 4.18 or newer") + } + initialized = true + }) + + Context("after operator upgrade", func() { + It("should remove machineconfigs when no SElinux policy annotation is present", func() { + updatedNROObj := &nropv1.NUMAResourcesOperator{} + + err := e2eclient.Client.Get(context.TODO(), objects.NROObjectKey(), updatedNROObj) + Expect(err).NotTo(HaveOccurred()) + + if annotations.IsCustomPolicyEnabled(updatedNROObj.Annotations) { + Skip("SElinux policy annotation is present") + } + mcps, err := nropmcp.GetListByNodeGroupsV1(context.TODO(), e2eclient.Client, updatedNROObj.Spec.NodeGroups) + Expect(err).NotTo(HaveOccurred()) + + for _, mcp := range mcps { + mc := &machineconfigv1.MachineConfig{} + // Check mc not created + mcKey := client.ObjectKey{ + Name: objectnames.GetMachineConfigName(updatedNROObj.Name, mcp.Name), + } + + err := e2eclient.Client.Get(context.TODO(), mcKey, mc) + Expect(err).ToNot(BeNil(), "MachineConfig %s is not expected to to be present", mcKey.String()) + Expect(errors.IsNotFound(err)).To(BeTrue(), "Unexpected error occurred while getting MachineConfig %s: %v", mcKey.String(), err) + } + }) + }) +}) diff --git a/test/utils/deploy/find.go b/test/utils/deploy/find.go new file mode 100644 index 000000000..eccc66eaa --- /dev/null +++ b/test/utils/deploy/find.go @@ -0,0 +1,62 @@ +/* + * Copyright 2024 Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package deploy + +import ( + "context" + "errors" + "fmt" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + + "sigs.k8s.io/controller-runtime/pkg/client" + + nropv1 "github.com/openshift-kni/numaresources-operator/api/numaresourcesoperator/v1" +) + +func FindNUMAResourcesOperatorPod(ctx context.Context, cli client.Client, nrop *nropv1.NUMAResourcesOperator) (*corev1.Pod, error) { + if len(nrop.Status.NodeGroups) < 1 { + return nil, errors.New("node groups not reported, nothing to do") + } + // nrop places all daemonsets in the same namespace on which it resides, so any group is fine + namespace := nrop.Status.NodeGroups[0].DaemonSet.Namespace // shortcut + klog.InfoS("NROP pod", "namespace", namespace) + + sel, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "numaresources-operator", + }, + }) + if err != nil { + return nil, err + } + klog.InfoS("NROP pod", "selector", sel.String()) + + podList := corev1.PodList{} + err = cli.List(ctx, &podList, &client.ListOptions{Namespace: namespace, LabelSelector: sel}) + if err != nil { + return nil, err + } + + if len(podList.Items) != 1 { + return nil, fmt.Errorf("unexpected number of pods found: %d", len(podList.Items)) + } + + return &podList.Items[0], nil +}