New operator tests w real tests

.golangci.yml

@@ -49,7 +49,7 @@ linters:
     - gocritic
     - gocyclo
     - godot
-    - godox
+    # - godox
     - gofmt
     - goheader
     - goimports

tests/globalhelper/pod.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"strings"
 	"time"
 
 	. "github.com/onsi/gomega"
@@ -159,3 +160,46 @@ func getRunningPod(client typedcorev1.CoreV1Interface, namespace, name string) (
 
 	return pod, nil
 }
+
+func GetControllerPodFromOperator(namespace, operatorName string) (*corev1.Pod, error) {
+	// Wait for the controller manager pod to come up
+	podsFound := false
+
+	var (
+		pods *corev1.PodList
+		err  error
+	)
+
+	// Try 10 times to find the pod
+	for i := 0; i < 10; i++ {
+		pods, err = GetListOfPodsInNamespace(namespace)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(pods.Items) == 0 {
+			fmt.Println("No pods found, retrying in 5 seconds...")
+			time.Sleep(5 * time.Second)
+
+			continue
+		} else {
+			podsFound = true
+
+			break
+		}
+	}
+
+	if !podsFound {
+		return nil, fmt.Errorf("no pods found in namespace %s", namespace)
+	}
+
+	for _, pod := range pods.Items {
+		fmt.Printf("Checking pod %s\n", pod.Name)
+
+		if strings.Contains(pod.Name, operatorName) {
+			return &pod, nil
+		}
+	}
+
+	return nil, fmt.Errorf("pod for operator %s not found", operatorName)
+}

tests/globalhelper/runhelper.go

@@ -101,6 +101,14 @@ func launchTestsViaImage(testCaseName string, tcNameForReport string, reportDir
 		"--label-filter", testCaseName,
 	}
 
+	// print the command
+	glog.V(5).Info(fmt.Sprintf("Running command: %s %s", containerEngine, strings.Join(certsuiteCmdArgs, " ")))
+
+	// fmt.Printf("Running command: %s %s", containerEngine, strings.Join(certsuiteCmdArgs, " "))
+
+	// fmt.Println("Sleeping for 5 minutes")
+	// time.Sleep(5 * time.Minute)
+
 	cmd := exec.Command(containerEngine, certsuiteCmdArgs...)
 
 	debugTnf, err := GetConfiguration().DebugTnf()

tests/operator/parameters/parameters.go

@@ -33,19 +33,20 @@ var (
 		testPodLabelPrefixName: testPodLabelValue,
 		"app":                  "test",
 	}
-	TnfTargetOperatorLabels  = fmt.Sprintf("%s: %s", "test-network-function.com/operator", "target")
-	TnfTargetCrdFilters      = []string{"charts.operatorhub.io"}
-	OperatorGroupName        = "operator-test-operator-group"
-	OperatorLabel            = map[string]string{"test-network-function.com/operator": "target"}
-	CertifiedOperatorGroup   = "certified-operators"
-	CommunityOperatorGroup   = "community-operators"
-	OperatorSourceNamespace  = "openshift-marketplace"
-	OperatorPrefixCloudbees  = "cloudbees-ci"
-	OperatorPrefixAnchore    = "anchore-engine"
-	OperatorPrefixQuay       = "quay-operator"
-	OperatorPrefixKiali      = "kiali-operator"
-	OperatorPrefixOpenvino   = "openvino-operator"
-	SubscriptionNameOpenvino = "ovms-operator-subscription"
+	TnfTargetOperatorLabels      = fmt.Sprintf("%s: %s", "test-network-function.com/operator", "target")
+	TnfTargetCrdFilters          = []string{"charts.operatorhub.io"}
+	OperatorGroupName            = "operator-test-operator-group"
+	OperatorLabel                = map[string]string{"test-network-function.com/operator": "target"}
+	CertifiedOperatorGroup       = "certified-operators"
+	CommunityOperatorGroup       = "community-operators"
+	OperatorSourceNamespace      = "openshift-marketplace"
+	OperatorPrefixCloudbees      = "cloudbees-ci"
+	OperatorPrefixAnchore        = "anchore-engine"
+	OperatorPrefixQuay           = "quay-operator"
+	OperatorPrefixKiali          = "kiali-operator"
+	OperatorPrefixOpenvino       = "openvino-operator"
+	CertifiedOperatorPrefixNginx = "nginx-ingress-operator"
+	SubscriptionNameOpenvino     = "ovms-operator-subscription"
 )
 
 const (
@@ -58,4 +59,8 @@ const (
 	TnfOperatorSemanticVersioning        = "operator-semantic-versioning"
 	TnfOperatorCrdVersioning             = "operator-crd-versioning"
 	TnfOperatorCrdOpenAPISchema          = "operator-crd-openapi-schema"
+	TnfOperatorNonRoot                   = "operator-run-as-non-root"
+	TnfOperatorReadOnlyFilesystem        = "operator-read-only-file-system"
+	TnfOperatorPodAutomountToken         = "operator-automount-tokens"
+	TnfOperatorPodRunAsUserID            = "operator-run-as-user-id"
 )

tests/operator/tests/operator_common.go

@@ -0,0 +1,38 @@
+package operator
+
+import (
+	"log"
+
+	. "github.com/onsi/gomega"
+
+	"github.com/operator-framework/api/pkg/operators/v1alpha1"
+	tshelper "github.com/test-network-function/cnfcert-tests-verification/tests/operator/helper"
+	tsparams "github.com/test-network-function/cnfcert-tests-verification/tests/operator/parameters"
+)
+
+func waitUntilOperatorIsReady(csvPrefix, namespace string) error {
+	var err error
+
+	var csv *v1alpha1.ClusterServiceVersion
+
+	Eventually(func() bool {
+		csv, err = tshelper.GetCsvByPrefix(csvPrefix, namespace)
+		if csv != nil && csv.Status.Phase != v1alpha1.CSVPhaseNone {
+			return csv.Status.Phase != "InstallReady" &&
+				csv.Status.Phase != "Deleting" &&
+				csv.Status.Phase != "Replacing" &&
+				csv.Status.Phase != "Unknown"
+		}
+
+		if err != nil {
+			log.Printf("Error getting csv: %s", err)
+
+			return false
+		}
+
+		return false
+	}, tsparams.Timeout, tsparams.PollingInterval).Should(Equal(true),
+		csvPrefix+" is not ready.")
+
+	return err
+}

tests/operator/tests/operator_non_root.go

@@ -0,0 +1,108 @@
+package operator
+
+import (
+	"fmt"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/operator-framework/api/pkg/operators/v1alpha1"
+	"github.com/test-network-function/cnfcert-tests-verification/tests/globalhelper"
+	"github.com/test-network-function/cnfcert-tests-verification/tests/globalparameters"
+	tshelper "github.com/test-network-function/cnfcert-tests-verification/tests/operator/helper"
+	tsparams "github.com/test-network-function/cnfcert-tests-verification/tests/operator/parameters"
+)
+
+var _ = Describe("Operator pods non-root", func() {
+	var randomNamespace string
+	var randomReportDir string
+	var randomTnfConfigDir string
+
+	BeforeEach(func() {
+		// Create random namespace and keep original report and TNF config directories
+		randomNamespace, randomReportDir, randomTnfConfigDir = globalhelper.BeforeEachSetupWithRandomNamespace(
+			tsparams.OperatorNamespace)
+
+		By("Define TNF config file")
+		err := globalhelper.DefineTnfConfig(
+			[]string{randomNamespace},
+			[]string{tsparams.TestPodLabel},
+			[]string{tsparams.TnfTargetOperatorLabels},
+			[]string{},
+			tsparams.TnfTargetCrdFilters, randomTnfConfigDir)
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		globalhelper.AfterEachCleanupWithRandomNamespace(randomNamespace, randomReportDir, randomTnfConfigDir, tsparams.Timeout)
+	})
+
+	It("Operator pods should not run as root", func() {
+		// TODO: Find an operator that runs completely as non-root
+	})
+
+	It("Operator pods should not run as root [negative]", func() {
+		// Deploy an operator that runs as root
+		By("Deploy operator group")
+		err := tshelper.DeployTestOperatorGroup(randomNamespace)
+		Expect(err).ToNot(HaveOccurred(), "Error deploying operator group")
+
+		By("Query the packagemanifest for the " + tsparams.CertifiedOperatorPrefixNginx)
+		version, err := globalhelper.QueryPackageManifestForVersion(tsparams.CertifiedOperatorPrefixNginx, randomNamespace)
+		Expect(err).ToNot(HaveOccurred(), "Error querying package manifest for nginx-ingress-operator")
+
+		By(fmt.Sprintf("Deploy nginx-ingress-operator%s for testing", "."+version))
+		// nginx-ingress-operator: in certified-operators group and version is certified
+		err = tshelper.DeployOperatorSubscription(
+			tsparams.CertifiedOperatorPrefixNginx,
+			"alpha",
+			randomNamespace,
+			tsparams.CertifiedOperatorGroup,
+			tsparams.OperatorSourceNamespace,
+			tsparams.CertifiedOperatorPrefixNginx+".v"+version,
+			v1alpha1.ApprovalAutomatic,
+		)
+		Expect(err).ToNot(HaveOccurred(), ErrorDeployOperatorStr+
+			tsparams.CertifiedOperatorPrefixNginx)
+
+		err = waitUntilOperatorIsReady(tsparams.CertifiedOperatorPrefixNginx,
+			randomNamespace)
+		Expect(err).ToNot(HaveOccurred(), "Operator "+tsparams.CertifiedOperatorPrefixNginx+".v"+version+
+			" is not ready")
+
+		By("Label operator")
+		Eventually(func() error {
+			return tshelper.AddLabelToInstalledCSV(
+				tsparams.CertifiedOperatorPrefixNginx,
+				randomNamespace,
+				tsparams.OperatorLabel)
+		}, tsparams.TimeoutLabelCsv, tsparams.PollingInterval).Should(Not(HaveOccurred()),
+			ErrorLabelingOperatorStr+tsparams.CertifiedOperatorPrefixNginx)
+
+		By("Assert that the manager pod is not running as root")
+		controllerPod, err := globalhelper.GetControllerPodFromOperator(randomNamespace, tsparams.CertifiedOperatorPrefixNginx)
+		Expect(err).ToNot(HaveOccurred(), "Error getting controller pod")
+
+		By(fmt.Sprintf("Checking if pod %s is not running as root", controllerPod.Name))
+		Expect(controllerPod.Spec.SecurityContext).ToNot(BeNil())
+		Expect(*controllerPod.Spec.SecurityContext.RunAsNonRoot).To(BeTrue())
+
+		for _, container := range controllerPod.Spec.Containers {
+			Expect(container.SecurityContext).ToNot(BeNil())
+			if container.SecurityContext.RunAsNonRoot != nil {
+				Expect(*container.SecurityContext.RunAsNonRoot).To(BeTrue())
+			}
+		}
+
+		By("Start test")
+		err = globalhelper.LaunchTests(
+			tsparams.TnfOperatorNonRoot,
+			globalhelper.ConvertSpecNameToFileName(CurrentSpecReport().FullText()), randomReportDir, randomTnfConfigDir)
+		Expect(err).ToNot(HaveOccurred())
+
+		By("Verify test case status in Claim report")
+		err = globalhelper.ValidateIfReportsAreValid(
+			tsparams.TnfOperatorNonRoot,
+			globalparameters.TestCaseFailed, randomReportDir)
+		Expect(err).ToNot(HaveOccurred())
+	})
+})

tests/operator/tests/operator_pod_automount_token.go

@@ -0,0 +1,109 @@
+package operator
+
+import (
+	"fmt"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/operator-framework/api/pkg/operators/v1alpha1"
+	"github.com/test-network-function/cnfcert-tests-verification/tests/globalhelper"
+	"github.com/test-network-function/cnfcert-tests-verification/tests/globalparameters"
+	tshelper "github.com/test-network-function/cnfcert-tests-verification/tests/operator/helper"
+	tsparams "github.com/test-network-function/cnfcert-tests-verification/tests/operator/parameters"
+)
+
+var _ = Describe("Operator pods automount token", func() {
+	var randomNamespace string
+	var randomReportDir string
+	var randomTnfConfigDir string
+
+	BeforeEach(func() {
+		// Create random namespace and keep original report and TNF config directories
+		randomNamespace, randomReportDir, randomTnfConfigDir = globalhelper.BeforeEachSetupWithRandomNamespace(
+			tsparams.OperatorNamespace)
+
+		By("Define TNF config file")
+		err := globalhelper.DefineTnfConfig(
+			[]string{randomNamespace},
+			[]string{tsparams.TestPodLabel},
+			[]string{tsparams.TnfTargetOperatorLabels},
+			[]string{},
+			tsparams.TnfTargetCrdFilters, randomTnfConfigDir)
+		Expect(err).ToNot(HaveOccurred())
+
+		By("Deploy operator group")
+		err = tshelper.DeployTestOperatorGroup(randomNamespace)
+		Expect(err).ToNot(HaveOccurred(), "Error deploying operator group")
+	})
+
+	AfterEach(func() {
+		globalhelper.AfterEachCleanupWithRandomNamespace(randomNamespace, randomReportDir, randomTnfConfigDir, tsparams.Timeout)
+	})
+
+	It("Operator pods should not have automount token", func() {
+		// Deploy an operator that does not have automount token
+		By("Deploy operator group")
+		err := tshelper.DeployTestOperatorGroup(randomNamespace)
+		Expect(err).ToNot(HaveOccurred(), "Error deploying operator group")
+
+		By("Query the packagemanifest for the " + tsparams.CertifiedOperatorPrefixNginx)
+		version, err := globalhelper.QueryPackageManifestForVersion(tsparams.CertifiedOperatorPrefixNginx, randomNamespace)
+		Expect(err).ToNot(HaveOccurred(), "Error querying package manifest for nginx-ingress-operator")
+
+		By(fmt.Sprintf("Deploy nginx-ingress-operator%s for testing", "."+version))
+		// nginx-ingress-operator: in certified-operators group and version is certified
+		err = tshelper.DeployOperatorSubscription(
+			tsparams.CertifiedOperatorPrefixNginx,
+			"alpha",
+			randomNamespace,
+			tsparams.CertifiedOperatorGroup,
+			tsparams.OperatorSourceNamespace,
+			tsparams.CertifiedOperatorPrefixNginx+".v"+version,
+			v1alpha1.ApprovalAutomatic,
+		)
+		Expect(err).ToNot(HaveOccurred(), ErrorDeployOperatorStr+
+			tsparams.CertifiedOperatorPrefixNginx)
+
+		err = waitUntilOperatorIsReady(tsparams.CertifiedOperatorPrefixNginx,
+			randomNamespace)
+		Expect(err).ToNot(HaveOccurred(), "Operator "+tsparams.CertifiedOperatorPrefixNginx+".v"+version+
+			" is not ready")
+
+		By("Label operator")
+		Eventually(func() error {
+			return tshelper.AddLabelToInstalledCSV(
+				tsparams.CertifiedOperatorPrefixNginx,
+				randomNamespace,
+				tsparams.OperatorLabel)
+		}, tsparams.TimeoutLabelCsv, tsparams.PollingInterval).Should(Not(HaveOccurred()),
+			ErrorLabelingOperatorStr+tsparams.CertifiedOperatorPrefixNginx)
+
+		By("Assert that the manager pod has automount token nil or false")
+		controllerPod, err := globalhelper.GetControllerPodFromOperator(randomNamespace, tsparams.CertifiedOperatorPrefixNginx)
+		Expect(err).ToNot(HaveOccurred(), "Error getting controller pod")
+
+		By(fmt.Sprintf("Checking if pod %s has automount token nil or false", controllerPod.Name))
+		if controllerPod.Spec.AutomountServiceAccountToken != nil {
+			Expect(*controllerPod.Spec.AutomountServiceAccountToken).To(BeFalse())
+		} else {
+			Expect(controllerPod.Spec.AutomountServiceAccountToken).To(BeNil())
+		}
+
+		By("Start test")
+		err = globalhelper.LaunchTests(
+			tsparams.TnfOperatorPodAutomountToken,
+			globalhelper.ConvertSpecNameToFileName(CurrentSpecReport().FullText()), randomReportDir, randomTnfConfigDir)
+		Expect(err).ToNot(HaveOccurred())
+
+		By("Verify test case status in Claim report")
+		err = globalhelper.ValidateIfReportsAreValid(
+			tsparams.TnfOperatorPodAutomountToken,
+			globalparameters.TestCasePassed, randomReportDir)
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	It("Operator pods have automount token [negative]", func() {
+		// Deploy an operator that explicitly has automount token
+		// TODO: Find an operator that has automount token set explicitly
+	})
+})