From 8bf3fce5eaa438b922471be37d9497b693268a4b Mon Sep 17 00:00:00 2001 From: Brandon Palm Date: Wed, 29 Nov 2023 08:23:13 -0600 Subject: [PATCH 1/3] Remove gotools; adjust expected/actual (#1667) --- cmd/tnf/claim/compare/compare_test.go | 2 +- cmd/tnf/claim/compare/testcases/testcases_test.go | 8 ++++---- cmd/tnf/claim/show/failures/failures_test.go | 10 +++++----- cmd/tnf/pkg/claim/claim_test.go | 2 +- go.mod | 1 - pkg/junit/convert_test.go | 4 ++-- pkg/scheduling/scheduling_test.go | 6 +++--- 7 files changed, 16 insertions(+), 17 deletions(-) diff --git a/cmd/tnf/claim/compare/compare_test.go b/cmd/tnf/claim/compare/compare_test.go index 08df17032..1b8f1e44c 100644 --- a/cmd/tnf/claim/compare/compare_test.go +++ b/cmd/tnf/claim/compare/compare_test.go @@ -87,7 +87,7 @@ func Test_claimCompareFilesfunc(t *testing.T) { } if err != nil { - assert.Equal(t, err.Error(), tc.expectedError) + assert.Equal(t, tc.expectedError, err.Error()) } else { // Read the output from the pipe. out, _ := io.ReadAll(r) diff --git a/cmd/tnf/claim/compare/testcases/testcases_test.go b/cmd/tnf/claim/compare/testcases/testcases_test.go index 25fa2b7ce..655ae473d 100644 --- a/cmd/tnf/claim/compare/testcases/testcases_test.go +++ b/cmd/tnf/claim/compare/testcases/testcases_test.go @@ -4,8 +4,8 @@ import ( "reflect" "testing" + "github.com/stretchr/testify/assert" "github.com/test-network-function/cnf-certification-test/cmd/tnf/pkg/claim" - "gotest.tools/v3/assert" ) func TestGetTestCasesResultsMap(t *testing.T) { @@ -84,7 +84,7 @@ func TestGetTestCasesResultsMap(t *testing.T) { for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { resultsMap := getTestCasesResultsMap(tc.results) - assert.Equal(t, true, reflect.DeepEqual(tc.expectedTestCasesResultsMap, resultsMap)) + assert.True(t, reflect.DeepEqual(tc.expectedTestCasesResultsMap, resultsMap)) }) } } @@ -143,7 +143,7 @@ func TestGetMergedTestCasesNames(t *testing.T) { for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { tcMergedNamesList := getMergedTestCasesNames(tc.claim1Results, tc.claim2Results) - assert.Equal(t, true, reflect.DeepEqual(tc.expectedMergedTcNames, tcMergedNamesList)) + assert.True(t, reflect.DeepEqual(tc.expectedMergedTcNames, tcMergedNamesList)) }) } } @@ -308,7 +308,7 @@ func TestGetDiffReport(t *testing.T) { // Check test case results differences t.Logf("diffs: %+v", diffReport.TestCases) - assert.Equal(t, true, reflect.DeepEqual(tc.expectedDiffReport.TestCases, diffReport.TestCases)) + assert.True(t, reflect.DeepEqual(tc.expectedDiffReport.TestCases, diffReport.TestCases)) // Check count assert.Equal(t, tc.expectedDiffReport.DifferentTestCasesResults, diffReport.DifferentTestCasesResults) diff --git a/cmd/tnf/claim/show/failures/failures_test.go b/cmd/tnf/claim/show/failures/failures_test.go index c57a747c8..27337cf35 100644 --- a/cmd/tnf/claim/show/failures/failures_test.go +++ b/cmd/tnf/claim/show/failures/failures_test.go @@ -4,8 +4,8 @@ import ( "fmt" "testing" + "github.com/stretchr/testify/assert" "github.com/test-network-function/cnf-certification-test/cmd/tnf/pkg/claim" - "gotest.tools/v3/assert" ) func TestParseTargetTestSuitesFlag(t *testing.T) { @@ -50,7 +50,7 @@ func TestParseTargetTestSuitesFlag(t *testing.T) { for _, tc := range testCases { testSuitesFlag = tc.flag parsedTestSuites := parseTargetTestSuitesFlag() - assert.DeepEqual(t, tc.expectedTestSuites, parsedTestSuites) + assert.Equal(t, tc.expectedTestSuites, parsedTestSuites) } } @@ -265,7 +265,7 @@ func TestGetNonCompliantObjectsFromFailureReason(t *testing.T) { assert.Equal(t, tc.expectedError, err.Error()) } - assert.DeepEqual(t, tc.expectedNonCompliantObjects, nonCompliantObjects) + assert.Equal(t, tc.expectedNonCompliantObjects, nonCompliantObjects) } } @@ -376,7 +376,7 @@ func TestGetFailedTestCasesByTestSuite(t *testing.T) { for _, tc := range testCases { claimScheme, err := claim.Parse(tc.claimFilePath) - assert.NilError(t, err) + assert.Nil(t, err) // Order test case results by test suite, using a helper map. resultsByTestSuite := map[string][]*claim.TestCaseResult{} @@ -393,6 +393,6 @@ func TestGetFailedTestCasesByTestSuite(t *testing.T) { map[string]bool{tc.targetTestSuite: true}, ) fmt.Printf("%#v\n\n", testSuites) - assert.DeepEqual(t, tc.expectedFailedTestSuites, testSuites) + assert.Equal(t, tc.expectedFailedTestSuites, testSuites) } } diff --git a/cmd/tnf/pkg/claim/claim_test.go b/cmd/tnf/pkg/claim/claim_test.go index c7fe57e66..a91c9f346 100644 --- a/cmd/tnf/pkg/claim/claim_test.go +++ b/cmd/tnf/pkg/claim/claim_test.go @@ -3,7 +3,7 @@ package claim import ( "testing" - "gotest.tools/v3/assert" + "github.com/stretchr/testify/assert" ) func TestIsClaimFormatVersionSupported(t *testing.T) { diff --git a/go.mod b/go.mod index 605a309e3..386160c3c 100644 --- a/go.mod +++ b/go.mod @@ -219,7 +219,6 @@ require ( github.com/test-network-function/oct v0.0.4 github.com/test-network-function/privileged-daemonset v1.0.15 gopkg.in/yaml.v3 v3.0.1 - gotest.tools/v3 v3.5.1 k8s.io/kubectl v0.28.4 ) diff --git a/pkg/junit/convert_test.go b/pkg/junit/convert_test.go index 42086efd0..16267eb80 100644 --- a/pkg/junit/convert_test.go +++ b/pkg/junit/convert_test.go @@ -38,7 +38,7 @@ func TestExtractTestSuiteResults(t *testing.T) { results, err := junit.ExtractTestSuiteResults(claim, testKey) assert.Nil(t, err) // positive test - assert.Equal(t, true, results["[It] operator Runs test on operators operator-install-status-CSV_INSTALLED"].Passed) + assert.True(t, results["[It] operator Runs test on operators operator-install-status-CSV_INSTALLED"].Passed) // negative test - assert.Equal(t, false, results["[It] platform-alteration platform-alteration-boot-params"].Passed) + assert.False(t, results["[It] platform-alteration platform-alteration-boot-params"].Passed) } diff --git a/pkg/scheduling/scheduling_test.go b/pkg/scheduling/scheduling_test.go index e55324633..af833c269 100644 --- a/pkg/scheduling/scheduling_test.go +++ b/pkg/scheduling/scheduling_test.go @@ -470,9 +470,9 @@ func TestExistsSchedulingPolicyAndPriority(t *testing.T) { for _, tc := range testCases { policy, priority, err := parseSchedulingPolicyAndPriority(tc.outputString) - assert.Equal(t, policy, tc.expectedPolicy) - assert.Equal(t, priority, tc.expectedPriority) - assert.Equal(t, err, tc.expectedError) + assert.Equal(t, tc.expectedPolicy, policy) + assert.Equal(t, tc.expectedPriority, priority) + assert.Equal(t, tc.expectedError, err) } } From 887a378c0f223dd0d19e1fd6ead77cca323fdf15 Mon Sep 17 00:00:00 2001 From: David Elie-Dit-Cosaque <86730676+edcdavid@users.noreply.github.com> Date: Wed, 29 Nov 2023 09:03:04 -0600 Subject: [PATCH 2/3] Graceful exit when no kubeconfig is configured (#1668) --- internal/clientsholder/clientsholder.go | 7 ++++++- main.go | 10 +++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/internal/clientsholder/clientsholder.go b/internal/clientsholder/clientsholder.go index 0aee59c60..066f28930 100644 --- a/internal/clientsholder/clientsholder.go +++ b/internal/clientsholder/clientsholder.go @@ -19,6 +19,7 @@ package clientsholder import ( "errors" "fmt" + "os" "time" clientconfigv1 "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1" @@ -146,10 +147,14 @@ func ClearTestClientsHolder() { // GetClientsHolder returns the singleton ClientsHolder object. func GetClientsHolder(filenames ...string) *ClientsHolder { + const exitUsage = 2 if clientsHolder.ready { return &clientsHolder } - + if len(filenames) == 0 { + logrus.Errorf("Please provide a valid kubeconfig. Either set the KUBECONFIG environment variable or alternatively copy a kube config to $HOME/.kube/config") + os.Exit(exitUsage) + } clientsHolder, err := newClientsHolder(filenames...) if err != nil { logrus.Panic("Failed to create k8s clients holder: ", err) diff --git a/main.go b/main.go index ccd3da137..bdc7e7f60 100644 --- a/main.go +++ b/main.go @@ -110,11 +110,19 @@ func getK8sClientsConfigFileNames() []string { params := configuration.GetTestParameters() fileNames := []string{} if params.Kubeconfig != "" { + // Add the kubeconfig path fileNames = append(fileNames, params.Kubeconfig) } if params.Home != "" { kubeConfigFilePath := filepath.Join(params.Home, ".kube", "config") - fileNames = append(fileNames, kubeConfigFilePath) + // Check if the kubeconfig path exists + if _, err := os.Stat(kubeConfigFilePath); err == nil { + log.Infof("kubeconfig path %s is present", kubeConfigFilePath) + // Only add the kubeconfig to the list of paths if it exists, since it is not added by the user + fileNames = append(fileNames, kubeConfigFilePath) + } else { + log.Infof("kubeconfig path %s is not present", kubeConfigFilePath) + } } return fileNames From a3a6492b19f70653574f53074eaf733ddf1144f0 Mon Sep 17 00:00:00 2001 From: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> Date: Wed, 29 Nov 2023 17:33:11 +0100 Subject: [PATCH 3/3] Transition platform-alteration suite to ginkgoless. (#1658) Also added all the needed skip functions to the testhelper package. --- cnf-certification-test/platform/suite.go | 325 +++++++++++++---------- generated_policy.json | 10 + pkg/certsuite/certsuite.go | 2 + pkg/testhelper/testhelper.go | 44 +++ 4 files changed, 238 insertions(+), 143 deletions(-) diff --git a/cnf-certification-test/platform/suite.go b/cnf-certification-test/platform/suite.go index 2442d86df..5509cac81 100644 --- a/cnf-certification-test/platform/suite.go +++ b/cnf-certification-test/platform/suite.go @@ -23,6 +23,7 @@ import ( "github.com/sirupsen/logrus" clientsholder "github.com/test-network-function/cnf-certification-test/internal/clientsholder" + "github.com/test-network-function/cnf-certification-test/pkg/checksdb" "github.com/test-network-function/cnf-certification-test/pkg/compatibility" "github.com/test-network-function/cnf-certification-test/pkg/provider" "github.com/test-network-function/cnf-certification-test/pkg/testhelper" @@ -38,141 +39,175 @@ import ( "github.com/test-network-function/cnf-certification-test/cnf-certification-test/platform/operatingsystem" "github.com/test-network-function/cnf-certification-test/cnf-certification-test/platform/sysctlconfig" - "github.com/onsi/ginkgo/v2" "github.com/test-network-function/cnf-certification-test/cnf-certification-test/platform/nodetainted" ) -// All actual test code belongs below here. Utilities belong above. -var _ = ginkgo.Describe(common.PlatformAlterationTestKey, func() { - logrus.Debugf("Entering %s suite", common.PlatformAlterationTestKey) - var env provider.TestEnvironment - ginkgo.BeforeEach(func() { +var ( + env provider.TestEnvironment + + beforeEachFn = func(check *checksdb.Check) error { + logrus.Infof("Check %s: getting test environment.", check.ID) env = provider.GetTestEnvironment() - }) + return nil + } +) + +//nolint:funlen +func LoadChecks() { + logrus.Debugf("Entering %s suite", common.PlatformAlterationTestKey) + + checksGroup := checksdb.NewChecksGroup(common.PlatformAlterationTestKey). + WithBeforeEachFn(beforeEachFn) testID, tags := identifiers.GetGinkgoTestIDAndLabels(identifiers.TestHyperThreadEnable) - ginkgo.It(testID, ginkgo.Label(tags...), func() { - testhelper.SkipIfEmptyAny(ginkgo.Skip, testhelper.NewSkipObject(env.GetBaremetalNodes(), "env.GetBaremetalNodes()")) - testHyperThreadingEnabled(&env) - }) + check := checksdb.NewCheck(testID, tags). + WithSkipCheckFn(testhelper.GetNoBareMetalNodesSkipFn(&env)). + WithCheckFn(func(c *checksdb.Check) error { + testHyperThreadingEnabled(c, &env) + return nil + }) + + checksGroup.Add(check) + testID, tags = identifiers.GetGinkgoTestIDAndLabels(identifiers.TestUnalteredBaseImageIdentifier) - ginkgo.It(testID, ginkgo.Label(tags...), func() { - if !provider.IsOCPCluster() { - ginkgo.Skip("Non-OCP cluster found, skipping testContainersFsDiff") - } - testhelper.SkipIfEmptyAny(ginkgo.Skip, testhelper.NewSkipObject(env.Containers, "env.Containers")) - if env.DaemonsetFailedToSpawn { - ginkgo.Skip("Debug Daemonset failed to spawn skipping testContainersFsDiff") - } - testContainersFsDiff(&env) - }) + check = checksdb.NewCheck(testID, tags). + WithSkipCheckFn( + testhelper.GetNonOCPClusterSkipFn(), + testhelper.GetDaemonSetFailedToSpawnSkipFn(&env), + testhelper.GetNoContainersUnderTestSkipFn(&env)). + WithCheckFn(func(c *checksdb.Check) error { + testContainersFsDiff(c, &env) + return nil + }) + + checksGroup.Add(check) testID, tags = identifiers.GetGinkgoTestIDAndLabels(identifiers.TestNonTaintedNodeKernelsIdentifier) - ginkgo.It(testID, ginkgo.Label(tags...), func() { - testhelper.SkipIfEmptyAny(ginkgo.Skip, testhelper.NewSkipObject(env.DebugPods, "env.DebugPods")) - if env.DaemonsetFailedToSpawn { - ginkgo.Skip("Debug Daemonset failed to spawn skipping testTainted") - } - testTainted(&env) // Kind tainted kernels are allowed via config - }) + check = checksdb.NewCheck(testID, tags). + WithSkipCheckFn(testhelper.GetDaemonSetFailedToSpawnSkipFn(&env)). + WithCheckFn(func(c *checksdb.Check) error { + testTainted(c, &env) + return nil + }) + + checksGroup.Add(check) testID, tags = identifiers.GetGinkgoTestIDAndLabels(identifiers.TestIsRedHatReleaseIdentifier) - ginkgo.It(testID, ginkgo.Label(tags...), func() { - testhelper.SkipIfEmptyAny(ginkgo.Skip, testhelper.NewSkipObject(env.Containers, "env.Containers")) - testIsRedHatRelease(&env) - }) + check = checksdb.NewCheck(testID, tags). + WithSkipCheckFn(testhelper.GetNoContainersUnderTestSkipFn(&env)). + WithCheckFn(func(c *checksdb.Check) error { + testIsRedHatRelease(c, &env) + return nil + }) + + checksGroup.Add(check) testID, tags = identifiers.GetGinkgoTestIDAndLabels(identifiers.TestIsSELinuxEnforcingIdentifier) - ginkgo.It(testID, ginkgo.Label(tags...), func() { - if !provider.IsOCPCluster() { - ginkgo.Skip("Non-OCP cluster found, skipping testIsSELinuxEnforcing") - } - testhelper.SkipIfEmptyAny(ginkgo.Skip, testhelper.NewSkipObject(env.DebugPods, "env.DebugPods")) - if env.DaemonsetFailedToSpawn { - ginkgo.Skip("Debug Daemonset failed to spawn skipping testIsSELinuxEnforcing") - } - testIsSELinuxEnforcing(&env) - }) + check = checksdb.NewCheck(testID, tags). + WithSkipCheckFn( + testhelper.GetNonOCPClusterSkipFn(), + testhelper.GetDaemonSetFailedToSpawnSkipFn(&env)). + WithCheckFn(func(c *checksdb.Check) error { + testIsSELinuxEnforcing(c, &env) + return nil + }) + + checksGroup.Add(check) testID, tags = identifiers.GetGinkgoTestIDAndLabels(identifiers.TestHugepagesNotManuallyManipulated) - ginkgo.It(testID, ginkgo.Label(tags...), func() { - if !provider.IsOCPCluster() { - ginkgo.Skip("Non-OCP cluster found, skipping testHugepages") - } - testhelper.SkipIfEmptyAny(ginkgo.Skip, testhelper.NewSkipObject(env.DebugPods, "env.DebugPods")) - if env.DaemonsetFailedToSpawn { - ginkgo.Skip("Debug Daemonset failed to spawn skipping testHugepages") - } - testHugepages(&env) - }) + check = checksdb.NewCheck(testID, tags). + WithSkipCheckFn( + testhelper.GetNonOCPClusterSkipFn(), + testhelper.GetDaemonSetFailedToSpawnSkipFn(&env)). + WithCheckFn(func(c *checksdb.Check) error { + testHugepages(c, &env) + return nil + }) + + checksGroup.Add(check) testID, tags = identifiers.GetGinkgoTestIDAndLabels(identifiers.TestUnalteredStartupBootParamsIdentifier) - ginkgo.It(testID, ginkgo.Label(tags...), func() { - if !provider.IsOCPCluster() { - ginkgo.Skip("Non-OCP cluster found, skipping testUnalteredBootParams") - } - testhelper.SkipIfEmptyAny(ginkgo.Skip, testhelper.NewSkipObject(env.DebugPods, "env.DebugPods")) - if env.DaemonsetFailedToSpawn { - ginkgo.Skip("Debug Daemonset failed to spawn skipping testUnalteredBootParams") - } - testUnalteredBootParams(&env) - }) + check = checksdb.NewCheck(testID, tags). + WithSkipCheckFn( + testhelper.GetNonOCPClusterSkipFn(), + testhelper.GetDaemonSetFailedToSpawnSkipFn(&env)). + WithCheckFn(func(c *checksdb.Check) error { + testUnalteredBootParams(c, &env) + return nil + }) + + checksGroup.Add(check) testID, tags = identifiers.GetGinkgoTestIDAndLabels(identifiers.TestSysctlConfigsIdentifier) - ginkgo.It(testID, ginkgo.Label(tags...), func() { - if !provider.IsOCPCluster() { - ginkgo.Skip("Non-OCP cluster found, skipping testSysctlConfigs") - } - testhelper.SkipIfEmptyAny(ginkgo.Skip, testhelper.NewSkipObject(env.DebugPods, "env.DebugPods")) - if env.DaemonsetFailedToSpawn { - ginkgo.Skip("Debug Daemonset failed to spawn skipping testSysctlConfigs") - } - testSysctlConfigs(&env) - }) + check = checksdb.NewCheck(testID, tags). + WithSkipCheckFn( + testhelper.GetNonOCPClusterSkipFn(), + testhelper.GetDaemonSetFailedToSpawnSkipFn(&env)). + WithCheckFn(func(c *checksdb.Check) error { + testSysctlConfigs(c, &env) + return nil + }) + + checksGroup.Add(check) testID, tags = identifiers.GetGinkgoTestIDAndLabels(identifiers.TestServiceMeshIdentifier) - ginkgo.It(testID, ginkgo.Label(tags...), func() { - testhelper.SkipIfEmptyAny(ginkgo.Skip, testhelper.NewSkipObject(env.Pods, "env.Pods")) - testServiceMesh(&env) - }) + check = checksdb.NewCheck(testID, tags). + WithSkipCheckFn( + testhelper.GetNoIstioSkipFn(&env), + testhelper.GetNoPodsUnderTestSkipFn(&env)). + WithCheckFn(func(c *checksdb.Check) error { + testServiceMesh(c, &env) + return nil + }) + + checksGroup.Add(check) testID, tags = identifiers.GetGinkgoTestIDAndLabels(identifiers.TestOCPLifecycleIdentifier) - ginkgo.It(testID, ginkgo.Label(tags...), func() { - if !provider.IsOCPCluster() { - ginkgo.Skip("Non-OCP cluster found, skipping testOCPStatus") - } - testOCPStatus(&env) - }) + check = checksdb.NewCheck(testID, tags). + WithSkipCheckFn(testhelper.GetNonOCPClusterSkipFn()). + WithCheckFn(func(c *checksdb.Check) error { + testOCPStatus(c, &env) + return nil + }) + + checksGroup.Add(check) testID, tags = identifiers.GetGinkgoTestIDAndLabels(identifiers.TestNodeOperatingSystemIdentifier) - ginkgo.It(testID, ginkgo.Label(tags...), func() { - if !provider.IsOCPCluster() { - ginkgo.Skip("Non-OCP cluster found, skipping testNodeOperatingSystemStatus") - } - testNodeOperatingSystemStatus(&env) - }) + check = checksdb.NewCheck(testID, tags). + WithSkipCheckFn(testhelper.GetNonOCPClusterSkipFn()). + WithCheckFn(func(c *checksdb.Check) error { + testNodeOperatingSystemStatus(c, &env) + return nil + }) + + checksGroup.Add(check) testID, tags = identifiers.GetGinkgoTestIDAndLabels(identifiers.TestPodHugePages2M) - ginkgo.It(testID, ginkgo.Label(tags...), func() { - if !provider.IsOCPCluster() { - ginkgo.Skip("Non-OCP cluster found, skipping testPodHugePagesSize2M") - } - testhelper.SkipIfEmptyAny(ginkgo.Skip, testhelper.NewSkipObject(env.GetHugepagesPods(), "env.GetHugepagesPods()")) - testPodHugePagesSize(&env, provider.HugePages2Mi) - }) + check = checksdb.NewCheck(testID, tags). + WithSkipCheckFn( + testhelper.GetNonOCPClusterSkipFn(), + testhelper.GetNoHugepagesPodsSkipFn(&env)). + WithCheckFn(func(c *checksdb.Check) error { + testPodHugePagesSize(c, &env, provider.HugePages2Mi) + return nil + }) - testID, tags = identifiers.GetGinkgoTestIDAndLabels(identifiers.TestPodHugePages1G) - ginkgo.It(testID, ginkgo.Label(tags...), func() { - if !provider.IsOCPCluster() { - ginkgo.Skip("Non-OCP cluster found, skipping testPodHugePagesSize1G") - } - testhelper.SkipIfEmptyAny(ginkgo.Skip, testhelper.NewSkipObject(env.GetHugepagesPods(), "env.GetHugepagesPods()")) - testPodHugePagesSize(&env, provider.HugePages1Gi) - }) + checksGroup.Add(check) -}) + testID, tags = identifiers.GetGinkgoTestIDAndLabels(identifiers.TestPodHugePages1G) + check = checksdb.NewCheck(testID, tags). + WithSkipCheckFn( + testhelper.GetNonOCPClusterSkipFn(), + testhelper.GetNoHugepagesPodsSkipFn(&env)). + WithCheckFn(func(c *checksdb.Check) error { + testPodHugePagesSize(c, &env, provider.HugePages1Gi) + return nil + }) + + checksGroup.Add(check) +} -func testHyperThreadingEnabled(env *provider.TestEnvironment) { +func testHyperThreadingEnabled(check *checksdb.Check, env *provider.TestEnvironment) { var compliantObjects []*testhelper.ReportObject var nonCompliantObjects []*testhelper.ReportObject baremetalNodes := env.GetBaremetalNodes() @@ -188,17 +223,10 @@ func testHyperThreadingEnabled(env *provider.TestEnvironment) { nonCompliantObjects = append(nonCompliantObjects, testhelper.NewNodeReportObject(nodeName, "Node has hyperthreading disabled ", false)) } } - testhelper.AddTestResultReason(compliantObjects, nonCompliantObjects, tnf.ClaimFilePrintf, ginkgo.Fail) + check.SetResult(compliantObjects, nonCompliantObjects) } -func testServiceMesh(env *provider.TestEnvironment) { - // check if istio is installed - if !env.IstioServiceMeshFound { - tnf.ClaimFilePrintf("Istio is not installed") - ginkgo.Skip("No service mesh detected.") - } - tnf.ClaimFilePrintf("Istio is installed") - +func testServiceMesh(check *checksdb.Check, env *provider.TestEnvironment) { var compliantObjects []*testhelper.ReportObject var nonCompliantObjects []*testhelper.ReportObject for _, put := range env.Pods { @@ -217,19 +245,18 @@ func testServiceMesh(env *provider.TestEnvironment) { compliantObjects = append(compliantObjects, testhelper.NewPodReportObject(put.Namespace, put.Name, "Pod found with service mesh container", true)) } } - testhelper.AddTestResultReason(compliantObjects, nonCompliantObjects, tnf.ClaimFilePrintf, ginkgo.Fail) + + check.SetResult(compliantObjects, nonCompliantObjects) } // testContainersFsDiff test that all CUT did not install new packages are starting -func testContainersFsDiff(env *provider.TestEnvironment) { +func testContainersFsDiff(check *checksdb.Check, env *provider.TestEnvironment) { var compliantObjects []*testhelper.ReportObject var nonCompliantObjects []*testhelper.ReportObject for _, cut := range env.Containers { logrus.Debug(fmt.Sprintf("%s should not install new packages after starting", cut.String())) debugPod := env.DebugPods[cut.NodeName] - if debugPod == nil { - ginkgo.Fail(fmt.Sprintf("Debug pod not found on Node: %s", cut.NodeName)) - } + ctxt := clientsholder.NewContext(debugPod.Namespace, debugPod.Name, debugPod.Spec.Containers[0].Name) fsDiffTester := cnffsdiff.NewFsDiffTester(clientsholder.GetClientsHolder(), ctxt) fsDiffTester.RunTest(cut.UID) @@ -249,11 +276,11 @@ func testContainersFsDiff(env *provider.TestEnvironment) { } } - testhelper.AddTestResultReason(compliantObjects, nonCompliantObjects, tnf.ClaimFilePrintf, ginkgo.Fail) + check.SetResult(compliantObjects, nonCompliantObjects) } //nolint:funlen -func testTainted(env *provider.TestEnvironment) { +func testTainted(check *checksdb.Check, env *provider.TestEnvironment) { var compliantObjects []*testhelper.ReportObject var nonCompliantObjects []*testhelper.ReportObject @@ -281,7 +308,7 @@ func testTainted(env *provider.TestEnvironment) { for _, dp := range env.DebugPods { nodeName := dp.Spec.NodeName - ginkgo.By(fmt.Sprintf("Checking kernel taints of node %s", nodeName)) + logrus.Infof("Checking kernel taints of node %s", nodeName) ocpContext := clientsholder.NewContext(dp.Namespace, dp.Name, dp.Spec.Containers[0].Name) tf := nodetainted.NewNodeTaintedTester(&ocpContext, nodeName) @@ -374,15 +401,15 @@ func testTainted(env *provider.TestEnvironment) { logrus.Info("Nodes have been found to be tainted. Check claim log for more details.") } - testhelper.AddTestResultReason(compliantObjects, nonCompliantObjects, tnf.ClaimFilePrintf, ginkgo.Fail) + check.SetResult(compliantObjects, nonCompliantObjects) } -func testIsRedHatRelease(env *provider.TestEnvironment) { - ginkgo.By("should report a proper Red Hat version") +func testIsRedHatRelease(check *checksdb.Check, env *provider.TestEnvironment) { + logrus.Info("should report a proper Red Hat version") var compliantObjects []*testhelper.ReportObject var nonCompliantObjects []*testhelper.ReportObject for _, cut := range env.Containers { - ginkgo.By(fmt.Sprintf("%s is checked for Red Hat version", cut)) + logrus.Infof("%s is checked for Red Hat version", cut) baseImageTester := isredhat.NewBaseImageTester(clientsholder.GetClientsHolder(), clientsholder.NewContext(cut.Namespace, cut.Podname, cut.Name)) result, err := baseImageTester.TestContainerIsRedHatRelease() @@ -397,10 +424,10 @@ func testIsRedHatRelease(env *provider.TestEnvironment) { } } - testhelper.AddTestResultReason(compliantObjects, nonCompliantObjects, tnf.ClaimFilePrintf, ginkgo.Fail) + check.SetResult(compliantObjects, nonCompliantObjects) } -func testIsSELinuxEnforcing(env *provider.TestEnvironment) { +func testIsSELinuxEnforcing(check *checksdb.Check, env *provider.TestEnvironment) { const ( getenforceCommand = `chroot /host getenforce` enforcingString = "Enforcing\n" @@ -434,10 +461,10 @@ func testIsSELinuxEnforcing(env *provider.TestEnvironment) { logrus.Infof(fmt.Sprintf("Failed because %d nodes are not running selinux", nodesFailed)) } - testhelper.AddTestResultReason(compliantObjects, nonCompliantObjects, tnf.ClaimFilePrintf, ginkgo.Fail) + check.SetResult(compliantObjects, nonCompliantObjects) } -func testHugepages(env *provider.TestEnvironment) { +func testHugepages(check *checksdb.Check, env *provider.TestEnvironment) { var compliantObjects []*testhelper.ReportObject var nonCompliantObjects []*testhelper.ReportObject for i := range env.Nodes { @@ -468,10 +495,10 @@ func testHugepages(env *provider.TestEnvironment) { } } - testhelper.AddTestResultReason(compliantObjects, nonCompliantObjects, tnf.ClaimFilePrintf, ginkgo.Fail) + check.SetResult(compliantObjects, nonCompliantObjects) } -func testUnalteredBootParams(env *provider.TestEnvironment) { +func testUnalteredBootParams(check *checksdb.Check, env *provider.TestEnvironment) { var compliantObjects []*testhelper.ReportObject var nonCompliantObjects []*testhelper.ReportObject alreadyCheckedNodes := map[string]bool{} @@ -494,10 +521,10 @@ func testUnalteredBootParams(env *provider.TestEnvironment) { } } - testhelper.AddTestResultReason(compliantObjects, nonCompliantObjects, tnf.ClaimFilePrintf, ginkgo.Fail) + check.SetResult(compliantObjects, nonCompliantObjects) } -func testSysctlConfigs(env *provider.TestEnvironment) { +func testSysctlConfigs(check *checksdb.Check, env *provider.TestEnvironment) { var compliantObjects []*testhelper.ReportObject var nonCompliantObjects []*testhelper.ReportObject @@ -537,17 +564,18 @@ func testSysctlConfigs(env *provider.TestEnvironment) { } } - testhelper.AddTestResultReason(compliantObjects, nonCompliantObjects, tnf.ClaimFilePrintf, ginkgo.Fail) + check.SetResult(compliantObjects, nonCompliantObjects) } -func testOCPStatus(env *provider.TestEnvironment) { - ginkgo.By("Testing the OCP Version for lifecycle status") +func testOCPStatus(check *checksdb.Check, env *provider.TestEnvironment) { + logrus.Infof("Testing the OCP Version for lifecycle status") + clusterIsInEOL := false switch env.OCPStatus { case compatibility.OCPStatusEOL: msg := fmt.Sprintf("OCP Version %s has been found to be in end of life", env.OpenshiftVersion) tnf.ClaimFilePrintf(msg) - ginkgo.Fail(msg) + clusterIsInEOL = true case compatibility.OCPStatusMS: msg := fmt.Sprintf("OCP Version %s has been found to be in maintenance support", env.OpenshiftVersion) tnf.ClaimFilePrintf(msg) @@ -561,11 +589,22 @@ func testOCPStatus(env *provider.TestEnvironment) { msg := fmt.Sprintf("OCP Version %s was unable to be found in the lifecycle compatibility matrix", env.OpenshiftVersion) tnf.ClaimFilePrintf(msg) } + + var compliantObjects []*testhelper.ReportObject + var nonCompliantObjects []*testhelper.ReportObject + + if clusterIsInEOL { + nonCompliantObjects = []*testhelper.ReportObject{testhelper.NewClusterVersionReportObject(env.OpenshiftVersion, "Openshift Version is in End Of Life (EOL)", false)} + } else { + compliantObjects = []*testhelper.ReportObject{testhelper.NewClusterVersionReportObject(env.OpenshiftVersion, "Openshift Version is not in End Of Life (EOL)", true)} + } + + check.SetResult(compliantObjects, nonCompliantObjects) } //nolint:funlen -func testNodeOperatingSystemStatus(env *provider.TestEnvironment) { - ginkgo.By("Testing the control-plane and workers in the cluster for Operating System compatibility") +func testNodeOperatingSystemStatus(check *checksdb.Check, env *provider.TestEnvironment) { + logrus.Info("Testing the control-plane and workers in the cluster for Operating System compatibility") logrus.Debug(fmt.Sprintf("There are %d nodes to process for Operating System compatibility.", len(env.Nodes))) @@ -674,10 +713,10 @@ func testNodeOperatingSystemStatus(env *provider.TestEnvironment) { tnf.ClaimFilePrintf(errMsg) } - testhelper.AddTestResultReason(compliantObjects, nonCompliantObjects, tnf.ClaimFilePrintf, ginkgo.Fail) + check.SetResult(compliantObjects, nonCompliantObjects) } -func testPodHugePagesSize(env *provider.TestEnvironment, size string) { +func testPodHugePagesSize(check *checksdb.Check, env *provider.TestEnvironment, size string) { var compliantObjects []*testhelper.ReportObject var nonCompliantObjects []*testhelper.ReportObject for _, put := range env.GetHugepagesPods() { @@ -688,5 +727,5 @@ func testPodHugePagesSize(env *provider.TestEnvironment, size string) { compliantObjects = append(compliantObjects, testhelper.NewPodReportObject(put.Namespace, put.Name, "Pod has been found to be running with a correct hugepages size", true)) } } - testhelper.AddTestResultReason(compliantObjects, nonCompliantObjects, tnf.ClaimFilePrintf, ginkgo.Fail) + check.SetResult(compliantObjects, nonCompliantObjects) } diff --git a/generated_policy.json b/generated_policy.json index 3c99927f5..c96b32077 100644 --- a/generated_policy.json +++ b/generated_policy.json @@ -270,6 +270,16 @@ "id": "performance-max-resources-exec-probes", "suite": "performance", "tags": "faredge" + }, + { + "id": "platform-alteration-isredhat-release", + "suite": "platform-alteration", + "tags": "common" + }, + { + "id": "platform-alteration-tainted-node-kernel", + "suite": "platform-alteration", + "tags": "common" } ], "gradeName": "good" diff --git a/pkg/certsuite/certsuite.go b/pkg/certsuite/certsuite.go index 1108c304a..d31000fc8 100644 --- a/pkg/certsuite/certsuite.go +++ b/pkg/certsuite/certsuite.go @@ -13,6 +13,7 @@ import ( "github.com/test-network-function/cnf-certification-test/cnf-certification-test/networking" "github.com/test-network-function/cnf-certification-test/cnf-certification-test/observability" "github.com/test-network-function/cnf-certification-test/cnf-certification-test/performance" + "github.com/test-network-function/cnf-certification-test/cnf-certification-test/platform" "github.com/test-network-function/cnf-certification-test/cnf-certification-test/results" "github.com/test-network-function/cnf-certification-test/pkg/checksdb" "github.com/test-network-function/cnf-certification-test/pkg/claimhelper" @@ -29,6 +30,7 @@ func LoadChecksDB() { networking.LoadChecks() observability.LoadChecks() performance.LoadChecks() + platform.LoadChecks() } func Run(labelsFilter, outputFolder string, timeout time.Duration) { diff --git a/pkg/testhelper/testhelper.go b/pkg/testhelper/testhelper.go index f28dbc1cd..a09ba1e1a 100644 --- a/pkg/testhelper/testhelper.go +++ b/pkg/testhelper/testhelper.go @@ -178,6 +178,8 @@ const ( ContainerType = "Container" ContainerImageType = "Container Image" NodeType = "Node" + OCPClusterType = "OCP Cluster" + OCPClusterVersionType = "OCP Cluster Version" ContainerProcessType = "ContainerProcess" ContainerCategory = "ContainerCategory" ServiceType = "Service" @@ -235,6 +237,12 @@ func NewNodeReportObject(aNodeName, aReason string, isCompliant bool) (out *Repo return out } +func NewClusterVersionReportObject(version, aReason string, isCompliant bool) (out *ReportObject) { + out = NewReportObject(aReason, OCPClusterType, isCompliant) + out.AddField(OCPClusterVersionType, version) + return out +} + func NewTaintReportObject(taintBit, nodeName, aReason string, isCompliant bool) (out *ReportObject) { out = NewReportObject(aReason, TaintType, isCompliant) out.AddField(NodeType, nodeName) @@ -326,6 +334,15 @@ func ResultToString(result int) (str string) { return "" } +func GetNonOCPClusterSkipFn() func() (bool, string) { + return func() (bool, string) { + if !provider.IsOCPCluster() { + return true, "non-OCP cluster detected" + } + return false, "" + } +} + func GetNoServicesUnderTestSkipFn(env *provider.TestEnvironment) func() (bool, string) { return func() (bool, string) { if len(env.Services) == 0 { @@ -529,6 +546,33 @@ func GetNoPersistentVolumeClaimsSkipFn(env *provider.TestEnvironment) func() (bo } } +func GetNoBareMetalNodesSkipFn(env *provider.TestEnvironment) func() (bool, string) { + return func() (bool, string) { + if len(env.GetBaremetalNodes()) == 0 { + return true, "no baremetal nodes found" + } + return false, "" + } +} + +func GetNoIstioSkipFn(env *provider.TestEnvironment) func() (bool, string) { + return func() (bool, string) { + if !env.IstioServiceMeshFound { + return true, "no istio service mesh found" + } + return false, "" + } +} + +func GetNoHugepagesPodsSkipFn(env *provider.TestEnvironment) func() (bool, string) { + return func() (bool, string) { + if len(env.GetHugepagesPods()) == 0 { + return true, "no pods requesting hugepages found" + } + return false, "" + } +} + func SkipIfEmptyAny(skip func(string, ...int), object ...[2]interface{}) { for _, o := range object { s := reflect.ValueOf(o[0])