Merge branch 'main' into cnfweb

.github/workflows/pre-main.yaml

@@ -67,7 +67,7 @@ jobs:
       - name: Golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.54
+          version: v1.55
           args: --timeout 10m0s
 
       - name: Checkmake
@@ -355,6 +355,18 @@ jobs:
         env:
           IMAGE_TAG: ${TNF_IMAGE_TAG}
 
+      # Prepare collector to be used when running smoke tests
+      - name: Check out `Collector`
+        uses: actions/checkout@v4
+        with:
+          repository: test-network-function/collector
+          path: collector
+
+      - name: Deploy collector and mysql
+        uses: ./collector/.github/actions/prepare-collector-for-use
+        with:
+          working_directory: collector
+
       # Clean up unused container image layers. We need to filter out a possible error return code
       # from docker with "|| true" as some images might still be used by running kind containers and
       # won't be removed.
@@ -376,23 +388,11 @@ jobs:
       - name: 'Test: Run Smoke Tests in a TNF container'
         run: TNF_LOG_LEVEL=${TNF_SMOKE_TESTS_LOG_LEVEL} TNF_ENABLE_DATA_COLLECTION=true ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }} -l "${SMOKE_TESTS_GINKGO_LABELS_FILTER}"
 
-      # Prepare collector to be used when running smoke tests
-      - name: Check out `Collector`
-        uses: actions/checkout@v4
-        with:
-          repository: test-network-function/collector
-          path: collector
-
-      - name: Deploy collector and mysql
-        uses: ./collector/.github/actions/prepare-collector-for-use
+      - name: Run sanity check on collector
+        uses: ./collector/.github/actions/run-sanity-check
         with:
           working_directory: collector
 
-      # - name: Run sanity check on collector
-      #   uses: ./collector/.github/actions/run-sanity-check
-      #   with:
-      #     working_directory: collector
-
       - name: Upload container test results as an artifact
         uses: actions/upload-artifact@v3
         if: always()

.github/workflows/preflight.yml

@@ -0,0 +1,49 @@
+---
+name: Preflight
+'on':
+  schedule:
+    - cron: '0 0 * * *'
+  # pull_request:
+  #   branches: [ main ]
+  workflow_dispatch:
+
+jobs:
+  preflight-unstable:
+    runs-on: ubuntu-22.04
+    env:
+      SHELL: /bin/bash
+      IMAGE_NAME: quay.io/testnetworkfunction/cnf-certification-test:unstable
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go 1.21
+        uses: actions/setup-go@v4
+        with:
+          go-version: 1.21.3
+
+      - name: Disable default go problem matcher
+        run: echo "::remove-matcher owner=go::"
+
+      - name: Clone the preflight repository
+        uses: actions/checkout@v4
+        with:
+          repository: redhat-openshift-ecosystem/openshift-preflight
+          path: openshift-preflight
+
+      - name: Run preflight
+        working-directory: openshift-preflight
+        run: |
+          make build
+          ./preflight check container ${{ env.IMAGE_NAME }} --artifacts ${GITHUB_WORKSPACE}/openshift-preflight/artifacts
+
+          ALLOWED_FAILURES=("HasLicense" "RunAsNonRoot")
+          FAILURES=$(cat ${GITHUB_WORKSPACE}/openshift-preflight/artifacts/amd64/results.json | jq -r '.results.failed[].name')
+
+          for FAILURE in $FAILURES; do
+            FAILURE=$(echo $FAILURE | tr -d '"')
+            if [[ ! "${ALLOWED_FAILURES[@]}" =~ "${FAILURE}" ]]; then
+              echo >&2 "ERROR: $FAILURE is not an allowed failure"
+              exit 1
+            fi
+          done

.github/workflows/qe-ocp-intrusive.yaml

@@ -0,0 +1,66 @@
+name: QE OCP Intrusive Testing
+
+on:
+  # pull_request:
+  #   branches: [ main ]
+  workflow_dispatch:
+  # Schedule a daily cron at midnight UTC
+  schedule:
+    - cron: '0 0 * * *'
+env:
+  QE_REPO: test-network-function/cnfcert-tests-verification
+
+jobs:
+  qe-ocp-intrusive-testing:
+    runs-on: qe-ocp
+    strategy:
+      fail-fast: false
+      matrix: 
+        # Add more suites if more intrusive tests are added to the QE repo
+        suite: [lifecycle]
+    env:
+      SHELL: /bin/bash
+      KUBECONFIG: '/home/labuser/.kube/config'
+      PFLT_DOCKERCONFIG: '/home/labuser/.docker/config'
+      TEST_TNF_IMAGE_NAME: quay.io/testnetworkfunction/cnf-certification-test
+      TEST_TNF_IMAGE_TAG: localtest
+      DOCKER_CONFIG_DIR: '/home/labuser/.docker'
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.sha }}
+
+      - name: Run initial setup
+        uses: ./.github/actions/setup
+
+      - name: Preemptively prune docker resources
+        run: docker system prune -f --volumes
+
+      - name: Build the test image
+        run: make build-image-local # quay.io/testnetworkfunction/cnf-certification-test:localtest
+
+      - name: Show pods
+        run: oc get pods -A
+
+      - name: Clone the QE repository
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ env.QE_REPO }}
+          path: cnfcert-tests-verification
+
+      - name: Preemptively potential QE namespaces
+        run: ./scripts/delete-namespaces.sh
+        working-directory: cnfcert-tests-verification
+
+      - name: Preemptively delete report and config folders
+        shell: bash
+        run: |
+          sudo rm -rf /tmp/tnf_config/
+          sudo rm -rf /tmp/tnf_report/
+
+      # Setup is complete.  Time to run the QE tests.
+      - name: Run the tests
+        run: FEATURES=${{matrix.suite}} TNF_REPO_PATH=${GITHUB_WORKSPACE} TNF_IMAGE=${{env.TEST_TNF_IMAGE_NAME}} TNF_IMAGE_TAG=${{env.TEST_TNF_IMAGE_TAG}} DISABLE_INTRUSIVE_TESTS=false ENABLE_PARALLEL=false ENABLE_FLAKY_RETRY=true make test-features
+        working-directory: cnfcert-tests-verification

.github/workflows/qe-ocp.yaml

@@ -34,6 +34,9 @@ jobs:
       - name: Run initial setup
         uses: ./.github/actions/setup
 
+      - name: Preemptively prune docker resources
+        run: docker system prune -f --volumes
+
       - name: Build the test image
         run: make build-image-local # quay.io/testnetworkfunction/cnf-certification-test:localtest
 

.golangci.yml

@@ -121,6 +121,6 @@ issues:
 # golangci.com configuration
 # https://github.com/golangci/golangci/wiki/Configuration
 service:
-  golangci-lint-version: 1.54.x # use the fixed version to not introduce new linters unexpectedly
+  golangci-lint-version: 1.55.x # use the fixed version to not introduce new linters unexpectedly
   prepare:
     - echo "here I can run custom commands, but no preparation needed for this repo"

Makefile

@@ -49,7 +49,7 @@ GIT_COMMIT=$(shell script/create-version-files.sh)
 GIT_RELEASE=$(shell script/get-git-release.sh)
 GIT_PREVIOUS_RELEASE=$(shell script/get-git-previous-release.sh)
 CLAIM_FORMAT_VERSION=$(shell script/get-claim-version.sh)
-GOLANGCI_VERSION=v1.54.2
+GOLANGCI_VERSION=v1.55.1
 LINKER_TNF_RELEASE_FLAGS=-X github.com/test-network-function/cnf-certification-test/cnf-certification-test.GitCommit=${GIT_COMMIT}
 LINKER_TNF_RELEASE_FLAGS+= -X github.com/test-network-function/cnf-certification-test/cnf-certification-test.GitRelease=${GIT_RELEASE}
 LINKER_TNF_RELEASE_FLAGS+= -X github.com/test-network-function/cnf-certification-test/cnf-certification-test.GitPreviousRelease=${GIT_PREVIOUS_RELEASE}

_typos.toml

@@ -1,5 +1,6 @@
 [default.extend-words]
 iif = "iif"
+ono = "ono"
 
 [files]
-extend-exclude = ["go.mod", "results.html", "cmd/tnf/claim/compare/testdata", "docs/assets/images/*.svg"]
+extend-exclude = ["depends-on.json", "go.mod", "results.html", "cmd/tnf/claim/compare/testdata", "docs/assets/images/*.svg"]

cnf-certification-test/identifiers/identifiers.go

@@ -48,6 +48,10 @@ const (
 	TagPreflight = "preflight"
 )
 
+const (
+	NotApplicableSNO = ` Not applicable to SNO applications.`
+)
+
 func init() {
 	InitCatalog()
 }
@@ -925,7 +929,7 @@ tag. (2) It does not have any of the following prefixes: default, openshift-, is
 		common.LifecycleTestKey,
 		`Ensures that CNF Pods specify podAntiAffinity rules and replica value is set to more than 1.`,
 		PodHighAvailabilityBestPracticesRemediation,
-		NoDocumentedProcess+` Not applicable to SNO applications.`,
+		NoDocumentedProcess+NotApplicableSNO,
 		TestPodHighAvailabilityBestPracticesDocLink,
 		true,
 		map[string]string{
@@ -973,7 +977,7 @@ tag. (2) It does not have any of the following prefixes: default, openshift-, is
 		common.LifecycleTestKey,
 		`Tests that CNF deployments support scale in/out operations. First, the test starts getting the current replicaCount (N) of the deployment/s with the Pod Under Test. Then, it executes the scale-in oc command for (N-1) replicas. Lastly, it executes the scale-out oc command, restoring the original replicaCount of the deployment/s. In case of deployments that are managed by HPA the test is changing the min and max value to deployment Replica - 1 during scale-in and the original replicaCount again for both min/max during the scale-out stage. Lastly its restoring the original min/max replica of the deployment/s`, //nolint:lll
 		DeploymentScalingRemediation,
-		NoDocumentedProcess+` Not applicable to SNO applications.`,
+		NoDocumentedProcess+NotApplicableSNO,
 		TestDeploymentScalingIdentifierDocLink,
 		true,
 		map[string]string{
@@ -989,7 +993,7 @@ tag. (2) It does not have any of the following prefixes: default, openshift-, is
 		common.LifecycleTestKey,
 		`Tests that CNF statefulsets support scale in/out operations. First, the test starts getting the current replicaCount (N) of the statefulset/s with the Pod Under Test. Then, it executes the scale-in oc command for (N-1) replicas. Lastly, it executes the scale-out oc command, restoring the original replicaCount of the statefulset/s. In case of statefulsets that are managed by HPA the test is changing the min and max value to statefulset Replica - 1 during scale-in and the original replicaCount again for both min/max during the scale-out stage. Lastly its restoring the original min/max replica of the statefulset/s`, //nolint:lll
 		StatefulSetScalingRemediation,
-		NoDocumentedProcess+` Not applicable to SNO applications.`,
+		NoDocumentedProcess+NotApplicableSNO,
 		TestStateFulSetScalingIdentifierDocLink,
 		true,
 		map[string]string{
@@ -1357,7 +1361,7 @@ tag. (2) It does not have any of the following prefixes: default, openshift-, is
 		common.AccessControlTestKey,
 		`Check that all containers under test have only one process running`,
 		OneProcessPerContainerRemediation,
-		NoExceptionProcessForExtendedTests+` Not applicable to SNO applications.`,
+		NoExceptionProcessForExtendedTests+NotApplicableSNO,
 		TestOneProcessPerContainerIdentifierDocLink,
 		true,
 		map[string]string{
@@ -1549,7 +1553,7 @@ tag. (2) It does not have any of the following prefixes: default, openshift-, is
 		common.LifecycleTestKey,
 		`Tests that CNF crd support scale in/out operations. First, the test starts getting the current replicaCount (N) of the crd/s with the Pod Under Test. Then, it executes the scale-in oc command for (N-1) replicas. Lastly, it executes the scale-out oc command, restoring the original replicaCount of the crd/s. In case of crd that are managed by HPA the test is changing the min and max value to crd Replica - 1 during scale-in and the original replicaCount again for both min/max during the scale-out stage. Lastly its restoring the original min/max replica of the crd/s`, //nolint:lll
 		CrdScalingRemediation,
-		NoDocumentedProcess+` Not applicable to SNO applications.`,
+		NoDocumentedProcess+NotApplicableSNO,
 		TestCrdScalingIdentifierDocLink,
 		true,
 		map[string]string{

cnf-certification-test/performance/suite.go

@@ -19,6 +19,7 @@ package performance
 import (
 	"fmt"
 	"strconv"
+	"strings"
 
 	"github.com/onsi/ginkgo/v2"
 	"github.com/sirupsen/logrus"
@@ -232,30 +233,54 @@ func testSchedulingPolicyInCPUPool(env *provider.TestEnvironment,
 	testhelper.AddTestResultReason(compliantContainersPids, nonCompliantContainersPids, tnf.ClaimFilePrintf, ginkgo.Fail)
 }
 
+const noProcessFoundErrMsg = "No such process"
+
 func testRtAppsNoExecProbes(env *provider.TestEnvironment, cuts []*provider.Container) {
 	var compliantObjects []*testhelper.ReportObject
 	var nonCompliantObjects []*testhelper.ReportObject
 	for _, cut := range cuts {
+		if !cut.HasExecProbes() {
+			compliantObjects = append(compliantObjects, testhelper.NewContainerReportObject(cut.Namespace, cut.Podname, cut.Name, "Container does not define exec probes", true))
+			continue
+		}
+
 		processes, err := crclient.GetContainerProcesses(cut, env)
 		if err != nil {
 			tnf.ClaimFilePrintf("Could not determine the processes pids for container %s, err: %v", cut, err)
 			nonCompliantObjects = append(nonCompliantObjects, testhelper.NewContainerReportObject(cut.Namespace, cut.Podname, cut.Name, "Could not determine the processes pids for container", false))
 			break
 		}
+
+		allProcessesCompliant := true
 		for _, p := range processes {
 			schedPolicy, _, err := scheduling.GetProcessCPUScheduling(p.Pid, cut)
 			if err != nil {
+				// If the process does not exist anymore it means that it has finished since the time the process list
+				// was retrieved. In this case, just ignore the error and continue processing the rest of the processes.
+				if strings.Contains(err.Error(), noProcessFoundErrMsg) {
+					compliantObjects = append(compliantObjects, testhelper.NewContainerReportObject(cut.Namespace, cut.Podname, cut.Name, "Container process disappeared", true).
+						AddField(testhelper.ProcessID, strconv.Itoa(p.Pid)).
+						AddField(testhelper.ProcessCommandLine, p.Args))
+					continue
+				}
 				tnf.ClaimFilePrintf("Could not determine the scheduling policy for container %s (pid=%v), err: %v", cut, p.Pid, err)
 				nonCompliantObjects = append(nonCompliantObjects, testhelper.NewContainerReportObject(cut.Namespace, cut.Podname, cut.Name, "Could not determine the scheduling policy for container", false).
-					AddField(testhelper.ProcessID, strconv.Itoa(p.Pid)))
-				break
+					AddField(testhelper.ProcessID, strconv.Itoa(p.Pid)).
+					AddField(testhelper.ProcessCommandLine, p.Args))
+				allProcessesCompliant = false
+				continue
 			}
-			if scheduling.PolicyIsRT(schedPolicy) && cut.HasExecProbes() {
+			if scheduling.PolicyIsRT(schedPolicy) {
 				tnf.ClaimFilePrintf("Pod %s/Container %s defines exec probes while having a RT scheduling policy for pid %d", cut.Podname, cut, p.Pid)
 				nonCompliantObjects = append(nonCompliantObjects, testhelper.NewContainerReportObject(cut.Namespace, cut.Podname, cut.Name, "Container defines exec probes while having a RT scheduling policy", false).
 					AddField(testhelper.ProcessID, strconv.Itoa(p.Pid)))
+				allProcessesCompliant = false
 			}
 		}
+
+		if allProcessesCompliant {
+			compliantObjects = append(compliantObjects, testhelper.NewContainerReportObject(cut.Namespace, cut.Podname, cut.Name, "Container defines exec probes but does not have a RT scheduling policy", true))
+		}
 	}
 	testhelper.AddTestResultReason(compliantObjects, nonCompliantObjects, tnf.ClaimFilePrintf, ginkgo.Fail)
 }

cnf-certification-test/platform/cnffsdiff/fsdiff_test.go

@@ -23,7 +23,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	. "github.com/test-network-function/cnf-certification-test/cnf-certification-test/platform/cnffsdiff"
+	"github.com/test-network-function/cnf-certification-test/cnf-certification-test/platform/cnffsdiff"
 	"github.com/test-network-function/cnf-certification-test/internal/clientsholder"
 	"github.com/test-network-function/cnf-certification-test/pkg/testhelper"
 )
@@ -105,7 +105,7 @@ func TestRunTest(t *testing.T) {
 			err:    tc.clientErr,
 		}
 
-		fsdiff := NewFsDiffTester(chm, clientsholder.Context{})
+		fsdiff := cnffsdiff.NewFsDiffTester(chm, clientsholder.Context{})
 		fsdiff.RunTest("fakeUID")
 		assert.Equal(t, tc.expectedResult, fsdiff.GetResults())
 	}
@@ -181,7 +181,7 @@ func TestRunTestMountFolderErrors(t *testing.T) {
 	}
 
 	for _, tc := range testCases {
-		fsdiff := NewFsDiffTester(tc.mockedClientshHolder, clientsholder.Context{})
+		fsdiff := cnffsdiff.NewFsDiffTester(tc.mockedClientshHolder, clientsholder.Context{})
 		fsdiff.RunTest("fakeUID")
 		assert.Equal(t, testhelper.ERROR, fsdiff.GetResults())
 		assert.Equal(t, fsdiff.Error.Error(), tc.expectedError)
@@ -263,7 +263,7 @@ func TestRunTestUnmountFolderErrors(t *testing.T) {
 	}
 
 	for _, tc := range testCases {
-		fsdiff := NewFsDiffTester(tc.mockedClientshHolder, clientsholder.Context{})
+		fsdiff := cnffsdiff.NewFsDiffTester(tc.mockedClientshHolder, clientsholder.Context{})
 		fsdiff.RunTest("fakeUID")
 		assert.Equal(t, testhelper.ERROR, fsdiff.GetResults())
 		assert.Equal(t, fsdiff.Error.Error(), tc.expectedError)

cnf-certification-test/platform/operatingsystem/files/rhcos_version_map

@@ -214,6 +214,7 @@
 4.13.16 / 413.92.202310100714-0
 4.13.17 / 413.92.202310141129-0
 4.13.18 / 413.92.202310170351-0
+4.13.19 / 413.92.202310210500-0
 4.13.2 / 413.92.202305302312-0
 4.13.3 / 413.92.202306070210-0
 4.13.4 / 413.92.202306141213-0

docs/test-standalone.md

@@ -34,7 +34,7 @@ make install-tools
 Dependency|Minimum Version
 ---|---
 [GoLang](https://golang.org/dl/)|1.21
-[golangci-lint](https://golangci-lint.run/usage/install/)|1.54.2
+[golangci-lint](https://golangci-lint.run/usage/install/)|1.55.1
 [jq](https://stedolan.github.io/jq/)|1.6
 [OpenShift Client](https://mirror.openshift.com/pub/openshift-v4/clients/ocp/)|4.12