diff --git a/.github/workflows/pre-main.yaml b/.github/workflows/pre-main.yaml index 5c470ef02..72edceda3 100644 --- a/.github/workflows/pre-main.yaml +++ b/.github/workflows/pre-main.yaml @@ -28,6 +28,7 @@ env: TERM: xterm-color CM_BIN: /usr/local/bin/checkmake CM_URL_LINUX: https://github.com/mrtazz/checkmake/releases/download/0.2.2/checkmake-0.2.2.linux.amd64 # yamllint disable-line + REDHAT_CI: true concurrency: group: ${{ github.workflow }}-${{ github.event.number || github.ref }} diff --git a/.github/workflows/qe-ocp-414-intrusive.yaml b/.github/workflows/qe-ocp-414-intrusive.yaml index 9b66ef823..d49e289ef 100644 --- a/.github/workflows/qe-ocp-414-intrusive.yaml +++ b/.github/workflows/qe-ocp-414-intrusive.yaml @@ -13,6 +13,7 @@ permissions: env: QE_REPO: redhat-best-practices-for-k8s/certsuite-qe + REDHAT_CI: true jobs: pull-unstable-image: diff --git a/.github/workflows/qe-ocp-414.yaml b/.github/workflows/qe-ocp-414.yaml index 6bb9eeb7f..b33d0c9d0 100644 --- a/.github/workflows/qe-ocp-414.yaml +++ b/.github/workflows/qe-ocp-414.yaml @@ -13,6 +13,7 @@ permissions: env: QE_REPO: redhat-best-practices-for-k8s/certsuite-qe + REDHAT_CI: true jobs: pull-unstable-image: diff --git a/.github/workflows/qe-ocp-415-intrusive.yaml b/.github/workflows/qe-ocp-415-intrusive.yaml index ccf3b8094..09c86bd30 100644 --- a/.github/workflows/qe-ocp-415-intrusive.yaml +++ b/.github/workflows/qe-ocp-415-intrusive.yaml @@ -13,6 +13,7 @@ permissions: env: QE_REPO: redhat-best-practices-for-k8s/certsuite-qe + REDHAT_CI: true jobs: pull-unstable-image: diff --git a/.github/workflows/qe-ocp-415.yaml b/.github/workflows/qe-ocp-415.yaml index 566a65be2..459431743 100644 --- a/.github/workflows/qe-ocp-415.yaml +++ b/.github/workflows/qe-ocp-415.yaml @@ -13,6 +13,7 @@ permissions: env: QE_REPO: redhat-best-practices-for-k8s/certsuite-qe + REDHAT_CI: true jobs: pull-unstable-image: diff --git a/.github/workflows/qe-ocp-416-intrusive.yaml b/.github/workflows/qe-ocp-416-intrusive.yaml index 1f173a63b..c5468a945 100644 --- a/.github/workflows/qe-ocp-416-intrusive.yaml +++ b/.github/workflows/qe-ocp-416-intrusive.yaml @@ -13,6 +13,7 @@ permissions: env: QE_REPO: redhat-best-practices-for-k8s/certsuite-qe + REDHAT_CI: true jobs: pull-unstable-image: diff --git a/.github/workflows/qe-ocp-416.yaml b/.github/workflows/qe-ocp-416.yaml index 82fa986b1..10f27d8e2 100644 --- a/.github/workflows/qe-ocp-416.yaml +++ b/.github/workflows/qe-ocp-416.yaml @@ -13,6 +13,7 @@ permissions: env: QE_REPO: redhat-best-practices-for-k8s/certsuite-qe + REDHAT_CI: true jobs: pull-unstable-image: diff --git a/.github/workflows/qe-ocp-417-intrusive.yaml b/.github/workflows/qe-ocp-417-intrusive.yaml index 431a09500..72970056c 100644 --- a/.github/workflows/qe-ocp-417-intrusive.yaml +++ b/.github/workflows/qe-ocp-417-intrusive.yaml @@ -13,6 +13,7 @@ permissions: env: QE_REPO: redhat-best-practices-for-k8s/certsuite-qe + REDHAT_CI: true jobs: pull-unstable-image: diff --git a/.github/workflows/qe-ocp-417.yaml b/.github/workflows/qe-ocp-417.yaml index b90bc6731..ab88501b7 100644 --- a/.github/workflows/qe-ocp-417.yaml +++ b/.github/workflows/qe-ocp-417.yaml @@ -13,6 +13,7 @@ permissions: env: QE_REPO: redhat-best-practices-for-k8s/certsuite-qe + REDHAT_CI: true jobs: pull-unstable-image: diff --git a/.github/workflows/qe-ocp-arm-416.yaml b/.github/workflows/qe-ocp-arm-416.yaml index 5b2d61091..6d7a44943 100644 --- a/.github/workflows/qe-ocp-arm-416.yaml +++ b/.github/workflows/qe-ocp-arm-416.yaml @@ -22,6 +22,7 @@ env: DOCKER_CONFIG_DIR: '/home/labuser/.docker' CERTSUITE_CONFIG_DIR: '/home/labuser/certsuite_config' CERTSUITE_REPORT_DIR: '/home/labuser/tnf_report' + REDHAT_CI: true jobs: build-arm-image-for-qe: diff --git a/.github/workflows/qe-ocp-pre-main.yaml b/.github/workflows/qe-ocp-pre-main.yaml index dc76771af..3c2c2034a 100644 --- a/.github/workflows/qe-ocp-pre-main.yaml +++ b/.github/workflows/qe-ocp-pre-main.yaml @@ -18,6 +18,7 @@ env: DOCKER_CONFIG_DIR: '/home/labuser3/.docker' CERTSUITE_CONFIG_DIR: '/home/labuser3/certsuite_config' CERTSUITE_REPORT_DIR: '/home/labuser3/tnf_report' + REDHAT_CI: true jobs: # Build the image used for testing first, then pass the reference to the QE tests. diff --git a/CATALOG.md b/CATALOG.md index 6c80a42df..5bb7d0929 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -36,11 +36,11 @@ Depending on the workload type, not all tests are required to pass to satisfy be |---|---| |8|1| -### Non-Telco specific tests only: 69 +### Non-Telco specific tests only: 65 |Mandatory|Optional| |---|---| -|44|25| +|42|23| ### Telco specific tests only: 27 @@ -1006,22 +1006,6 @@ Tags|telco,networking |Non-Telco|Optional| |Telco|Mandatory| -#### networking-network-attachment-definition-sriov-mtu - -Property|Description ----|--- -Unique ID|networking-network-attachment-definition-sriov-mtu -Description|Ensures that MTU values are set correctly in NetworkAttachmentDefinitions for SRIOV network interfaces. -Suggested Remediation|Ensure that the MTU of the SR-IOV network attachment definition is set explicitly. -Best Practice Reference|No Doc Link - Far Edge -Exception Process|There is no documented exception process for this. -Tags|faredge,networking -|**Scenario**|**Optional/Mandatory**| -|Extended|Mandatory| -|Far-Edge|Mandatory| -|Non-Telco|Mandatory| -|Telco|Mandatory| - #### networking-network-policy-deny-all Property|Description @@ -1186,22 +1170,6 @@ Tags|telco,observability ### operator -#### operator-catalogsource-bundle-count - -Property|Description ----|--- -Unique ID|operator-catalogsource-bundle-count -Description|Tests operator catalog source bundle count is less than 1000 -Suggested Remediation|Ensure that the Operator's catalog source has a valid bundle count less than 1000. -Best Practice Reference|https://redhat-best-practices-for-k8s.github.io/guide/#redhat-best-practices-for-k8s-cnf-operator-requirements -Exception Process|No exceptions -Tags|common,operator -|**Scenario**|**Optional/Mandatory**| -|Extended|Mandatory| -|Far-Edge|Mandatory| -|Non-Telco|Mandatory| -|Telco|Mandatory| - #### operator-crd-openapi-schema Property|Description @@ -1282,54 +1250,6 @@ Tags|common,operator |Non-Telco|Mandatory| |Telco|Mandatory| -#### operator-multiple-same-operators - -Property|Description ----|--- -Unique ID|operator-multiple-same-operators -Description|Tests whether multiple instances of the same Operator CSV are installed. -Suggested Remediation|Ensure that only one Operator of the same type is installed in the cluster. -Best Practice Reference|https://redhat-best-practices-for-k8s.github.io/guide/#redhat-best-practices-for-k8s-cnf-operator-requirements -Exception Process|No exceptions -Tags|common,operator -|**Scenario**|**Optional/Mandatory**| -|Extended|Mandatory| -|Far-Edge|Mandatory| -|Non-Telco|Mandatory| -|Telco|Mandatory| - -#### operator-olm-skip-range - -Property|Description ----|--- -Unique ID|operator-olm-skip-range -Description|Test that checks the operator has a valid olm skip range. -Suggested Remediation|Ensure that the Operator has a valid OLM skip range. If the operator does not have another version to "skip", then ignore the result of this test. -Best Practice Reference|https://redhat-best-practices-for-k8s.github.io/guide/#redhat-best-practices-for-k8s-cnf-operator-requirements -Exception Process|If there is not a version of the operator that needs to be skipped, then an exception will be granted. -Tags|common,operator -|**Scenario**|**Optional/Mandatory**| -|Extended|Optional| -|Far-Edge|Optional| -|Non-Telco|Optional| -|Telco|Optional| - -#### operator-pods-no-hugepages - -Property|Description ----|--- -Unique ID|operator-pods-no-hugepages -Description|Tests that the pods do not have hugepages enabled. -Suggested Remediation|Ensure that the pods are not using hugepages -Best Practice Reference|https://redhat-best-practices-for-k8s.github.io/guide/#redhat-best-practices-for-k8s-cnf-operator-requirements -Exception Process|No exceptions -Tags|common,operator -|**Scenario**|**Optional/Mandatory**| -|Extended|Optional| -|Far-Edge|Optional| -|Non-Telco|Optional| -|Telco|Optional| - #### operator-semantic-versioning Property|Description diff --git a/cmd/certsuite/generate/catalog/catalog.go b/cmd/certsuite/generate/catalog/catalog.go index 3ceb76935..5784c966a 100644 --- a/cmd/certsuite/generate/catalog/catalog.go +++ b/cmd/certsuite/generate/catalog/catalog.go @@ -251,6 +251,11 @@ func outputTestCases() (outString string, summary catalogSummary) { //nolint:fun // Every paragraph starts with a new line. + // Skip the test if it has the "waiting-for-release" tag. + if strings.Contains(tags, "waiting-for-release") { + continue + } + outString += fmt.Sprintf("\n#### %s\n\n", k.testName) outString += "Property|Description\n" outString += "---|---\n" diff --git a/expected_results.yaml b/expected_results.yaml index a3a6c30e1..e62ac9ef6 100644 --- a/expected_results.yaml +++ b/expected_results.yaml @@ -77,16 +77,16 @@ testCases: - networking-dpdk-cpu-pinning-exec-probe - networking-icmpv6-connectivity - networking-restart-on-reboot-sriov-pod - - networking-network-attachment-definition-sriov-mtu + # - networking-network-attachment-definition-sriov-mtu # Re-enable when new releasing new minor version. - operator-install-source - operator-install-status-no-privileges - operator-install-status-succeeded - - operator-olm-skip-range + # - operator-olm-skip-range # Re-enable when new releasing new minor version. - operator-semantic-versioning - operator-single-crd-owner - - operator-pods-no-hugepages - - operator-multiple-same-operators - - operator-catalogsource-bundle-count + # - operator-pods-no-hugepages # Re-enable when new releasing new minor version. + # - operator-multiple-same-operators # Re-enable when new releasing new minor version. + # - operator-catalogsource-bundle-count # Re-enable when new releasing new minor version. - performance-exclusive-cpu-pool-rt-scheduling-policy - performance-isolated-cpu-pool-rt-scheduling-policy - performance-shared-cpu-pool-non-rt-scheduling-policy diff --git a/pkg/checksdb/checksgroup.go b/pkg/checksdb/checksgroup.go index 2dc76150a..037f4d11e 100644 --- a/pkg/checksdb/checksgroup.go +++ b/pkg/checksdb/checksgroup.go @@ -3,11 +3,13 @@ package checksdb import ( "errors" "fmt" + "os" "runtime/debug" "strings" "github.com/redhat-best-practices-for-k8s/certsuite/internal/cli" "github.com/redhat-best-practices-for-k8s/certsuite/internal/log" + "github.com/redhat-best-practices-for-k8s/certsuite/pkg/stringhelper" ) const ( @@ -76,7 +78,11 @@ func (group *ChecksGroup) Add(check *Check) { dbLock.Lock() defer dbLock.Unlock() - group.checks = append(group.checks, check) + // Only add the check to the group if its not "waiting-for-release" or if the REDHAT_CI env var is set. + if !stringhelper.StringInSlice(check.Labels, "waiting-for-release", false) || + os.Getenv("REDHAT_CI") == "true" { + group.checks = append(group.checks, check) + } } func skipCheck(check *Check, reason string) { diff --git a/pkg/provider/catalogsources.go b/pkg/provider/catalogsources.go index fbf48c0f8..777d97de4 100644 --- a/pkg/provider/catalogsources.go +++ b/pkg/provider/catalogsources.go @@ -22,7 +22,7 @@ func GetCatalogSourceBundleCount(env *TestEnvironment, cs *olmv1Alpha.CatalogSou ) // Check if the cluster is running an OCP version <= 4.12 - if env.OpenshiftVersion != "" { + if env.OpenshiftVersion != "" && env.OpenshiftVersion != "0.0.0" { log.Info("Cluster is determined to be running Openshift version %q.", env.OpenshiftVersion) version, err := semver.NewVersion(env.OpenshiftVersion) if err != nil { diff --git a/tests/identifiers/identifiers.go b/tests/identifiers/identifiers.go index c19eb0e58..8a7e8a20b 100644 --- a/tests/identifiers/identifiers.go +++ b/tests/identifiers/identifiers.go @@ -35,17 +35,18 @@ for instance the Performance Addon Operator (PAO) or istio.` ) const ( - TagCommon = "common" - TagExtended = "extended" - TagTelco = "telco" - TagFarEdge = "faredge" - FarEdge = "FarEdge" - Telco = "Telco" - NonTelco = "NonTelco" - Extended = "Extended" - Optional = "Optional" - Mandatory = "Mandatory" - TagPreflight = "preflight" + TagCommon = "common" + TagExtended = "extended" + TagTelco = "telco" + TagFarEdge = "faredge" + FarEdge = "FarEdge" + Telco = "Telco" + NonTelco = "NonTelco" + Extended = "Extended" + Optional = "Optional" + Mandatory = "Mandatory" + TagPreflight = "preflight" + TagWaitingForRelease = "waiting-for-release" ) const ( @@ -598,7 +599,7 @@ func InitCatalog() map[claim.Identifier]claim.TestCaseDescription { NonTelco: Mandatory, Extended: Mandatory, }, - TagFarEdge) + TagFarEdge, TagWaitingForRelease) TestSecConNonRootUserIDIdentifier = AddCatalogEntry( "security-context-non-root-user-id-check", @@ -1002,7 +1003,7 @@ that Node's kernel may not have the same hacks.'`, NonTelco: Optional, Extended: Optional, }, - TagCommon) + TagCommon, TagWaitingForRelease) TestOperatorCrdVersioningIdentifier = AddCatalogEntry( "crd-versioning", @@ -1066,7 +1067,7 @@ that Node's kernel may not have the same hacks.'`, NonTelco: Optional, Extended: Optional, }, - TagCommon) + TagCommon, TagWaitingForRelease) TestOperatorCatalogSourceBundleCountIdentifier = AddCatalogEntry( "catalogsource-bundle-count", @@ -1082,7 +1083,7 @@ that Node's kernel may not have the same hacks.'`, NonTelco: Mandatory, Extended: Mandatory, }, - TagCommon) + TagCommon, TagWaitingForRelease) TestMultipleSameOperatorsIdentifier = AddCatalogEntry( "multiple-same-operators", @@ -1098,7 +1099,7 @@ that Node's kernel may not have the same hacks.'`, NonTelco: Mandatory, Extended: Mandatory, }, - TagCommon) + TagCommon, TagWaitingForRelease) TestPodNodeSelectorAndAffinityBestPractices = AddCatalogEntry( "pod-scheduling", diff --git a/tests/operator/suite.go b/tests/operator/suite.go index 363e86ecb..c8f93cde4 100644 --- a/tests/operator/suite.go +++ b/tests/operator/suite.go @@ -466,7 +466,7 @@ func testOperatorCatalogSourceBundleCount(check *checksdb.Check, env *provider.T ocp412Skip := false // Check if the cluster is running an OCP version <= 4.12 - if env.OpenshiftVersion != "" { + if env.OpenshiftVersion != "" && env.OpenshiftVersion != "0.0.0" { log.Info("Cluster is determined to be running Openshift version %q.", env.OpenshiftVersion) version, err := semver.NewVersion(env.OpenshiftVersion) if err != nil {