Add WaitingForRelease feature flag

.github/workflows/pre-main.yaml

@@ -28,6 +28,7 @@ env:
   TERM: xterm-color
   CM_BIN: /usr/local/bin/checkmake
   CM_URL_LINUX: https://github.com/mrtazz/checkmake/releases/download/0.2.2/checkmake-0.2.2.linux.amd64 # yamllint disable-line
+  REDHAT_CI: true
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.number || github.ref }}

.github/workflows/qe-ocp-414-intrusive.yaml

@@ -13,6 +13,7 @@ permissions:
 
 env:
   QE_REPO: redhat-best-practices-for-k8s/certsuite-qe
+  REDHAT_CI: true
 
 jobs:
   pull-unstable-image:

.github/workflows/qe-ocp-414.yaml

@@ -13,6 +13,7 @@ permissions:
 
 env:
   QE_REPO: redhat-best-practices-for-k8s/certsuite-qe
+  REDHAT_CI: true
 
 jobs:
   pull-unstable-image:

.github/workflows/qe-ocp-415-intrusive.yaml

@@ -13,6 +13,7 @@ permissions:
 
 env:
   QE_REPO: redhat-best-practices-for-k8s/certsuite-qe
+  REDHAT_CI: true
 
 jobs:
   pull-unstable-image:

.github/workflows/qe-ocp-415.yaml

@@ -13,6 +13,7 @@ permissions:
 
 env:
   QE_REPO: redhat-best-practices-for-k8s/certsuite-qe
+  REDHAT_CI: true
 
 jobs:
   pull-unstable-image:

.github/workflows/qe-ocp-416-intrusive.yaml

@@ -13,6 +13,7 @@ permissions:
 
 env:
   QE_REPO: redhat-best-practices-for-k8s/certsuite-qe
+  REDHAT_CI: true
 
 jobs:
   pull-unstable-image:

.github/workflows/qe-ocp-416.yaml

@@ -13,6 +13,7 @@ permissions:
 
 env:
   QE_REPO: redhat-best-practices-for-k8s/certsuite-qe
+  REDHAT_CI: true
 
 jobs:
   pull-unstable-image:

.github/workflows/qe-ocp-417-intrusive.yaml

@@ -13,6 +13,7 @@ permissions:
 
 env:
   QE_REPO: redhat-best-practices-for-k8s/certsuite-qe
+  REDHAT_CI: true
 
 jobs:
   pull-unstable-image:

.github/workflows/qe-ocp-417.yaml

@@ -13,6 +13,7 @@ permissions:
 
 env:
   QE_REPO: redhat-best-practices-for-k8s/certsuite-qe
+  REDHAT_CI: true
 
 jobs:
   pull-unstable-image:

.github/workflows/qe-ocp-arm-416.yaml

@@ -22,6 +22,7 @@ env:
   DOCKER_CONFIG_DIR: '/home/labuser/.docker'
   CERTSUITE_CONFIG_DIR: '/home/labuser/certsuite_config'
   CERTSUITE_REPORT_DIR: '/home/labuser/tnf_report'
+  REDHAT_CI: true
 
 jobs:
   build-arm-image-for-qe:

.github/workflows/qe-ocp-pre-main.yaml

@@ -18,6 +18,7 @@ env:
   DOCKER_CONFIG_DIR: '/home/labuser3/.docker'
   CERTSUITE_CONFIG_DIR: '/home/labuser3/certsuite_config'
   CERTSUITE_REPORT_DIR: '/home/labuser3/tnf_report'
+  REDHAT_CI: true
 
 jobs:
   # Build the image used for testing first, then pass the reference to the QE tests.

CATALOG.md

@@ -36,11 +36,11 @@ Depending on the workload type, not all tests are required to pass to satisfy be
 |---|---|
 |8|1|
 
-### Non-Telco specific tests only: 69
+### Non-Telco specific tests only: 65
 
 |Mandatory|Optional|
 |---|---|
-|44|25|
+|42|23|
 
 ### Telco specific tests only: 27
 
@@ -1006,22 +1006,6 @@ Tags|telco,networking
 |Non-Telco|Optional|
 |Telco|Mandatory|
 
-#### networking-network-attachment-definition-sriov-mtu
-
-Property|Description
----|---
-Unique ID|networking-network-attachment-definition-sriov-mtu
-Description|Ensures that MTU values are set correctly in NetworkAttachmentDefinitions for SRIOV network interfaces.
-Suggested Remediation|Ensure that the MTU of the SR-IOV network attachment definition is set explicitly.
-Best Practice Reference|No Doc Link - Far Edge
-Exception Process|There is no documented exception process for this.
-Tags|faredge,networking
-|**Scenario**|**Optional/Mandatory**|
-|Extended|Mandatory|
-|Far-Edge|Mandatory|
-|Non-Telco|Mandatory|
-|Telco|Mandatory|
-
 #### networking-network-policy-deny-all
 
 Property|Description
@@ -1186,22 +1170,6 @@ Tags|telco,observability
 
 ### operator
 
-#### operator-catalogsource-bundle-count
-
-Property|Description
----|---
-Unique ID|operator-catalogsource-bundle-count
-Description|Tests operator catalog source bundle count is less than 1000
-Suggested Remediation|Ensure that the Operator's catalog source has a valid bundle count less than 1000.
-Best Practice Reference|https://redhat-best-practices-for-k8s.github.io/guide/#redhat-best-practices-for-k8s-cnf-operator-requirements
-Exception Process|No exceptions
-Tags|common,operator
-|**Scenario**|**Optional/Mandatory**|
-|Extended|Mandatory|
-|Far-Edge|Mandatory|
-|Non-Telco|Mandatory|
-|Telco|Mandatory|
-
 #### operator-crd-openapi-schema
 
 Property|Description
@@ -1282,54 +1250,6 @@ Tags|common,operator
 |Non-Telco|Mandatory|
 |Telco|Mandatory|
 
-#### operator-multiple-same-operators
-
-Property|Description
----|---
-Unique ID|operator-multiple-same-operators
-Description|Tests whether multiple instances of the same Operator CSV are installed.
-Suggested Remediation|Ensure that only one Operator of the same type is installed in the cluster.
-Best Practice Reference|https://redhat-best-practices-for-k8s.github.io/guide/#redhat-best-practices-for-k8s-cnf-operator-requirements
-Exception Process|No exceptions
-Tags|common,operator
-|**Scenario**|**Optional/Mandatory**|
-|Extended|Mandatory|
-|Far-Edge|Mandatory|
-|Non-Telco|Mandatory|
-|Telco|Mandatory|
-
-#### operator-olm-skip-range
-
-Property|Description
----|---
-Unique ID|operator-olm-skip-range
-Description|Test that checks the operator has a valid olm skip range.
-Suggested Remediation|Ensure that the Operator has a valid OLM skip range. If the operator does not have another version to "skip", then ignore the result of this test.
-Best Practice Reference|https://redhat-best-practices-for-k8s.github.io/guide/#redhat-best-practices-for-k8s-cnf-operator-requirements
-Exception Process|If there is not a version of the operator that needs to be skipped, then an exception will be granted.
-Tags|common,operator
-|**Scenario**|**Optional/Mandatory**|
-|Extended|Optional|
-|Far-Edge|Optional|
-|Non-Telco|Optional|
-|Telco|Optional|
-
-#### operator-pods-no-hugepages
-
-Property|Description
----|---
-Unique ID|operator-pods-no-hugepages
-Description|Tests that the pods do not have hugepages enabled.
-Suggested Remediation|Ensure that the pods are not using hugepages
-Best Practice Reference|https://redhat-best-practices-for-k8s.github.io/guide/#redhat-best-practices-for-k8s-cnf-operator-requirements
-Exception Process|No exceptions
-Tags|common,operator
-|**Scenario**|**Optional/Mandatory**|
-|Extended|Optional|
-|Far-Edge|Optional|
-|Non-Telco|Optional|
-|Telco|Optional|
-
 #### operator-semantic-versioning
 
 Property|Description

cmd/certsuite/generate/catalog/catalog.go

@@ -251,6 +251,11 @@ func outputTestCases() (outString string, summary catalogSummary) { //nolint:fun
 
 			// Every paragraph starts with a new line.
 
+			// Skip the test if it has the "waiting-for-release" tag.
+			if strings.Contains(tags, "waiting-for-release") {
+				continue
+			}
+
 			outString += fmt.Sprintf("\n#### %s\n\n", k.testName)
 			outString += "Property|Description\n"
 			outString += "---|---\n"

expected_results.yaml

@@ -77,16 +77,16 @@ testCases:
     - networking-dpdk-cpu-pinning-exec-probe
     - networking-icmpv6-connectivity
     - networking-restart-on-reboot-sriov-pod
-    - networking-network-attachment-definition-sriov-mtu
+    # - networking-network-attachment-definition-sriov-mtu # Re-enable when new releasing new minor version.
     - operator-install-source
     - operator-install-status-no-privileges
     - operator-install-status-succeeded
-    - operator-olm-skip-range
+    # - operator-olm-skip-range # Re-enable when new releasing new minor version.
     - operator-semantic-versioning
     - operator-single-crd-owner
-    - operator-pods-no-hugepages
-    - operator-multiple-same-operators
-    - operator-catalogsource-bundle-count
+    # - operator-pods-no-hugepages # Re-enable when new releasing new minor version.
+    # - operator-multiple-same-operators # Re-enable when new releasing new minor version.
+    # - operator-catalogsource-bundle-count # Re-enable when new releasing new minor version.
     - performance-exclusive-cpu-pool-rt-scheduling-policy
     - performance-isolated-cpu-pool-rt-scheduling-policy
     - performance-shared-cpu-pool-non-rt-scheduling-policy

pkg/checksdb/checksgroup.go

@@ -3,11 +3,13 @@ package checksdb
 import (
 	"errors"
 	"fmt"
+	"os"
 	"runtime/debug"
 	"strings"
 
 	"github.com/redhat-best-practices-for-k8s/certsuite/internal/cli"
 	"github.com/redhat-best-practices-for-k8s/certsuite/internal/log"
+	"github.com/redhat-best-practices-for-k8s/certsuite/pkg/stringhelper"
 )
 
 const (
@@ -76,7 +78,11 @@ func (group *ChecksGroup) Add(check *Check) {
 	dbLock.Lock()
 	defer dbLock.Unlock()
 
-	group.checks = append(group.checks, check)
+	// Only add the check to the group if its not "waiting-for-release" or if the REDHAT_CI env var is set.
+	if !stringhelper.StringInSlice(check.Labels, "waiting-for-release", false) ||
+		os.Getenv("REDHAT_CI") == "true" {
+		group.checks = append(group.checks, check)
+	}
 }
 
 func skipCheck(check *Check, reason string) {

pkg/provider/catalogsources.go

@@ -22,7 +22,7 @@ func GetCatalogSourceBundleCount(env *TestEnvironment, cs *olmv1Alpha.CatalogSou
 	)
 
 	// Check if the cluster is running an OCP version <= 4.12
-	if env.OpenshiftVersion != "" {
+	if env.OpenshiftVersion != "" && env.OpenshiftVersion != "0.0.0" {
 		log.Info("Cluster is determined to be running Openshift version %q.", env.OpenshiftVersion)
 		version, err := semver.NewVersion(env.OpenshiftVersion)
 		if err != nil {

tests/identifiers/identifiers.go

@@ -35,17 +35,18 @@ for instance the Performance Addon Operator (PAO) or istio.`
 )
 
 const (
-	TagCommon    = "common"
-	TagExtended  = "extended"
-	TagTelco     = "telco"
-	TagFarEdge   = "faredge"
-	FarEdge      = "FarEdge"
-	Telco        = "Telco"
-	NonTelco     = "NonTelco"
-	Extended     = "Extended"
-	Optional     = "Optional"
-	Mandatory    = "Mandatory"
-	TagPreflight = "preflight"
+	TagCommon            = "common"
+	TagExtended          = "extended"
+	TagTelco             = "telco"
+	TagFarEdge           = "faredge"
+	FarEdge              = "FarEdge"
+	Telco                = "Telco"
+	NonTelco             = "NonTelco"
+	Extended             = "Extended"
+	Optional             = "Optional"
+	Mandatory            = "Mandatory"
+	TagPreflight         = "preflight"
+	TagWaitingForRelease = "waiting-for-release"
 )
 
 const (
@@ -598,7 +599,7 @@ func InitCatalog() map[claim.Identifier]claim.TestCaseDescription {
 			NonTelco: Mandatory,
 			Extended: Mandatory,
 		},
-		TagFarEdge)
+		TagFarEdge, TagWaitingForRelease)
 
 	TestSecConNonRootUserIDIdentifier = AddCatalogEntry(
 		"security-context-non-root-user-id-check",
@@ -1002,7 +1003,7 @@ that Node's kernel may not have the same hacks.'`,
 			NonTelco: Optional,
 			Extended: Optional,
 		},
-		TagCommon)
+		TagCommon, TagWaitingForRelease)
 
 	TestOperatorCrdVersioningIdentifier = AddCatalogEntry(
 		"crd-versioning",
@@ -1066,7 +1067,7 @@ that Node's kernel may not have the same hacks.'`,
 			NonTelco: Optional,
 			Extended: Optional,
 		},
-		TagCommon)
+		TagCommon, TagWaitingForRelease)
 
 	TestOperatorCatalogSourceBundleCountIdentifier = AddCatalogEntry(
 		"catalogsource-bundle-count",
@@ -1082,7 +1083,7 @@ that Node's kernel may not have the same hacks.'`,
 			NonTelco: Mandatory,
 			Extended: Mandatory,
 		},
-		TagCommon)
+		TagCommon, TagWaitingForRelease)
 
 	TestMultipleSameOperatorsIdentifier = AddCatalogEntry(
 		"multiple-same-operators",
@@ -1098,7 +1099,7 @@ that Node's kernel may not have the same hacks.'`,
 			NonTelco: Mandatory,
 			Extended: Mandatory,
 		},
-		TagCommon)
+		TagCommon, TagWaitingForRelease)
 
 	TestPodNodeSelectorAndAffinityBestPractices = AddCatalogEntry(
 		"pod-scheduling",

tests/operator/suite.go

@@ -466,7 +466,7 @@ func testOperatorCatalogSourceBundleCount(check *checksdb.Check, env *provider.T
 
 	ocp412Skip := false
 	// Check if the cluster is running an OCP version <= 4.12
-	if env.OpenshiftVersion != "" {
+	if env.OpenshiftVersion != "" && env.OpenshiftVersion != "0.0.0" {
 		log.Info("Cluster is determined to be running Openshift version %q.", env.OpenshiftVersion)
 		version, err := semver.NewVersion(env.OpenshiftVersion)
 		if err != nil {