Use tags

CATALOG.md

@@ -36,11 +36,11 @@ Depending on the workload type, not all tests are required to pass to satisfy be
 |---|---|
 |8|1|
 
-### Non-Telco specific tests only: 69
+### Non-Telco specific tests only: 65
 
 |Mandatory|Optional|
 |---|---|
-|44|25|
+|42|23|
 
 ### Telco specific tests only: 27
 
@@ -1006,22 +1006,6 @@ Tags|telco,networking
 |Non-Telco|Optional|
 |Telco|Mandatory|
 
-#### networking-network-attachment-definition-sriov-mtu
-
-Property|Description
----|---
-Unique ID|networking-network-attachment-definition-sriov-mtu
-Description|Ensures that MTU values are set correctly in NetworkAttachmentDefinitions for SRIOV network interfaces.
-Suggested Remediation|Ensure that the MTU of the SR-IOV network attachment definition is set explicitly.
-Best Practice Reference|No Doc Link - Far Edge
-Exception Process|There is no documented exception process for this.
-Tags|faredge,networking
-|**Scenario**|**Optional/Mandatory**|
-|Extended|Mandatory|
-|Far-Edge|Mandatory|
-|Non-Telco|Mandatory|
-|Telco|Mandatory|
-
 #### networking-network-policy-deny-all
 
 Property|Description
@@ -1186,22 +1170,6 @@ Tags|telco,observability
 
 ### operator
 
-#### operator-catalogsource-bundle-count
-
-Property|Description
----|---
-Unique ID|operator-catalogsource-bundle-count
-Description|Tests operator catalog source bundle count is less than 1000
-Suggested Remediation|Ensure that the Operator's catalog source has a valid bundle count less than 1000.
-Best Practice Reference|https://redhat-best-practices-for-k8s.github.io/guide/#redhat-best-practices-for-k8s-cnf-operator-requirements
-Exception Process|No exceptions
-Tags|common,operator
-|**Scenario**|**Optional/Mandatory**|
-|Extended|Mandatory|
-|Far-Edge|Mandatory|
-|Non-Telco|Mandatory|
-|Telco|Mandatory|
-
 #### operator-crd-openapi-schema
 
 Property|Description
@@ -1282,54 +1250,6 @@ Tags|common,operator
 |Non-Telco|Mandatory|
 |Telco|Mandatory|
 
-#### operator-multiple-same-operators
-
-Property|Description
----|---
-Unique ID|operator-multiple-same-operators
-Description|Tests whether multiple instances of the same Operator CSV are installed.
-Suggested Remediation|Ensure that only one Operator of the same type is installed in the cluster.
-Best Practice Reference|https://redhat-best-practices-for-k8s.github.io/guide/#redhat-best-practices-for-k8s-cnf-operator-requirements
-Exception Process|No exceptions
-Tags|common,operator
-|**Scenario**|**Optional/Mandatory**|
-|Extended|Mandatory|
-|Far-Edge|Mandatory|
-|Non-Telco|Mandatory|
-|Telco|Mandatory|
-
-#### operator-olm-skip-range
-
-Property|Description
----|---
-Unique ID|operator-olm-skip-range
-Description|Test that checks the operator has a valid olm skip range.
-Suggested Remediation|Ensure that the Operator has a valid OLM skip range. If the operator does not have another version to "skip", then ignore the result of this test.
-Best Practice Reference|https://redhat-best-practices-for-k8s.github.io/guide/#redhat-best-practices-for-k8s-cnf-operator-requirements
-Exception Process|If there is not a version of the operator that needs to be skipped, then an exception will be granted.
-Tags|common,operator
-|**Scenario**|**Optional/Mandatory**|
-|Extended|Optional|
-|Far-Edge|Optional|
-|Non-Telco|Optional|
-|Telco|Optional|
-
-#### operator-pods-no-hugepages
-
-Property|Description
----|---
-Unique ID|operator-pods-no-hugepages
-Description|Tests that the pods do not have hugepages enabled.
-Suggested Remediation|Ensure that the pods are not using hugepages
-Best Practice Reference|https://redhat-best-practices-for-k8s.github.io/guide/#redhat-best-practices-for-k8s-cnf-operator-requirements
-Exception Process|No exceptions
-Tags|common,operator
-|**Scenario**|**Optional/Mandatory**|
-|Extended|Optional|
-|Far-Edge|Optional|
-|Non-Telco|Optional|
-|Telco|Optional|
-
 #### operator-semantic-versioning
 
 Property|Description

cmd/certsuite/generate/catalog/catalog.go

@@ -251,6 +251,11 @@ func outputTestCases() (outString string, summary catalogSummary) { //nolint:fun
 
 			// Every paragraph starts with a new line.
 
+			// Skip the test if it has the "waiting-for-release" tag.
+			if strings.Contains(tags, "waiting-for-release") {
+				continue
+			}
+
 			outString += fmt.Sprintf("\n#### %s\n\n", k.testName)
 			outString += "Property|Description\n"
 			outString += "---|---\n"

pkg/checksdb/check.go

@@ -56,8 +56,6 @@ type Check struct {
 	Timeout            time.Duration
 	Error              error
 	abortChan          chan string
-
-	enabled bool
 }
 
 func NewCheck(id string, labels []string) *Check {
@@ -74,19 +72,6 @@ func NewCheck(id string, labels []string) *Check {
 	return check
 }
 
-// SetDisabled disables the check.
-func (check *Check) SetDisabled() *Check {
-	// Check if the environment variable REDHAT_CI is set.
-	// If it is set, the check is enabled.
-	if os.Getenv("REDHAT_CI") != "" {
-		log.Info("Check %s is enabled because the REDHAT_CI environment variable is set.", check.ID)
-		check.enabled = true
-	} else {
-		check.enabled = false
-	}
-	return check
-}
-
 func (check *Check) Abort(reason string) {
 	check.mutex.Lock()
 	defer check.mutex.Unlock()

pkg/checksdb/checksgroup.go

@@ -3,11 +3,13 @@ package checksdb
 import (
 	"errors"
 	"fmt"
+	"os"
 	"runtime/debug"
 	"strings"
 
 	"github.com/redhat-best-practices-for-k8s/certsuite/internal/cli"
 	"github.com/redhat-best-practices-for-k8s/certsuite/internal/log"
+	"github.com/redhat-best-practices-for-k8s/certsuite/pkg/stringhelper"
 )
 
 const (
@@ -76,7 +78,12 @@ func (group *ChecksGroup) Add(check *Check) {
 	dbLock.Lock()
 	defer dbLock.Unlock()
 
-	group.checks = append(group.checks, check)
+	// Only add the check to the group if its not "waiting-for-release" or if the REDHAT_CI env var is set.
+	if !stringhelper.StringInSlice(check.Labels, "waiting-for-release", false) ||
+		os.Getenv("REDHAT_CI") == "true" {
+		group.checks = append(group.checks, check)
+	}
+
 }
 
 func skipCheck(check *Check, reason string) {
@@ -213,11 +220,6 @@ func shouldSkipCheck(check *Check) (skip bool, reasons []string) {
 		return false, []string{}
 	}
 
-	// If the check is disabled, skip it.
-	if !check.enabled {
-		return true, []string{"check is disabled in this release"}
-	}
-
 	// Save the skipFn index in case it panics so it can be used in the log trace.
 	currentSkipFnIndex := 0
 

tests/identifiers/identifiers.go

@@ -35,17 +35,18 @@ for instance the Performance Addon Operator (PAO) or istio.`
 )
 
 const (
-	TagCommon    = "common"
-	TagExtended  = "extended"
-	TagTelco     = "telco"
-	TagFarEdge   = "faredge"
-	FarEdge      = "FarEdge"
-	Telco        = "Telco"
-	NonTelco     = "NonTelco"
-	Extended     = "Extended"
-	Optional     = "Optional"
-	Mandatory    = "Mandatory"
-	TagPreflight = "preflight"
+	TagCommon            = "common"
+	TagExtended          = "extended"
+	TagTelco             = "telco"
+	TagFarEdge           = "faredge"
+	FarEdge              = "FarEdge"
+	Telco                = "Telco"
+	NonTelco             = "NonTelco"
+	Extended             = "Extended"
+	Optional             = "Optional"
+	Mandatory            = "Mandatory"
+	TagPreflight         = "preflight"
+	TagWaitingForRelease = "waiting-for-release"
 )
 
 const (
@@ -598,7 +599,7 @@ func InitCatalog() map[claim.Identifier]claim.TestCaseDescription {
 			NonTelco: Mandatory,
 			Extended: Mandatory,
 		},
-		TagFarEdge)
+		TagFarEdge, TagWaitingForRelease)
 
 	TestSecConNonRootUserIDIdentifier = AddCatalogEntry(
 		"security-context-non-root-user-id-check",
@@ -1002,7 +1003,7 @@ that Node's kernel may not have the same hacks.'`,
 			NonTelco: Optional,
 			Extended: Optional,
 		},
-		TagCommon)
+		TagCommon, TagWaitingForRelease)
 
 	TestOperatorCrdVersioningIdentifier = AddCatalogEntry(
 		"crd-versioning",
@@ -1066,7 +1067,7 @@ that Node's kernel may not have the same hacks.'`,
 			NonTelco: Optional,
 			Extended: Optional,
 		},
-		TagCommon)
+		TagCommon, TagWaitingForRelease)
 
 	TestOperatorCatalogSourceBundleCountIdentifier = AddCatalogEntry(
 		"catalogsource-bundle-count",
@@ -1082,7 +1083,7 @@ that Node's kernel may not have the same hacks.'`,
 			NonTelco: Mandatory,
 			Extended: Mandatory,
 		},
-		TagCommon)
+		TagCommon, TagWaitingForRelease)
 
 	TestMultipleSameOperatorsIdentifier = AddCatalogEntry(
 		"multiple-same-operators",
@@ -1098,7 +1099,7 @@ that Node's kernel may not have the same hacks.'`,
 			NonTelco: Mandatory,
 			Extended: Mandatory,
 		},
-		TagCommon)
+		TagCommon, TagWaitingForRelease)
 
 	TestPodNodeSelectorAndAffinityBestPractices = AddCatalogEntry(
 		"pod-scheduling",

tests/networking/suite.go

@@ -158,7 +158,6 @@ func LoadChecks() {
 	checksGroup.Add(checksdb.NewCheck(identifiers.GetTestIDAndLabels(
 		identifiers.TestNetworkAttachmentDefinitionSRIOVUsingMTU)).
 		WithSkipCheckFn(testhelper.GetNoSRIOVPodsSkipFn(&env)).
-		SetDisabled(). // Remove in the next release.
 		WithCheckFn(func(c *checksdb.Check) error {
 			sriovPods, err := env.GetPodsUsingSRIOV()
 			if err != nil {

tests/operator/suite.go

@@ -101,31 +101,27 @@ func LoadChecks() {
 
 	checksGroup.Add(checksdb.NewCheck(identifiers.GetTestIDAndLabels(identifiers.TestOperatorPodsNoHugepages)).
 		WithSkipCheckFn(testhelper.GetNoOperatorsSkipFn(&env), testhelper.GetNoOperatorPodsSkipFn(&env)).
-		SetDisabled(). // Remove in the next release.
 		WithCheckFn(func(c *checksdb.Check) error {
 			testOperatorPodsNoHugepages(c, &env)
 			return nil
 		}))
 
 	checksGroup.Add(checksdb.NewCheck(identifiers.GetTestIDAndLabels(identifiers.TestOperatorOlmSkipRange)).
 		WithSkipCheckFn(testhelper.GetNoOperatorsSkipFn(&env)).
-		SetDisabled(). // Remove in the next release.
 		WithCheckFn(func(c *checksdb.Check) error {
 			testOperatorOlmSkipRange(c, &env)
 			return nil
 		}))
 
 	checksGroup.Add(checksdb.NewCheck(identifiers.GetTestIDAndLabels(identifiers.TestMultipleSameOperatorsIdentifier)).
 		WithSkipCheckFn(testhelper.GetNoOperatorsSkipFn(&env)).
-		SetDisabled(). // Remove in the next release.
 		WithCheckFn(func(c *checksdb.Check) error {
 			testMultipleSameOperators(c, &env)
 			return nil
 		}))
 
 	checksGroup.Add(checksdb.NewCheck(identifiers.GetTestIDAndLabels(identifiers.TestOperatorCatalogSourceBundleCountIdentifier)).
 		WithSkipCheckFn(testhelper.GetNoCatalogSourcesSkipFn(&env)).
-		SetDisabled(). // Remove in the next release.
 		WithCheckFn(func(c *checksdb.Check) error {
 			testOperatorCatalogSourceBundleCount(c, &env)
 			return nil