Propagate errors from preflight in the claim file

.github/workflows/pre-main.yaml

@@ -52,6 +52,11 @@ jobs:
         with:
           ref: ${{ github.sha }}
 
+      - name: Install yaml dependency
+        run: pip3 install pyyaml
+        env:
+          SHELL: /bin/bash
+
       - name: Extract dependent Pull Requests
         uses: depends-on/depends-on-action@main
         with:
@@ -124,6 +129,11 @@ jobs:
         with:
           ref: ${{ github.sha }}
 
+      - name: Install yaml dependency
+        run: pip3 install pyyaml
+        env:
+          SHELL: /bin/bash
+
       - name: Extract dependent Pull Requests
         uses: depends-on/depends-on-action@main
         with:
@@ -178,6 +188,11 @@ jobs:
         with:
           ref: ${{ github.sha }}
 
+      - name: Install yaml dependency
+        run: pip3 install pyyaml
+        env:
+          SHELL: /bin/bash
+
       - name: Extract dependent Pull Requests
         uses: depends-on/depends-on-action@main
         with:
@@ -215,7 +230,7 @@ jobs:
         with:
           timeout_minutes: 90
           max_attempts: 3
-          command: cd ${GITHUB_WORKSPACE}/cnf-certification-test-partner; make bootstrap-cluster; make make bootstrap-docker-ubuntu-local; make bootstrap-python-ubuntu-local
+          command: cd ${GITHUB_WORKSPACE}/cnf-certification-test-partner; make bootstrap-cluster; make make bootstrap-docker-ubuntu-local; make bootstrap-python-ubuntu-local; pip3 install pyyaml
 
       - name: Run 'make rebuild-cluster'
         uses: nick-fields/retry@v2
@@ -310,7 +325,7 @@ jobs:
         with:
           timeout_minutes: 90
           max_attempts: 3
-          command: cd ${GITHUB_WORKSPACE}/cnf-certification-test-partner; make bootstrap-cluster; make make bootstrap-docker-ubuntu-local; make bootstrap-python-ubuntu-local
+          command: cd ${GITHUB_WORKSPACE}/cnf-certification-test-partner; make bootstrap-cluster; make make bootstrap-docker-ubuntu-local; make bootstrap-python-ubuntu-local; pip3 install pyyaml
 
       - name: Run 'make rebuild-cluster'
         uses: nick-fields/retry@v2

.github/workflows/qe-hosted.yml

@@ -62,7 +62,7 @@ jobs:
         with:
           timeout_minutes: 90
           max_attempts: 3
-          command: cd ${GITHUB_WORKSPACE}/cnf-certification-test-partner; make bootstrap-cluster; make make bootstrap-docker-ubuntu-local; make bootstrap-python-ubuntu-local
+          command: cd ${GITHUB_WORKSPACE}/cnf-certification-test-partner; make bootstrap-cluster; make make bootstrap-docker-ubuntu-local; make bootstrap-python-ubuntu-local; pip3 install pyyaml
 
       - name: Run 'make rebuild-cluster'
         uses: nick-fields/retry@v2
@@ -97,6 +97,9 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Build the test image
+        run: make build-image-local # quay.io/testnetworkfunction/cnf-certification-test:localtest
+
       - name: Run the tests
         uses: nick-fields/retry@v2
         with:

cmd/tnf/generate/catalog/catalog.go

@@ -17,12 +17,19 @@
 package catalog
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"os"
 	"sort"
 	"strings"
 
+	"github.com/redhat-openshift-ecosystem/openshift-preflight/artifacts"
+	plibContainer "github.com/redhat-openshift-ecosystem/openshift-preflight/container"
+	plibOperator "github.com/redhat-openshift-ecosystem/openshift-preflight/operator"
+	"github.com/sirupsen/logrus"
+
+	"github.com/test-network-function/cnf-certification-test/cnf-certification-test/common"
 	"github.com/test-network-function/cnf-certification-test/cnf-certification-test/identifiers"
 	"github.com/test-network-function/cnf-certification-test/internal/log"
 	"github.com/test-network-function/cnf-certification-test/pkg/arrayhelper"
@@ -126,8 +133,55 @@ func scenarioIDToText(id string) (text string) {
 	return text
 }
 
+func addPreflightTestsToCatalog() {
+	const dummy = "dummy"
+	// Create artifacts handler
+	artifactsWriter, err := artifacts.NewMapWriter()
+	if err != nil {
+		logrus.Errorf("error creating artifact, failed to add preflight tests to catalog")
+		return
+	}
+	ctx := artifacts.ContextWithWriter(context.TODO(), artifactsWriter)
+	optsOperator := []plibOperator.Option{}
+	optsContainer := []plibContainer.Option{}
+	checkOperator := plibOperator.NewCheck(dummy, dummy, []byte(""), optsOperator...)
+	checkContainer := plibContainer.NewCheck(dummy, optsContainer...)
+	_, checksOperator, err := checkOperator.List(ctx)
+	if err != nil {
+		logrus.Errorf("error getting preflight operator tests.")
+	}
+	_, checksContainer, err := checkContainer.List(ctx)
+	if err != nil {
+		logrus.Errorf("error getting preflight container tests.")
+	}
+
+	allChecks := checksOperator
+	allChecks = append(allChecks, checksContainer...)
+
+	for _, c := range allChecks {
+		_ = identifiers.AddCatalogEntry(
+			c.Name(),
+			common.PreflightTestKey,
+			c.Metadata().Description,
+			c.Help().Suggestion,
+			identifiers.NoDocumentedProcess,
+			identifiers.NoDocLink,
+			true,
+			map[string]string{
+				identifiers.FarEdge:  identifiers.Optional,
+				identifiers.Telco:    identifiers.Optional,
+				identifiers.NonTelco: identifiers.Optional,
+				identifiers.Extended: identifiers.Optional,
+			},
+			identifiers.TagCommon)
+	}
+}
+
 // outputTestCases outputs the Markdown representation for test cases from the catalog to stdout.
 func outputTestCases() (outString string, summary catalogSummary) { //nolint:funlen
+	// Adds Preflight tests to catalog
+	addPreflightTestsToCatalog()
+
 	// Building a separate data structure to store the key order for the map
 	keys := make([]claim.Identifier, 0, len(identifiers.Catalog))
 	for k := range identifiers.Catalog {

cnf-certification-test/identifiers/doclinks.go

@@ -5,6 +5,7 @@ const (
 	NoDocLinkExtended = "No Doc Link - Extended"
 	NoDocLinkFarEdge  = "No Doc Link - Far Edge"
 	NoDocLinkTelco    = "No Doc Link - Telco"
+	NoDocLink         = "No Doc Link"
 
 	// Networking Suite
 	TestICMPv4ConnectivityIdentifierDocLink         = "https://test-network-function.github.io/cnf-best-practices-guide/#cnf-best-practices-ipv4-&-ipv6"

cnf-certification-test/preflight/suite.go

@@ -17,6 +17,7 @@
 package preflight
 
 import (
+	"fmt"
 	"os"
 	"strings"
 
@@ -56,6 +57,7 @@ func labelsAllowTestRun(labelFilter string, allowedLabels []string) bool {
 // time to finish. When they're finished, a checksdb.Check is created for each preflight lib's
 // check that has run. The CheckFn will simply store the result.
 func ShouldRun(labelsExpr string) bool {
+	env = provider.GetTestEnvironment()
 	preflightAllowedLabels := []string{common.PreflightTestKey, identifiers.TagPreflight}
 
 	if !labelsAllowTestRun(labelsExpr, preflightAllowedLabels) {
@@ -65,8 +67,8 @@ func ShouldRun(labelsExpr string) bool {
 	// Add safeguard against running the preflight tests if the docker config does not exist.
 	preflightDockerConfigFile := configuration.GetTestParameters().PfltDockerconfig
 	if preflightDockerConfigFile == "" || preflightDockerConfigFile == "NA" {
-		log.Warn("Skipping the Preflight suite because the Docker Config file is not provided.")
-		return false
+		log.Warn("Skipping the preflight suite because the Docker Config file is not provided.")
+		env.SkipPreflight = true
 	}
 
 	return true
@@ -169,7 +171,7 @@ func generatePreflightContainerCnfCertTest(checksGroup *checksdb.ChecksGroup, te
 				for _, r := range cut.PreflightResults.Errors {
 					if r.Name() == testName {
 						check.LogError("Container %q has errored Preflight test %q", cut, testName)
-						nonCompliantObjects = append(nonCompliantObjects, testhelper.NewContainerReportObject(cut.Namespace, cut.Podname, cut.Name, "Container has errored preflight test "+testName, false))
+						nonCompliantObjects = append(nonCompliantObjects, testhelper.NewContainerReportObject(cut.Namespace, cut.Podname, cut.Name, fmt.Sprintf("Container has errored preflight test %s, err=%v", testName, r.Error()), false))
 					}
 				}
 			}

go.mod

@@ -5,7 +5,7 @@ go 1.21.6
 require (
 	github.com/Masterminds/semver/v3 v3.2.1
 	github.com/basgys/goxml2json v1.1.0
-	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.8.4
 	github.com/test-network-function/test-network-function-claim v1.0.34
@@ -18,7 +18,6 @@ require k8s.io/client-go v0.29.1
 require (
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/mittwald/go-helm-client v0.12.7
-	github.com/onsi/ginkgo/v2 v2.13.2 // indirect
 	github.com/openshift/api v0.0.1
 	github.com/openshift/client-go v0.0.1
 	github.com/operator-framework/api v0.21.0
@@ -53,7 +52,7 @@ require (
 	github.com/containerd/log v0.1.0 // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/distribution/reference v0.5.0 // indirect
 	github.com/docker/cli v24.0.7+incompatible // indirect
 	github.com/docker/distribution v2.8.3+incompatible // indirect
@@ -69,7 +68,7 @@ require (
 	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
-	github.com/glebarez/go-sqlite v1.21.2 // indirect
+	github.com/glebarez/go-sqlite v1.22.0 // indirect
 	github.com/go-errors/errors v1.5.1 // indirect
 	github.com/go-gorp/gorp/v3 v3.1.0 // indirect
 	github.com/go-openapi/jsonpointer v0.20.2 // indirect
@@ -84,7 +83,7 @@ require (
 	github.com/google/cel-go v0.17.7 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
-	github.com/google/go-containerregistry v0.15.2 // indirect
+	github.com/google/go-containerregistry v0.17.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20230323073829-e72429f035bd // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
@@ -131,9 +130,9 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0-rc5 // indirect
 	github.com/operator-framework/operator-manifest-tools v0.4.0 // indirect
-	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
+	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_golang v1.18.0 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/common v0.45.0 // indirect
@@ -142,15 +141,17 @@ require (
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/rubenv/sql-migrate v1.6.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/sagikazarmark/locafero v0.4.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 // indirect
-	github.com/spf13/afero v1.10.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
 	github.com/spf13/cast v1.6.0 // indirect
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/spf13/viper v1.16.0 // indirect
+	github.com/spf13/viper v1.18.2 // indirect
 	github.com/stoewer/go-strcase v1.3.0 // indirect
-	github.com/subosito/gotenv v1.4.2 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/vbatts/tar-split v0.11.3 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
@@ -164,6 +165,7 @@ require (
 	go.opentelemetry.io/otel/trace v1.21.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
 	go.starlark.net v0.0.0-20231121155337-90ade8b19d09 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.17.0 // indirect
 	golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect
 	golang.org/x/net v0.19.0 // indirect
@@ -172,7 +174,7 @@ require (
 	golang.org/x/sys v0.16.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20231211222908-989df2bf70f3 // indirect
@@ -187,10 +189,10 @@ require (
 	k8s.io/component-base v0.29.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20240103195357-a9f8850cb432 // indirect
 	k8s.io/utils v0.0.0-20240102154912-e7106e64919e // indirect
-	modernc.org/libc v1.22.5 // indirect
-	modernc.org/mathutil v1.5.0 // indirect
-	modernc.org/memory v1.5.0 // indirect
-	modernc.org/sqlite v1.23.1 // indirect
+	modernc.org/libc v1.37.6 // indirect
+	modernc.org/mathutil v1.6.0 // indirect
+	modernc.org/memory v1.7.2 // indirect
+	modernc.org/sqlite v1.28.0 // indirect
 	oras.land/oras-go v1.2.4 // indirect
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 // indirect
 	sigs.k8s.io/controller-runtime v0.16.3 // indirect
@@ -223,4 +225,4 @@ require (
 	k8s.io/kubectl v0.29.1
 )
 
-replace github.com/redhat-openshift-ecosystem/openshift-preflight => github.com/redhat-openshift-ecosystem/openshift-preflight v0.0.0-20231218171045-b6bcc3506c0d
+replace github.com/redhat-openshift-ecosystem/openshift-preflight => github.com/redhat-openshift-ecosystem/openshift-preflight v0.0.0-20240124141856-950b000116dc