Propagate errors from preflight in the claim file

.github/workflows/pre-main.yaml

@@ -51,6 +51,11 @@ jobs:
         with:
           ref: ${{ github.sha }}
 
+      - name: Install yaml dependency
+        run: pip3 install pyyaml
+        env:
+          SHELL: /bin/bash
+
       - name: Extract dependent Pull Requests
         uses: depends-on/depends-on-action@main
         with:
@@ -123,6 +128,11 @@ jobs:
         with:
           ref: ${{ github.sha }}
 
+      - name: Install yaml dependency
+        run: pip3 install pyyaml
+        env:
+          SHELL: /bin/bash
+
       - name: Extract dependent Pull Requests
         uses: depends-on/depends-on-action@main
         with:
@@ -189,6 +199,11 @@ jobs:
         with:
           ref: ${{ github.sha }}
 
+      - name: Install yaml dependency
+        run: pip3 install pyyaml
+        env:
+          SHELL: /bin/bash
+
       - name: Extract dependent Pull Requests
         uses: depends-on/depends-on-action@main
         with:
@@ -229,7 +244,7 @@ jobs:
         with:
           timeout_minutes: 90
           max_attempts: 3
-          command: cd ${GITHUB_WORKSPACE}/cnf-certification-test-partner; make bootstrap-cluster; make make bootstrap-docker-ubuntu-local; make bootstrap-python-ubuntu-local
+          command: cd ${GITHUB_WORKSPACE}/cnf-certification-test-partner; make bootstrap-cluster; make make bootstrap-docker-ubuntu-local; make bootstrap-python-ubuntu-local; pip3 install pyyaml
 
       - name: Run 'make rebuild-cluster'
         uses: nick-fields/retry@v2
@@ -336,7 +351,7 @@ jobs:
         with:
           timeout_minutes: 90
           max_attempts: 3
-          command: cd ${GITHUB_WORKSPACE}/cnf-certification-test-partner; make bootstrap-cluster; make make bootstrap-docker-ubuntu-local; make bootstrap-python-ubuntu-local
+          command: cd ${GITHUB_WORKSPACE}/cnf-certification-test-partner; make bootstrap-cluster; make make bootstrap-docker-ubuntu-local; make bootstrap-python-ubuntu-local; pip3 install pyyaml
 
       - name: Run 'make rebuild-cluster'
         uses: nick-fields/retry@v2

.github/workflows/qe-hosted.yml

@@ -55,9 +55,6 @@ jobs:
           sudo apt-get install -y python3-pip
           sudo pip3 install j2cli
 
-      - name: Build the test image
-        run: make build-image-local # quay.io/testnetworkfunction/cnf-certification-test:localtest
-
       # Create a Kind cluster for testing.
       - name: Check out `cnf-certification-test-partner`
         uses: actions/checkout@v4
@@ -70,7 +67,7 @@ jobs:
         with:
           timeout_minutes: 90
           max_attempts: 3
-          command: cd ${GITHUB_WORKSPACE}/cnf-certification-test-partner; make bootstrap-cluster; make make bootstrap-docker-ubuntu-local; make bootstrap-python-ubuntu-local
+          command: cd ${GITHUB_WORKSPACE}/cnf-certification-test-partner; make bootstrap-cluster; make make bootstrap-docker-ubuntu-local; make bootstrap-python-ubuntu-local; pip3 install pyyaml
 
       - name: Run 'make rebuild-cluster'
         uses: nick-fields/retry@v2
@@ -105,6 +102,9 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Build the test image
+        run: make build-image-local # quay.io/testnetworkfunction/cnf-certification-test:localtest
+
       - name: Run the tests
         uses: nick-fields/retry@v2
         with:

cmd/tnf/generate/catalog/catalog.go

@@ -17,12 +17,19 @@
 package catalog
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"os"
 	"sort"
 	"strings"
 
+	"github.com/redhat-openshift-ecosystem/openshift-preflight/artifacts"
+	plibContainer "github.com/redhat-openshift-ecosystem/openshift-preflight/container"
+	plibOperator "github.com/redhat-openshift-ecosystem/openshift-preflight/operator"
+	"github.com/sirupsen/logrus"
+
+	"github.com/test-network-function/cnf-certification-test/cnf-certification-test/common"
 	"github.com/test-network-function/cnf-certification-test/cnf-certification-test/identifiers"
 	"github.com/test-network-function/cnf-certification-test/internal/log"
 	"github.com/test-network-function/cnf-certification-test/pkg/arrayhelper"
@@ -126,8 +133,55 @@ func scenarioIDToText(id string) (text string) {
 	return text
 }
 
+func addPreflightTestsToCatalog() {
+	const dummy = "dummy"
+	// Create artifacts handler
+	artifactsWriter, err := artifacts.NewMapWriter()
+	if err != nil {
+		logrus.Errorf("error creating artifact, failed to add preflight tests to catalog")
+		return
+	}
+	ctx := artifacts.ContextWithWriter(context.TODO(), artifactsWriter)
+	optsOperator := []plibOperator.Option{}
+	optsContainer := []plibContainer.Option{}
+	checkOperator := plibOperator.NewCheck(dummy, dummy, []byte(""), optsOperator...)
+	checkContainer := plibContainer.NewCheck(dummy, optsContainer...)
+	_, checksOperator, err := checkOperator.List(ctx)
+	if err != nil {
+		logrus.Errorf("error getting preflight operator tests.")
+	}
+	_, checksContainer, err := checkContainer.List(ctx)
+	if err != nil {
+		logrus.Errorf("error getting preflight container tests.")
+	}
+
+	allChecks := checksOperator
+	allChecks = append(allChecks, checksContainer...)
+
+	for _, c := range allChecks {
+		_ = identifiers.AddCatalogEntry(
+			c.Name(),
+			common.PreflightTestKey,
+			c.Metadata().Description,
+			c.Help().Suggestion,
+			identifiers.NoDocumentedProcess,
+			identifiers.NoDocLink,
+			true,
+			map[string]string{
+				identifiers.FarEdge:  identifiers.Optional,
+				identifiers.Telco:    identifiers.Optional,
+				identifiers.NonTelco: identifiers.Optional,
+				identifiers.Extended: identifiers.Optional,
+			},
+			identifiers.TagCommon)
+	}
+}
+
 // outputTestCases outputs the Markdown representation for test cases from the catalog to stdout.
 func outputTestCases() (outString string, summary catalogSummary) { //nolint:funlen
+	// Adds Preflight tests to catalog
+	addPreflightTestsToCatalog()
+
 	// Building a separate data structure to store the key order for the map
 	keys := make([]claim.Identifier, 0, len(identifiers.Catalog))
 	for k := range identifiers.Catalog {

cnf-certification-test/identifiers/doclinks.go

@@ -5,6 +5,7 @@ const (
 	NoDocLinkExtended = "No Doc Link - Extended"
 	NoDocLinkFarEdge  = "No Doc Link - Far Edge"
 	NoDocLinkTelco    = "No Doc Link - Telco"
+	NoDocLink         = "No Doc Link"
 
 	// Networking Suite
 	TestICMPv4ConnectivityIdentifierDocLink         = "https://test-network-function.github.io/cnf-best-practices-guide/#cnf-best-practices-ipv4-&-ipv6"

cnf-certification-test/preflight/suite.go

@@ -17,6 +17,7 @@
 package preflight
 
 import (
+	"fmt"
 	"os"
 	"strings"
 
@@ -57,6 +58,7 @@ func labelsAllowTestRun(labelFilter string, allowedLabels []string) bool {
 // time to finish. When they're finished, a checksdb.Check is created for each preflight lib's
 // check that has run. The CheckFn will simply store the result.
 func ShouldRun(labelsExpr string) bool {
+	env = provider.GetTestEnvironment()
 	preflightAllowedLabels := []string{common.PreflightTestKey, identifiers.TagPreflight}
 
 	if !labelsAllowTestRun(labelsExpr, preflightAllowedLabels) {
@@ -67,7 +69,7 @@ func ShouldRun(labelsExpr string) bool {
 	preflightDockerConfigFile := configuration.GetTestParameters().PfltDockerconfig
 	if preflightDockerConfigFile == "" || preflightDockerConfigFile == "NA" {
 		log.Warn("Skipping the preflight suite because the Docker Config file is not provided.")
-		return false
+		env.SkipPreflight = true
 	}
 
 	return true
@@ -170,7 +172,7 @@ func generatePreflightContainerGinkgoTest(checksGroup *checksdb.ChecksGroup, tes
 				}
 				for _, r := range cut.PreflightResults.Errors {
 					if r.Name() == testName {
-						nonCompliantObjects = append(nonCompliantObjects, testhelper.NewContainerReportObject(cut.Namespace, cut.Podname, cut.Name, "Container has errored preflight test "+testName, false))
+						nonCompliantObjects = append(nonCompliantObjects, testhelper.NewContainerReportObject(cut.Namespace, cut.Podname, cut.Name, fmt.Sprintf("Container has errored preflight test %s, err=%v", testName, r.Err), false))
 						log.Error("%s has errored preflight test: %s", cut, testName)
 					}
 				}

pkg/provider/containers.go

@@ -119,10 +119,19 @@
 	ctx = logr.NewContext(ctx, logger)
 
 	check := plibContainer.NewCheck(c.Image, opts...)
+
 	results, runtimeErr := check.Run(ctx)
 	if runtimeErr != nil {
-		log.Error("%v", runtimeErr)
-		return runtimeErr
+		_, checks, err := check.List(ctx)
+		if err != nil {
+			return fmt.Errorf("could not get preflight container test list")
+		}
+
+		results.TestedImage = c.Image
+		for _, c := range checks {
+			results.PassedOverall = false
+			results.Errors = append(results.Errors, plibRuntime.Result{Check: c, ElapsedTime: 0, Err: runtimeErr})
+		}
 	}
 
 	// Take all of the preflight logs and stick them into our log.

pkg/provider/operators.go

@@ -99,10 +99,17 @@
 	ctx = logr.NewContext(ctx, logger)
 
 	check := plibOperator.NewCheck(bundleImage, indexImage, oc.KubeConfig, opts...)
+
 	results, runtimeErr := check.Run(ctx)
 	if runtimeErr != nil {
-		log.Error("%v", runtimeErr)
-		return runtimeErr
+		_, checks, err := check.List(ctx)
+		if err != nil {
+			return fmt.Errorf("could not get preflight container test list")
+		}
+		for _, c := range checks {
+			results.PassedOverall = false
+			results.Errors = append(results.Errors, plibRuntime.Result{Check: c, ElapsedTime: 0, Err: runtimeErr})
+		}
 	}
 
 	// Take all of the preflight logs and stick them into our log.

pkg/provider/provider.go

@@ -121,6 +121,7 @@ type TestEnvironment struct { // rename this with testTarget
 	ExecutedBy             string
 	PartnerName            string
 	CollectorAppPassword   string
+	SkipPreflight          bool
 }
 
 type MachineConfig struct {