re-add logs in claim file

cmd/tnf/claim/show/csv/csv.go

@@ -146,7 +146,7 @@ func buildCSV(claimScheme *claim.Schema, cnfType string, catalogMap map[string]c
 			"CNFName", "testID", "Suite",
 			"Description", "State",
 			"StartTime", "EndTime",
-			"FailureReason", "Output",
+			"SkipReason", "CheckDetails", "Output",
 			"Remediation", "CNFType",
 			"Mandatory/Optional",
 		})
@@ -164,7 +164,8 @@ func buildCSV(claimScheme *claim.Schema, cnfType string, catalogMap map[string]c
 			claimScheme.Claim.Results[testID].State,
 			claimScheme.Claim.Results[testID].StartTime,
 			claimScheme.Claim.Results[testID].EndTime,
-			claimScheme.Claim.Results[testID].FailureReason,
+			claimScheme.Claim.Results[testID].SkipReason,
+			claimScheme.Claim.Results[testID].CheckDetails,
 			claimScheme.Claim.Results[testID].CapturedTestOutput,
 			catalogMap[testID].Remediation,
 			cnfType, // Append the CNF type

cmd/tnf/claim/show/failures/failures.go

@@ -157,17 +157,17 @@ func parseOutputFormatFlag() (string, error) {
 	return "", fmt.Errorf("invalid output format flag %q - available formats: %v", outputFormatFlag, availableOutputFormats)
 }
 
-// Parses the claim's test case's failureReason field and creates a list
+// Parses the claim's test case's checkDetails field and creates a list
 // of NonCompliantObject's.
-func getNonCompliantObjectsFromFailureReason(failureReason string) ([]NonCompliantObject, error) {
+func getNonCompliantObjectsFromFailureReason(checkDetails string) ([]NonCompliantObject, error) {
 	objects := struct {
 		Compliant    []testhelper.ReportObject `json:"CompliantObjectsOut"`
 		NonCompliant []testhelper.ReportObject `json:"NonCompliantObjectsOut"`
 	}{}
 
-	err := json.Unmarshal([]byte(failureReason), &objects)
+	err := json.Unmarshal([]byte(checkDetails), &objects)
 	if err != nil {
-		return nil, fmt.Errorf("failed to decode failureReason %s: %v", failureReason, err)
+		return nil, fmt.Errorf("failed to decode checkDetails %s: %v", checkDetails, err)
 	}
 
 	// Now let's create a list of our NonCompliantObject-type items.
@@ -194,7 +194,7 @@ func printFailuresText(testSuites []FailedTestSuite) {
 
 			// In case this tc was not using report objects, just print the failure reason string.
 			if len(tc.NonCompliantObjects) == 0 {
-				fmt.Printf("    Failure reason: %s\n", tc.FailureReason)
+				fmt.Printf("    Failure reason: %s\n", tc.CheckDetails)
 				continue
 			}
 
@@ -253,12 +253,12 @@ func getFailedTestCasesByTestSuite(claimResultsByTestSuite map[string][]*claim.T
 				TestCaseDescription: tc.CatalogInfo.Description,
 			}
 
-			nonCompliantObjects, err := getNonCompliantObjectsFromFailureReason(tc.FailureReason)
+			nonCompliantObjects, err := getNonCompliantObjectsFromFailureReason(tc.CheckDetails)
 			if err != nil {
 				// This means the test case doesn't use the report objects yet. Just use the raw failure reason instead.
 				// Also, send the error into stderr, so it can be filtered out with "2>/errors.txt" or "2>/dev/null".
 				fmt.Fprintf(os.Stderr, "Failed to parse non compliant objects from test case %s (test suite %s): %v", tc.TestID.ID, testSuite, err)
-				failingTc.FailureReason = tc.FailureReason
+				failingTc.CheckDetails = tc.CheckDetails
 			} else {
 				failingTc.NonCompliantObjects = nonCompliantObjects
 			}

cmd/tnf/claim/show/failures/failures_test.go

@@ -90,22 +90,22 @@ func TestParseOutputFormatFlag(t *testing.T) {
 
 func TestGetNonCompliantObjectsFromFailureReason(t *testing.T) {
 	testCases := []struct {
-		failureReason               string
+		checkDetails                string
 		expectedNonCompliantObjects []NonCompliantObject
 		expectedError               string
 	}{
 		{
-			failureReason:               "",
+			checkDetails:                "",
 			expectedNonCompliantObjects: nil,
-			expectedError:               `failed to decode failureReason : unexpected end of JSON input`,
+			expectedError:               `failed to decode checkDetails : unexpected end of JSON input`,
 		},
 		{
-			failureReason:               `{"CompliantObjectsOut": null, "NonCompliantObjectsOut": null}`,
+			checkDetails:                `{"CompliantObjectsOut": null, "NonCompliantObjectsOut": null}`,
 			expectedNonCompliantObjects: []NonCompliantObject{},
 		},
 		// One container failed the SYS_ADMIN check.
 		{
-			failureReason: `{
+			checkDetails: `{
 				"CompliantObjectsOut": null,
 				"NonCompliantObjectsOut": [
 				  {
@@ -159,7 +159,7 @@ func TestGetNonCompliantObjectsFromFailureReason(t *testing.T) {
 		},
 		// Two containers failed the SYS_ADMIN check.
 		{
-			failureReason: `{
+			checkDetails: `{
 				"CompliantObjectsOut": null,
 				"NonCompliantObjectsOut": [
 				  {
@@ -260,7 +260,7 @@ func TestGetNonCompliantObjectsFromFailureReason(t *testing.T) {
 	}
 
 	for _, tc := range testCases {
-		nonCompliantObjects, err := getNonCompliantObjectsFromFailureReason(tc.failureReason)
+		nonCompliantObjects, err := getNonCompliantObjectsFromFailureReason(tc.checkDetails)
 		if err != nil {
 			assert.Equal(t, tc.expectedError, err.Error())
 		}
@@ -271,7 +271,7 @@ func TestGetNonCompliantObjectsFromFailureReason(t *testing.T) {
 
 // Uses claim files in testdata folder:
 // claim1.json -> Two test suites, access-control & platform-alteration. One failed test case in the access-control ts.
-// claim2.json -> Same as clam1.json, but the failureReason is a simple string, not using report objects yet.
+// claim2.json -> Same as clam1.json, but the checkDetails is a simple string, not using report objects yet.
 func TestGetFailedTestCasesByTestSuite(t *testing.T) {
 	testCases := []struct {
 		claimFilePath            string
@@ -295,7 +295,7 @@ func TestGetFailedTestCasesByTestSuite(t *testing.T) {
 						{
 							TestCaseName:        "access-control-sys-admin-capability-check",
 							TestCaseDescription: "Ensures that containers do not use SYS_ADMIN capability",
-							FailureReason:       "pod xxx ns yyy container zzz uses SYS_ADMIN",
+							CheckDetails:        "pod xxx ns yyy container zzz uses SYS_ADMIN",
 						},
 					},
 				},

cmd/tnf/claim/show/failures/testdata/claim1.json

@@ -7,7 +7,7 @@
           "endTime": "2023-07-18 03:37:42.095508375 -0500 CDT m=+23.133713410",
           "failureLineContent": "\t\tfail(string(bytes))",
           "failureLocation": "/home/greyerof/github/tnf/pkg/testhelper/testhelper.go:352",
-          "failureReason": "{\"CompliantObjectsOut\":null,\"NonCompliantObjectsOut\":[{\"ObjectType\":\"Container\",\"ObjectFieldsKeys\":[\"Reason For Non Compliance\",\"Namespace\",\"Pod Name\",\"Container Name\",\"SCC Capability\"],\"ObjectFieldsValues\":[\"Non compliant capability detected in container\",\"tnf\",\"test-887998557-8gwwm\",\"test\",\"SYS_ADMIN\"]},{\"ObjectType\":\"Container\",\"ObjectFieldsKeys\":[\"Reason For Non Compliance\",\"Namespace\",\"Pod Name\",\"Container Name\",\"SCC Capability\"],\"ObjectFieldsValues\":[\"Non compliant capability detected in container\",\"tnf\",\"test-887998557-pr2w5\",\"test\",\"SYS_ADMIN\"]}]}",
+          "checkDetails": "{\"CompliantObjectsOut\":null,\"NonCompliantObjectsOut\":[{\"ObjectType\":\"Container\",\"ObjectFieldsKeys\":[\"Reason For Non Compliance\",\"Namespace\",\"Pod Name\",\"Container Name\",\"SCC Capability\"],\"ObjectFieldsValues\":[\"Non compliant capability detected in container\",\"tnf\",\"test-887998557-8gwwm\",\"test\",\"SYS_ADMIN\"]},{\"ObjectType\":\"Container\",\"ObjectFieldsKeys\":[\"Reason For Non Compliance\",\"Namespace\",\"Pod Name\",\"Container Name\",\"SCC Capability\"],\"ObjectFieldsValues\":[\"Non compliant capability detected in container\",\"tnf\",\"test-887998557-pr2w5\",\"test\",\"SYS_ADMIN\"]}]}",
           "startTime": "2023-07-18 03:37:42.095225914 -0500 CDT m=+23.133430956",
           "state": "failed",
           "testID": {
@@ -34,7 +34,7 @@
           "endTime": "2023-07-18 03:37:44.324268378 -0500 CDT m=+25.362473413",
           "failureLineContent": "",
           "failureLocation": ":0",
-          "failureReason": "",
+          "checkDetails": "",
           "startTime": "2023-07-18 03:37:44.324023044 -0500 CDT m=+25.362228078",
           "state": "passed",
           "testID": {
@@ -61,7 +61,7 @@
           "endTime": "0001-01-01 00:00:00 +0000 UTC",
           "failureLineContent": "",
           "failureLocation": ":0",
-          "failureReason": "",
+          "checkDetails": "",
           "startTime": "2023-07-18 03:37:46.483797163 -0500 CDT m=+27.522002219",
           "state": "skipped",
           "testID": {
@@ -88,7 +88,7 @@
           "endTime": "0001-01-01 00:00:00 +0000 UTC",
           "failureLineContent": "",
           "failureLocation": ":0",
-          "failureReason": "",
+          "checkDetails": "",
           "startTime": "2023-07-18 03:37:46.483566421 -0500 CDT m=+27.521771494",
           "state": "skipped",
           "testID": {

cmd/tnf/claim/show/failures/testdata/claim2.json

@@ -7,7 +7,7 @@
           "endTime": "2023-07-18 03:37:42.095508375 -0500 CDT m=+23.133713410",
           "failureLineContent": "\t\tfail(string(bytes))",
           "failureLocation": "/home/greyerof/github/tnf/pkg/testhelper/testhelper.go:352",
-          "failureReason": "pod xxx ns yyy container zzz uses SYS_ADMIN",
+          "checkDetails": "pod xxx ns yyy container zzz uses SYS_ADMIN",
           "startTime": "2023-07-18 03:37:42.095225914 -0500 CDT m=+23.133430956",
           "state": "failed",
           "testID": {
@@ -34,7 +34,7 @@
           "endTime": "2023-07-18 03:37:44.324268378 -0500 CDT m=+25.362473413",
           "failureLineContent": "",
           "failureLocation": ":0",
-          "failureReason": "",
+          "checkDetails": "",
           "startTime": "2023-07-18 03:37:44.324023044 -0500 CDT m=+25.362228078",
           "state": "passed",
           "testID": {
@@ -61,7 +61,7 @@
           "endTime": "0001-01-01 00:00:00 +0000 UTC",
           "failureLineContent": "",
           "failureLocation": ":0",
-          "failureReason": "",
+          "checkDetails": "",
           "startTime": "2023-07-18 03:37:46.483797163 -0500 CDT m=+27.522002219",
           "state": "skipped",
           "testID": {
@@ -88,7 +88,7 @@
           "endTime": "0001-01-01 00:00:00 +0000 UTC",
           "failureLineContent": "",
           "failureLocation": ":0",
-          "failureReason": "",
+          "checkDetails": "",
           "startTime": "2023-07-18 03:37:46.483566421 -0500 CDT m=+27.521771494",
           "state": "skipped",
           "testID": {

cmd/tnf/claim/show/failures/types.go

@@ -3,7 +3,7 @@ package failures
 import "fmt"
 
 // Custom object type needed to provide a different JSON serialization than
-// the one in claim's test cases' failureReason field.
+// the one in claim's test cases' skipReason field.
 type NonCompliantObject struct {
 	Type   string     `json:"type"`
 	Reason string     `json:"reason"`
@@ -42,7 +42,7 @@ func (spec *ObjectSpec) MarshalJSON() ([]byte, error) {
 type FailedTestCase struct {
 	TestCaseName        string               `json:"name"`
 	TestCaseDescription string               `json:"description"`
-	FailureReason       string               `json:"failureReason,omitempty"`
+	CheckDetails        string               `json:"checkDetails,omitempty"`
 	NonCompliantObjects []NonCompliantObject `json:"nonCompliantObjects,omitempty"`
 }
 

cmd/tnf/pkg/claim/claim.go

@@ -10,7 +10,7 @@ import (
 )
 
 const (
-	supportedClaimFormatVersion = "v0.2.0"
+	supportedClaimFormatVersion = "v0.3.0"
 )
 
 const (
@@ -43,7 +43,8 @@ type TestCaseResult struct {
 	EndTime                string            `json:"endTime"`
 	FailureLineContent     string            `json:"failureLineContent"`
 	FailureLocation        string            `json:"failureLocation"`
-	FailureReason          string            `json:"failureReason"`
+	SkipReason             string            `json:"skipReason"`
+	CheckDetails           string            `json:"checkDetails"`
 	StartTime              string            `json:"startTime"`
 	State                  string            `json:"state"`
 	TestID                 struct {

cmd/tnf/pkg/claim/claim_test.go

@@ -22,14 +22,14 @@ func TestIsClaimFormatVersionSupported(t *testing.T) {
 		},
 		{
 			claimFormatVersion: "v0.0.0",
-			expectedError:      "claim format version v0.0.0 is not supported. Supported version is v0.2.0",
+			expectedError:      "claim format version v0.0.0 is not supported. Supported version is v0.3.0",
 		},
 		{
 			claimFormatVersion: "v0.0.1",
-			expectedError:      "claim format version v0.0.1 is not supported. Supported version is v0.2.0",
+			expectedError:      "claim format version v0.0.1 is not supported. Supported version is v0.3.0",
 		},
 		{
-			claimFormatVersion: "v0.2.0",
+			claimFormatVersion: "v0.3.0",
 			expectedError:      "",
 		},
 	}

go.mod

@@ -1,6 +1,6 @@
 module github.com/test-network-function/cnf-certification-test
 
-go 1.21.4
+go 1.21.5
 
 require (
 	github.com/Masterminds/semver/v3 v3.2.1
@@ -17,13 +17,13 @@ require k8s.io/client-go v0.28.4
 require (
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/mittwald/go-helm-client v0.12.3
-	github.com/onsi/ginkgo/v2 v2.13.1
+	github.com/onsi/ginkgo/v2 v2.13.1 // indirect
 	github.com/openshift/api v0.0.1
 	github.com/openshift/client-go v0.0.1
 	github.com/operator-framework/api v0.19.0
 	github.com/operator-framework/operator-lifecycle-manager v0.20.0
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/test-network-function/test-network-function-claim v1.0.31
+	github.com/test-network-function/test-network-function-claim v1.0.32
 	helm.sh/helm/v3 v3.13.2
 	k8s.io/api v0.28.4
 	k8s.io/apimachinery v0.28.4
@@ -74,7 +74,6 @@ require (
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/go-sql-driver/mysql v1.7.0 // indirect
-	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@@ -126,6 +125,7 @@ require (
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/onsi/gomega v1.29.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0-rc5 // indirect
 	github.com/operator-framework/operator-manifest-tools v0.4.0 // indirect
@@ -165,14 +165,14 @@ require (
 	go.uber.org/atomic v1.11.0 // indirect
 	golang.org/x/crypto v0.14.0 // indirect
 	golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea // indirect
+	golang.org/x/mod v0.13.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/oauth2 v0.10.0 // indirect
 	golang.org/x/sync v0.4.0 // indirect
 	golang.org/x/sys v0.14.0 // indirect
 	golang.org/x/term v0.13.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.14.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect

go.sum

@@ -577,8 +577,8 @@ github.com/test-network-function/oct v0.0.4 h1:rU4kps/gbAHkR0rc5WzVtTOcJt/NBcse8
 github.com/test-network-function/oct v0.0.4/go.mod h1:oOPuUMnX6YR+cl3usBJfwCllsv7Hphw9jVi7VtniAzo=
 github.com/test-network-function/privileged-daemonset v1.0.15 h1:Jgjf3sa4d9OuhZRTj3oLhaaGV7PtQLVeLK/LSd9YgdE=
 github.com/test-network-function/privileged-daemonset v1.0.15/go.mod h1:rDiFimleKbW2E501cNgHMYCrR52+w5Sg0a6trF2HZTo=
-github.com/test-network-function/test-network-function-claim v1.0.31 h1:Yqb9/8QPEEZO0LAIeuw65uPzDPnKSG8z/njpXAN2CJs=
-github.com/test-network-function/test-network-function-claim v1.0.31/go.mod h1:itpxi9Ehhv9oNC9MiSAt52SKFtJBbQ/T1njTXspl1Hk=
+github.com/test-network-function/test-network-function-claim v1.0.32 h1:GeUwbHYaXL5Yx785NmbuSQbqby8LVPEWHeW3bFEpQ9g=
+github.com/test-network-function/test-network-function-claim v1.0.32/go.mod h1:+0c6DMF/ycFmEH3EB5mJ9rSQ+3T/d48NuqmY2aXjrqQ=
 github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8=
 github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck=
 github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY=

pkg/checksdb/check.go

@@ -45,7 +45,8 @@ type Check struct {
 
 	Result         CheckResult
 	CapturedOutput string
-	FailureReason  string
+	details        string
+	skipReason     string
 
 	logger     *log.Logger
 	logArchive *strings.Builder
@@ -189,7 +190,7 @@ func (check *Check) SetResult(compliantObjects, nonCompliantObjects []*testhelpe
 		check.LogError("Failed to get result objects string for check %s: %v", check.ID, err)
 	}
 
-	check.CapturedOutput = resultObjectsStr
+	check.details = resultObjectsStr
 
 	// If an error/panic happened before, do not change the result.
 	if check.Result == CheckResultError {
@@ -198,27 +199,15 @@ func (check *Check) SetResult(compliantObjects, nonCompliantObjects []*testhelpe
 
 	if len(nonCompliantObjects) > 0 {
 		check.Result = CheckResultFailed
-		check.FailureReason = resultObjectsStr
+		check.skipReason = ""
 	} else if len(compliantObjects) == 0 {
 		// Mark this check as skipped.
 		check.LogWarn("Check %s marked as skipped as both compliant and non-compliant objects lists are empty.", check.ID)
-		check.FailureReason = "Compliant and non-compliant objects lists are empty."
+		check.skipReason = "Compliant and non-compliant objects lists are empty."
 		check.Result = CheckResultSkipped
 	}
 }
 
-func (check *Check) SetResultFailed(reason string) {
-	check.mutex.Lock()
-	defer check.mutex.Unlock()
-
-	if check.Result == CheckResultAborted {
-		return
-	}
-
-	check.Result = CheckResultFailed
-	check.FailureReason = reason
-}
-
 func (check *Check) SetResultSkipped(reason string) {
 	check.mutex.Lock()
 	defer check.mutex.Unlock()
@@ -228,7 +217,7 @@ func (check *Check) SetResultSkipped(reason string) {
 	}
 
 	check.Result = CheckResultSkipped
-	check.FailureReason = reason
+	check.skipReason = reason
 }
 
 func (check *Check) SetResultError(reason string) {
@@ -244,15 +233,15 @@ func (check *Check) SetResultError(reason string) {
 		return
 	}
 	check.Result = CheckResultError
-	check.FailureReason = reason
+	check.skipReason = reason
 }
 
 func (check *Check) SetResultAborted(reason string) {
 	check.mutex.Lock()
 	defer check.mutex.Unlock()
 
 	check.Result = CheckResultAborted
-	check.FailureReason = reason
+	check.skipReason = reason
 }
 
 func (check *Check) Run() error {