Add new tnf command to check the test results (#1864)

* Add new tnf command to check the test results

The command compares the test results of a CNFCERT execution using the
log file with those provided in a YAML template to check that they match.
If not, a table showing the expected vs actual results will be printed
on stdout.

To generate the test results YAML template from a log file:

* tnf check results --generate-template

To check the test results against a reference template:

* tnf check results --template <expected_results.yaml> --log-file
  <cnf-certsuite.log>

cmd/tnf/check/check.go

@@ -3,17 +3,19 @@ package check
 import (
 	"github.com/spf13/cobra"
 	imagecert "github.com/test-network-function/cnf-certification-test/cmd/tnf/check/image_cert_status"
+	"github.com/test-network-function/cnf-certification-test/cmd/tnf/check/results"
 )
 
 var (
 	checkCmd = &cobra.Command{
 		Use:   "check",
-		Short: "check the status of CNF resources.",
+		Short: "check the status of CNF resources or artifacts.",
 	}
 )
 
 func NewCommand() *cobra.Command {
 	checkCmd.AddCommand(imagecert.NewCommand())
+	checkCmd.AddCommand(results.NewCommand())
 
 	return checkCmd
 }

cmd/tnf/check/results/results.go

@@ -0,0 +1,202 @@
+// Copyright (C) 2020-2024 Red Hat, Inc.
+
+package results
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v2"
+)
+
+const (
+	TestResultsTemplateFileName        = "expected_results.yaml"
+	TestResultsTemplateFilePermissions = 0o644
+)
+
+const (
+	resultPass = "PASSED"
+	resultSkip = "SKIPPED"
+	resultFail = "FAILED"
+	resultMiss = "MISSING"
+)
+
+type TestCaseList struct {
+	Pass []string `yaml:"pass"`
+	Fail []string `yaml:"fail"`
+	Skip []string `yaml:"skip"`
+}
+
+type TestResults struct {
+	TestCaseList `yaml:"testCases"`
+}
+
+var checkResultsCmd = &cobra.Command{
+	Use:   "results",
+	Short: "Verifies that the actual CNFCERT results match the ones found in a reference template",
+	RunE:  checkResults,
+}
+
+func checkResults(cmd *cobra.Command, _ []string) error {
+	templateFileName, _ := cmd.Flags().GetString("template")
+	generateTemplate, _ := cmd.Flags().GetBool("generate-template")
+	logFileName, _ := cmd.Flags().GetString("log-file")
+
+	// Build a database with the test results from the log file
+	actualTestResults, err := getTestResultsDB(logFileName)
+	if err != nil {
+		return fmt.Errorf("could not get the test results DB, err: %v", err)
+	}
+
+	// Generate a reference YAML template with the test results if required
+	if generateTemplate {
+		return generateTemplateFile(actualTestResults)
+	}
+
+	// Get the expected test results from the reference YAML template
+	expectedTestResults, err := getExpectedTestResults(templateFileName)
+	if err != nil {
+		return fmt.Errorf("could not get the expected test results, err: %v", err)
+	}
+
+	// Match the results between the test results DB and the reference YAML template
+	var mismatchedTestCases []string
+	for testCase, testResult := range actualTestResults {
+		if testResult != expectedTestResults[testCase] {
+			mismatchedTestCases = append(mismatchedTestCases, testCase)
+		}
+	}
+
+	// Verify that there are no unmatched expected test results
+	for testCase := range expectedTestResults {
+		if _, exists := actualTestResults[testCase]; !exists {
+			mismatchedTestCases = append(mismatchedTestCases, testCase)
+		}
+	}
+
+	if len(mismatchedTestCases) > 0 {
+		fmt.Println("Expected results DO NOT match actual results")
+		printTestResultsMismatch(mismatchedTestCases, actualTestResults, expectedTestResults)
+		os.Exit(1)
+	}
+
+	fmt.Println("Expected results and actual results match")
+
+	return nil
+}
+
+func getTestResultsDB(logFileName string) (map[string]string, error) {
+	resultsDB := make(map[string]string)
+
+	file, err := os.Open(logFileName)
+	if err != nil {
+		return nil, fmt.Errorf("could not open file %q, err: %v", logFileName, err)
+	}
+	defer file.Close()
+
+	re := regexp.MustCompile(`.*\[(.*?)\]\s+Recording result\s+"(.*?)"`)
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		match := re.FindStringSubmatch(line)
+		if match != nil {
+			testCaseName := match[1]
+			result := match[2]
+			resultsDB[testCaseName] = result
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("error scanning file, err: %v", err)
+	}
+
+	return resultsDB, nil
+}
+
+func getExpectedTestResults(templateFileName string) (map[string]string, error) {
+	templateFile, err := os.ReadFile(templateFileName)
+	if err != nil {
+		return nil, fmt.Errorf("could not open template file %q, err: %v", templateFileName, err)
+	}
+
+	var expectedTestResultsList TestResults
+	err = yaml.Unmarshal(templateFile, &expectedTestResultsList)
+	if err != nil {
+		return nil, fmt.Errorf("could not parse the template YAML file, err: %v", err)
+	}
+
+	expectedTestResults := make(map[string]string)
+	for _, testCase := range expectedTestResultsList.Pass {
+		expectedTestResults[testCase] = resultPass
+	}
+	for _, testCase := range expectedTestResultsList.Skip {
+		expectedTestResults[testCase] = resultSkip
+	}
+	for _, testCase := range expectedTestResultsList.Fail {
+		expectedTestResults[testCase] = resultFail
+	}
+
+	return expectedTestResults, nil
+}
+
+func printTestResultsMismatch(mismatchedTestCases []string, actualResults, expectedResults map[string]string) {
+	fmt.Printf("\n")
+	fmt.Println(strings.Repeat("-", 96)) //nolint:gomnd // table line
+	fmt.Printf("| %-58s %-19s %s |\n", "TEST_CASE", "EXPECTED_RESULT", "ACTUAL_RESULT")
+	fmt.Println(strings.Repeat("-", 96)) //nolint:gomnd // table line
+	for _, testCase := range mismatchedTestCases {
+		expectedResult, exist := expectedResults[testCase]
+		if !exist {
+			expectedResult = resultMiss
+		}
+		actualResult, exist := actualResults[testCase]
+		if !exist {
+			actualResult = resultMiss
+		}
+		fmt.Printf("| %-54s %19s %17s |\n", testCase, expectedResult, actualResult)
+		fmt.Println(strings.Repeat("-", 96)) //nolint:gomnd // table line
+	}
+}
+
+func generateTemplateFile(resultsDB map[string]string) error {
+	var modelTemplate TestResults
+	for testCase, result := range resultsDB {
+		switch result {
+		case resultPass:
+			modelTemplate.Pass = append(modelTemplate.Pass, testCase)
+		case resultSkip:
+			modelTemplate.Skip = append(modelTemplate.Skip, testCase)
+		case resultFail:
+			modelTemplate.Fail = append(modelTemplate.Fail, testCase)
+		default:
+			return fmt.Errorf("unknown test case result %q", result)
+		}
+	}
+
+	modelOut, err := yaml.Marshal(&modelTemplate)
+	if err != nil {
+		return fmt.Errorf("could not marshal template yaml, err: %v", err)
+	}
+
+	err = os.WriteFile(TestResultsTemplateFileName, modelOut, TestResultsTemplateFilePermissions)
+	if err != nil {
+		return fmt.Errorf("could not write to file %q: %v", TestResultsTemplateFileName, err)
+	}
+
+	return nil
+}
+
+func NewCommand() *cobra.Command {
+	checkResultsCmd.PersistentFlags().String("template", "expected_results.yaml", "reference YAML template with the expected results")
+	checkResultsCmd.PersistentFlags().String("log-file", "cnf-certification-test/cnf-certsuite.log", "log file of the CNFCERT execution")
+	checkResultsCmd.PersistentFlags().Bool("generate-template", false, "generate a reference YAML template from the log file")
+
+	checkResultsCmd.MarkFlagsMutuallyExclusive("template", "generate-template")
+
+	return checkResultsCmd
+}