From 98a520ef635fc05e8fb56d671ab5cb1edc421fa9 Mon Sep 17 00:00:00 2001 From: jmontesi Date: Thu, 8 Feb 2024 16:11:13 +0100 Subject: [PATCH 1/2] Add new tnf command to check the test results The command compares the test results of a CNFCERT execution using the log file with those provided in a YAML template to check that they match. If not, a table showing the expected vs actual results will be printed on stdout. To generate the test results YAML template from a log file: * tnf check results --generate-template To check the test results against a reference template: * tnf check results --template --log-file --- cmd/tnf/check/check.go | 4 +- cmd/tnf/check/results/results.go | 216 +++++++++++++++++++++++++++++++ 2 files changed, 219 insertions(+), 1 deletion(-) create mode 100644 cmd/tnf/check/results/results.go diff --git a/cmd/tnf/check/check.go b/cmd/tnf/check/check.go index 8468acdcf..ba34bfb14 100644 --- a/cmd/tnf/check/check.go +++ b/cmd/tnf/check/check.go @@ -3,17 +3,19 @@ package check import ( "github.com/spf13/cobra" imagecert "github.com/test-network-function/cnf-certification-test/cmd/tnf/check/image_cert_status" + "github.com/test-network-function/cnf-certification-test/cmd/tnf/check/results" ) var ( checkCmd = &cobra.Command{ Use: "check", - Short: "check the status of CNF resources.", + Short: "check the status of CNF resources or artifacts.", } ) func NewCommand() *cobra.Command { checkCmd.AddCommand(imagecert.NewCommand()) + checkCmd.AddCommand(results.NewCommand()) return checkCmd } diff --git a/cmd/tnf/check/results/results.go b/cmd/tnf/check/results/results.go new file mode 100644 index 000000000..85bbb241b --- /dev/null +++ b/cmd/tnf/check/results/results.go @@ -0,0 +1,216 @@ +// Copyright (C) 2020-2024 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package results + +import ( + "bufio" + "fmt" + "os" + "regexp" + "strings" + + "github.com/spf13/cobra" + "gopkg.in/yaml.v2" +) + +const ( + TestResultsTemplateFileName = "expected_results.yaml" + TestResultsTemplateFilePermissions = 0o644 +) + +const ( + resultPass = "PASSED" + resultSkip = "SKIPPED" + resultFail = "FAILED" + resultMiss = "MISSING" +) + +type TestCaseList struct { + Pass []string `yaml:"pass"` + Fail []string `yaml:"fail"` + Skip []string `yaml:"skip"` +} + +type TestResults struct { + TestCaseList `yaml:"testCases"` +} + +var checkResultsCmd = &cobra.Command{ + Use: "results", + Short: "Verifies that the actual CNFCERT results match the ones found in a reference template", + RunE: checkResults, +} + +func checkResults(cmd *cobra.Command, _ []string) error { + templateFileName, _ := cmd.Flags().GetString("template") + generateTemplate, _ := cmd.Flags().GetBool("generate-template") + logFileName, _ := cmd.Flags().GetString("log-file") + + // Build a database with the test results from the log file + actualTestResults, err := getTestResultsDB(logFileName) + if err != nil { + return fmt.Errorf("could not get the test results DB, err: %v", err) + } + + // Generate a reference YAML template with the test results if required + if generateTemplate { + return generateTemplateFile(actualTestResults) + } + + // Get the expected test results from the reference YAML template + expectedTestResults, err := getExpectedTestResults(templateFileName) + if err != nil { + return fmt.Errorf("could not get the expected test results, err: %v", err) + } + + // Match the results between the test results DB and the reference YAML template + var mismatchedTestCases []string + for testCase, testResult := range actualTestResults { + if testResult != expectedTestResults[testCase] { + mismatchedTestCases = append(mismatchedTestCases, testCase) + } + } + + // Verify that there are no unmatched expected test results + for testCase := range expectedTestResults { + if _, exists := actualTestResults[testCase]; !exists { + mismatchedTestCases = append(mismatchedTestCases, testCase) + } + } + + if len(mismatchedTestCases) > 0 { + fmt.Println("Expected results DO NOT match actual results") + printTestResultsMismatch(mismatchedTestCases, actualTestResults, expectedTestResults) + os.Exit(1) + } + + fmt.Println("Expected results and actual results match") + + return nil +} + +func getTestResultsDB(logFileName string) (map[string]string, error) { + resultsDB := make(map[string]string) + + file, err := os.Open(logFileName) + if err != nil { + return nil, fmt.Errorf("could not open file %q, err: %v", logFileName, err) + } + defer file.Close() + + re := regexp.MustCompile(`.*\[(.*?)\]\s+Recording result\s+"(.*?)"`) + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + match := re.FindStringSubmatch(line) + if match != nil { + testCaseName := match[1] + result := match[2] + resultsDB[testCaseName] = result + } + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("error scanning file, err: %v", err) + } + + return resultsDB, nil +} + +func getExpectedTestResults(templateFileName string) (map[string]string, error) { + templateFile, err := os.ReadFile(templateFileName) + if err != nil { + return nil, fmt.Errorf("could not open template file %q, err: %v", templateFileName, err) + } + + var expectedTestResultsList TestResults + err = yaml.Unmarshal(templateFile, &expectedTestResultsList) + if err != nil { + return nil, fmt.Errorf("could not parse the template YAML file, err: %v", err) + } + + expectedTestResults := make(map[string]string) + for _, testCase := range expectedTestResultsList.Pass { + expectedTestResults[testCase] = resultPass + } + for _, testCase := range expectedTestResultsList.Skip { + expectedTestResults[testCase] = resultSkip + } + for _, testCase := range expectedTestResultsList.Fail { + expectedTestResults[testCase] = resultFail + } + + return expectedTestResults, nil +} + +func printTestResultsMismatch(mismatchedTestCases []string, actualResults, expectedResults map[string]string) { + fmt.Printf("\n") + fmt.Println(strings.Repeat("-", 96)) //nolint:gomnd // table line + fmt.Printf("| %-58s %-19s %s |\n", "TEST_CASE", "EXPECTED_RESULT", "ACTUAL_RESULT") + fmt.Println(strings.Repeat("-", 96)) //nolint:gomnd // table line + for _, testCase := range mismatchedTestCases { + expectedResult, exist := expectedResults[testCase] + if !exist { + expectedResult = resultMiss + } + actualResult, exist := actualResults[testCase] + if !exist { + actualResult = resultMiss + } + fmt.Printf("| %-54s %19s %17s |\n", testCase, expectedResult, actualResult) + fmt.Println(strings.Repeat("-", 96)) //nolint:gomnd // table line + } +} + +func generateTemplateFile(resultsDB map[string]string) error { + var modelTemplate TestResults + for testCase, result := range resultsDB { + switch result { + case resultPass: + modelTemplate.Pass = append(modelTemplate.Pass, testCase) + case resultSkip: + modelTemplate.Skip = append(modelTemplate.Skip, testCase) + case resultFail: + modelTemplate.Fail = append(modelTemplate.Fail, testCase) + default: + return fmt.Errorf("unknown test case result %q", result) + } + } + + modelOut, err := yaml.Marshal(&modelTemplate) + if err != nil { + return fmt.Errorf("could not marshal template yaml, err: %v", err) + } + + err = os.WriteFile(TestResultsTemplateFileName, modelOut, TestResultsTemplateFilePermissions) + if err != nil { + return fmt.Errorf("could not write to file %q: %v", TestResultsTemplateFileName, err) + } + + return nil +} + +func NewCommand() *cobra.Command { + checkResultsCmd.PersistentFlags().String("template", "expected_results.yaml", "reference YAML template with the expected results") + checkResultsCmd.PersistentFlags().String("log-file", "cnf-certification-test/cnf-certsuite.log", "log file of the CNFCERT execution") + checkResultsCmd.PersistentFlags().Bool("generate-template", false, "generate a reference YAML template from the log file") + + checkResultsCmd.MarkFlagsMutuallyExclusive("template", "generate-template") + + return checkResultsCmd +} From eeb107c9f554290b7483a63ef01014d2b37789b8 Mon Sep 17 00:00:00 2001 From: jmontesi Date: Mon, 12 Feb 2024 10:07:02 +0100 Subject: [PATCH 2/2] Simplify copyright message --- cmd/tnf/check/results/results.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/cmd/tnf/check/results/results.go b/cmd/tnf/check/results/results.go index 85bbb241b..874e8c60a 100644 --- a/cmd/tnf/check/results/results.go +++ b/cmd/tnf/check/results/results.go @@ -1,18 +1,4 @@ // Copyright (C) 2020-2024 Red Hat, Inc. -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. package results