diff --git a/docs/configuration.md b/docs/configuration.md index aa7565ea1e..6d3dfce9ab 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -95,6 +95,18 @@ managedStatefulsets: - name: jack ``` +#### JUnit XML File Creation + +The test suite has the ability to create the JUNit XML File output containing the test ID and the corresponding test result. + +To enable this, set: + +``` +export ENABLE_XML_CREATION=true +``` + +This will create a file named `cnf-certification-test/cnf-certification-tests_junit.xml`. + ### Exceptions These options allow adding exceptions to skip several checks for different resources. The exceptions must be justified in order to pass the CNF Certification. diff --git a/docs/test-output.md b/docs/test-output.md index ff5f3d208e..eed0cfcce2 100644 --- a/docs/test-output.md +++ b/docs/test-output.md @@ -61,7 +61,7 @@ The "20230620-110654" sample prefix means "June-20th 2023, 11:06:54" This is the content of the tar.gz file: * claim.json -* cnf-certification-tests_junit.xml +* cnf-certification-tests_junit.xml (Only if enabled via `ENABLE_XML_CREATION` environment variable) * claimjson.js * classification.js * results.html @@ -69,7 +69,7 @@ This is the content of the tar.gz file: This file serves two different purposes: 1. Make it easier to store and send the test results for review. -2. View the results in the html web page. In addition, the web page (either results-embed.thml or results.html) has a selector for workload type and allows the parter to introduce feedback for each of the failing test cases for later review from Red Hat. It's important to note that this web page needs the `claimjson.js` and `classification.js` files to be in the same folder as the html files to work properly. +2. View the results in the html web page. In addition, the web page (either results-embed.html or results.html) has a selector for workload type and allows the partner to introduce feedback for each of the failing test cases for later review from Red Hat. It's important to note that this web page needs the `claimjson.js` and `classification.js` files to be in the same folder as the html files to work properly. ## Show Results after running the test code @@ -79,13 +79,13 @@ https://github.com/test-network-function/parser ## Compare claim files from two different CNF Certification Suite runs -Parters can use the `tnf claim compare` tool in order to compare two claim files. The differences are shown in a table per section. +Partners can use the `tnf claim compare` tool in order to compare two claim files. The differences are shown in a table per section. This tool can be helpful when the result of some test cases is different between two (consecutive) runs, as it shows configuration differences in both the CNF Cert Suite config and the cluster nodes that could be the root cause for some of the test cases results discrepancy. All the compared sections, except the test cases results are compared blindly, traversing the whole json tree and -substrees to get a list of all the fields and their values. Three tables are shown: +sub-trees to get a list of all the fields and their values. Three tables are shown: * Differences: same fields with different values. * Fields in claim 1 only: json fields in claim file 1 that don't exist in claim 2. diff --git a/pkg/certsuite/certsuite.go b/pkg/certsuite/certsuite.go index a494bb3a24..3694232f87 100644 --- a/pkg/certsuite/certsuite.go +++ b/pkg/certsuite/certsuite.go @@ -68,7 +68,11 @@ func Run(labelsFilter, outputFolder string, timeout time.Duration) { // Marshal the claim and output to file claimBuilder.Build(claimOutputFile) - claimBuilder.ToJUnitXML(junitXMLOutputFile, startTime, endTime) + + if configuration.GetTestParameters().EnableXMLCreation { + logrus.Infof("XML file creation is enabled. Creating JUnit XML file: %s", junitXMLOutputFile) + claimBuilder.ToJUnitXML(junitXMLOutputFile, startTime, endTime) + } // Send claim file to the collector if specified by env var if configuration.GetTestParameters().EnableDataCollection { diff --git a/pkg/checksdb/checksdb.go b/pkg/checksdb/checksdb.go index c1dea72684..2d43aacd2b 100644 --- a/pkg/checksdb/checksdb.go +++ b/pkg/checksdb/checksdb.go @@ -196,6 +196,8 @@ func printFailedChecksLog() { } } } +} + func GetResults() map[string]claim.Result { return resultsDB } diff --git a/pkg/configuration/configuration.go b/pkg/configuration/configuration.go index aaec22a47f..7b81c58ffc 100644 --- a/pkg/configuration/configuration.go +++ b/pkg/configuration/configuration.go @@ -104,4 +104,5 @@ type TestParameters struct { IncludeWebFilesInOutputFolder bool `split_words:"true" default:"false"` OmitArtifactsZipFile bool `split_words:"true" default:"false"` EnableDataCollection bool `split_words:"true" envconfig:"ENABLE_DATA_COLLECTION" default:"false"` + EnableXMLCreation bool `split_words:"true" envconfig:"ENABLE_XML_CREATION" default:"false"` } diff --git a/script/run-container.sh b/script/run-container.sh index a3c6e92548..3346c5f772 100755 --- a/script/run-container.sh +++ b/script/run-container.sh @@ -31,6 +31,7 @@ TNF_OMIT_ARTIFACTS_ZIP_FILE_DEFAULT=false TNF_INCLUDE_WEB_FILES_IN_OUTPUT_FOLDER_DEFAULT=false ON_DEMAND_DEBUG_PODS_DEFAULT=false TNF_ENABLE_DATA_COLLECTION_DEFAULT=false +TNF_ENABLE_XML_CREATION=false get_container_tnf_kubeconfig_path_from_index() { local local_path_index="$1" @@ -126,6 +127,7 @@ ON_DEMAND_DEBUG_PODS="${ON_DEMAND_DEBUG_PODS:-$ON_DEMAND_DEBUG_PODS_DEFAULT}" TNF_OMIT_ARTIFACTS_ZIP_FILE="${TNF_OMIT_ARTIFACTS_ZIP_FILE:-$TNF_OMIT_ARTIFACTS_ZIP_FILE_DEFAULT}" TNF_INCLUDE_WEB_FILES_IN_OUTPUT_FOLDER="${TNF_INCLUDE_WEB_FILES_IN_OUTPUT_FOLDER:-$TNF_INCLUDE_WEB_FILES_IN_OUTPUT_FOLDER_DEFAULT}" TNF_ENABLE_DATA_COLLECTION="${TNF_ENABLE_DATA_COLLECTION:-$TNF_ENABLE_DATA_COLLECTION_DEFAULT}" +TNF_ENABLE_XML_CREATION="${TNF_ENABLE_XML_CREATION:-$TNF_ENABLE_XML_CREATION_DEFAULT}" display_config_summary # Construct new $KUBECONFIG env variable containing all paths to kubeconfigs mounted to the container. @@ -180,6 +182,7 @@ ${TNF_CONTAINER_CLIENT} run --rm $DNS_ARG \ -e TNF_OMIT_ARTIFACTS_ZIP_FILE=$TNF_OMIT_ARTIFACTS_ZIP_FILE \ -e TNF_INCLUDE_WEB_FILES_IN_OUTPUT_FOLDER=$TNF_INCLUDE_WEB_FILES_IN_OUTPUT_FOLDER \ -e TNF_ENABLE_DATA_COLLECTION=$TNF_ENABLE_DATA_COLLECTION \ + -e TNF_ENABLE_XML_CREATION=$TNF_ENABLE_XML_CREATION \ -e PATH=/usr/bin:/usr/local/oc/bin \ $TNF_IMAGE \ $TNF_CMD $OUTPUT_ARG $CONTAINER_TNF_DIR/claim $LABEL_ARG $TNF_LABEL "$@"