Improve scs-compliance-check report even further (#701)

Signed-off-by: Kurt Garloff <[email protected]>
Signed-off-by: Matthias Büchse <[email protected]>

Tests/scs-compatible-iaas.yaml

@@ -19,8 +19,8 @@ modules:
     testcases:
       - id: flavor-name-check
         tags: [mandatory]
-        description: |
-          The environment fulfills all requirements of
+        description: >
+          Must fulfill all requirements of
           https://docs.scs.community/standards/scs-0100-v1-flavor-naming
   - id: scs-0100-v2
     name: Flavor naming v2
@@ -31,8 +31,8 @@ modules:
     testcases:
       - id: flavor-name-check
         tags: [mandatory]
-        description: |
-          The environment fulfills all requirements of
+        description: >
+          Must fulfill all requirements of
           https://docs.scs.community/standards/scs-0100-v2-flavor-naming
   - id: scs-0100-v3.1
     name: Flavor naming v3.1
@@ -44,8 +44,8 @@ modules:
     testcases:
       - id: flavor-name-check
         tags: [mandatory]
-        description: |
-          The environment fulfills all requirements of
+        description: >
+          Must fulfill all requirements of
           https://docs.scs.community/standards/scs-0100-v3-flavor-naming -- plus the list of mandatory
           and recommended flavors found in https://docs.scs.community/standards/scs-0100-v2-flavor-naming#standard-scs-flavors
   - id: scs-0100-v3.2
@@ -58,8 +58,8 @@ modules:
     testcases:
       - id: flavor-name-check
         tags: [mandatory]
-        description: |
-          The environment fulfills all requirements of
+        description: >
+          Must fulfill all requirements of
           https://docs.scs.community/standards/scs-0100-v3-flavor-naming
   - id: scs-0101-v1
     name: Entropy v1
@@ -80,8 +80,8 @@ modules:
         tags: []  # don't use this testcase, but list it anyway because the script will output a result
       - id: entropy-check
         tags: [mandatory]
-        description: |
-          The environment fulfills all requirements of
+        description: >
+          Must fulfill all requirements of
           https://docs.scs.community/standards/scs-0101-v1-entropy
   - id: scs-0101-v1.1
     name: Entropy v1
@@ -92,28 +92,28 @@ modules:
     testcases:
       - id: entropy-check-flavor-properties
         tags: [recommended]
-        description: |
-          The environment has all flavor properties recommended in
+        description: >
+          Must have all flavor properties recommended in
           https://docs.scs.community/standards/scs-0101-w1-entropy-implementation-testing#warnings
       - id: entropy-check-image-properties
         tags: [recommended]
-        description: |
-          The environment has all image properties recommended in
+        description: >
+          Must have all image properties recommended in
           https://docs.scs.community/standards/scs-0101-w1-entropy-implementation-testing#warnings
       - id: entropy-check-rngd
         tags: [mandatory]
-        description: |
-          The images of the test sample have the service `rngd`; see
+        description: >
+          Images of the test sample must have the service `rngd`; see
           https://docs.scs.community/standards/scs-0101-w1-entropy-implementation-testing#errors
       - id: entropy-check-entropy-avail
         tags: [mandatory]
-        description: |
-          A test instance has the correct `entropy_avail`; see
+        description: >
+          A test instance must have the correct `entropy_avail`; see
           https://docs.scs.community/standards/scs-0101-w1-entropy-implementation-testing#errors
       - id: entropy-check-fips-test
         tags: [mandatory]
-        description: |
-          A test instance passes the "FIPS test"; see
+        description: >
+          A test instance must pass the "FIPS test"; see
           https://docs.scs.community/standards/scs-0101-w1-entropy-implementation-testing#errors
       - id: entropy-check
         tags: []  # don't use this testcase, but list it anyway because the script will output a result
@@ -128,8 +128,8 @@ modules:
     testcases:
       - id: image-metadata-check
         tags: [mandatory]
-        description: |
-          The environment fulfills all requirements of https://docs.scs.community/standards/scs-0102-v1-image-metadata
+        description: >
+          Must fulfill all requirements of https://docs.scs.community/standards/scs-0102-v1-image-metadata
   - id: scs-0103-v1
     name: Standard flavors
     url: https://docs.scs.community/standards/scs-0103-v1-standard-flavors
@@ -139,8 +139,8 @@ modules:
     testcases:
       - id: standard-flavors-check
         tags: [mandatory]
-        description: |
-          The environment fulfills all requirements of https://docs.scs.community/standards/scs-0103-v1-standard-flavors
+        description: >
+          Must fulfill all requirements of https://docs.scs.community/standards/scs-0103-v1-standard-flavors
   - id: scs-0104-v1
     name: Standard images
     url: https://docs.scs.community/standards/scs-0104-v1-standard-images
@@ -152,8 +152,8 @@ modules:
     testcases:
       - id: standard-images-check
         tags: [mandatory]
-        description: |
-          The environment fulfills all requirements of https://docs.scs.community/standards/scs-0104-v1-standard-images
+        description: >
+          Must fulfill all requirements of https://docs.scs.community/standards/scs-0104-v1-standard-images
 timeline:
   - date: 2024-07-31
     versions:

Tests/scs-compliance-check.py

@@ -199,6 +199,7 @@ def __init__(self, cwd, assignment, verbosity=0):
         self.num_abort = 0
         self.num_error = 0
         self.verbosity = verbosity
+        self.spamminess = 0
 
     def run(self, check):
         parameters = check.get('parameters')
@@ -221,9 +222,11 @@ def run(self, check):
             }
             if self.verbosity > 1 and invocation["stdout"]:
                 print("\n".join(invocation["stdout"]))
+                self.spamminess += 1
             # the following check used to be "> 0", but this is quite verbose...
             if invocation['rc'] or self.verbosity > 1 and invocation["stderr"]:
                 print("\n".join(invocation["stderr"]))
+                self.spamminess += 1
             self.memo[memo_key] = invocation
         logger.debug(f".. rc {invocation['rc']}, {invocation['critical']} critical, {invocation['error']} error")
         self.num_abort += invocation["critical"]
@@ -260,18 +263,18 @@ def finalize(self, permissible_ids=None):
         return final
 
 
-def run_suite(suite: TestSuite, runner: CheckRunner, results: ResultBuilder):
-    """run all checks of `suite` using `runner`, collecting `results`"""
+def run_suite(suite: TestSuite, runner: CheckRunner):
+    """run all checks of `suite` using `runner`, returning results dict via `ResultBuilder`"""
     suite.check_sanity()
+    builder = ResultBuilder(suite.name)
     for check in suite.checks:
         invocation = runner.run(check)
         for id_, value in invocation["results"].items():
-            results.record(id_, result=value, invocation=invocation['id'])
+            builder.record(id_, result=value, invocation=invocation['id'])
+    return builder.finalize(permissible_ids=suite.ids)
 
 
-def print_report(subject: str, suite: TestSuite, targets: dict, results: dict, verbose=False):
-    if verbose:
-        print("********" * 10)
+def print_report(subject: str, suite: TestSuite, targets: dict, results: dict):
     print(f"{subject} {suite.name}:")
     for tname, target_spec in targets.items():
         by_result = suite.select(tname, target_spec).evaluate(results)
@@ -287,10 +290,13 @@ def print_report(subject: str, suite: TestSuite, targets: dict, results: dict, v
         for offenders, category in ((failed, 'FAILED'), (missing, 'MISSING')):
             if category == 'MISSING' and suite.partial:
                 continue  # do not report each missing testcase if a filter was used
+            if not offenders:
+                continue
+            print(f"  - {category}:")
             for testcase in offenders:
-                print(f"  - {category} {testcase['id']}")
+                print(f"    - {testcase['id']}:")
                 if 'description' in testcase:  # used to be `verbose and ...`, but users need the URL!
-                    print('    ' + testcase['description'])
+                    print(f"      > {testcase['description'].strip()}")
 
 
 def create_report(argv, config, spec, versions, invocations):
@@ -344,6 +350,8 @@ def main(argv):
     check_cwd = os.path.dirname(config.arg0) or os.getcwd()
     runner = CheckRunner(check_cwd, config.assignment, verbosity=config.verbose and 2 or not config.quiet)
     version_report = {}
+    # collect report data as tuples (version, suite, results) before printing them
+    report_data = []
     for version in versions:
         vname = version['version']
         suite = compile_suite(
@@ -352,12 +360,16 @@ def main(argv):
             config.sections,
             config.tests,
         )
-        builder = ResultBuilder(suite.name)
-        run_suite(suite, runner, builder)
-        results = version_report[vname] = builder.finalize(permissible_ids=suite.ids)
-        if not config.quiet:
-            print_report(config.subject, suite, version['targets'], results, verbose=config.verbose)
+        report_data.append((version, suite, run_suite(suite, runner)))
+    # now report: to console if requested, and likewise for yaml output
+    if not config.quiet:
+        # print a horizontal line if we had any script output
+        if runner.spamminess:
+            print("********" * 10)  # 80 characters
+        for version, suite, results in report_data:
+            print_report(config.subject, suite, version['targets'], results)
     if config.output:
+        version_report = {version['version']: results for version, _, results in report_data}
         report = create_report(argv, config, spec, version_report, runner.get_invocations())
         with open(config.output, 'w', encoding='UTF-8') as fileobj:
             yaml.safe_dump(report, fileobj, default_flow_style=False, sort_keys=False)