HTML report improvements and related unit tests (#433)

* Lint improvement * Improve HTML reports. * More unit tests for transcript and HTML generation. Closes #332 Closes #232 --------- Co-authored-by: Johannes Ernst <[email protected]>
fediverse-devnet · Dec 17, 2024 · 056dd08 · 056dd08
1 parent 5993cc0
commit 056dd08
Show file tree

Hide file tree

Showing 10 changed files with 412 additions and 23 deletions.
diff --git a/pyproject.toml b/pyproject.toml
@@ -22,11 +22,15 @@ dependencies = [
   "multidict",
   "jinja2",
   "pyhamcrest",
-  "pytest",
   "requests",
   "types-requests",
-  "pre-commit"
+  "pre-commit",
+
+  # For testing: not sure how to specify this just for testing
+  "pytest",
+  "beautifulsoup4"
 ]
+
 description = "Test framework to test distributed, heterogeneous systems with complex protocols such as the Fediverse"
 readme = "README-PyPI.md"
 

diff --git a/src/feditest/testrun.py b/src/feditest/testrun.py
@@ -496,6 +496,7 @@ def transcribe(self) -> TestRunTranscript:
 
             trans_sessions.append(TestRunSessionTranscript(
                     run_session.plan_constellation_index,
+                    len(self.run_sessions),
                     cast(datetime, run_session.started),
                     cast(datetime, run_session.ended),
                     trans_constellation,

diff --git a/src/feditest/testruntranscript.py b/src/feditest/testruntranscript.py
@@ -296,6 +296,7 @@ class TestRunSessionTranscript(msgspec.Struct):
     Captures information about the run of a single session in a transcript.
     """
     run_session_index: int
+    total_sessions: int
     started : datetime
     ended : datetime
     constellation: TestRunConstellationTranscript
@@ -315,7 +316,8 @@ def build_summary(self, augment_this: TestRunTranscriptSummary | None = None ):
 
 
     def __str__(self):
-        return f"Session {self.run_session_index}"
+        return f"Test Run Session { self.run_session_index + 1 }/{ self.total_sessions }"
+        # +1 for what humans expect when it says 1/2
 
 
 class TestRunTranscript(msgspec.Struct):
@@ -383,6 +385,6 @@ def has_compatible_version(self):
 
     def __str__(self):
         if self.plan.name:
-            return f'{ self.id } ({ self.plan.name })'
-        return self.id
+            return str(self.plan.name)
+        return 'Test Run'
 
diff --git a/src/feditest/testruntranscriptserializer/html.py b/src/feditest/testruntranscriptserializer/html.py
@@ -54,7 +54,7 @@ class HtmlRunTranscriptSerializer(TestRunTranscriptSerializer):
     A CSS file will be written to the provided destination with an extra extension.
     """
 
-    def __init__(self, template_path: str):
+    def __init__(self, template_path: str | None = None):
         if template_path:
             self.template_path = [ t.strip() for t in template_path.split(",") ]
         else:

diff --git a/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/matrix.jinja2 b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/matrix.jinja2
@@ -2,11 +2,11 @@
 <html lang="en">
  <head>
 {%  include "partials/shared/head.jinja2" %}
-  <title>{{ transcript.plan.name | e }} | Feditest</title>
+  <title>{{ transcript | e }} ({{ transcript.id }}) | Feditest</title>
  </head>
  <body>
   <header class="feditest title">
-   <h1><span class="prefix"><a href="https://feditest.org/">Feditest</a> Summary Report:</span> {{ transcript.plan.name | e }}</h1>
+   <h1><span class="prefix"><a href="https://feditest.org/">Feditest</a>:</span> {{ transcript | e }}</h1>
    <p class="id">{{ transcript.id }}</p>
   </header>
 {%  include "partials/shared/mobile.jinja2" %}

diff --git a/...untranscriptserializer/templates/testplantranscript_default/partials/matrix/matrix.jinja2 b/...untranscriptserializer/templates/testplantranscript_default/partials/matrix/matrix.jinja2
@@ -5,14 +5,13 @@
 {%-         set constellation = run_session.constellation %}
         <th>
          <div class="title">
-          <a href="{{ session_file_path(run_session) }}">
-           <dl class="roles">
+          <p><a href="{{ session_file_path(run_session) }}">{{ run_session }}</a></p>
+          <dl class="roles">
 {%-         for role, node in constellation.nodes.items() %}
-             <dt>{{ role | e }}</dt>
-             <dd>{{ node.node_driver | e }}</dd>
+           <dt>{{ role | e }}</dt>
+           <dd>{{ node.appdata['app'] | e }}</dd>
 {%-         endfor %}
-           </dl>
-          </a>
+          </dl>
          </div>
         </th>
 {%-     endfor %}

diff --git a/...struntranscriptserializer/templates/testplantranscript_default/session_with_matrix.jinja2 b/...struntranscriptserializer/templates/testplantranscript_default/session_with_matrix.jinja2
@@ -2,15 +2,14 @@
 <html lang="en">
  <head>
 {%  include "partials/shared/head.jinja2" %}
-  <title>{{ run_session.name | e }} | Feditest</title>
+  <title>{{ run_session | e }} ({{ transcript.id }}) | Feditest</title>
  </head>
  <body>
-    <header class="feditest title" style="text-align: center">
-      <h1><span class="prefix"><a href="https://feditest.org/">Feditest</a> Session Report:</span> {{ run_session | e }}</h1>
-      <p class="id">{{ transcript.id }} [<a href="{{ matrix_file_path }}">Summary</a>]</p>
-    </header>
-
-    {%  include "partials/shared/mobile.jinja2" %}
+  <header class="feditest title" style="text-align: center">
+   <h1><span class="prefix"><a href="https://feditest.org/">Feditest</a>:</span> {{ run_session | e }}</h1>
+   <p class="id">{{ transcript.id }} <a href="{{ matrix_file_path }}">&#x21e7;</a></p>
+  </header>
+{%  include "partials/shared/mobile.jinja2" %}
   <nav class="feditest matrix">
    <ul>
     <li><a href="#summary">Summary</a></li>
@@ -19,14 +18,14 @@
    </ul>
   </nav>
   <div id="summary" class="feditest module summary">
-   <h2>Test Run Summary</h2>
+   <h2>Test Run Session Summary</h2>
 {%  include "partials/shared/summary.jinja2" %}
   </div>
   <div id="results" class="feditest module results">
 {%  include "partials/shared_session/results.jinja2" %}
   </div>
   <div id="metadata" class="feditest module metadata">
-   <h2>Test Run Metadata</h2>
+   <h2>Test Run Session Metadata</h2>
 {%  include "partials/shared_session/metadata.jinja2" %}
   </div>
 {%  include "partials/shared/footer.jinja2" %}

diff --git a/tests.unit/dummy.py b/tests.unit/dummy.py
@@ -0,0 +1,16 @@
+#
+# Dummy classes for testing
+#
+
+from feditest.nodedrivers import AccountManager, Node, NodeConfiguration, NodeDriver
+
+
+class DummyNode(Node):
+    pass
+
+
+class DummyNodeDriver(NodeDriver):
+    # Python 3.12 @Override
+    def _provision_node(self, rolename: str, config: NodeConfiguration, account_manager: AccountManager | None) -> Node:
+        return DummyNode(rolename, config, account_manager)
+
diff --git a/tests.unit/test_50_transcript.py b/tests.unit/test_50_transcript.py
@@ -0,0 +1,154 @@
+#
+# Test the transcript has all the expected values
+#
+
+import pytest
+
+import feditest
+from feditest import nodedriver, test
+from feditest.utils import FEDITEST_VERSION
+from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanConstellationNode, TestPlanSessionTemplate, TestPlanTestSpec
+from feditest.testrun import TestRun
+from feditest.testruncontroller import AutomaticTestRunController
+from feditest.testruntranscript import TestRunResultTranscript
+
+from dummy import DummyNodeDriver
+
+APP_NAMES = [
+    'APP_0',
+    'APP_1',
+    'APP_2',
+    'APP_3'
+]
+driver_names = []
+
+@pytest.fixture(scope="module", autouse=True)
+def init_node_drivers():
+    global driver_names
+
+    """ Keep these isolated to this module """
+    feditest.all_node_drivers = {}
+    feditest._loading_node_drivers = True
+
+    @nodedriver
+    class NodeDriver1(DummyNodeDriver):
+        pass
+
+    @nodedriver
+    class NodeDriver2(DummyNodeDriver):
+        pass
+
+    feditest._loading_node_drivers = False
+
+    driver_names = list(feditest.all_node_drivers.keys())
+
+
+@pytest.fixture(scope="module", autouse=True)
+def init_tests():
+    """
+    Cleanly define some tests.
+    """
+    feditest.all_tests = {}
+    feditest._registered_as_test = {}
+    feditest._registered_as_test_step = {}
+    feditest._loading_tests = True
+
+    ##
+    ## FediTest tests start here
+    ##
+
+    @test
+    def passes() -> None:
+        """
+        This test always passes.
+        """
+        return
+
+    ##
+    ## FediTest tests end here
+    ## (Don't forget the next two lines)
+    ##
+
+    feditest._loading_tests = False
+    feditest._load_tests_pass2()
+
+
+@pytest.fixture(autouse=True)
+def test_plan_fixture() -> TestPlan:
+    """
+    The test plan tests all known tests.
+    """
+    constellations = [
+        TestPlanConstellation(
+            {
+                'node1a': TestPlanConstellationNode(
+                    driver_names[0],
+                    {
+                        'app' : APP_NAMES[0]
+                    }
+                ),
+                'node2a': TestPlanConstellationNode(
+                    driver_names[1],
+                    {
+                        'app' : APP_NAMES[1]
+                    }
+                )
+            },
+            'constellation-1'),
+        TestPlanConstellation(
+            {
+                'node1b': TestPlanConstellationNode(
+                    driver_names[0],
+                    {
+                        'app' : APP_NAMES[2]
+                    }
+                ),
+                'node2b': TestPlanConstellationNode(
+                    driver_names[1],
+                    {
+                        'app' : APP_NAMES[3]
+                    }
+                )
+            },
+            'constellation-2')
+    ]
+    tests = [ TestPlanTestSpec(name) for name in sorted(feditest.all_tests.keys()) if feditest.all_tests.get(name) is not None ]
+    session = TestPlanSessionTemplate(tests, "Test a test that passes")
+    ret = TestPlan(session, constellations)
+    ret.properties_validate()
+    # ret.print()
+    return ret
+
+
+@pytest.fixture
+def transcript(test_plan_fixture: TestPlan) -> TestRunResultTranscript:
+    test_plan_fixture.check_can_be_executed()
+
+    test_run = TestRun(test_plan_fixture)
+    controller = AutomaticTestRunController(test_run)
+    test_run.run(controller)
+
+    ret = test_run.transcribe()
+    return ret
+
+
+def test_transcript(transcript: TestRunResultTranscript):
+    assert transcript.plan
+    assert transcript.id
+    assert transcript.started
+    assert transcript.ended
+
+    assert len(transcript.sessions) == 2
+    assert len(transcript.test_meta) == 1
+    assert transcript.result is None
+    assert transcript.type == 'feditest-testrun-transcript'
+    assert transcript.feditest_version == FEDITEST_VERSION
+
+    for i in range(0, 1):
+        assert transcript.sessions[i].started
+        assert transcript.sessions[i].ended
+        assert len(transcript.sessions[i].run_tests) == 1
+        assert transcript.sessions[i].run_tests[0].started
+        assert transcript.sessions[i].run_tests[0].ended
+        assert transcript.sessions[i].run_tests[0].result is None
+