Parallel runner implementation (#59)

* Toc Update (#30)

* Release v0.1.34 (#52)

* Improved the error message when the call to the parent class constructor is missing in a test fixture

* Fixing the Environment variale setting documentation for Windows Powershell

* Apply suggestions from code review

Co-authored-by: Omri Mendels <[email protected]>

* Feature: Discover test files with '_test' suffix (#47)

* Enable test discovery for test names with suffix 'test'

* Combine redundant suffix tests

* Remove old suffix tests

* Encapsulate test name parsing and have nuttercli call test name validation from api

* Have api client results call _is_valid_test_name from api

Co-authored-by: Quan Nguyen <[email protected]>

* Invalid State response is retriable (#49)

* fixed import error and refactoring

* invalid state is retriable, pull sleep is 5 seconds

* Poll wait time as flag (#51)

* poll wait time as flag

* lint fixes

Co-authored-by: RobBagby <[email protected]>
Co-authored-by: Prakash Kudkuli Vishnu <[email protected]>
Co-authored-by: Omri Mendels <[email protected]>
Co-authored-by: quanuw <[email protected]>
Co-authored-by: Quan Nguyen <[email protected]>

* Update README.md

* Parallel runner

* Rename helper class

* Fix collect results

* Rename execute method

* Use new execute method

* Introduce add_test_fixture() method

Co-authored-by: Jesus Aguilar <[email protected]>
Co-authored-by: RobBagby <[email protected]>
Co-authored-by: Prakash Kudkuli Vishnu <[email protected]>
Co-authored-by: Omri Mendels <[email protected]>
Co-authored-by: quanuw <[email protected]>
Co-authored-by: Quan Nguyen <[email protected]>

runtime/runner.py

@@ -0,0 +1,56 @@
+"""
+Copyright (c) Microsoft Corporation.
+
+Licensed under the MIT license.
+"""
+
+import common.scheduler as scheduler
+from common.testexecresults import TestExecResults
+from common.testresult import TestResults
+
+from runtime.nutterfixture import NutterFixture
+
+
+class NutterFixtureParallelRunner(object):
+    """Helper class to execute tests in parallel."""
+
+    def __init__(self, num_of_workers=1):
+        """Initialize the runner.
+
+        Args:
+            num_of_workers (int): number of parallel workers.
+        """
+        self.tests = []
+        self.num_of_workers = num_of_workers
+
+    def add_test_fixture(self, fixture):
+        """Add a test to the list of tests to run.
+
+        Args:
+            fixture (NutterFixture): the test to add.
+        """
+        if not isinstance(fixture, NutterFixture):
+            raise TypeError("fixture must be of type NutterFixture")
+        self.tests.append(fixture)
+
+    def execute(self):
+        """Execute the tests."""
+        sched = scheduler.get_scheduler(self.num_of_workers)
+
+        for i in self.tests:
+            sched.add_function(i.execute_tests)
+
+        results = sched.run_and_wait()
+
+        return self._collect_results(results)
+
+    def _collect_results(self, results):
+        """Collect all results in a single TestExecResults object."""
+        all_results = TestResults()
+
+        for funcres in results:
+            if funcres.func_result is not None:
+                for testres in funcres.func_result.test_results.results:
+                    all_results.append(testres)
+
+        return TestExecResults(all_results)

tests/runtime/test_runner.py

@@ -0,0 +1,142 @@
+"""
+Copyright (c) Microsoft Corporation.
+
+Licensed under the MIT license.
+"""
+
+import time
+
+import pytest
+
+from runtime.nutterfixture import NutterFixture
+from runtime.runner import NutterFixtureParallelRunner
+
+test_cases = [
+    (1, 1),
+    (1, 2),
+    (2, 1),
+    (2, 2),
+    (3, 1),
+    (3, 2),
+    (3, 3)
+]
+
+@pytest.mark.parametrize('num_of_tests, num_of_workers', test_cases)
+def test__execute_tests__x_tests_x_workers__results_ok(num_of_tests, num_of_workers):
+    # Assemble list of tests
+    runner = NutterFixtureParallelRunner(num_of_workers)
+    for i in range(num_of_tests):
+        test_case = RunnerTestFixture()
+        runner.add_test_fixture(test_case)
+
+    # Execute tests
+    results = runner.execute()
+
+    # Assert results
+    assert len(results.test_results.results) == num_of_tests
+    assert results.test_results.passed() == True
+
+def test__execute_tests__3_tests_in_sequence_with_failed_assertion__results_ok():
+    # Arrange
+    tests = [
+        RunnerTestFixture(),
+        RunnerTestFixtureFailAssert(),
+        RunnerTestFixture()
+    ]
+
+    runner = NutterFixtureParallelRunner()
+    for i in tests:
+        runner.add_test_fixture(i)
+
+    # Act
+    results = runner.execute()
+
+    # Assert
+    assert len(results.test_results.results) == len(tests)
+    assert results.test_results.results[0].passed == True
+    assert results.test_results.results[1].passed == False
+    assert results.test_results.results[2].passed == True
+
+def test__execute_tests__3_tests_in_sequence_with_run_exception__results_ok():
+    # Arrange
+    tests = [
+        RunnerTestFixture(),
+        RunnerTestFixtureRunException(),
+        RunnerTestFixture()
+    ]
+
+    runner = NutterFixtureParallelRunner()
+    for i in tests:
+        runner.add_test_fixture(i)
+
+    # Act
+    results = runner.execute()
+
+    # Assert
+    assert len(results.test_results.results) == len(tests)
+    assert results.test_results.results[0].passed == True
+    assert results.test_results.results[1].passed == False
+    assert results.test_results.results[2].passed == True
+
+def test__execute_tests__3_tests_in_sequence_with_exec_exception__results_ok():
+    # Arrange
+    tests = [
+        RunnerTestFixture(),
+        RunnerTestFixtureExecuteException(),
+        RunnerTestFixture()
+    ]
+
+    runner = NutterFixtureParallelRunner()
+    for i in tests:
+        runner.add_test_fixture(i)
+
+    # Act
+    results = runner.execute()
+
+    # Assert
+    assert len(results.test_results.results) == len(tests) - 1
+    assert results.test_results.results[0].passed == True
+    assert results.test_results.results[1].passed == True
+
+class RunnerTestFixture(NutterFixture):
+    def before_test(self):
+        pass
+
+    def run_test(self):
+        pass
+
+    def assertion_test(self):
+        assert 1 == 1
+
+    def after_test(self):
+        pass
+
+class RunnerTestFixtureFailAssert(NutterFixture):
+    def before_test(self):
+        pass
+
+    def run_test(self):
+        pass
+
+    def assertion_test(self):
+        assert 1 != 1
+
+    def after_test(self):
+        pass
+
+class RunnerTestFixtureRunException(NutterFixture):
+    def before_test(self):
+        pass
+
+    def run_test(self):
+        raise(Exception())
+
+    def assertion_test(self):
+        assert 1 == 1
+
+    def after_test(self):
+        pass
+
+class RunnerTestFixtureExecuteException(NutterFixture):
+    def execute_tests(self):
+        raise(Exception())