diff --git a/runtime/runner.py b/runtime/runner.py new file mode 100644 index 0000000..c80c852 --- /dev/null +++ b/runtime/runner.py @@ -0,0 +1,56 @@ +""" +Copyright (c) Microsoft Corporation. + +Licensed under the MIT license. +""" + +import common.scheduler as scheduler +from common.testexecresults import TestExecResults +from common.testresult import TestResults + +from runtime.nutterfixture import NutterFixture + + +class NutterFixtureParallelRunner(object): + """Helper class to execute tests in parallel.""" + + def __init__(self, num_of_workers=1): + """Initialize the runner. + + Args: + num_of_workers (int): number of parallel workers. + """ + self.tests = [] + self.num_of_workers = num_of_workers + + def add_test_fixture(self, fixture): + """Add a test to the list of tests to run. + + Args: + fixture (NutterFixture): the test to add. + """ + if not isinstance(fixture, NutterFixture): + raise TypeError("fixture must be of type NutterFixture") + self.tests.append(fixture) + + def execute(self): + """Execute the tests.""" + sched = scheduler.get_scheduler(self.num_of_workers) + + for i in self.tests: + sched.add_function(i.execute_tests) + + results = sched.run_and_wait() + + return self._collect_results(results) + + def _collect_results(self, results): + """Collect all results in a single TestExecResults object.""" + all_results = TestResults() + + for funcres in results: + if funcres.func_result is not None: + for testres in funcres.func_result.test_results.results: + all_results.append(testres) + + return TestExecResults(all_results) diff --git a/tests/runtime/test_runner.py b/tests/runtime/test_runner.py new file mode 100644 index 0000000..febdea9 --- /dev/null +++ b/tests/runtime/test_runner.py @@ -0,0 +1,142 @@ +""" +Copyright (c) Microsoft Corporation. + +Licensed under the MIT license. +""" + +import time + +import pytest + +from runtime.nutterfixture import NutterFixture +from runtime.runner import NutterFixtureParallelRunner + +test_cases = [ + (1, 1), + (1, 2), + (2, 1), + (2, 2), + (3, 1), + (3, 2), + (3, 3) +] + +@pytest.mark.parametrize('num_of_tests, num_of_workers', test_cases) +def test__execute_tests__x_tests_x_workers__results_ok(num_of_tests, num_of_workers): + # Assemble list of tests + runner = NutterFixtureParallelRunner(num_of_workers) + for i in range(num_of_tests): + test_case = RunnerTestFixture() + runner.add_test_fixture(test_case) + + # Execute tests + results = runner.execute() + + # Assert results + assert len(results.test_results.results) == num_of_tests + assert results.test_results.passed() == True + +def test__execute_tests__3_tests_in_sequence_with_failed_assertion__results_ok(): + # Arrange + tests = [ + RunnerTestFixture(), + RunnerTestFixtureFailAssert(), + RunnerTestFixture() + ] + + runner = NutterFixtureParallelRunner() + for i in tests: + runner.add_test_fixture(i) + + # Act + results = runner.execute() + + # Assert + assert len(results.test_results.results) == len(tests) + assert results.test_results.results[0].passed == True + assert results.test_results.results[1].passed == False + assert results.test_results.results[2].passed == True + +def test__execute_tests__3_tests_in_sequence_with_run_exception__results_ok(): + # Arrange + tests = [ + RunnerTestFixture(), + RunnerTestFixtureRunException(), + RunnerTestFixture() + ] + + runner = NutterFixtureParallelRunner() + for i in tests: + runner.add_test_fixture(i) + + # Act + results = runner.execute() + + # Assert + assert len(results.test_results.results) == len(tests) + assert results.test_results.results[0].passed == True + assert results.test_results.results[1].passed == False + assert results.test_results.results[2].passed == True + +def test__execute_tests__3_tests_in_sequence_with_exec_exception__results_ok(): + # Arrange + tests = [ + RunnerTestFixture(), + RunnerTestFixtureExecuteException(), + RunnerTestFixture() + ] + + runner = NutterFixtureParallelRunner() + for i in tests: + runner.add_test_fixture(i) + + # Act + results = runner.execute() + + # Assert + assert len(results.test_results.results) == len(tests) - 1 + assert results.test_results.results[0].passed == True + assert results.test_results.results[1].passed == True + +class RunnerTestFixture(NutterFixture): + def before_test(self): + pass + + def run_test(self): + pass + + def assertion_test(self): + assert 1 == 1 + + def after_test(self): + pass + +class RunnerTestFixtureFailAssert(NutterFixture): + def before_test(self): + pass + + def run_test(self): + pass + + def assertion_test(self): + assert 1 != 1 + + def after_test(self): + pass + +class RunnerTestFixtureRunException(NutterFixture): + def before_test(self): + pass + + def run_test(self): + raise(Exception()) + + def assertion_test(self): + assert 1 == 1 + + def after_test(self): + pass + +class RunnerTestFixtureExecuteException(NutterFixture): + def execute_tests(self): + raise(Exception())