From 9e96c0c4bc7978ae16daced03d39f590f2d6c260 Mon Sep 17 00:00:00 2001 From: zhenyu-ms <111329301+zhenyu-ms@users.noreply.github.com> Date: Tue, 10 Oct 2023 11:41:41 +0800 Subject: [PATCH] adjust part slicing; add newsfrag; update tests --- .../2626_changed.pattern_with_parts.rst | 5 ++ testplan/testing/multitest/base.py | 5 +- .../testing/multitest/test_multitest_parts.py | 82 +++++++++++++++---- .../testplan/testing/test_filtering.py | 15 +++- 4 files changed, 88 insertions(+), 19 deletions(-) create mode 100644 doc/newsfragments/2626_changed.pattern_with_parts.rst diff --git a/doc/newsfragments/2626_changed.pattern_with_parts.rst b/doc/newsfragments/2626_changed.pattern_with_parts.rst new file mode 100644 index 000000000..f05ba25c0 --- /dev/null +++ b/doc/newsfragments/2626_changed.pattern_with_parts.rst @@ -0,0 +1,5 @@ +Filtering Patterns now are easier to use with MultiTest with parts. + + * Both patterns with part specified or not could be used to filter MultiTest with parts. + * Applying patterns won't cause certain testcase appear in some different part now. + * Part feature has been tuned to generate more even parts in term of number of testcases. \ No newline at end of file diff --git a/testplan/testing/multitest/base.py b/testplan/testing/multitest/base.py index bae9e34cb..45630fdcb 100644 --- a/testplan/testing/multitest/base.py +++ b/testplan/testing/multitest/base.py @@ -376,15 +376,18 @@ def get_test_context(self): ctx = [] sorted_suites = self.cfg.test_sorter.sorted_testsuites(self.cfg.suites) + g_offset = 0 for suite in sorted_suites: testcases = suite.get_testcases() if self.cfg.part and self.cfg.part[1] > 1: + offset = len(testcases) testcases = [ testcase for (idx, testcase) in enumerate(testcases) - if idx % self.cfg.part[1] == self.cfg.part[0] + if (idx + g_offset) % self.cfg.part[1] == self.cfg.part[0] ] + g_offset += offset sorted_testcases = ( testcases diff --git a/tests/functional/testplan/testing/multitest/test_multitest_parts.py b/tests/functional/testplan/testing/multitest/test_multitest_parts.py index 530fe347b..8825de6fd 100644 --- a/tests/functional/testplan/testing/multitest/test_multitest_parts.py +++ b/tests/functional/testplan/testing/multitest/test_multitest_parts.py @@ -1,11 +1,11 @@ import itertools -from testplan.testing.multitest import MultiTest, testsuite, testcase - from testplan import TestplanMock +from testplan.common.utils.testing import check_report_context +from testplan.report import Status from testplan.runners.pools.base import Pool as ThreadPool from testplan.runners.pools.tasks import Task -from testplan.report import Status +from testplan.testing.multitest import MultiTest, testcase, testsuite @testsuite @@ -76,19 +76,69 @@ def test_multi_parts_not_merged(): assert plan.run().run is True - assert len(plan.report.entries) == 3 - assert plan.report.entries[0].name == "MTest - part(0/3)" - assert plan.report.entries[1].name == "MTest - part(1/3)" - assert plan.report.entries[2].name == "MTest - part(2/3)" - assert len(plan.report.entries[0].entries) == 2 # 2 suites - assert plan.report.entries[0].entries[0].name == "Suite1" - assert plan.report.entries[0].entries[1].name == "Suite2" - assert len(plan.report.entries[0].entries[0].entries) == 1 # param group - assert plan.report.entries[0].entries[0].entries[0].name == "test_true" - assert len(plan.report.entries[0].entries[1].entries) == 1 # param group - assert plan.report.entries[0].entries[1].entries[0].name == "test_false" - assert len(plan.report.entries[0].entries[0].entries[0].entries) == 4 - assert len(plan.report.entries[0].entries[1].entries[0].entries) == 1 + check_report_context( + plan.report, + [ + ( + "MTest - part(0/3)", + [ + ( + "Suite1", + [ + ( + "test_true", + [ + "test_true ", + "test_true ", + "test_true ", + "test_true ", + ], + ) + ], + ), + ("Suite2", [("test_false", ["test_false "])]), + ], + ), + ( + "MTest - part(1/3)", + [ + ( + "Suite1", + [ + ( + "test_true", + [ + "test_true ", + "test_true ", + "test_true ", + ], + ) + ], + ), + ("Suite2", [("test_false", ["test_false "])]), + ], + ), + ( + "MTest - part(2/3)", + [ + ( + "Suite1", + [ + ( + "test_true", + [ + "test_true ", + "test_true ", + "test_true ", + ], + ) + ], + ), + ("Suite2", [("test_false", ["test_false "])]), + ], + ), + ], + ) def test_multi_parts_merged(): diff --git a/tests/functional/testplan/testing/test_filtering.py b/tests/functional/testplan/testing/test_filtering.py index ab75f7804..9551d54fc 100644 --- a/tests/functional/testplan/testing/test_filtering.py +++ b/tests/functional/testplan/testing/test_filtering.py @@ -197,6 +197,7 @@ def test_programmatic_filtering(filter_obj, report_ctx): @pytest.mark.parametrize( "filter_obj, report_ctx", ( + # Case 1, part not specified ( filtering.Pattern("XXX:Alpha:test_one") | filtering.Pattern("XXX:Alpha:test_two"), @@ -205,6 +206,7 @@ def test_programmatic_filtering(filter_obj, report_ctx): ("XXX - part(1/3)", [("Alpha", ["test_two"])]), ], ), + # Case 2, part specified ( filtering.Pattern("XXX - part(0/3):Alpha:test_one") | filtering.Pattern("XXX - part(0/3):Beta:test_three"), @@ -212,15 +214,24 @@ def test_programmatic_filtering(filter_obj, report_ctx): ("XXX - part(0/3)", [("Alpha", ["test_one"])]), ], ), + # Case 3, unix filename pattern in part ( - filtering.Pattern("XXX - part([01]/3):Alpha") + filtering.Pattern("XXX - part([012]/*):Alpha") | filtering.Pattern("XXX:Beta:test_three"), [ ("XXX - part(0/3)", [("Alpha", ["test_one"])]), ("XXX - part(1/3)", [("Alpha", ["test_two"])]), - ("XXX - part(2/3)", [("Beta", ["test_three"])]), + ( + "XXX - part(2/3)", + [("Alpha", ["test_three"]), ("Beta", ["test_three"])], + ), ], ), + # Case 4, ill-formed part + ( + filtering.Pattern("XXX - part*"), + [], + ), ), ) def test_programmatic_filtering_with_parts(filter_obj, report_ctx):