a revision on parts

- have consistent parts no matter filters presented
- apply certain constraints on test names
- fix a testing utility bug

examples/Multitest/Tagging and Filtering/Composite Filters/test_plan_command_line.py

@@ -4,10 +4,9 @@
 """
 import sys
 
-from testplan.testing.multitest import MultiTest, testsuite, testcase
-
 from testplan import test_plan
 from testplan.report.testing.styles import Style
+from testplan.testing.multitest import MultiTest, testcase, testsuite
 
 
 @testsuite
@@ -96,8 +95,8 @@ def test_2(self, env, result):
 
 # Run all tests: tagged with `server`
 # AND (belong to `Gamma` multitest OR has the name `test_3`)
-# command line: `--tags server --pattern Gamma *:*:test_3`
-# command line (alt.): `--tags server --pattern Gamma --pattern *:*:test_3`
+# command line: `--tags server --patterns Gamma *:*:test_3`
+# command line (alt.): `--tags server --patterns Gamma --patterns *:*:test_3`
 
 
 @test_plan(

testplan/common/utils/testing.py

@@ -1,22 +1,22 @@
 """This module contains utilites for testing Testplan itself."""
 
-import sys
+import collections
 import functools
+import io
 import logging
-import pprint
 import os
-import io
+import pprint
+import sys
 import warnings
+from contextlib import contextmanager
+from typing import Collection
 
 from lxml import objectify
 
-from contextlib import contextmanager
-from typing import Collection
 from testplan.runners.pools.tasks.base import Task
 
 from ..report.base import Report, ReportGroup
 from ..utils.comparison import is_regex
-import collections
 
 null_handler = logging.NullHandler()
 
@@ -287,6 +287,7 @@ def check_report_context(report, ctx):
     interested in report contents, just the existence of reports
     with matching names, with the correct order.
     """
+    assert len(report) == len(ctx)
     for mt_report, (multitest_name, suite_ctx) in zip(report, ctx):
         assert mt_report.name == multitest_name
         assert len(mt_report) == len(suite_ctx)

testplan/exporters/testing/coverage/__init__.py

@@ -17,10 +17,10 @@
 from testplan.exporters.testing.base import Exporter
 from testplan.report.testing.base import (
     ReportCategories,
-    TestCaseReport,
     TestGroupReport,
     TestReport,
 )
+from testplan.testing.common import TEST_PART_FORMAT_STRING
 
 
 class CoveredTestsExporter(Exporter):
@@ -83,9 +83,12 @@ def _append_mt_coverage(
         Here we use an OrderedDict as an ordered set.
         """
 
-        mt_pat = report.instance_name
         if report.part is not None:
-            mt_pat += f" - part({report.part[0]}/{report.part[1]})"
+            mt_pat = TEST_PART_FORMAT_STRING.format(
+                report.instance_name, report.part[0], report.part[1]
+            )
+        else:
+            mt_pat = report.instance_name
 
         if report.covered_lines:
             result[(mt_pat,)] = None

testplan/parser.py

@@ -198,7 +198,7 @@ def generate_parser(self) -> HelpParser:
 
 --patterns <Multitest Name>
 --patterns <Multitest Name 1> <Multitest Name 2>
---patterns <Multitest Name 1> --pattern <Multitest Name 2>
+--patterns <Multitest Name 1> --patterns <Multitest Name 2>
 --patterns <Multitest Name>:<Suite Name>
 --patterns <Multitest Name>:<Suite Name>:<Testcase name>
 --patterns <Multitest Name>:*:<Testcase name>

testplan/runnable/base.py

@@ -57,6 +57,7 @@
 from testplan.runners.pools.tasks.base import is_task_target
 from testplan.testing import filtering, listing, ordering, tagging
 from testplan.testing.base import Test, TestResult
+from testplan.testing.common import TEST_PART_FORMAT_STRING
 from testplan.testing.multitest import MultiTest
 
 
@@ -1126,7 +1127,7 @@ def _merge_reports(
                 placeholder_report._index = {}
                 placeholder_report.status_override = Status.ERROR
                 for _, report in result:
-                    report.name = "{} - part({}/{})".format(
+                    report.name = TEST_PART_FORMAT_STRING.format(
                         report.name, report.part[0], report.part[1]
                     )
                     report.uid = strings.uuid4()  # considered as error report

testplan/testing/base.py

@@ -42,6 +42,16 @@
 ASSERTION_INDENT = 8
 
 
+def test_name_sanity_check(name):
+    for s in [" - part", ":"]:
+        if s in name:
+            raise ValueError(
+                f'"{s}" is specially treated by Testplan, '
+                "it cannot be used in Test names."
+            )
+    return True
+
+
 class TestConfig(RunnableConfig):
     """Configuration object for :py:class:`~testplan.testing.base.Test`."""
 
@@ -53,7 +63,9 @@ def get_options(cls):
 
         return {
             "name": And(
-                str, lambda s: len(s) <= defaults.MAX_TEST_NAME_LENGTH
+                str,
+                lambda s: len(s) <= defaults.MAX_TEST_NAME_LENGTH,
+                test_name_sanity_check,
             ),
             ConfigOption("description", default=None): Or(str, None),
             ConfigOption("environment", default=[]): [
@@ -411,22 +423,7 @@ def dry_run(self):
         Return an empty report skeleton for this test including all
         testsuites, testcases etc. hierarchy. Does not run any tests.
         """
-        suites_to_run = self.test_context
-        self.result.report = self._new_test_report()
-
-        for testsuite, testcases in suites_to_run:
-            testsuite_report = TestGroupReport(
-                name=testsuite,
-                category=ReportCategories.TESTSUITE,
-            )
-
-            for testcase in testcases:
-                testcase_report = TestCaseReport(name=testcase)
-                testsuite_report.append(testcase_report)
-
-            self.result.report.append(testsuite_report)
-
-        return self.result
+        raise NotImplementedError
 
     def set_discover_path(self, path: str) -> None:
         """

testplan/testing/common.py

@@ -0,0 +1,8 @@
+import re
+
+TEST_PART_FORMAT_STRING = "{} - part({}/{})"
+
+# NOTE: no rigorous check performed before passed to fnmatch
+TEST_PART_REGEX = re.compile(
+    r"^(.*) - part\(([\!0-9\[\]\?\*]+)/([\!0-9\[\]\?\*]+)\)$"
+)

testplan/testing/filtering.py

@@ -4,10 +4,15 @@
 import collections
 import fnmatch
 import operator
-from enum import Enum
-from typing import Callable, List, Type
+import re
+from enum import Enum, IntEnum, auto
+from typing import TYPE_CHECKING, Callable, List, Type
 
 from testplan.testing import tagging
+from testplan.testing.common import TEST_PART_REGEX
+
+if TYPE_CHECKING:
+    from testplan.testing.base import Test
 
 
 class FilterLevel(Enum):
@@ -23,6 +28,12 @@ class FilterLevel(Enum):
     TESTCASE = "testcase"
 
 
+class FilterCategory(IntEnum):
+    COMMON = auto()
+    PATTERN = auto()
+    TAG = auto()
+
+
 class BaseFilter:
     """
     Base class for filters, supports bitwise
@@ -54,7 +65,7 @@ class Filter(BaseFilter):
     to apply the filtering logic.
     """
 
-    category = "common"
+    category = FilterCategory.COMMON
 
     def filter_test(self, test) -> bool:
         return True
@@ -66,18 +77,17 @@ def filter_case(self, case) -> bool:
         return True
 
     def filter(self, test, suite, case):
-
-        filter_levels = test.get_filter_levels()
-        results = []
-
-        if FilterLevel.TEST in filter_levels:
-            results.append(self.filter_test(test))
-        if FilterLevel.TESTSUITE in filter_levels:
-            results.append(self.filter_suite(suite))
-        if FilterLevel.TESTCASE in filter_levels:
-            results.append(self.filter_case(case))
-
-        return all(results)
+        res = True
+        for level in test.get_filter_levels():
+            if level is FilterLevel.TEST:
+                res = res and self.filter_test(test)
+            elif level is FilterLevel.TESTSUITE:
+                res = res and self.filter_suite(suite)
+            elif level is FilterLevel.TESTCASE:
+                res = res and self.filter_case(case)
+            if not res:
+                return False
+        return True
 
 
 def flatten_filters(
@@ -136,8 +146,6 @@ def composed_filter(self, _test, _suite, _case) -> bool:
         raise NotImplementedError
 
     def filter(self, test, suite, case):
-        # we might intentionally use another overriden method name
-        # to distinguish MetaFilter from Filter, or shall we?
         return self.composed_filter(test, suite, case)
 
 
@@ -186,7 +194,7 @@ def filter(self, test, suite, case):
 class BaseTagFilter(Filter):
     """Base filter class for tag based filtering."""
 
-    category = "tag"
+    category = FilterCategory.TAG
 
     def __init__(self, tags):
         self.tags_orig = tags
@@ -256,13 +264,13 @@ class Pattern(Filter):
     DELIMITER = ":"
     ALL_MATCH = "*"
 
-    category = "pattern"
+    category = FilterCategory.PATTERN
 
     def __init__(self, pattern, match_definition=False):
         self.pattern = pattern
         self.match_definition = match_definition
-        patterns = self.parse_pattern(pattern)
-        self.test_pattern, self.suite_pattern, self.case_pattern = patterns
+        self.parse_pattern(pattern)
+        # self.test_pattern, self.suite_pattern, self.case_pattern = patterns
 
     def __eq__(self, other):
         return (
@@ -285,10 +293,61 @@ def parse_pattern(self, pattern: str) -> List[str]:
                 )
             )
 
-        return patterns + ([self.ALL_MATCH] * (self.MAX_LEVEL - len(patterns)))
+        test_level, suite_level, case_level = patterns + (
+            [self.ALL_MATCH] * (self.MAX_LEVEL - len(patterns))
+        )
 
-    def filter_test(self, test):
-        return fnmatch.fnmatch(test.name, self.test_pattern)
+        m = re.match(TEST_PART_REGEX, test_level)
+        if m:
+            test_name_p = m.group(1)
+            test_cur_part_p = m.group(2)
+            test_ttl_part_p = m.group(3)
+
+            try:
+                test_cur_part_p, test_ttl_part_p = int(test_cur_part_p), int(
+                    test_ttl_part_p
+                )
+            except ValueError:
+                pass
+            else:
+                if test_ttl_part_p <= test_cur_part_p:
+                    raise ValueError(
+                        f"Meaningless part specified for {test_name_p}, "
+                        f"we cannot cut a pizza by {test_ttl_part_p} and then take "
+                        f"the {test_cur_part_p}-th slice, and we count from 0."
+                    )
+                test_cur_part_p, test_ttl_part_p = str(test_cur_part_p), str(
+                    test_ttl_part_p
+                )
+            self.test_pattern = (
+                test_name_p,
+                (test_cur_part_p, test_ttl_part_p),
+            )
+        else:
+            self.test_pattern = test_level
+
+        self.suite_pattern = suite_level
+        self.case_pattern = case_level
+
+    def filter_test(self, test: "Test"):
+        if isinstance(self.test_pattern, tuple):
+            if not hasattr(test.cfg, "part"):
+                raise ValueError(
+                    f"Invalid pattern, Part feature not implemented for {type(test).__qualname__}."
+                )
+
+            name_p, (cur_part_p, ttl_part_p) = self.test_pattern
+
+            cur_part: int
+            ttl_part: int
+            cur_part, ttl_part = test.cfg.part or (0, 1)
+            return (
+                fnmatch.fnmatch(test.name, name_p)
+                and fnmatch.fnmatch(str(cur_part), cur_part_p)
+                and fnmatch.fnmatch(str(ttl_part), ttl_part_p)
+            )
+        else:
+            return fnmatch.fnmatch(test.name, self.test_pattern)
 
     def filter_suite(self, suite):
         # For test suite uid is the same as name, just like that of Multitest
@@ -332,7 +391,7 @@ class PatternAction(argparse.Action):
 
     .. code-block:: bash
 
-        --pattern foo bar --pattern baz
+        --patterns foo bar --patterns baz
 
     Out:
 
@@ -424,7 +483,7 @@ def parse_filter_args(parsed_args, arg_names):
 
     .. code-block:: bash
 
-        --pattern my_pattern --tags foo --tags-all bar baz
+        --patterns my_pattern --tags foo --tags-all bar baz
 
     Out: