diff --git a/doc/newsfragments/2626_changed.pattern_with_parts.rst b/doc/newsfragments/2626_changed.pattern_with_parts.rst new file mode 100644 index 000000000..8826fea4a --- /dev/null +++ b/doc/newsfragments/2626_changed.pattern_with_parts.rst @@ -0,0 +1,4 @@ +Filtering Patterns now are easier to use with MultiTest with parts. + + * Support filtering MultiTest with both part-specified patterns and part-unspecified patterns. + * Applying filtering patterns will not change testcase shuffling - the same testcase will always end up in the same part of a MultiTest. \ No newline at end of file diff --git a/examples/Multitest/Tagging and Filtering/Composite Filters/test_plan_command_line.py b/examples/Multitest/Tagging and Filtering/Composite Filters/test_plan_command_line.py index 50a46ac9a..b9945bd78 100755 --- a/examples/Multitest/Tagging and Filtering/Composite Filters/test_plan_command_line.py +++ b/examples/Multitest/Tagging and Filtering/Composite Filters/test_plan_command_line.py @@ -4,10 +4,9 @@ """ import sys -from testplan.testing.multitest import MultiTest, testsuite, testcase - from testplan import test_plan from testplan.report.testing.styles import Style +from testplan.testing.multitest import MultiTest, testcase, testsuite @testsuite @@ -96,8 +95,8 @@ def test_2(self, env, result): # Run all tests: tagged with `server` # AND (belong to `Gamma` multitest OR has the name `test_3`) -# command line: `--tags server --pattern Gamma *:*:test_3` -# command line (alt.): `--tags server --pattern Gamma --pattern *:*:test_3` +# command line: `--tags server --patterns Gamma *:*:test_3` +# command line (alt.): `--tags server --patterns Gamma --patterns *:*:test_3` @test_plan( diff --git a/testplan/common/report/base.py b/testplan/common/report/base.py index 5dc5caad5..c1fb5db0e 100644 --- a/testplan/common/report/base.py +++ b/testplan/common/report/base.py @@ -7,16 +7,16 @@ Later on these reports would be merged together to build the final report as the testplan result. """ -import copy -import time -import uuid import collections +import copy import dataclasses import itertools - +import time +import uuid from typing import Dict, List, Optional, Union from testplan.common.utils import strings + from .log import create_logging_adapter @@ -57,11 +57,17 @@ class Report: exception_logger = ExceptionLogger def __init__( - self, name, description=None, uid=None, entries=None, parent_uids=None + self, + name: str, + description: Optional[str] = None, + definition_name: Optional[str] = None, + uid: Optional[str] = None, + entries: Optional[list] = None, + parent_uids: Optional[List[str]] = None, ): self.name = name self.description = description - + self.definition_name = definition_name or name self.uid = uid or name self.entries = entries or [] @@ -77,12 +83,12 @@ def __init__( self.parent_uids = parent_uids or [] def __str__(self): - return '{kls}(name="{name}", id="{uid}")'.format( + return '{kls}(name="{name}", uid="{uid}")'.format( kls=self.__class__.__name__, name=self.name, uid=self.uid ) def __repr__(self): - return '{kls}(name="{name}", id="{uid}", entries={entries})'.format( + return '{kls}(name="{name}", uid="{uid}", entries={entries})'.format( kls=self.__class__.__name__, name=self.name, uid=self.uid, @@ -131,15 +137,15 @@ def logged_exceptions(self, *exception_classes, **kwargs): def _check_report(self, report): """ - Utility method for checking `report` `type` and `uid`. + Utility method for checking `report` `type` and `definition_name`. """ msg = "Report check failed for `{}` and `{}`. ".format(self, report) - if report.uid != self.uid: + if report.definition_name != self.definition_name: raise AttributeError( msg - + "`uid` attributes (`{}`, `{}`) do not match.".format( - self.uid, report.uid + + "`definition_name` attributes (`{}`, `{}`) do not match.".format( + self.definition_name, report.definition_name ) ) diff --git a/testplan/common/report/schemas.py b/testplan/common/report/schemas.py index a814bc276..79c12a74c 100644 --- a/testplan/common/report/schemas.py +++ b/testplan/common/report/schemas.py @@ -4,10 +4,10 @@ from marshmallow import Schema, fields, post_load from marshmallow.utils import EXCLUDE -from testplan.common.serialization import schemas, fields as custom_fields - -from .base import Report, ReportGroup, EventRecorder +from testplan.common.serialization import fields as custom_fields +from testplan.common.serialization import schemas +from .base import EventRecorder, Report, ReportGroup __all__ = ["ReportLogSchema", "ReportSchema", "ReportGroupSchema"] @@ -36,6 +36,9 @@ class Meta: name = fields.String() description = fields.String(allow_none=True) + definition_name = fields.String( + allow_none=True + ) # otherwise new tpr cannot process old report entries = fields.List(custom_fields.NativeOrPretty()) parent_uids = fields.List(fields.String()) diff --git a/testplan/common/utils/interface.py b/testplan/common/utils/interface.py index 68e9aa293..ddbdeb6e5 100644 --- a/testplan/common/utils/interface.py +++ b/testplan/common/utils/interface.py @@ -1,7 +1,7 @@ """Validates methods signature.""" from inspect import signature -from typing import Union, Optional +from typing import Union class NoSuchMethodInClass(Exception): diff --git a/testplan/common/utils/testing.py b/testplan/common/utils/testing.py index c25f105ad..eae27cc2e 100644 --- a/testplan/common/utils/testing.py +++ b/testplan/common/utils/testing.py @@ -1,22 +1,22 @@ """This module contains utilites for testing Testplan itself.""" -import sys +import collections import functools +import io import logging -import pprint import os -import io +import pprint +import sys import warnings +from contextlib import contextmanager +from typing import Collection from lxml import objectify -from contextlib import contextmanager -from typing import Collection from testplan.runners.pools.tasks.base import Task from ..report.base import Report, ReportGroup from ..utils.comparison import is_regex -import collections null_handler = logging.NullHandler() @@ -287,6 +287,7 @@ def check_report_context(report, ctx): interested in report contents, just the existence of reports with matching names, with the correct order. """ + assert len(report) == len(ctx) for mt_report, (multitest_name, suite_ctx) in zip(report, ctx): assert mt_report.name == multitest_name assert len(mt_report) == len(suite_ctx) diff --git a/testplan/exporters/testing/coverage/__init__.py b/testplan/exporters/testing/coverage/__init__.py index 0de7b9600..ba0642b8e 100644 --- a/testplan/exporters/testing/coverage/__init__.py +++ b/testplan/exporters/testing/coverage/__init__.py @@ -7,19 +7,20 @@ import sys from collections import OrderedDict from contextlib import contextmanager -from typing import Dict, Generator, Mapping, TextIO, Tuple, Optional +from typing import Dict, Generator, Mapping, Optional, TextIO, Tuple from testplan.common.exporters import ( - ExporterConfig, ExportContext, + ExporterConfig, verify_export_context, ) from testplan.exporters.testing.base import Exporter from testplan.report.testing.base import ( - TestCaseReport, + ReportCategories, TestGroupReport, TestReport, ) +from testplan.testing.common import TEST_PART_PATTERN_FORMAT_STRING class CoveredTestsExporter(Exporter): @@ -52,8 +53,11 @@ def export( # here we use an OrderedDict as an ordered set results = OrderedDict() for entry in source.entries: - if isinstance(entry, TestGroupReport): - self._append_covered_group_n_case(entry, [], results) + if ( + isinstance(entry, TestGroupReport) + and entry.category == ReportCategories.MULTITEST + ): + self._append_mt_coverage(entry, results) if results: with _custom_open(self.cfg.tracing_tests_output) as ( f, @@ -68,27 +72,44 @@ def export( return None return None - def _append_covered_group_n_case( + def _append_mt_coverage( self, report: TestGroupReport, - path: Tuple[str, ...], result: Mapping[Tuple[str, ...], None], ): """ - Recursively add test group or test case with covered_lines set to - the result ordered set. + Add test entity with covered_lines set to the result ordered set. Here we use an OrderedDict as an ordered set. """ - curr_path = (*path, report.name) + + if report.part is not None: + mt_pat = TEST_PART_PATTERN_FORMAT_STRING.format( + report.definition_name, report.part[0], report.part[1] + ) + else: + mt_pat = report.definition_name + if report.covered_lines: - result[curr_path] = None - for entry in report.entries: - if isinstance(entry, TestGroupReport): - self._append_covered_group_n_case(entry, curr_path, result) - elif isinstance(entry, TestCaseReport): - if entry.covered_lines: - result[(*curr_path, entry.name)] = None + result[(mt_pat,)] = None + for st in report.entries: + if st.covered_lines: + result[(mt_pat, st.definition_name)] = None + for tc in st.entries: + if tc.category == ReportCategories.PARAMETRIZATION: + for sub_tc in tc.entries: + if sub_tc.covered_lines: + result[ + ( + mt_pat, + st.definition_name, + sub_tc.definition_name, + ) + ] = None + elif tc.covered_lines: + result[ + (mt_pat, st.definition_name, tc.definition_name) + ] = None @contextmanager diff --git a/testplan/exporters/testing/pdf/renderers/reports.py b/testplan/exporters/testing/pdf/renderers/reports.py index f14974420..0848db625 100644 --- a/testplan/exporters/testing/pdf/renderers/reports.py +++ b/testplan/exporters/testing/pdf/renderers/reports.py @@ -3,25 +3,26 @@ """ import logging from collections import OrderedDict -from typing import Optional, Tuple +from typing import Optional, Tuple, Union from reportlab.lib import colors, styles from reportlab.platypus import Paragraph from testplan.common.exporters.pdf import RowStyle from testplan.common.utils.registry import Registry -from testplan.common.utils.strings import format_description, wrap, split_text +from testplan.common.utils.strings import format_description, split_text, wrap from testplan.report import ( + ReportCategories, Status, - TestReport, - TestGroupReport, TestCaseReport, - ReportCategories, + TestGroupReport, + TestReport, ) from testplan.report.testing.styles import StyleFlag from testplan.testing import tagging + from . import constants as const -from .base import format_duration, RowData, BaseRowRenderer, MetadataMixin +from .base import BaseRowRenderer, MetadataMixin, RowData, format_duration class ReportRendererRegistry(Registry): @@ -260,6 +261,9 @@ def get_header( [ - ][][][] + This method is also used by its subclass, where source will be of type + ``TestCaseReport``. + :param source: Source object for the renderer. :param depth: Depth of the source object on report tree. Used for indentation. :param row_idx: Index of the current table row to be rendered. @@ -417,7 +421,7 @@ def get_header_linestyle(self) -> Tuple[int, colors.HexColor]: def get_header( self, - source: TestCaseReport, + source: TestGroupReport, depth: int, row_idx: int, ) -> RowData: diff --git a/testplan/parser.py b/testplan/parser.py index 5ef35febc..a671d0022 100644 --- a/testplan/parser.py +++ b/testplan/parser.py @@ -6,9 +6,11 @@ import copy import json import sys +import warnings from typing import Dict, List import schema + from testplan import defaults from testplan.common.utils import logger from testplan.report.testing import ( @@ -219,7 +221,7 @@ def generate_parser(self) -> HelpParser: --patterns --patterns ---patterns --pattern +--patterns --patterns --patterns : --patterns :: --patterns :*: @@ -511,6 +513,15 @@ def process_args(self, namespace: argparse.Namespace) -> Dict: if args["list"] and not args["test_lister"]: args["test_lister"] = listing.NameLister() + if ( + args["interactive_port"] is not None + and args["tracing_tests"] is not None + ): + warnings.warn( + "Tracing tests feature not available in interactive mode." + ) + args["tracing_tests"] = None + return args diff --git a/testplan/runnable/base.py b/testplan/runnable/base.py index c3a0529bc..383b4eec7 100644 --- a/testplan/runnable/base.py +++ b/testplan/runnable/base.py @@ -13,6 +13,7 @@ from typing import ( Any, Callable, + Collection, Dict, List, MutableMapping, @@ -20,7 +21,6 @@ Pattern, Tuple, Union, - Collection, ) import pytz @@ -34,11 +34,7 @@ RunnableResult, RunnableStatus, ) -from testplan.common.exporters import ( - BaseExporter, - ExportContext, - run_exporter, -) +from testplan.common.exporters import BaseExporter, ExportContext, run_exporter from testplan.common.remote.remote_service import RemoteService from testplan.common.report import MergeError from testplan.common.utils import logger, strings @@ -57,18 +53,18 @@ from testplan.report.testing.styles import Style from testplan.runnable.interactive import TestRunnerIHandler from testplan.runners.base import Executor +from testplan.runners.pools.base import Pool from testplan.runners.pools.tasks import Task, TaskResult from testplan.runners.pools.tasks.base import ( - is_task_target, TaskTargetInformation, get_task_target_information, + is_task_target, ) from testplan.testing import filtering, listing, ordering, tagging from testplan.testing.base import Test, TestResult +from testplan.testing.common import TEST_PART_PATTERN_FORMAT_STRING from testplan.testing.listing import Lister from testplan.testing.multitest import MultiTest -from testplan.runners.pools.base import Pool - TestTask = Union[Test, Task, Callable] @@ -1105,12 +1101,11 @@ def _create_result(self): report.category != ReportCategories.TASK_RERUN and self.cfg.merge_scheduled_parts ): - report.uid = report.name # Save the report temporarily and later will merge it - test_rep_lookup.setdefault(report.uid, []).append( - (test_results[uid].run, report) - ) - if report.uid not in test_report.entry_uids: + test_rep_lookup.setdefault( + report.definition_name, [] + ).append((test_results[uid].run, report)) + if report.definition_name not in test_report.entry_uids: # Create a placeholder for merging sibling reports if isinstance(resource_result, TaskResult): # `runnable` must be an instance of MultiTest since @@ -1125,16 +1120,11 @@ def _create_result(self): else: report = report.__class__( - report.name, - uid=report.uid, + report.definition_name, category=report.category, ) else: continue # Wait all sibling reports collected - else: - # If do not want to merge sibling reports, then display - # them with different names. (e.g. `MTest - part(0/3)`) - report.name = report.uid test_report.append(report) step_result = step_result and run is True # boolean or exception @@ -1223,7 +1213,7 @@ def _merge_reports( placeholder_report._index = {} placeholder_report.status_override = Status.ERROR for _, report in result: - report.name = "{} - part({}/{})".format( + report.name = TEST_PART_PATTERN_FORMAT_STRING.format( report.name, report.part[0], report.part[1] ) report.uid = strings.uuid4() # considered as error report diff --git a/testplan/runnable/interactive/base.py b/testplan/runnable/interactive/base.py index a3eda394d..97e879b52 100644 --- a/testplan/runnable/interactive/base.py +++ b/testplan/runnable/interactive/base.py @@ -7,18 +7,19 @@ import threading import warnings from concurrent import futures -from typing import Union, Awaitable, Dict, Optional +from typing import Awaitable, Dict, Optional, Union from testplan.common import config, entity from testplan.common.report import Report from testplan.common.utils.networking import get_hostname_access_url -from testplan.runnable.interactive import http, reloader, resource_loader from testplan.report import ( - TestReport, - TestGroupReport, - Status, + ReportCategories, RuntimeStatus, + Status, + TestGroupReport, + TestReport, ) +from testplan.runnable.interactive import http, reloader, resource_loader def _exclude_assertions_filter(obj: object) -> bool: @@ -567,7 +568,7 @@ def case_filter(obj): if obj.uid == case_uid: return True return obj.uid == suite_uid or ( - obj.category == "parametrization" + obj.category == ReportCategories.PARAMETRIZATION and any(entry.uid == case_uid for entry in obj.entries) ) except Exception: diff --git a/testplan/runners/pools/tasks/base.py b/testplan/runners/pools/tasks/base.py index 8b5a03baf..451a8ab2e 100644 --- a/testplan/runners/pools/tasks/base.py +++ b/testplan/runners/pools/tasks/base.py @@ -23,11 +23,11 @@ from testplan.common.entity import Runnable -from testplan.testing.base import Test, TestResult from testplan.common.serialization import SelectiveSerializable from testplan.common.utils import strings from testplan.common.utils.package import import_tmp_module from testplan.common.utils.path import is_subdir, pwd, rebase_path +from testplan.testing.base import Test, TestResult from testplan.testing.multitest import MultiTest @@ -79,24 +79,21 @@ def __init__( self._kwargs = kwargs or dict() self._uid = uid or strings.uuid4() self._aborted = False - self._max_rerun_limit = ( - self.MAX_RERUN_LIMIT - if rerun > self.MAX_RERUN_LIMIT - else int(rerun) - ) self._assign_for_rerun = 0 self._executors = OrderedDict() self.priority = -weight - if self._max_rerun_limit < 0: + if rerun < 0: raise ValueError("Value of `rerun` cannot be negative.") - elif self._max_rerun_limit > self.MAX_RERUN_LIMIT: + elif rerun > self.MAX_RERUN_LIMIT: warnings.warn( "Value of `rerun` cannot exceed {}".format( self.MAX_RERUN_LIMIT ) ) self._max_rerun_limit = self.MAX_RERUN_LIMIT + else: + self._max_rerun_limit = rerun self._part = part @@ -167,9 +164,9 @@ def reassign_cnt(self) -> int: def reassign_cnt(self, value: int): if value < 0: raise ValueError("Value of `reassign_cnt` cannot be negative") - elif value > self.MAX_RERUN_LIMIT: + elif value > self._max_rerun_limit: raise ValueError( - f"Value of `reassign_cnt` cannot exceed {self.MAX_RERUN_LIMIT}" + f"Value of `reassign_cnt` cannot exceed {self._max_rerun_limit}" ) self._assign_for_rerun = value diff --git a/testplan/testing/base.py b/testplan/testing/base.py index 0d2005b89..03e32eaa2 100644 --- a/testplan/testing/base.py +++ b/testplan/testing/base.py @@ -3,7 +3,7 @@ import subprocess import sys import warnings -from typing import List, Optional, Dict, Generator +from typing import Dict, Generator, List, Optional from schema import And, Or, Use @@ -44,6 +44,16 @@ ASSERTION_INDENT = 8 +def test_name_sanity_check(name): + for s in [" - part", ":"]: + if s in name: + raise ValueError( + f'"{s}" is specially treated by Testplan, ' + "it cannot be used in Test names." + ) + return True + + class TestConfig(RunnableConfig): """Configuration object for :py:class:`~testplan.testing.base.Test`.""" @@ -55,7 +65,9 @@ def get_options(cls): return { "name": And( - str, lambda s: len(s) <= defaults.MAX_TEST_NAME_LENGTH + str, + lambda s: len(s) <= defaults.MAX_TEST_NAME_LENGTH, + test_name_sanity_check, ), ConfigOption("description", default=None): Or(str, None), ConfigOption("environment", default=[]): [ @@ -191,7 +203,7 @@ def get_filter_levels(self): @property def name(self): - """Instance name. Also uid.""" + """Instance name.""" return self.cfg.name @property @@ -255,9 +267,9 @@ def should_log_test_result(self, depth, test_obj, style): if isinstance(test_obj, TestGroupReport): if depth == 0: return style.display_test, TEST_INST_INDENT - elif test_obj.category == "testsuite": + elif test_obj.category == ReportCategories.TESTSUITE: return style.display_testsuite, SUITE_INDENT - elif test_obj.category == "parametrization": + elif test_obj.category == ReportCategories.PARAMETRIZATION: return False, 0 # DO NOT display else: raise ValueError( @@ -416,22 +428,7 @@ def dry_run(self): Return an empty report skeleton for this test including all testsuites, testcases etc. hierarchy. Does not run any tests. """ - suites_to_run = self.test_context - self.result.report = self._new_test_report() - - for testsuite, testcases in suites_to_run: - testsuite_report = TestGroupReport( - name=testsuite, - category=ReportCategories.TESTSUITE, - ) - - for testcase in testcases: - testcase_report = TestCaseReport(name=testcase) - testsuite_report.append(testcase_report) - - self.result.report.append(testsuite_report) - - return self.result + raise NotImplementedError def set_discover_path(self, path: str) -> None: """ diff --git a/testplan/testing/common.py b/testplan/testing/common.py new file mode 100644 index 000000000..e117ff496 --- /dev/null +++ b/testplan/testing/common.py @@ -0,0 +1,8 @@ +import re + +TEST_PART_PATTERN_FORMAT_STRING = "{} - part({}/{})" + +# NOTE: no rigorous check performed before passed to fnmatch +TEST_PART_PATTERN_REGEX = re.compile( + r"^(.*) - part\(([\!0-9\[\]\?\*]+)/([\!0-9\[\]\?\*]+)\)$" +) diff --git a/testplan/testing/filtering.py b/testplan/testing/filtering.py index 8b86e74f5..b1cafcdbb 100644 --- a/testplan/testing/filtering.py +++ b/testplan/testing/filtering.py @@ -4,10 +4,15 @@ import collections import fnmatch import operator -from enum import Enum -from typing import Callable, List, Type +import re +from enum import Enum, IntEnum, auto +from typing import TYPE_CHECKING, Callable, List, Type from testplan.testing import tagging +from testplan.testing.common import TEST_PART_PATTERN_REGEX + +if TYPE_CHECKING: + from testplan.testing.base import Test class FilterLevel(Enum): @@ -23,6 +28,12 @@ class FilterLevel(Enum): TESTCASE = "testcase" +class FilterCategory(IntEnum): + COMMON = auto() + PATTERN = auto() + TAG = auto() + + class BaseFilter: """ Base class for filters, supports bitwise @@ -54,7 +65,7 @@ class Filter(BaseFilter): to apply the filtering logic. """ - category = "common" + category = FilterCategory.COMMON def filter_test(self, test) -> bool: return True @@ -66,18 +77,17 @@ def filter_case(self, case) -> bool: return True def filter(self, test, suite, case): - - filter_levels = test.get_filter_levels() - results = [] - - if FilterLevel.TEST in filter_levels: - results.append(self.filter_test(test)) - if FilterLevel.TESTSUITE in filter_levels: - results.append(self.filter_suite(suite)) - if FilterLevel.TESTCASE in filter_levels: - results.append(self.filter_case(case)) - - return all(results) + res = True + for level in test.get_filter_levels(): + if level is FilterLevel.TEST: + res = res and self.filter_test(test) + elif level is FilterLevel.TESTSUITE: + res = res and self.filter_suite(suite) + elif level is FilterLevel.TESTCASE: + res = res and self.filter_case(case) + if not res: + return False + return True def flatten_filters( @@ -136,8 +146,6 @@ def composed_filter(self, _test, _suite, _case) -> bool: raise NotImplementedError def filter(self, test, suite, case): - # we might intentionally use another overriden method name - # to distinguish MetaFilter from Filter, or shall we? return self.composed_filter(test, suite, case) @@ -186,7 +194,7 @@ def filter(self, test, suite, case): class BaseTagFilter(Filter): """Base filter class for tag based filtering.""" - category = "tag" + category = FilterCategory.TAG def __init__(self, tags): self.tags_orig = tags @@ -253,22 +261,20 @@ class Pattern(Filter): """ MAX_LEVEL = 3 - DELIMITER = ":" ALL_MATCH = "*" - category = "pattern" + category = FilterCategory.PATTERN - def __init__(self, pattern, match_definition=False): + def __init__(self, pattern, match_uid=False): self.pattern = pattern - self.match_definition = match_definition - patterns = self.parse_pattern(pattern) - self.test_pattern, self.suite_pattern, self.case_pattern = patterns + self.match_uid = match_uid + self.parse_pattern(pattern) def __eq__(self, other): return ( isinstance(other, self.__class__) and self.pattern == other.pattern - and self.match_definition == other.match_definition + and self.match_uid == other.match_uid ) def __repr__(self): @@ -276,7 +282,9 @@ def __repr__(self): def parse_pattern(self, pattern: str) -> List[str]: # ":" or "::" can be used as delimiter - patterns = [s for s in pattern.split(self.DELIMITER) if s] + patterns = ( + pattern.split("::") if "::" in pattern else pattern.split(":") + ) if len(patterns) > self.MAX_LEVEL: raise ValueError( @@ -285,18 +293,68 @@ def parse_pattern(self, pattern: str) -> List[str]: ) ) - return patterns + ([self.ALL_MATCH] * (self.MAX_LEVEL - len(patterns))) + test_level, suite_level, case_level = patterns + ( + [self.ALL_MATCH] * (self.MAX_LEVEL - len(patterns)) + ) + + # structural pattern for test level + m = re.match(TEST_PART_PATTERN_REGEX, test_level) + if m: + test_name_p = m.group(1) + test_cur_part_p = m.group(2) + test_ttl_part_p = m.group(3) + + try: + test_cur_part_p_, test_ttl_part_p_ = int(test_cur_part_p), int( + test_ttl_part_p + ) + except ValueError: + pass + else: + if test_ttl_part_p_ <= test_cur_part_p_: + raise ValueError( + f"Meaningless part specified for {test_name_p}, " + f"we cannot cut a pizza by {test_ttl_part_p_} and then take " + f"the {test_cur_part_p_}-th slice, and we count from 0." + ) + self.test_pattern = ( + test_name_p, + (test_cur_part_p, test_ttl_part_p), + ) + else: + self.test_pattern = test_level + + self.suite_pattern = suite_level + self.case_pattern = case_level + + def filter_test(self, test: "Test"): + # uid and structural pattern may differ under certain circumstances + if self.match_uid: + return fnmatch.fnmatch(test.uid(), self.test_pattern) + + if isinstance(self.test_pattern, tuple): + if not hasattr(test.cfg, "part"): + return False + + name_p, (cur_part_p, ttl_part_p) = self.test_pattern + cur_part: int + ttl_part: int + cur_part, ttl_part = test.cfg.part or (0, 1) + return ( + fnmatch.fnmatch(test.name, name_p) + and fnmatch.fnmatch(str(cur_part), cur_part_p) + and fnmatch.fnmatch(str(ttl_part), ttl_part_p) + ) - def filter_test(self, test): return fnmatch.fnmatch(test.name, self.test_pattern) def filter_suite(self, suite): - # For test suite uid is the same as name, just like that of Multitest + # For test suite uid is the same as name return fnmatch.fnmatch(suite.name, self.suite_pattern) def filter_case(self, case): name_match = fnmatch.fnmatch( - case.__name__ if self.match_definition else case.name, + case.__name__ if self.match_uid else case.name, self.case_pattern, ) @@ -332,7 +390,7 @@ class PatternAction(argparse.Action): .. code-block:: bash - --pattern foo bar --pattern baz + --patterns foo bar --patterns baz Out: @@ -424,7 +482,7 @@ def parse_filter_args(parsed_args, arg_names): .. code-block:: bash - --pattern my_pattern --tags foo --tags-all bar baz + --patterns my_pattern --tags foo --tags-all bar baz Out: diff --git a/testplan/testing/listing.py b/testplan/testing/listing.py index 0208c5617..1ddf93b8a 100644 --- a/testplan/testing/listing.py +++ b/testplan/testing/listing.py @@ -7,16 +7,19 @@ from argparse import Action, ArgumentParser, Namespace from enum import Enum from os import PathLike -from typing import List, Union, Sequence, Any, Tuple +from typing import TYPE_CHECKING, List, Tuple, Union from urllib.parse import urlparse -from testplan.common.utils.parser import ArgMixin from testplan.common.utils.logger import TESTPLAN_LOGGER - +from testplan.common.utils.parser import ArgMixin from testplan.testing import tagging +from testplan.testing.common import TEST_PART_PATTERN_FORMAT_STRING from testplan.testing.multitest import MultiTest from testplan.testing.multitest.test_metadata import TestPlanMetadata +if TYPE_CHECKING: + from testplan.testing.base import Test + INDENT = " " MAX_TESTCASES = 25 @@ -71,7 +74,7 @@ class ExpandedNameLister(BaseLister): DESCRIPTION = "List tests in readable format." def format_instance(self, instance): - return instance.name + return instance.uid() def format_suite(self, instance, suite): return suite if isinstance(suite, str) else suite.name @@ -111,6 +114,18 @@ def get_output(self, instance): return result +def test_pattern(test_instance: "Test") -> str: + if hasattr(test_instance.cfg, "part") and isinstance( + test_instance.cfg.part, tuple + ): + return TEST_PART_PATTERN_FORMAT_STRING.format( + test_instance.name, + test_instance.cfg.part[0], + test_instance.cfg.part[1], + ) + return test_instance.name + + class ExpandedPatternLister(ExpandedNameLister): """ Lists the items in test context in a copy-pasta friendly format @@ -132,7 +147,7 @@ class ExpandedPatternLister(ExpandedNameLister): DESCRIPTION = "List tests in `--patterns` / `--tags` compatible format." def format_instance(self, instance): - return instance.name + return test_pattern(instance) def apply_tag_label(self, pattern, obj): if obj.__tags__: @@ -143,17 +158,18 @@ def apply_tag_label(self, pattern, obj): def format_suite(self, instance, suite): if not isinstance(instance, MultiTest): - return "{}::{}".format(instance.name, suite) + return "{}:{}".format(test_pattern(instance), suite) - pattern = "{}::{}".format(instance.name, suite.name) + pattern = "{}:{}".format(test_pattern(instance), suite.name) return self.apply_tag_label(pattern, suite) def format_testcase(self, instance, suite, testcase): - if not isinstance(instance, MultiTest): - return "{}::{}::{}".format(instance.name, suite, testcase) + return "{}:{}:{}".format(test_pattern(instance), suite, testcase) - pattern = "{}::{}::{}".format(instance.name, suite.name, testcase.name) + pattern = "{}:{}:{}".format( + test_pattern(instance), suite.name, testcase.name + ) return self.apply_tag_label(pattern, testcase) @@ -223,7 +239,7 @@ def get_output(self, instance): " suite{num_suites_plural}," " {num_testcases}" " testcase{num_testcases_plural})".format( - instance_name=instance.name, + instance_name=instance.uid(), num_suites=len(suites), num_suites_plural="s" if len(suites) > 1 else "", num_testcases=total_testcases, diff --git a/testplan/testing/multitest/base.py b/testplan/testing/multitest/base.py index efbad48fc..51f3588a0 100644 --- a/testplan/testing/multitest/base.py +++ b/testplan/testing/multitest/base.py @@ -5,7 +5,7 @@ import functools import itertools import os -from typing import Callable, Optional, Tuple, Dict, List, Generator +from typing import Callable, Dict, Generator, List, Optional, Tuple from schema import And, Or, Use @@ -29,6 +29,7 @@ ) from testplan.testing import base as testing_base from testplan.testing import filtering, tagging +from testplan.testing.common import TEST_PART_PATTERN_FORMAT_STRING from testplan.testing.multitest import result from testplan.testing.multitest import suite as mtest_suite from testplan.testing.multitest.entries import base as entries_base @@ -346,7 +347,7 @@ def uid(self): return ( self.cfg.multi_part_uid(self.cfg.name, self.cfg.part) if self.cfg.multi_part_uid - else "{} - part({}/{})".format( + else TEST_PART_PATTERN_FORMAT_STRING.format( self.cfg.name, self.cfg.part[0], self.cfg.part[1] ) ) @@ -361,11 +362,7 @@ def setup(self): """ # watch line features depends on configuration from the outside world - if ( - self.cfg.parent is not None - and self.cfg.tracing_tests is not None - and self.cfg.interactive_port is None - ): + if self.cfg.parent is not None and self.cfg.tracing_tests is not None: self.watcher.set_watching_lines(self.cfg.tracing_tests) def get_test_context(self): @@ -381,6 +378,7 @@ def get_test_context(self): for suite in sorted_suites: testcases = suite.get_testcases() + sorted_testcases = ( testcases if getattr(suite, "strict_order", False) @@ -388,6 +386,13 @@ def get_test_context(self): else self.cfg.test_sorter.sorted_testcases(suite, testcases) ) + if self.cfg.part: + sorted_testcases = [ + testcase + for (idx, testcase) in enumerate(sorted_testcases) + if (idx) % self.cfg.part[1] == self.cfg.part[0] + ] + testcases_to_run = [ case for case in sorted_testcases @@ -403,13 +408,6 @@ def get_test_context(self): ) < len(sorted_testcases): testcases_to_run = sorted_testcases - if self.cfg.part and self.cfg.part[1] > 1: - testcases_to_run = [ - testcase - for (idx, testcase) in enumerate(testcases_to_run) - if idx % self.cfg.part[1] == self.cfg.part[0] - ] - if self.cfg.testcase_report_target: testcases_to_run = [ report_target( @@ -429,7 +427,7 @@ def get_test_context(self): testcase_instance = ":".join( [ self.name, - suite.__class__.__name__, + suite.name, testcase.name, ] ) @@ -532,7 +530,7 @@ def run_testcases_iter( if shallow_report is None: test_filter = filtering.Pattern( pattern="*:{}:{}".format(testsuite_pattern, testcase_pattern), - match_definition=True, + match_uid=True, ) else: test_targets = _extract_test_targets(shallow_report) @@ -723,7 +721,7 @@ def stop_test_resources(self): def get_metadata(self) -> TestMetadata: return TestMetadata( - name=self.name, + name=self.uid(), description=self.cfg.description, test_suites=[get_suite_metadata(suite) for suite in self.suites], ) @@ -812,8 +810,9 @@ def _new_test_report(self): :return: A new and empty test report object for this MultiTest. """ return TestGroupReport( - name=self.cfg.name, + name=self.uid(), description=self.cfg.description, + definition_name=self.cfg.name, uid=self.uid(), category=ReportCategories.MULTITEST, tags=self.cfg.tags, @@ -829,6 +828,7 @@ def _new_testsuite_report(self, testsuite): return TestGroupReport( name=testsuite.name, description=strings.get_docstring(testsuite.__class__), + definition_name=testsuite.name, uid=testsuite.uid(), category=ReportCategories.TESTSUITE, tags=testsuite.__tags__, @@ -842,6 +842,7 @@ def _new_testcase_report(self, testcase): return TestCaseReport( name=testcase.name, description=strings.get_docstring(testcase), + definition_name=testcase.name, uid=testcase.__name__, tags=testcase.__tags__, ) @@ -855,6 +856,7 @@ def _new_parametrized_group_report(self, param_template, param_method): return TestGroupReport( name=param_method.name, description=strings.get_docstring(param_method), + definition_name=param_template, uid=param_template, category=ReportCategories.PARAMETRIZATION, tags=param_method.__tags__, diff --git a/testplan/testing/multitest/parametrization.py b/testplan/testing/multitest/parametrization.py index fe72373ca..528e8de93 100644 --- a/testplan/testing/multitest/parametrization.py +++ b/testplan/testing/multitest/parametrization.py @@ -6,9 +6,8 @@ import re import warnings -from testplan.common.utils import convert -from testplan.common.utils import interface from testplan.common.utils import callable as callable_utils +from testplan.common.utils import convert, interface from testplan.testing import tagging # Although any string will be processed as normal, it's a good @@ -182,7 +181,7 @@ def _generate_kwarg_list(parameters, args, required_args, default_args): def _generate_func( - function, name, name_func, tag_func, docstring_func, tag_dict, kwargs + function, name, name_func, idx, tag_func, docstring_func, tag_dict, kwargs ): """ Generates a new function using the original function, name generation @@ -205,7 +204,11 @@ def _generated(self, env, result): _generated.__name__ = _parametrization_name_func_wrapper( func_name=function.__name__, kwargs=kwargs ) - _generated.name = name_func(name, kwargs) if name_func else name + # Users request the feature that when `name_func` set to `None`, + # then simply append integer suffixes to the names of testcases + _generated.name = ( + name_func(name, kwargs) if name_func is not None else f"{name} {idx}" + ) if hasattr(function, "__xfail__"): _generated.__xfail__ = function.__xfail__ @@ -271,18 +274,18 @@ def _parametrization_name_func_wrapper(func_name, kwargs): If somehow a 'bad' function name is generated, will just return the original ``func_name`` instead (which will later on be suffixed with an - integer by :py:func:`_ensure_unique_testcase_names`) + integer by :py:func:`_ensure_unique_generated_testcase_names`) """ generated_name = parametrization_name_func(func_name, kwargs) if not PYTHON_VARIABLE_REGEX.match(generated_name): - # Generated method name is not a valid Python attribute name - # e.g. "{func_name}_1", "{func_name}_2" will be used. + # Generated method name is not a valid Python attribute name. + # Index suffixed names, e.g. "{func_name}__1", "{func_name}__2", will be used. return func_name if len(generated_name) > MAX_METHOD_NAME_LENGTH: - # Generated method name is a bit too long. Simple index suffixes - # e.g. "{func_name}_1", "{func_name}_2" will be used. + # Generated method name is a bit too long. + # Index suffixed names, e.g. "{func_name}__1", "{func_name}__2", will be used. return func_name return generated_name @@ -408,6 +411,7 @@ def generate_functions( raise ParametrizationError('"parameters" cannot be a empty.') _check_name_func(name_func) + _check_tag_func(tag_func) argspec = callable_utils.getargspec(function) args = argspec.args[3:] # get rid of self, env, result @@ -420,30 +424,24 @@ def generate_functions( parameters, args, required_args, default_args ) - functions = [ - _generate_func( + functions = [] + for idx, kwargs in enumerate(kwarg_list): + func = _generate_func( function=function, name=name, name_func=name_func, + idx=idx, tag_func=tag_func, docstring_func=docstring_func, tag_dict=tag_dict, kwargs=kwargs, ) - for kwargs in kwarg_list - ] - - for idx, func in enumerate(functions): - # Users request the feature that when `name_func` set to `None`, - # then simply append integer suffixes to the names of testcases - if name_func is None: - func.name = "{} {}".format(func.name, idx) - func.summarize = summarize func.summarize_num_passing = num_passing func.summarize_num_failing = num_failing func.summarize_key_combs_limit = key_combs_limit func.execution_group = execution_group func.timeout = timeout + functions.append(func) return functions diff --git a/testplan/testing/multitest/suite.py b/testplan/testing/multitest/suite.py index 7322944c9..0eaab2c2b 100644 --- a/testplan/testing/multitest/suite.py +++ b/testplan/testing/multitest/suite.py @@ -7,7 +7,7 @@ import itertools import types import warnings -from typing import Optional, Callable +from typing import Callable, Optional from testplan import defaults from testplan.common.utils import interface, strings @@ -16,10 +16,10 @@ from . import parametrization from .test_metadata import ( LocationMetadata, + TestCaseMetadata, + TestCaseStaticMetadata, TestSuiteMetadata, TestSuiteStaticMetadata, - TestCaseStaticMetadata, - TestCaseMetadata, ) # Global variables @@ -356,9 +356,8 @@ def _ensure_unique_generated_testcase_names(names, functions): for func in functions: name = func.__name__ if name in dupe_names or name in valid_names: - count = dupe_counter[name] while True: - func.__name__ = "{}__{}".format(name, count) + func.__name__ = "{}__{}".format(name, dupe_counter[name]) dupe_counter[name] += 1 if ( func.__name__ not in dupe_names @@ -367,14 +366,17 @@ def _ensure_unique_generated_testcase_names(names, functions): valid_names.add(func.__name__) break else: - valid_names.add(name) + valid_names.add(func.__name__) # Functions should have different __name__ attributes after the step above name_counts = collections.Counter( itertools.chain(names, (func.__name__ for func in functions)) ) dupe_names = {k for k, v in name_counts.items() if v > 1} - assert len(dupe_names) == 0 + if len(dupe_names): + raise RuntimeError( + f"Internal error, duplicate case names found: {dupe_names}" + ) def _testsuite(klass): diff --git a/testplan/web_ui/testing/src/Nav/NavList.js b/testplan/web_ui/testing/src/Nav/NavList.js index d473952b4..5728af21c 100644 --- a/testplan/web_ui/testing/src/Nav/NavList.js +++ b/testplan/web_ui/testing/src/Nav/NavList.js @@ -19,7 +19,7 @@ const NavList = (props) => { (entry) => ( ", + "logs": [], + "name": "basic_case ", + "parent_uids": [ + "InteractivePlan", + "Test1", + "BasicSuite", + "basic_case" + ], + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, + "timer": {}, + "type": "TestCaseReport", + "uid": "basic_case__arg_0" + }, + { + "category": "testcase", + "counter": { + "failed": 0, + "passed": 1, + "total": 1 + }, + "description": null, + "entries": [ + { + "category": "DEFAULT", + "description": "Passing assertion", + "file_path": "", + "first": 1, + "flag": "DEFAULT", + "label": "==", + "line_no": 0, + "machine_time": "", + "meta_type": "assertion", + "passed": true, + "second": 1, + "type": "Equal", + "type_actual": "int", + "type_expected": "int", + "utc_time": "" + } + ], + "hash": 0, + "definition_name": "basic_case ", + "logs": [], + "name": "basic_case ", + "parent_uids": [ + "InteractivePlan", + "Test1", + "BasicSuite", + "basic_case" + ], + "runtime_status": "finished", + "status": "passed", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, + "timer": { + "run": { + "end": "", + "start": "" + } + }, + "type": "TestCaseReport", + "uid": "basic_case__arg_1" + }, + { + "category": "testcase", + "counter": { + "failed": 0, + "passed": 0, + "total": 1, + "unknown": 1 + }, + "description": null, + "entries": [], + "hash": 0, + "definition_name": "basic_case ", + "logs": [], + "name": "basic_case ", + "parent_uids": [ + "InteractivePlan", + "Test1", + "BasicSuite", + "basic_case" + ], + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, + "timer": {}, + "type": "TestCaseReport", + "uid": "basic_case__arg_2" + } + ], + "env_status": null, + "events": {}, + "fix_spec_path": null, + "hash": 0, + "host": null, + "definition_name": "basic_case", + "logs": [], + "name": "basic_case", + "parent_uids": [ + "InteractivePlan", + "Test1", + "BasicSuite" + ], + "part": null, + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, + "timer": {}, + "type": "TestGroupReport", + "uid": "basic_case" + } + ], + "env_status": null, + "events": {}, + "fix_spec_path": null, + "hash": 0, + "host": null, + "definition_name": "BasicSuite", + "logs": [], + "name": "BasicSuite", + "parent_uids": [ + "InteractivePlan", + "Test1" + ], + "part": null, + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, + "timer": {}, + "type": "TestGroupReport", + "uid": "BasicSuite" + }, + { + "category": "testsuite", + "counter": { + "failed": 0, + "passed": 0, + "total": 1, + "unknown": 1 + }, + "description": null, + "entries": [ + { + "category": "testcase", + "counter": { + "failed": 0, + "passed": 0, + "total": 1, + "unknown": 1 + }, + "description": "Client sends a message, server received and responds back.", + "entries": [], + "hash": 0, + "definition_name": "send_and_receive_msg", + "logs": [], + "name": "send_and_receive_msg", + "parent_uids": [ + "InteractivePlan", + "Test1", + "Custom_0" + ], + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, + "timer": {}, + "type": "TestCaseReport", + "uid": "send_and_receive_msg" + } + ], + "env_status": null, + "events": {}, + "fix_spec_path": null, + "hash": 0, + "host": null, + "definition_name": "Custom_0", + "logs": [], + "name": "Custom_0", + "parent_uids": [ + "InteractivePlan", + "Test1" + ], + "part": null, + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, + "timer": {}, + "type": "TestGroupReport", + "uid": "Custom_0" + }, + { + "category": "testsuite", + "counter": { + "failed": 0, + "passed": 0, + "total": 1, + "unknown": 1 + }, + "description": null, + "entries": [ + { + "category": "testcase", + "counter": { + "failed": 0, + "passed": 0, + "total": 1, + "unknown": 1 + }, + "description": "Client sends a message, server received and responds back.", + "entries": [], + "hash": 0, + "definition_name": "send_and_receive_msg", + "logs": [], + "name": "send_and_receive_msg", + "parent_uids": [ + "InteractivePlan", + "Test1", + "Custom_1" + ], + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, + "timer": {}, + "type": "TestCaseReport", + "uid": "send_and_receive_msg" + } + ], + "env_status": null, + "events": {}, + "fix_spec_path": null, + "hash": 0, + "host": null, + "definition_name": "Custom_1", + "logs": [], + "name": "Custom_1", + "parent_uids": [ + "InteractivePlan", + "Test1" + ], + "part": null, + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, + "timer": {}, + "type": "TestGroupReport", + "uid": "Custom_1" + } + ], + "env_status": "STARTED", + "events": {}, + "fix_spec_path": null, + "hash": 0, + "host": null, + "definition_name": "Test1", + "logs": [], + "name": "Test1", + "parent_uids": [ + "InteractivePlan" + ], + "part": null, + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, + "timer": {}, + "type": "TestGroupReport", + "uid": "Test1" +} \ No newline at end of file diff --git a/tests/functional/testplan/runnable/interactive/reports/basic_run_case_test1.py b/tests/functional/testplan/runnable/interactive/reports/basic_run_case_test1.py deleted file mode 100644 index 98092ce60..000000000 --- a/tests/functional/testplan/runnable/interactive/reports/basic_run_case_test1.py +++ /dev/null @@ -1,287 +0,0 @@ -from collections import Counter - -REPORT = { - "status_reason": None, - "counter": Counter({"total": 5, "unknown": 4, "passed": 1, "failed": 0}), - "runtime_status": "ready", - "tags": {}, - "timer": {}, - "category": "multitest", - "status_override": None, - "logs": [], - "part": None, - "status": "unknown", - "entries": [ - { - "status_reason": None, - "counter": Counter( - {"total": 3, "unknown": 2, "passed": 1, "failed": 0} - ), - "runtime_status": "ready", - "tags": {}, - "timer": {}, - "category": "testsuite", - "status_override": None, - "logs": [], - "part": None, - "status": "unknown", - "entries": [ - { - "status_reason": None, - "counter": Counter( - {"total": 3, "unknown": 2, "passed": 1, "failed": 0} - ), - "runtime_status": "ready", - "tags": {}, - "timer": {}, - "category": "parametrization", - "status_override": None, - "logs": [], - "part": None, - "status": "unknown", - "entries": [ - { - "status_reason": None, - "suite_related": False, - "counter": Counter( - { - "total": 1, - "unknown": 1, - "passed": 0, - "failed": 0, - } - ), - "description": None, - "runtime_status": "ready", - "tags": {}, - "timer": {}, - "parent_uids": [ - "InteractivePlan", - "Test1", - "BasicSuite", - "basic_case", - ], - "category": "testcase", - "status_override": None, - "logs": [], - "uid": "basic_case__arg_0", - "name": "basic_case ", - "status": "unknown", - "type": "TestCaseReport", - "hash": 0, - "entries": [], - }, - { - "status_reason": None, - "suite_related": False, - "counter": Counter( - {"passed": 1, "total": 1, "failed": 0} - ), - "description": None, - "runtime_status": "finished", - "tags": {}, - "timer": { - "run": { - "end": "2021-07-27T06:49:52.078141+00:00", - "start": "2021-07-27T06:49:52.069275+00:00", - } - }, - "parent_uids": [ - "InteractivePlan", - "Test1", - "BasicSuite", - "basic_case", - ], - "category": "testcase", - "status_override": None, - "logs": [], - "uid": "basic_case__arg_1", - "name": "basic_case ", - "status": "passed", - "type": "TestCaseReport", - "hash": 0, - "entries": [ - { - "flag": "DEFAULT", - "machine_time": "", - "meta_type": "assertion", - "label": "==", - "description": "Passing assertion", - "type_expected": "int", - "passed": True, - "type_actual": "int", - "category": "DEFAULT", - "second": 1, - "type": "Equal", - "first": 1, - "utc_time": "", - "line_no": 0, - } - ], - }, - { - "status_reason": None, - "suite_related": False, - "counter": Counter( - { - "total": 1, - "unknown": 1, - "passed": 0, - "failed": 0, - } - ), - "description": None, - "runtime_status": "ready", - "tags": {}, - "timer": {}, - "parent_uids": [ - "InteractivePlan", - "Test1", - "BasicSuite", - "basic_case", - ], - "category": "testcase", - "status_override": None, - "logs": [], - "uid": "basic_case__arg_2", - "name": "basic_case ", - "status": "unknown", - "type": "TestCaseReport", - "hash": 0, - "entries": [], - }, - ], - "env_status": None, - "fix_spec_path": None, - "host": None, - "events": {}, - "parent_uids": ["InteractivePlan", "Test1", "BasicSuite"], - "uid": "basic_case", - "strict_order": False, - "hash": 0, - "description": None, - "name": "basic_case", - "type": "TestGroupReport", - } - ], - "env_status": None, - "fix_spec_path": None, - "host": None, - "events": {}, - "parent_uids": ["InteractivePlan", "Test1"], - "uid": "BasicSuite", - "strict_order": False, - "hash": 0, - "description": None, - "name": "BasicSuite", - "type": "TestGroupReport", - }, - { - "status_reason": None, - "counter": Counter( - {"total": 1, "unknown": 1, "passed": 0, "failed": 0} - ), - "runtime_status": "ready", - "tags": {}, - "timer": {}, - "category": "testsuite", - "status_override": None, - "logs": [], - "part": None, - "status": "unknown", - "entries": [ - { - "status_reason": None, - "suite_related": False, - "counter": Counter( - {"total": 1, "unknown": 1, "passed": 0, "failed": 0} - ), - "description": "Client sends a message, server received and responds back.", - "runtime_status": "ready", - "tags": {}, - "timer": {}, - "parent_uids": ["InteractivePlan", "Test1", "Custom_0"], - "category": "testcase", - "status_override": None, - "logs": [], - "uid": "send_and_receive_msg", - "name": "send_and_receive_msg", - "status": "unknown", - "type": "TestCaseReport", - "hash": 0, - "entries": [], - } - ], - "env_status": None, - "fix_spec_path": None, - "host": None, - "events": {}, - "parent_uids": ["InteractivePlan", "Test1"], - "uid": "Custom_0", - "strict_order": False, - "hash": 0, - "description": None, - "name": "Custom_0", - "type": "TestGroupReport", - }, - { - "status_reason": None, - "counter": Counter( - {"total": 1, "unknown": 1, "passed": 0, "failed": 0} - ), - "runtime_status": "ready", - "tags": {}, - "timer": {}, - "category": "testsuite", - "status_override": None, - "logs": [], - "part": None, - "status": "unknown", - "entries": [ - { - "status_reason": None, - "suite_related": False, - "counter": Counter( - {"total": 1, "unknown": 1, "passed": 0, "failed": 0} - ), - "description": "Client sends a message, server received and responds back.", - "runtime_status": "ready", - "tags": {}, - "timer": {}, - "parent_uids": ["InteractivePlan", "Test1", "Custom_1"], - "category": "testcase", - "status_override": None, - "logs": [], - "uid": "send_and_receive_msg", - "name": "send_and_receive_msg", - "status": "unknown", - "type": "TestCaseReport", - "hash": 0, - "entries": [], - } - ], - "env_status": None, - "fix_spec_path": None, - "host": None, - "events": {}, - "parent_uids": ["InteractivePlan", "Test1"], - "uid": "Custom_1", - "strict_order": False, - "hash": 0, - "description": None, - "name": "Custom_1", - "type": "TestGroupReport", - }, - ], - "env_status": "STARTED", - "fix_spec_path": None, - "host": None, - "events": {}, - "parent_uids": ["InteractivePlan"], - "uid": "Test1", - "strict_order": False, - "hash": 0, - "description": None, - "name": "Test1", - "type": "TestGroupReport", -} diff --git a/tests/functional/testplan/runnable/interactive/reports/basic_run_suite_test2.data b/tests/functional/testplan/runnable/interactive/reports/basic_run_suite_test2.data new file mode 100644 index 000000000..e1aab0d98 --- /dev/null +++ b/tests/functional/testplan/runnable/interactive/reports/basic_run_suite_test2.data @@ -0,0 +1,318 @@ +{ + "category": "multitest", + "counter": { + "failed": 0, + "passed": 0, + "total": 5, + "unknown": 5 + }, + "description": null, + "entries": [ + { + "category": "testsuite", + "counter": { + "failed": 0, + "passed": 0, + "total": 3, + "unknown": 3 + }, + "description": null, + "entries": [ + { + "category": "parametrization", + "counter": { + "failed": 0, + "passed": 0, + "total": 3, + "unknown": 3 + }, + "description": null, + "entries": [ + { + "category": "testcase", + "counter": { + "failed": 0, + "passed": 0, + "total": 1, + "unknown": 1 + }, + "description": null, + "entries": [], + "hash": 0, + "definition_name": "basic_case ", + "logs": [], + "name": "basic_case ", + "parent_uids": [ + "InteractivePlan", + "Test2", + "BasicSuite", + "basic_case" + ], + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, + "timer": {}, + "type": "TestCaseReport", + "uid": "basic_case__arg_0" + }, + { + "category": "testcase", + "counter": { + "failed": 0, + "passed": 0, + "total": 1, + "unknown": 1 + }, + "description": null, + "entries": [], + "hash": 0, + "definition_name": "basic_case ", + "logs": [], + "name": "basic_case ", + "parent_uids": [ + "InteractivePlan", + "Test2", + "BasicSuite", + "basic_case" + ], + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, + "timer": {}, + "type": "TestCaseReport", + "uid": "basic_case__arg_1" + }, + { + "category": "testcase", + "counter": { + "failed": 0, + "passed": 0, + "total": 1, + "unknown": 1 + }, + "description": null, + "entries": [], + "hash": 0, + "definition_name": "basic_case ", + "logs": [], + "name": "basic_case ", + "parent_uids": [ + "InteractivePlan", + "Test2", + "BasicSuite", + "basic_case" + ], + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, + "timer": {}, + "type": "TestCaseReport", + "uid": "basic_case__arg_2" + } + ], + "env_status": null, + "events": {}, + "fix_spec_path": null, + "hash": 0, + "host": null, + "definition_name": "basic_case", + "logs": [], + "name": "basic_case", + "parent_uids": [ + "InteractivePlan", + "Test2", + "BasicSuite" + ], + "part": null, + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, + "timer": {}, + "type": "TestGroupReport", + "uid": "basic_case" + } + ], + "env_status": null, + "events": {}, + "fix_spec_path": null, + "hash": 0, + "host": null, + "definition_name": "BasicSuite", + "logs": [], + "name": "BasicSuite", + "parent_uids": [ + "InteractivePlan", + "Test2" + ], + "part": null, + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, + "timer": {}, + "type": "TestGroupReport", + "uid": "BasicSuite" + }, + { + "category": "testsuite", + "counter": { + "failed": 0, + "passed": 0, + "total": 1, + "unknown": 1 + }, + "description": null, + "entries": [ + { + "category": "testcase", + "counter": { + "failed": 0, + "passed": 0, + "total": 1, + "unknown": 1 + }, + "description": "Client sends a message, server received and responds back.", + "entries": [], + "hash": 0, + "definition_name": "send_and_receive_msg", + "logs": [], + "name": "send_and_receive_msg", + "parent_uids": [ + "InteractivePlan", + "Test2", + "Custom_0" + ], + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, + "timer": {}, + "type": "TestCaseReport", + "uid": "send_and_receive_msg" + } + ], + "env_status": null, + "events": {}, + "fix_spec_path": null, + "hash": 0, + "host": null, + "definition_name": "Custom_0", + "logs": [], + "name": "Custom_0", + "parent_uids": [ + "InteractivePlan", + "Test2" + ], + "part": null, + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, + "timer": {}, + "type": "TestGroupReport", + "uid": "Custom_0" + }, + { + "category": "testsuite", + "counter": { + "failed": 0, + "passed": 0, + "total": 1, + "unknown": 1 + }, + "description": null, + "entries": [ + { + "category": "testcase", + "counter": { + "failed": 0, + "passed": 0, + "total": 1, + "unknown": 1 + }, + "description": "Client sends a message, server received and responds back.", + "entries": [], + "hash": 0, + "definition_name": "send_and_receive_msg", + "logs": [], + "name": "send_and_receive_msg", + "parent_uids": [ + "InteractivePlan", + "Test2", + "Custom_1" + ], + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, + "timer": {}, + "type": "TestCaseReport", + "uid": "send_and_receive_msg" + } + ], + "env_status": null, + "events": {}, + "fix_spec_path": null, + "hash": 0, + "host": null, + "definition_name": "Custom_1", + "logs": [], + "name": "Custom_1", + "parent_uids": [ + "InteractivePlan", + "Test2" + ], + "part": null, + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, + "timer": {}, + "type": "TestGroupReport", + "uid": "Custom_1" + } + ], + "env_status": "STARTED", + "events": {}, + "fix_spec_path": null, + "hash": 0, + "host": null, + "definition_name": "Test2", + "logs": [], + "name": "Test2", + "parent_uids": [ + "InteractivePlan" + ], + "part": null, + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, + "timer": {}, + "type": "TestGroupReport", + "uid": "Test2" +} \ No newline at end of file diff --git a/tests/functional/testplan/runnable/interactive/reports/basic_run_suite_test2.py b/tests/functional/testplan/runnable/interactive/reports/basic_run_suite_test2.py deleted file mode 100644 index 6f570492a..000000000 --- a/tests/functional/testplan/runnable/interactive/reports/basic_run_suite_test2.py +++ /dev/null @@ -1,270 +0,0 @@ -from collections import Counter - -REPORT = { - "strict_order": False, - "status_override": None, - "env_status": "STARTED", - "status_reason": None, - "fix_spec_path": None, - "host": None, - "events": {}, - "uid": "Test2", - "entries": [ - { - "strict_order": False, - "status_override": None, - "env_status": None, - "status_reason": None, - "fix_spec_path": None, - "host": None, - "events": {}, - "uid": "BasicSuite", - "entries": [ - { - "strict_order": False, - "status_override": None, - "env_status": None, - "status_reason": None, - "fix_spec_path": None, - "host": None, - "events": {}, - "uid": "basic_case", - "entries": [ - { - "logs": [], - "status_override": None, - "suite_related": False, - "runtime_status": "ready", - "timer": {}, - "tags": {}, - "description": None, - "parent_uids": [ - "InteractivePlan", - "Test2", - "BasicSuite", - "basic_case", - ], - "counter": Counter( - { - "total": 1, - "unknown": 1, - "passed": 0, - "failed": 0, - } - ), - "status": "unknown", - "category": "testcase", - "status_reason": None, - "name": "basic_case ", - "uid": "basic_case__arg_0", - "type": "TestCaseReport", - "entries": [], - "hash": 0, - }, - { - "logs": [], - "status_override": None, - "suite_related": False, - "runtime_status": "ready", - "timer": {}, - "tags": {}, - "description": None, - "parent_uids": [ - "InteractivePlan", - "Test2", - "BasicSuite", - "basic_case", - ], - "counter": Counter( - { - "total": 1, - "unknown": 1, - "passed": 0, - "failed": 0, - } - ), - "status": "unknown", - "category": "testcase", - "status_reason": None, - "name": "basic_case ", - "uid": "basic_case__arg_1", - "type": "TestCaseReport", - "entries": [], - "hash": 0, - }, - { - "logs": [], - "status_override": None, - "suite_related": False, - "runtime_status": "ready", - "timer": {}, - "tags": {}, - "description": None, - "parent_uids": [ - "InteractivePlan", - "Test2", - "BasicSuite", - "basic_case", - ], - "counter": Counter( - { - "total": 1, - "unknown": 1, - "passed": 0, - "failed": 0, - } - ), - "status": "unknown", - "category": "testcase", - "status_reason": None, - "name": "basic_case ", - "uid": "basic_case__arg_2", - "type": "TestCaseReport", - "entries": [], - "hash": 0, - }, - ], - "counter": Counter( - {"total": 3, "unknown": 3, "passed": 0, "failed": 0} - ), - "part": None, - "status": "unknown", - "type": "TestGroupReport", - "timer": {}, - "runtime_status": "ready", - "description": None, - "category": "parametrization", - "name": "basic_case", - "logs": [], - "tags": {}, - "parent_uids": ["InteractivePlan", "Test2", "BasicSuite"], - "hash": 0, - } - ], - "counter": Counter( - {"total": 3, "unknown": 3, "passed": 0, "failed": 0} - ), - "part": None, - "status": "unknown", - "type": "TestGroupReport", - "timer": {}, - "runtime_status": "ready", - "description": None, - "category": "testsuite", - "name": "BasicSuite", - "logs": [], - "tags": {}, - "parent_uids": ["InteractivePlan", "Test2"], - "hash": 0, - }, - { - "strict_order": False, - "status_override": None, - "env_status": None, - "status_reason": None, - "fix_spec_path": None, - "host": None, - "events": {}, - "uid": "Custom_0", - "entries": [ - { - "logs": [], - "status_override": None, - "suite_related": False, - "runtime_status": "ready", - "timer": {}, - "tags": {}, - "description": "Client sends a message, server received and responds back.", - "parent_uids": ["InteractivePlan", "Test2", "Custom_0"], - "counter": Counter( - {"total": 1, "unknown": 1, "passed": 0, "failed": 0} - ), - "status": "unknown", - "category": "testcase", - "status_reason": None, - "name": "send_and_receive_msg", - "uid": "send_and_receive_msg", - "type": "TestCaseReport", - "entries": [], - "hash": 0, - } - ], - "counter": Counter( - {"total": 1, "unknown": 1, "passed": 0, "failed": 0} - ), - "part": None, - "status": "unknown", - "type": "TestGroupReport", - "timer": {}, - "runtime_status": "ready", - "description": None, - "category": "testsuite", - "name": "Custom_0", - "logs": [], - "tags": {}, - "parent_uids": ["InteractivePlan", "Test2"], - "hash": 0, - }, - { - "strict_order": False, - "status_override": None, - "env_status": None, - "status_reason": None, - "fix_spec_path": None, - "host": None, - "events": {}, - "uid": "Custom_1", - "entries": [ - { - "logs": [], - "status_override": None, - "suite_related": False, - "runtime_status": "ready", - "timer": {}, - "tags": {}, - "description": "Client sends a message, server received and responds back.", - "parent_uids": ["InteractivePlan", "Test2", "Custom_1"], - "counter": Counter( - {"total": 1, "unknown": 1, "passed": 0, "failed": 0} - ), - "status": "unknown", - "category": "testcase", - "status_reason": None, - "name": "send_and_receive_msg", - "uid": "send_and_receive_msg", - "type": "TestCaseReport", - "entries": [], - "hash": 0, - } - ], - "counter": Counter( - {"total": 1, "unknown": 1, "passed": 0, "failed": 0} - ), - "part": None, - "status": "unknown", - "type": "TestGroupReport", - "timer": {}, - "runtime_status": "ready", - "description": None, - "category": "testsuite", - "name": "Custom_1", - "logs": [], - "tags": {}, - "parent_uids": ["InteractivePlan", "Test2"], - "hash": 0, - }, - ], - "counter": Counter({"total": 5, "unknown": 5, "passed": 0, "failed": 0}), - "part": None, - "status": "unknown", - "type": "TestGroupReport", - "timer": {}, - "runtime_status": "ready", - "description": None, - "category": "multitest", - "name": "Test2", - "logs": [], - "tags": {}, - "parent_uids": ["InteractivePlan"], - "hash": 0, -} diff --git a/tests/functional/testplan/runnable/interactive/reports/basic_top_level.py b/tests/functional/testplan/runnable/interactive/reports/basic_top_level.data similarity index 62% rename from tests/functional/testplan/runnable/interactive/reports/basic_top_level.py rename to tests/functional/testplan/runnable/interactive/reports/basic_top_level.data index 0edd45025..3a7912aec 100644 --- a/tests/functional/testplan/runnable/interactive/reports/basic_top_level.py +++ b/tests/functional/testplan/runnable/interactive/reports/basic_top_level.data @@ -1,825 +1,939 @@ -from collections import Counter - -REPORT = { - "information": [], - "events": {}, - "status_override": None, - "counter": Counter({"passed": 10, "total": 10, "failed": 0}), - "name": "InteractivePlan", +{ + "attachments": {}, + "category": "testplan", + "counter": { + "failed": 0, + "passed": 10, + "total": 10 + }, + "description": null, "entries": [ { - "name": "Test1", - "status_reason": None, - "env_status": "STARTED", - "tags": {}, - "logs": [], "category": "multitest", - "status_override": None, + "counter": { + "failed": 0, + "passed": 5, + "total": 5 + }, + "description": null, "entries": [ { - "name": "BasicSuite", - "status_reason": None, - "env_status": None, - "tags": {}, - "logs": [], "category": "testsuite", - "status_override": None, + "counter": { + "failed": 0, + "passed": 3, + "total": 3 + }, + "description": null, "entries": [ { - "name": "basic_case", - "status_reason": None, - "env_status": None, - "tags": {}, - "logs": [], "category": "parametrization", - "status_override": None, + "counter": { + "failed": 0, + "passed": 3, + "total": 3 + }, + "description": null, "entries": [ { - "counter": Counter( - {"passed": 1, "total": 1, "failed": 0} - ), - "status_override": None, + "category": "testcase", + "counter": { + "failed": 0, + "passed": 1, + "total": 1 + }, + "description": null, "entries": [ { - "machine_time": "", - "passed": True, - "type_expected": "int", - "flag": "DEFAULT", + "category": "DEFAULT", "description": "Passing assertion", - "meta_type": "assertion", - "line_no": 0, + "file_path": "", "first": 1, - "utc_time": "", + "flag": "DEFAULT", "label": "==", + "line_no": 0, + "machine_time": "", + "meta_type": "assertion", + "passed": true, + "second": 1, "type": "Equal", "type_actual": "int", - "second": 1, - "category": "DEFAULT", + "type_expected": "int", + "utc_time": "" } ], - "name": "basic_case ", "hash": 0, - "status_reason": None, + "definition_name": "basic_case ", + "logs": [], + "name": "basic_case ", "parent_uids": [ "InteractivePlan", "Test1", "BasicSuite", - "basic_case", + "basic_case" ], - "description": None, + "runtime_status": "finished", + "status": "passed", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, "timer": { "run": { - "start": "", "end": "", + "start": "" } }, - "uid": "basic_case__arg_0", - "tags": {}, - "status": "passed", "type": "TestCaseReport", - "runtime_status": "finished", - "suite_related": False, - "logs": [], - "category": "testcase", + "uid": "basic_case__arg_0" }, { - "counter": Counter( - {"passed": 1, "total": 1, "failed": 0} - ), - "status_override": None, + "category": "testcase", + "counter": { + "failed": 0, + "passed": 1, + "total": 1 + }, + "description": null, "entries": [ { - "machine_time": "", - "passed": True, - "type_expected": "int", - "flag": "DEFAULT", + "category": "DEFAULT", "description": "Passing assertion", - "meta_type": "assertion", - "line_no": 0, + "file_path": "", "first": 1, - "utc_time": "", + "flag": "DEFAULT", "label": "==", + "line_no": 0, + "machine_time": "", + "meta_type": "assertion", + "passed": true, + "second": 1, "type": "Equal", "type_actual": "int", - "second": 1, - "category": "DEFAULT", + "type_expected": "int", + "utc_time": "" } ], - "name": "basic_case ", "hash": 0, - "status_reason": None, + "definition_name": "basic_case ", + "logs": [], + "name": "basic_case ", "parent_uids": [ "InteractivePlan", "Test1", "BasicSuite", - "basic_case", + "basic_case" ], - "description": None, + "runtime_status": "finished", + "status": "passed", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, "timer": { "run": { - "start": "", "end": "", + "start": "" } }, - "uid": "basic_case__arg_1", - "tags": {}, - "status": "passed", "type": "TestCaseReport", - "runtime_status": "finished", - "suite_related": False, - "logs": [], - "category": "testcase", + "uid": "basic_case__arg_1" }, { - "counter": Counter( - {"passed": 1, "total": 1, "failed": 0} - ), - "status_override": None, + "category": "testcase", + "counter": { + "failed": 0, + "passed": 1, + "total": 1 + }, + "description": null, "entries": [ { - "machine_time": "", - "passed": True, - "type_expected": "int", - "flag": "DEFAULT", + "category": "DEFAULT", "description": "Passing assertion", - "meta_type": "assertion", - "line_no": 0, + "file_path": "", "first": 1, - "utc_time": "", + "flag": "DEFAULT", "label": "==", + "line_no": 0, + "machine_time": "", + "meta_type": "assertion", + "passed": true, + "second": 1, "type": "Equal", "type_actual": "int", - "second": 1, - "category": "DEFAULT", + "type_expected": "int", + "utc_time": "" } ], + "hash": 0, + "definition_name": "basic_case ", + "logs": [], "name": "basic_case ", - "hash": 904824446777638636, - "status_reason": None, "parent_uids": [ "InteractivePlan", "Test1", "BasicSuite", - "basic_case", + "basic_case" ], - "description": None, + "runtime_status": "finished", + "status": "passed", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, "timer": { "run": { - "start": "", "end": "", + "start": "" } }, - "uid": "basic_case__arg_2", - "tags": {}, - "status": "passed", "type": "TestCaseReport", - "runtime_status": "finished", - "suite_related": False, - "logs": [], - "category": "testcase", - }, + "uid": "basic_case__arg_2" + } ], - "strict_order": False, - "status": "passed", - "runtime_status": "finished", - "part": None, - "fix_spec_path": None, - "host": None, + "env_status": null, "events": {}, + "fix_spec_path": null, + "hash": 0, + "host": null, + "definition_name": "basic_case", + "logs": [], + "name": "basic_case", "parent_uids": [ "InteractivePlan", "Test1", - "BasicSuite", + "BasicSuite" ], - "type": "TestGroupReport", - "counter": Counter( - {"passed": 3, "total": 3, "failed": 0} - ), - "hash": 0, - "description": None, + "part": null, + "runtime_status": "finished", + "status": "passed", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, "timer": {}, - "uid": "basic_case", + "type": "TestGroupReport", + "uid": "basic_case" } ], - "strict_order": False, - "status": "passed", - "runtime_status": "finished", - "part": None, - "fix_spec_path": None, - "host": None, + "env_status": null, "events": {}, - "parent_uids": ["InteractivePlan", "Test1"], - "type": "TestGroupReport", - "counter": Counter({"passed": 3, "total": 3, "failed": 0}), + "fix_spec_path": null, "hash": 0, - "description": None, + "host": null, + "definition_name": "BasicSuite", + "logs": [], + "name": "BasicSuite", + "parent_uids": [ + "InteractivePlan", + "Test1" + ], + "part": null, + "runtime_status": "finished", + "status": "passed", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, "timer": {}, - "uid": "BasicSuite", + "type": "TestGroupReport", + "uid": "BasicSuite" }, { - "name": "Custom_0", - "status_reason": None, - "env_status": None, - "tags": {}, - "logs": [], "category": "testsuite", - "status_override": None, + "counter": { + "failed": 0, + "passed": 1, + "total": 1 + }, + "description": null, "entries": [ { - "counter": Counter( - {"passed": 1, "total": 1, "failed": 0} - ), - "status_override": None, + "category": "testcase", + "counter": { + "failed": 0, + "passed": 1, + "total": 1 + }, + "description": "Client sends a message, server received and responds back.", "entries": [ { - "machine_time": "", - "passed": True, - "type_expected": "str", - "flag": "DEFAULT", + "category": "DEFAULT", "description": "Server received", - "meta_type": "assertion", - "line_no": 0, + "file_path": "", "first": "hello", - "utc_time": "", + "flag": "DEFAULT", "label": "==", + "line_no": 0, + "machine_time": "", + "meta_type": "assertion", + "passed": true, + "second": "hello", "type": "Equal", "type_actual": "str", - "second": "hello", - "category": "DEFAULT", + "type_expected": "str", + "utc_time": "" }, { - "machine_time": "", - "passed": True, - "type_expected": "str", - "flag": "DEFAULT", + "category": "DEFAULT", "description": "Client received", - "meta_type": "assertion", - "line_no": 0, + "file_path": "", "first": "world", - "utc_time": "", + "flag": "DEFAULT", "label": "==", + "line_no": 0, + "machine_time": "", + "meta_type": "assertion", + "passed": true, + "second": "world", "type": "Equal", "type_actual": "str", - "second": "world", - "category": "DEFAULT", - }, + "type_expected": "str", + "utc_time": "" + } ], - "name": "send_and_receive_msg", "hash": 0, - "status_reason": None, + "definition_name": "send_and_receive_msg", + "logs": [], + "name": "send_and_receive_msg", "parent_uids": [ "InteractivePlan", "Test1", - "Custom_0", + "Custom_0" ], - "description": "Client sends a message, server received and responds back.", + "runtime_status": "finished", + "status": "passed", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, "timer": { "run": { - "start": "", "end": "", + "start": "" } }, - "uid": "send_and_receive_msg", - "tags": {}, - "status": "passed", "type": "TestCaseReport", - "runtime_status": "finished", - "suite_related": False, - "logs": [], - "category": "testcase", + "uid": "send_and_receive_msg" } ], - "strict_order": False, - "status": "passed", - "runtime_status": "finished", - "part": None, - "fix_spec_path": None, - "host": None, + "env_status": null, "events": {}, - "parent_uids": ["InteractivePlan", "Test1"], - "type": "TestGroupReport", - "counter": Counter({"passed": 1, "total": 1, "failed": 0}), + "fix_spec_path": null, "hash": 0, - "description": None, + "host": null, + "definition_name": "Custom_0", + "logs": [], + "name": "Custom_0", + "parent_uids": [ + "InteractivePlan", + "Test1" + ], + "part": null, + "runtime_status": "finished", + "status": "passed", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, "timer": {}, - "uid": "Custom_0", + "type": "TestGroupReport", + "uid": "Custom_0" }, { - "name": "Custom_1", - "status_reason": None, - "env_status": None, - "tags": {}, - "logs": [], "category": "testsuite", - "status_override": None, + "counter": { + "failed": 0, + "passed": 1, + "total": 1 + }, + "description": null, "entries": [ { - "counter": Counter( - {"passed": 1, "total": 1, "failed": 0} - ), - "status_override": None, + "category": "testcase", + "counter": { + "failed": 0, + "passed": 1, + "total": 1 + }, + "description": "Client sends a message, server received and responds back.", "entries": [ { - "machine_time": "", - "passed": True, - "type_expected": "str", - "flag": "DEFAULT", + "category": "DEFAULT", "description": "Server received", - "meta_type": "assertion", - "line_no": 0, + "file_path": "", "first": "hello", - "utc_time": "", + "flag": "DEFAULT", "label": "==", + "line_no": 0, + "machine_time": "", + "meta_type": "assertion", + "passed": true, + "second": "hello", "type": "Equal", "type_actual": "str", - "second": "hello", - "category": "DEFAULT", + "type_expected": "str", + "utc_time": "" }, { - "machine_time": "", - "passed": True, - "type_expected": "str", - "flag": "DEFAULT", + "category": "DEFAULT", "description": "Client received", - "meta_type": "assertion", - "line_no": 0, + "file_path": "", "first": "world", - "utc_time": "", + "flag": "DEFAULT", "label": "==", + "line_no": 0, + "machine_time": "", + "meta_type": "assertion", + "passed": true, + "second": "world", "type": "Equal", "type_actual": "str", - "second": "world", - "category": "DEFAULT", - }, + "type_expected": "str", + "utc_time": "" + } ], - "name": "send_and_receive_msg", "hash": 0, - "status_reason": None, + "definition_name": "send_and_receive_msg", + "logs": [], + "name": "send_and_receive_msg", "parent_uids": [ "InteractivePlan", "Test1", - "Custom_1", + "Custom_1" ], - "description": "Client sends a message, server received and responds back.", + "runtime_status": "finished", + "status": "passed", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, "timer": { "run": { - "start": "", "end": "", + "start": "" } }, - "uid": "send_and_receive_msg", - "tags": {}, - "status": "passed", "type": "TestCaseReport", - "runtime_status": "finished", - "suite_related": False, - "logs": [], - "category": "testcase", + "uid": "send_and_receive_msg" } ], - "strict_order": False, - "status": "passed", - "runtime_status": "finished", - "part": None, - "fix_spec_path": None, - "host": None, + "env_status": null, "events": {}, - "parent_uids": ["InteractivePlan", "Test1"], - "type": "TestGroupReport", - "counter": Counter({"passed": 1, "total": 1, "failed": 0}), + "fix_spec_path": null, "hash": 0, - "description": None, + "host": null, + "definition_name": "Custom_1", + "logs": [], + "name": "Custom_1", + "parent_uids": [ + "InteractivePlan", + "Test1" + ], + "part": null, + "runtime_status": "finished", + "status": "passed", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, "timer": {}, - "uid": "Custom_1", - }, + "type": "TestGroupReport", + "uid": "Custom_1" + } ], - "strict_order": False, - "status": "passed", - "runtime_status": "finished", - "part": None, - "fix_spec_path": None, - "host": None, + "env_status": "STARTED", "events": {}, - "parent_uids": ["InteractivePlan"], - "type": "TestGroupReport", - "counter": Counter({"passed": 5, "total": 5, "failed": 0}), + "fix_spec_path": null, "hash": 0, - "description": None, + "host": null, + "definition_name": "Test1", + "logs": [], + "name": "Test1", + "parent_uids": [ + "InteractivePlan" + ], + "part": null, + "runtime_status": "finished", + "status": "passed", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, "timer": {}, - "uid": "Test1", + "type": "TestGroupReport", + "uid": "Test1" }, { - "name": "Test2", - "status_reason": None, - "env_status": "STARTED", - "tags": {}, - "logs": [], "category": "multitest", - "status_override": None, + "counter": { + "failed": 0, + "passed": 5, + "total": 5 + }, + "description": null, "entries": [ { - "name": "BasicSuite", - "status_reason": None, - "env_status": None, - "tags": {}, - "logs": [], "category": "testsuite", - "status_override": None, + "counter": { + "failed": 0, + "passed": 3, + "total": 3 + }, + "description": null, "entries": [ { - "name": "basic_case", - "status_reason": None, - "env_status": None, - "tags": {}, - "logs": [], "category": "parametrization", - "status_override": None, + "counter": { + "failed": 0, + "passed": 3, + "total": 3 + }, + "description": null, "entries": [ { - "counter": Counter( - {"passed": 1, "total": 1, "failed": 0} - ), - "status_override": None, + "category": "testcase", + "counter": { + "failed": 0, + "passed": 1, + "total": 1 + }, + "description": null, "entries": [ { - "machine_time": "", - "passed": True, - "type_expected": "int", - "flag": "DEFAULT", + "category": "DEFAULT", "description": "Passing assertion", - "meta_type": "assertion", - "line_no": 0, + "file_path": "", "first": 1, - "utc_time": "", + "flag": "DEFAULT", "label": "==", + "line_no": 0, + "machine_time": "", + "meta_type": "assertion", + "passed": true, + "second": 1, "type": "Equal", "type_actual": "int", - "second": 1, - "category": "DEFAULT", + "type_expected": "int", + "utc_time": "" } ], - "name": "basic_case ", "hash": 0, - "status_reason": None, + "definition_name": "basic_case ", + "logs": [], + "name": "basic_case ", "parent_uids": [ "InteractivePlan", "Test2", "BasicSuite", - "basic_case", + "basic_case" ], - "description": None, + "runtime_status": "finished", + "status": "passed", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, "timer": { "run": { - "start": "", "end": "", + "start": "" } }, - "uid": "basic_case__arg_0", - "tags": {}, - "status": "passed", "type": "TestCaseReport", - "runtime_status": "finished", - "suite_related": False, - "logs": [], - "category": "testcase", + "uid": "basic_case__arg_0" }, { - "counter": Counter( - {"passed": 1, "total": 1, "failed": 0} - ), - "status_override": None, + "category": "testcase", + "counter": { + "failed": 0, + "passed": 1, + "total": 1 + }, + "description": null, "entries": [ { - "machine_time": "", - "passed": True, - "type_expected": "int", - "flag": "DEFAULT", + "category": "DEFAULT", "description": "Passing assertion", - "meta_type": "assertion", - "line_no": 0, + "file_path": "", "first": 1, - "utc_time": "", + "flag": "DEFAULT", "label": "==", + "line_no": 0, + "machine_time": "", + "meta_type": "assertion", + "passed": true, + "second": 1, "type": "Equal", "type_actual": "int", - "second": 1, - "category": "DEFAULT", + "type_expected": "int", + "utc_time": "" } ], - "name": "basic_case ", "hash": 0, - "status_reason": None, + "definition_name": "basic_case ", + "logs": [], + "name": "basic_case ", "parent_uids": [ "InteractivePlan", "Test2", "BasicSuite", - "basic_case", + "basic_case" ], - "description": None, + "runtime_status": "finished", + "status": "passed", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, "timer": { "run": { - "start": "", "end": "", + "start": "" } }, - "uid": "basic_case__arg_1", - "tags": {}, - "status": "passed", "type": "TestCaseReport", - "runtime_status": "finished", - "suite_related": False, - "logs": [], - "category": "testcase", + "uid": "basic_case__arg_1" }, { - "counter": Counter( - {"passed": 1, "total": 1, "failed": 0} - ), - "status_override": None, + "category": "testcase", + "counter": { + "failed": 0, + "passed": 1, + "total": 1 + }, + "description": null, "entries": [ { - "machine_time": "", - "passed": True, - "type_expected": "int", - "flag": "DEFAULT", + "category": "DEFAULT", "description": "Passing assertion", - "meta_type": "assertion", - "line_no": 0, + "file_path": "", "first": 1, - "utc_time": "", + "flag": "DEFAULT", "label": "==", + "line_no": 0, + "machine_time": "", + "meta_type": "assertion", + "passed": true, + "second": 1, "type": "Equal", "type_actual": "int", - "second": 1, - "category": "DEFAULT", + "type_expected": "int", + "utc_time": "" } ], - "name": "basic_case ", "hash": 0, - "status_reason": None, + "definition_name": "basic_case ", + "logs": [], + "name": "basic_case ", "parent_uids": [ "InteractivePlan", "Test2", "BasicSuite", - "basic_case", + "basic_case" ], - "description": None, + "runtime_status": "finished", + "status": "passed", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, "timer": { "run": { - "start": "", "end": "", + "start": "" } }, - "uid": "basic_case__arg_2", - "tags": {}, - "status": "passed", "type": "TestCaseReport", - "runtime_status": "finished", - "suite_related": False, - "logs": [], - "category": "testcase", - }, + "uid": "basic_case__arg_2" + } ], - "strict_order": False, - "status": "passed", - "runtime_status": "finished", - "part": None, - "fix_spec_path": None, - "host": None, + "env_status": null, "events": {}, + "fix_spec_path": null, + "hash": 0, + "host": null, + "definition_name": "basic_case", + "logs": [], + "name": "basic_case", "parent_uids": [ "InteractivePlan", "Test2", - "BasicSuite", + "BasicSuite" ], - "type": "TestGroupReport", - "counter": Counter( - {"passed": 3, "total": 3, "failed": 0} - ), - "hash": 0, - "description": None, + "part": null, + "runtime_status": "finished", + "status": "passed", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, "timer": {}, - "uid": "basic_case", + "type": "TestGroupReport", + "uid": "basic_case" } ], - "strict_order": False, - "status": "passed", - "runtime_status": "finished", - "part": None, - "fix_spec_path": None, - "host": None, + "env_status": null, "events": {}, - "parent_uids": ["InteractivePlan", "Test2"], - "type": "TestGroupReport", - "counter": Counter({"passed": 3, "total": 3, "failed": 0}), + "fix_spec_path": null, "hash": 0, - "description": None, + "host": null, + "definition_name": "BasicSuite", + "logs": [], + "name": "BasicSuite", + "parent_uids": [ + "InteractivePlan", + "Test2" + ], + "part": null, + "runtime_status": "finished", + "status": "passed", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, "timer": {}, - "uid": "BasicSuite", + "type": "TestGroupReport", + "uid": "BasicSuite" }, { - "name": "Custom_0", - "status_reason": None, - "env_status": None, - "tags": {}, - "logs": [], "category": "testsuite", - "status_override": None, + "counter": { + "failed": 0, + "passed": 1, + "total": 1 + }, + "description": null, "entries": [ { - "counter": Counter( - {"passed": 1, "total": 1, "failed": 0} - ), - "status_override": None, + "category": "testcase", + "counter": { + "failed": 0, + "passed": 1, + "total": 1 + }, + "description": "Client sends a message, server received and responds back.", "entries": [ { - "machine_time": "", - "passed": True, - "type_expected": "str", - "flag": "DEFAULT", + "category": "DEFAULT", "description": "Server received", - "meta_type": "assertion", - "line_no": 0, + "file_path": "", "first": "hello", - "utc_time": "", + "flag": "DEFAULT", "label": "==", + "line_no": 0, + "machine_time": "", + "meta_type": "assertion", + "passed": true, + "second": "hello", "type": "Equal", "type_actual": "str", - "second": "hello", - "category": "DEFAULT", + "type_expected": "str", + "utc_time": "" }, { - "machine_time": "", - "passed": True, - "type_expected": "str", - "flag": "DEFAULT", + "category": "DEFAULT", "description": "Client received", - "meta_type": "assertion", - "line_no": 0, + "file_path": "", "first": "world", - "utc_time": "", + "flag": "DEFAULT", "label": "==", + "line_no": 0, + "machine_time": "", + "meta_type": "assertion", + "passed": true, + "second": "world", "type": "Equal", "type_actual": "str", - "second": "world", - "category": "DEFAULT", - }, + "type_expected": "str", + "utc_time": "" + } ], - "name": "send_and_receive_msg", "hash": 0, - "status_reason": None, + "definition_name": "send_and_receive_msg", + "logs": [], + "name": "send_and_receive_msg", "parent_uids": [ "InteractivePlan", "Test2", - "Custom_0", + "Custom_0" ], - "description": "Client sends a message, server received and responds back.", + "runtime_status": "finished", + "status": "passed", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, "timer": { "run": { - "start": "", "end": "", + "start": "" } }, - "uid": "send_and_receive_msg", - "tags": {}, - "status": "passed", "type": "TestCaseReport", - "runtime_status": "finished", - "suite_related": False, - "logs": [], - "category": "testcase", + "uid": "send_and_receive_msg" } ], - "strict_order": False, - "status": "passed", - "runtime_status": "finished", - "part": None, - "fix_spec_path": None, - "host": None, + "env_status": null, "events": {}, - "parent_uids": ["InteractivePlan", "Test2"], - "type": "TestGroupReport", - "counter": Counter({"passed": 1, "total": 1, "failed": 0}), + "fix_spec_path": null, "hash": 0, - "description": None, + "host": null, + "definition_name": "Custom_0", + "logs": [], + "name": "Custom_0", + "parent_uids": [ + "InteractivePlan", + "Test2" + ], + "part": null, + "runtime_status": "finished", + "status": "passed", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, "timer": {}, - "uid": "Custom_0", + "type": "TestGroupReport", + "uid": "Custom_0" }, { - "name": "Custom_1", - "status_reason": None, - "env_status": None, - "tags": {}, - "logs": [], "category": "testsuite", - "status_override": None, + "counter": { + "failed": 0, + "passed": 1, + "total": 1 + }, + "description": null, "entries": [ { - "counter": Counter( - {"passed": 1, "total": 1, "failed": 0} - ), - "status_override": None, + "category": "testcase", + "counter": { + "failed": 0, + "passed": 1, + "total": 1 + }, + "description": "Client sends a message, server received and responds back.", "entries": [ { - "machine_time": "", - "passed": True, - "type_expected": "str", - "flag": "DEFAULT", + "category": "DEFAULT", "description": "Server received", - "meta_type": "assertion", - "line_no": 0, + "file_path": "", "first": "hello", - "utc_time": "", + "flag": "DEFAULT", "label": "==", + "line_no": 0, + "machine_time": "", + "meta_type": "assertion", + "passed": true, + "second": "hello", "type": "Equal", "type_actual": "str", - "second": "hello", - "category": "DEFAULT", + "type_expected": "str", + "utc_time": "" }, { - "machine_time": "", - "passed": True, - "type_expected": "str", - "flag": "DEFAULT", + "category": "DEFAULT", "description": "Client received", - "meta_type": "assertion", - "line_no": 0, + "file_path": "", "first": "world", - "utc_time": "", + "flag": "DEFAULT", "label": "==", + "line_no": 0, + "machine_time": "", + "meta_type": "assertion", + "passed": true, + "second": "world", "type": "Equal", "type_actual": "str", - "second": "world", - "category": "DEFAULT", - }, + "type_expected": "str", + "utc_time": "" + } ], - "name": "send_and_receive_msg", "hash": 0, - "status_reason": None, + "definition_name": "send_and_receive_msg", + "logs": [], + "name": "send_and_receive_msg", "parent_uids": [ "InteractivePlan", "Test2", - "Custom_1", + "Custom_1" ], - "description": "Client sends a message, server received and responds back.", + "runtime_status": "finished", + "status": "passed", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, "timer": { "run": { - "start": "", "end": "", + "start": "" } }, - "uid": "send_and_receive_msg", - "tags": {}, - "status": "passed", "type": "TestCaseReport", - "runtime_status": "finished", - "suite_related": False, - "logs": [], - "category": "testcase", + "uid": "send_and_receive_msg" } ], - "strict_order": False, - "status": "passed", - "runtime_status": "finished", - "part": None, - "fix_spec_path": None, - "host": None, + "env_status": null, "events": {}, - "parent_uids": ["InteractivePlan", "Test2"], - "type": "TestGroupReport", - "counter": Counter({"passed": 1, "total": 1, "failed": 0}), + "fix_spec_path": null, "hash": 0, - "description": None, + "host": null, + "definition_name": "Custom_1", + "logs": [], + "name": "Custom_1", + "parent_uids": [ + "InteractivePlan", + "Test2" + ], + "part": null, + "runtime_status": "finished", + "status": "passed", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, "timer": {}, - "uid": "Custom_1", - }, + "type": "TestGroupReport", + "uid": "Custom_1" + } ], - "strict_order": False, - "status": "passed", - "runtime_status": "finished", - "part": None, - "fix_spec_path": None, - "host": None, + "env_status": "STARTED", "events": {}, - "parent_uids": ["InteractivePlan"], - "type": "TestGroupReport", - "counter": Counter({"passed": 5, "total": 5, "failed": 0}), + "fix_spec_path": null, "hash": 0, - "description": None, + "host": null, + "definition_name": "Test2", + "logs": [], + "name": "Test2", + "parent_uids": [ + "InteractivePlan" + ], + "part": null, + "runtime_status": "finished", + "status": "passed", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, "timer": {}, - "uid": "Test2", - }, + "type": "TestGroupReport", + "uid": "Test2" + } ], - "description": None, - "timeout": None, - "timer": {}, - "uid": "InteractivePlan", - "label": None, - "tags_index": {}, - "status": "passed", - "runtime_status": "finished", - "meta": {}, + "events": {}, + "information": [], + "label": null, "logs": [], - "attachments": {}, - "category": "testplan", -} + "meta": {}, + "name": "InteractivePlan", + "runtime_status": "finished", + "status": "passed", + "status_override": null, + "tags_index": {}, + "timeout": null, + "timer": {}, + "uid": "InteractivePlan" +} \ No newline at end of file diff --git a/tests/functional/testplan/runnable/interactive/reports/basic_top_level_reset.data b/tests/functional/testplan/runnable/interactive/reports/basic_top_level_reset.data new file mode 100644 index 000000000..3d71821ca --- /dev/null +++ b/tests/functional/testplan/runnable/interactive/reports/basic_top_level_reset.data @@ -0,0 +1,662 @@ +{ + "attachments": {}, + "category": "testplan", + "counter": { + "failed": 0, + "passed": 0, + "total": 10, + "unknown": 10 + }, + "description": null, + "entries": [ + { + "category": "multitest", + "counter": { + "failed": 0, + "passed": 0, + "total": 5, + "unknown": 5 + }, + "description": null, + "entries": [ + { + "category": "testsuite", + "counter": { + "failed": 0, + "passed": 0, + "total": 3, + "unknown": 3 + }, + "description": null, + "entries": [ + { + "category": "parametrization", + "counter": { + "failed": 0, + "passed": 0, + "total": 3, + "unknown": 3 + }, + "description": null, + "entries": [ + { + "category": "testcase", + "counter": { + "failed": 0, + "passed": 0, + "total": 1, + "unknown": 1 + }, + "description": null, + "entries": [], + "hash": 0, + "definition_name": "basic_case ", + "logs": [], + "name": "basic_case ", + "parent_uids": [ + "InteractivePlan", + "Test1", + "BasicSuite", + "basic_case" + ], + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, + "timer": {}, + "type": "TestCaseReport", + "uid": "basic_case__arg_0" + }, + { + "category": "testcase", + "counter": { + "failed": 0, + "passed": 0, + "total": 1, + "unknown": 1 + }, + "description": null, + "entries": [], + "hash": 0, + "definition_name": "basic_case ", + "logs": [], + "name": "basic_case ", + "parent_uids": [ + "InteractivePlan", + "Test1", + "BasicSuite", + "basic_case" + ], + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, + "timer": {}, + "type": "TestCaseReport", + "uid": "basic_case__arg_1" + }, + { + "category": "testcase", + "counter": { + "failed": 0, + "passed": 0, + "total": 1, + "unknown": 1 + }, + "description": null, + "entries": [], + "hash": 0, + "definition_name": "basic_case ", + "logs": [], + "name": "basic_case ", + "parent_uids": [ + "InteractivePlan", + "Test1", + "BasicSuite", + "basic_case" + ], + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, + "timer": {}, + "type": "TestCaseReport", + "uid": "basic_case__arg_2" + } + ], + "env_status": null, + "events": {}, + "fix_spec_path": null, + "hash": 0, + "host": null, + "definition_name": "basic_case", + "logs": [], + "name": "basic_case", + "parent_uids": [ + "InteractivePlan", + "Test1", + "BasicSuite" + ], + "part": null, + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, + "timer": {}, + "type": "TestGroupReport", + "uid": "basic_case" + } + ], + "env_status": null, + "events": {}, + "fix_spec_path": null, + "hash": 0, + "host": null, + "definition_name": "BasicSuite", + "logs": [], + "name": "BasicSuite", + "parent_uids": [ + "InteractivePlan", + "Test1" + ], + "part": null, + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, + "timer": {}, + "type": "TestGroupReport", + "uid": "BasicSuite" + }, + { + "category": "testsuite", + "counter": { + "failed": 0, + "passed": 0, + "total": 1, + "unknown": 1 + }, + "description": null, + "entries": [ + { + "category": "testcase", + "counter": { + "failed": 0, + "passed": 0, + "total": 1, + "unknown": 1 + }, + "description": "Client sends a message, server received and responds back.", + "entries": [], + "hash": 0, + "definition_name": "send_and_receive_msg", + "logs": [], + "name": "send_and_receive_msg", + "parent_uids": [ + "InteractivePlan", + "Test1", + "Custom_0" + ], + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, + "timer": {}, + "type": "TestCaseReport", + "uid": "send_and_receive_msg" + } + ], + "env_status": null, + "events": {}, + "fix_spec_path": null, + "hash": 0, + "host": null, + "definition_name": "Custom_0", + "logs": [], + "name": "Custom_0", + "parent_uids": [ + "InteractivePlan", + "Test1" + ], + "part": null, + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, + "timer": {}, + "type": "TestGroupReport", + "uid": "Custom_0" + }, + { + "category": "testsuite", + "counter": { + "failed": 0, + "passed": 0, + "total": 1, + "unknown": 1 + }, + "description": null, + "entries": [ + { + "category": "testcase", + "counter": { + "failed": 0, + "passed": 0, + "total": 1, + "unknown": 1 + }, + "description": "Client sends a message, server received and responds back.", + "entries": [], + "hash": 0, + "definition_name": "send_and_receive_msg", + "logs": [], + "name": "send_and_receive_msg", + "parent_uids": [ + "InteractivePlan", + "Test1", + "Custom_1" + ], + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, + "timer": {}, + "type": "TestCaseReport", + "uid": "send_and_receive_msg" + } + ], + "env_status": null, + "events": {}, + "fix_spec_path": null, + "hash": 0, + "host": null, + "definition_name": "Custom_1", + "logs": [], + "name": "Custom_1", + "parent_uids": [ + "InteractivePlan", + "Test1" + ], + "part": null, + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, + "timer": {}, + "type": "TestGroupReport", + "uid": "Custom_1" + } + ], + "env_status": "STOPPED", + "events": {}, + "fix_spec_path": null, + "hash": 0, + "host": null, + "definition_name": "Test1", + "logs": [], + "name": "Test1", + "parent_uids": [ + "InteractivePlan" + ], + "part": null, + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, + "timer": {}, + "type": "TestGroupReport", + "uid": "Test1" + }, + { + "category": "multitest", + "counter": { + "failed": 0, + "passed": 0, + "total": 5, + "unknown": 5 + }, + "description": null, + "entries": [ + { + "category": "testsuite", + "counter": { + "failed": 0, + "passed": 0, + "total": 3, + "unknown": 3 + }, + "description": null, + "entries": [ + { + "category": "parametrization", + "counter": { + "failed": 0, + "passed": 0, + "total": 3, + "unknown": 3 + }, + "description": null, + "entries": [ + { + "category": "testcase", + "counter": { + "failed": 0, + "passed": 0, + "total": 1, + "unknown": 1 + }, + "description": null, + "entries": [], + "hash": 0, + "definition_name": "basic_case ", + "logs": [], + "name": "basic_case ", + "parent_uids": [ + "InteractivePlan", + "Test2", + "BasicSuite", + "basic_case" + ], + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, + "timer": {}, + "type": "TestCaseReport", + "uid": "basic_case__arg_0" + }, + { + "category": "testcase", + "counter": { + "failed": 0, + "passed": 0, + "total": 1, + "unknown": 1 + }, + "description": null, + "entries": [], + "hash": 0, + "definition_name": "basic_case ", + "logs": [], + "name": "basic_case ", + "parent_uids": [ + "InteractivePlan", + "Test2", + "BasicSuite", + "basic_case" + ], + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, + "timer": {}, + "type": "TestCaseReport", + "uid": "basic_case__arg_1" + }, + { + "category": "testcase", + "counter": { + "failed": 0, + "passed": 0, + "total": 1, + "unknown": 1 + }, + "description": null, + "entries": [], + "hash": 0, + "definition_name": "basic_case ", + "logs": [], + "name": "basic_case ", + "parent_uids": [ + "InteractivePlan", + "Test2", + "BasicSuite", + "basic_case" + ], + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, + "timer": {}, + "type": "TestCaseReport", + "uid": "basic_case__arg_2" + } + ], + "env_status": null, + "events": {}, + "fix_spec_path": null, + "hash": 0, + "host": null, + "definition_name": "basic_case", + "logs": [], + "name": "basic_case", + "parent_uids": [ + "InteractivePlan", + "Test2", + "BasicSuite" + ], + "part": null, + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, + "timer": {}, + "type": "TestGroupReport", + "uid": "basic_case" + } + ], + "env_status": null, + "events": {}, + "fix_spec_path": null, + "hash": 0, + "host": null, + "definition_name": "BasicSuite", + "logs": [], + "name": "BasicSuite", + "parent_uids": [ + "InteractivePlan", + "Test2" + ], + "part": null, + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, + "timer": {}, + "type": "TestGroupReport", + "uid": "BasicSuite" + }, + { + "category": "testsuite", + "counter": { + "failed": 0, + "passed": 0, + "total": 1, + "unknown": 1 + }, + "description": null, + "entries": [ + { + "category": "testcase", + "counter": { + "failed": 0, + "passed": 0, + "total": 1, + "unknown": 1 + }, + "description": "Client sends a message, server received and responds back.", + "entries": [], + "hash": 0, + "definition_name": "send_and_receive_msg", + "logs": [], + "name": "send_and_receive_msg", + "parent_uids": [ + "InteractivePlan", + "Test2", + "Custom_0" + ], + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, + "timer": {}, + "type": "TestCaseReport", + "uid": "send_and_receive_msg" + } + ], + "env_status": null, + "events": {}, + "fix_spec_path": null, + "hash": 0, + "host": null, + "definition_name": "Custom_0", + "logs": [], + "name": "Custom_0", + "parent_uids": [ + "InteractivePlan", + "Test2" + ], + "part": null, + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, + "timer": {}, + "type": "TestGroupReport", + "uid": "Custom_0" + }, + { + "category": "testsuite", + "counter": { + "failed": 0, + "passed": 0, + "total": 1, + "unknown": 1 + }, + "description": null, + "entries": [ + { + "category": "testcase", + "counter": { + "failed": 0, + "passed": 0, + "total": 1, + "unknown": 1 + }, + "description": "Client sends a message, server received and responds back.", + "entries": [], + "hash": 0, + "definition_name": "send_and_receive_msg", + "logs": [], + "name": "send_and_receive_msg", + "parent_uids": [ + "InteractivePlan", + "Test2", + "Custom_1" + ], + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "suite_related": false, + "tags": {}, + "timer": {}, + "type": "TestCaseReport", + "uid": "send_and_receive_msg" + } + ], + "env_status": null, + "events": {}, + "fix_spec_path": null, + "hash": 0, + "host": null, + "definition_name": "Custom_1", + "logs": [], + "name": "Custom_1", + "parent_uids": [ + "InteractivePlan", + "Test2" + ], + "part": null, + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, + "timer": {}, + "type": "TestGroupReport", + "uid": "Custom_1" + } + ], + "env_status": "STOPPED", + "events": {}, + "fix_spec_path": null, + "hash": 0, + "host": null, + "definition_name": "Test2", + "logs": [], + "name": "Test2", + "parent_uids": [ + "InteractivePlan" + ], + "part": null, + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "status_reason": null, + "strict_order": false, + "tags": {}, + "timer": {}, + "type": "TestGroupReport", + "uid": "Test2" + } + ], + "events": {}, + "information": [], + "label": null, + "logs": [], + "meta": {}, + "name": "InteractivePlan", + "runtime_status": "ready", + "status": "unknown", + "status_override": null, + "tags_index": {}, + "timeout": null, + "timer": {}, + "uid": "InteractivePlan" +} \ No newline at end of file diff --git a/tests/functional/testplan/runnable/interactive/reports/basic_top_level_reset.py b/tests/functional/testplan/runnable/interactive/reports/basic_top_level_reset.py deleted file mode 100644 index b79344035..000000000 --- a/tests/functional/testplan/runnable/interactive/reports/basic_top_level_reset.py +++ /dev/null @@ -1,617 +0,0 @@ -from collections import Counter - -REPORT = { - "description": None, - "status_override": None, - "name": "InteractivePlan", - "information": [], - "attachments": {}, - "events": {}, - "uid": "InteractivePlan", - "logs": [], - "counter": Counter({"total": 10, "unknown": 10, "passed": 0, "failed": 0}), - "entries": [ - { - "counter": Counter( - {"total": 5, "unknown": 5, "passed": 0, "failed": 0} - ), - "category": "multitest", - "fix_spec_path": None, - "host": None, - "events": {}, - "name": "Test1", - "type": "TestGroupReport", - "status": "unknown", - "strict_order": False, - "logs": [], - "part": None, - "hash": 0, - "description": None, - "status_override": None, - "entries": [ - { - "counter": Counter( - {"total": 3, "unknown": 3, "passed": 0, "failed": 0} - ), - "category": "testsuite", - "fix_spec_path": None, - "host": None, - "events": {}, - "name": "BasicSuite", - "type": "TestGroupReport", - "status": "unknown", - "strict_order": False, - "logs": [], - "part": None, - "hash": 0, - "description": None, - "status_override": None, - "entries": [ - { - "counter": Counter( - { - "total": 3, - "unknown": 3, - "passed": 0, - "failed": 0, - } - ), - "category": "parametrization", - "fix_spec_path": None, - "host": None, - "events": {}, - "name": "basic_case", - "type": "TestGroupReport", - "status": "unknown", - "strict_order": False, - "logs": [], - "part": None, - "hash": 0, - "description": None, - "status_override": None, - "entries": [ - { - "description": None, - "status_override": None, - "name": "basic_case ", - "entries": [], - "logs": [], - "uid": "basic_case__arg_0", - "counter": Counter( - { - "total": 1, - "unknown": 1, - "passed": 0, - "failed": 0, - } - ), - "parent_uids": [ - "InteractivePlan", - "Test1", - "BasicSuite", - "basic_case", - ], - "category": "testcase", - "timer": {}, - "tags": {}, - "type": "TestCaseReport", - "status": "unknown", - "hash": 0, - "runtime_status": "ready", - "suite_related": False, - "status_reason": None, - }, - { - "description": None, - "status_override": None, - "name": "basic_case ", - "entries": [], - "logs": [], - "uid": "basic_case__arg_1", - "counter": Counter( - { - "total": 1, - "unknown": 1, - "passed": 0, - "failed": 0, - } - ), - "parent_uids": [ - "InteractivePlan", - "Test1", - "BasicSuite", - "basic_case", - ], - "category": "testcase", - "timer": {}, - "tags": {}, - "type": "TestCaseReport", - "status": "unknown", - "hash": 0, - "runtime_status": "ready", - "suite_related": False, - "status_reason": None, - }, - { - "description": None, - "status_override": None, - "name": "basic_case ", - "entries": [], - "logs": [], - "uid": "basic_case__arg_2", - "counter": Counter( - { - "total": 1, - "unknown": 1, - "passed": 0, - "failed": 0, - } - ), - "parent_uids": [ - "InteractivePlan", - "Test1", - "BasicSuite", - "basic_case", - ], - "category": "testcase", - "timer": {}, - "tags": {}, - "type": "TestCaseReport", - "status": "unknown", - "hash": 0, - "runtime_status": "ready", - "suite_related": False, - "status_reason": None, - }, - ], - "uid": "basic_case", - "parent_uids": [ - "InteractivePlan", - "Test1", - "BasicSuite", - ], - "timer": {}, - "tags": {}, - "env_status": None, - "runtime_status": "ready", - "status_reason": None, - } - ], - "uid": "BasicSuite", - "parent_uids": ["InteractivePlan", "Test1"], - "timer": {}, - "tags": {}, - "env_status": None, - "runtime_status": "ready", - "status_reason": None, - }, - { - "counter": Counter( - {"total": 1, "unknown": 1, "passed": 0, "failed": 0} - ), - "category": "testsuite", - "fix_spec_path": None, - "host": None, - "events": {}, - "name": "Custom_0", - "type": "TestGroupReport", - "status": "unknown", - "strict_order": False, - "logs": [], - "part": None, - "hash": 0, - "description": None, - "status_override": None, - "entries": [ - { - "description": "Client sends a message, server received and responds back.", - "status_override": None, - "name": "send_and_receive_msg", - "entries": [], - "logs": [], - "uid": "send_and_receive_msg", - "counter": Counter( - { - "total": 1, - "unknown": 1, - "passed": 0, - "failed": 0, - } - ), - "parent_uids": [ - "InteractivePlan", - "Test1", - "Custom_0", - ], - "category": "testcase", - "timer": {}, - "tags": {}, - "type": "TestCaseReport", - "status": "unknown", - "hash": 0, - "runtime_status": "ready", - "suite_related": False, - "status_reason": None, - } - ], - "uid": "Custom_0", - "parent_uids": ["InteractivePlan", "Test1"], - "timer": {}, - "tags": {}, - "env_status": None, - "runtime_status": "ready", - "status_reason": None, - }, - { - "counter": Counter( - {"total": 1, "unknown": 1, "passed": 0, "failed": 0} - ), - "category": "testsuite", - "fix_spec_path": None, - "host": None, - "events": {}, - "name": "Custom_1", - "type": "TestGroupReport", - "status": "unknown", - "strict_order": False, - "logs": [], - "part": None, - "hash": 0, - "description": None, - "status_override": None, - "entries": [ - { - "description": "Client sends a message, server received and responds back.", - "status_override": None, - "name": "send_and_receive_msg", - "entries": [], - "logs": [], - "uid": "send_and_receive_msg", - "counter": Counter( - { - "total": 1, - "unknown": 1, - "passed": 0, - "failed": 0, - } - ), - "parent_uids": [ - "InteractivePlan", - "Test1", - "Custom_1", - ], - "category": "testcase", - "timer": {}, - "tags": {}, - "type": "TestCaseReport", - "status": "unknown", - "hash": 0, - "runtime_status": "ready", - "suite_related": False, - "status_reason": None, - } - ], - "uid": "Custom_1", - "parent_uids": ["InteractivePlan", "Test1"], - "timer": {}, - "tags": {}, - "env_status": None, - "runtime_status": "ready", - "status_reason": None, - }, - ], - "uid": "Test1", - "parent_uids": ["InteractivePlan"], - "timer": {}, - "tags": {}, - "env_status": "STOPPED", - "runtime_status": "ready", - "status_reason": None, - }, - { - "counter": Counter( - {"total": 5, "unknown": 5, "passed": 0, "failed": 0} - ), - "category": "multitest", - "fix_spec_path": None, - "host": None, - "events": {}, - "name": "Test2", - "type": "TestGroupReport", - "status": "unknown", - "strict_order": False, - "logs": [], - "part": None, - "hash": 0, - "description": None, - "status_override": None, - "entries": [ - { - "counter": Counter( - {"total": 3, "unknown": 3, "passed": 0, "failed": 0} - ), - "category": "testsuite", - "fix_spec_path": None, - "host": None, - "events": {}, - "name": "BasicSuite", - "type": "TestGroupReport", - "status": "unknown", - "strict_order": False, - "logs": [], - "part": None, - "hash": 0, - "description": None, - "status_override": None, - "entries": [ - { - "counter": Counter( - { - "total": 3, - "unknown": 3, - "passed": 0, - "failed": 0, - } - ), - "category": "parametrization", - "fix_spec_path": None, - "host": None, - "events": {}, - "name": "basic_case", - "type": "TestGroupReport", - "status": "unknown", - "strict_order": False, - "logs": [], - "part": None, - "hash": 0, - "description": None, - "status_override": None, - "entries": [ - { - "description": None, - "status_override": None, - "name": "basic_case ", - "entries": [], - "logs": [], - "uid": "basic_case__arg_0", - "counter": Counter( - { - "total": 1, - "unknown": 1, - "passed": 0, - "failed": 0, - } - ), - "parent_uids": [ - "InteractivePlan", - "Test2", - "BasicSuite", - "basic_case", - ], - "category": "testcase", - "timer": {}, - "tags": {}, - "type": "TestCaseReport", - "status": "unknown", - "hash": 0, - "runtime_status": "ready", - "suite_related": False, - "status_reason": None, - }, - { - "description": None, - "status_override": None, - "name": "basic_case ", - "entries": [], - "logs": [], - "uid": "basic_case__arg_1", - "counter": Counter( - { - "total": 1, - "unknown": 1, - "passed": 0, - "failed": 0, - } - ), - "parent_uids": [ - "InteractivePlan", - "Test2", - "BasicSuite", - "basic_case", - ], - "category": "testcase", - "timer": {}, - "tags": {}, - "type": "TestCaseReport", - "status": "unknown", - "hash": 0, - "runtime_status": "ready", - "suite_related": False, - "status_reason": None, - }, - { - "description": None, - "status_override": None, - "name": "basic_case ", - "entries": [], - "logs": [], - "uid": "basic_case__arg_2", - "counter": Counter( - { - "total": 1, - "unknown": 1, - "passed": 0, - "failed": 0, - } - ), - "parent_uids": [ - "InteractivePlan", - "Test2", - "BasicSuite", - "basic_case", - ], - "category": "testcase", - "timer": {}, - "tags": {}, - "type": "TestCaseReport", - "status": "unknown", - "hash": 0, - "runtime_status": "ready", - "suite_related": False, - "status_reason": None, - }, - ], - "uid": "basic_case", - "parent_uids": [ - "InteractivePlan", - "Test2", - "BasicSuite", - ], - "timer": {}, - "tags": {}, - "env_status": None, - "runtime_status": "ready", - "status_reason": None, - } - ], - "uid": "BasicSuite", - "parent_uids": ["InteractivePlan", "Test2"], - "timer": {}, - "tags": {}, - "env_status": None, - "runtime_status": "ready", - "status_reason": None, - }, - { - "counter": Counter( - {"total": 1, "unknown": 1, "passed": 0, "failed": 0} - ), - "category": "testsuite", - "fix_spec_path": None, - "host": None, - "events": {}, - "name": "Custom_0", - "type": "TestGroupReport", - "status": "unknown", - "strict_order": False, - "logs": [], - "part": None, - "hash": 0, - "description": None, - "status_override": None, - "entries": [ - { - "description": "Client sends a message, server received and responds back.", - "status_override": None, - "name": "send_and_receive_msg", - "entries": [], - "logs": [], - "uid": "send_and_receive_msg", - "counter": Counter( - { - "total": 1, - "unknown": 1, - "passed": 0, - "failed": 0, - } - ), - "parent_uids": [ - "InteractivePlan", - "Test2", - "Custom_0", - ], - "category": "testcase", - "timer": {}, - "tags": {}, - "type": "TestCaseReport", - "status": "unknown", - "hash": 0, - "runtime_status": "ready", - "suite_related": False, - "status_reason": None, - } - ], - "uid": "Custom_0", - "parent_uids": ["InteractivePlan", "Test2"], - "timer": {}, - "tags": {}, - "env_status": None, - "runtime_status": "ready", - "status_reason": None, - }, - { - "counter": Counter( - {"total": 1, "unknown": 1, "passed": 0, "failed": 0} - ), - "category": "testsuite", - "fix_spec_path": None, - "host": None, - "events": {}, - "name": "Custom_1", - "type": "TestGroupReport", - "status": "unknown", - "strict_order": False, - "logs": [], - "part": None, - "hash": 0, - "description": None, - "status_override": None, - "entries": [ - { - "description": "Client sends a message, server received and responds back.", - "status_override": None, - "name": "send_and_receive_msg", - "entries": [], - "logs": [], - "uid": "send_and_receive_msg", - "counter": Counter( - { - "total": 1, - "unknown": 1, - "passed": 0, - "failed": 0, - } - ), - "parent_uids": [ - "InteractivePlan", - "Test2", - "Custom_1", - ], - "category": "testcase", - "timer": {}, - "tags": {}, - "type": "TestCaseReport", - "status": "unknown", - "hash": 0, - "runtime_status": "ready", - "suite_related": False, - "status_reason": None, - } - ], - "uid": "Custom_1", - "parent_uids": ["InteractivePlan", "Test2"], - "timer": {}, - "tags": {}, - "env_status": None, - "runtime_status": "ready", - "status_reason": None, - }, - ], - "uid": "Test2", - "parent_uids": ["InteractivePlan"], - "timer": {}, - "tags": {}, - "env_status": "STOPPED", - "runtime_status": "ready", - "status_reason": None, - }, - ], - "category": "testplan", - "timer": {}, - "tags_index": {}, - "timeout": None, - "status": "unknown", - "runtime_status": "ready", - "meta": {}, - "label": None, -} diff --git a/tests/functional/testplan/runnable/interactive/test_interactive.py b/tests/functional/testplan/runnable/interactive/test_interactive.py index 5830b7a2f..f2af46d5c 100644 --- a/tests/functional/testplan/runnable/interactive/test_interactive.py +++ b/tests/functional/testplan/runnable/interactive/test_interactive.py @@ -1,6 +1,8 @@ """Interactive mode tests.""" +import json import os +from pathlib import Path import requests from pytest_test_filters import skip_on_windows @@ -12,8 +14,8 @@ from testplan.common.utils.logger import USER_INFO from testplan.common.utils.timing import wait from testplan.environment import LocalEnvironment -from testplan.testing.multitest import MultiTest, testsuite, testcase -from testplan.testing.multitest.driver.tcp import TCPServer, TCPClient +from testplan.testing.multitest import MultiTest, testcase, testsuite +from testplan.testing.multitest.driver.tcp import TCPClient, TCPServer from tests.functional.testplan.runnable.interactive import ( wait_for_interactive_start, ) @@ -21,22 +23,9 @@ THIS_DIRECTORY = os.path.dirname(os.path.abspath(__file__)) -def _assert_http_response( - response, - operation, - mode, - result=None, - error=False, - metadata=None, - trace=None, -): - assert response == { - "result": result, - "error": error, - "metadata": metadata or {}, - "trace": trace, - "message": "{} operation performed: {}".format(mode, operation), - } +def load_from_json(path: Path) -> dict: + with open(path, "r") as f: + return json.load(f) class InteractivePlan: @@ -132,9 +121,11 @@ def test_top_level_tests(): assert resource.status == resource.STATUS.STOPPED # RESET REPORTS - plan.interactive.reset_all_tests() - from .reports.basic_top_level_reset import REPORT as BTLReset + plan.i.reset_all_tests() + BTLReset = load_from_json( + Path(__file__).parent / "reports" / "basic_top_level_reset.data" + ) assert ( compare( BTLReset, @@ -145,9 +136,11 @@ def test_top_level_tests(): ) # RUN ALL TESTS - plan.interactive.run_all_tests() - from .reports.basic_top_level import REPORT as BTLevel + plan.i.run_all_tests() + BTLevel = load_from_json( + Path(__file__).parent / "reports" / "basic_top_level.data" + ) assert ( compare( BTLevel, @@ -166,8 +159,7 @@ def test_top_level_tests(): ) # RESET REPORTS - plan.interactive.reset_all_tests() - from .reports.basic_top_level_reset import REPORT as BTLReset + plan.i.reset_all_tests() assert ( compare( @@ -179,22 +171,26 @@ def test_top_level_tests(): ) # RUN SINGLE TESTSUITE (CUSTOM NAME) - plan.interactive.run_test_suite("Test2", "TCPSuite - Custom_1") - from .reports.basic_run_suite_test2 import REPORT as BRSTest2 + plan.i.run_test_suite("Test2", "TCPSuite - Custom_1") + BRSTest2 = load_from_json( + Path(__file__).parent / "reports" / "basic_run_suite_test2.data" + ) assert ( compare( BRSTest2, - plan.interactive.test_report("Test2"), - ignore=["hash"], + plan.i.test_report("Test2"), + ignore=["hash", "information"], )[0] is True ) # RUN SINGLE TESTCASE - plan.interactive.run_test_case("Test1", "*", "basic_case__arg_1") - from .reports.basic_run_case_test1 import REPORT as BRCTest1 + plan.i.run_test_case("Test1", "*", "basic_case__arg_1") + BRCTest1 = load_from_json( + Path(__file__).parent / "reports" / "basic_run_case_test1.data" + ) assert ( compare( BRCTest1, diff --git a/tests/functional/testplan/testing/multitest/test_xfail.py b/tests/functional/testplan/testing/multitest/test_xfail.py index 27778129e..42bfaa785 100644 --- a/tests/functional/testplan/testing/multitest/test_xfail.py +++ b/tests/functional/testplan/testing/multitest/test_xfail.py @@ -37,6 +37,8 @@ def test_pass(self, env, result, val): class DynamicXfailSuite: """A test suite to check dynamic allocation of xfail decorator""" + name = "DynamicXfailSuiteAlias" + @testcase def test_1(self, env, result): result.equal(2, 3) @@ -60,11 +62,11 @@ def test_dynamic_xfail(): plan = TestplanMock( name="dynamic_xfail_test", xfail_tests={ - "Dummy:DynamicXfailSuite:test_1": { + "Dummy:DynamicXfailSuiteAlias:test_1": { "reason": "known flaky", "strict": False, }, - "Dummy:DynamicXfailSuite:test_2 ": { + "Dummy:DynamicXfailSuiteAlias:test_2 ": { "reason": "unknown non-flaky", "strict": True, }, diff --git a/tests/functional/testplan/testing/test_filtering.py b/tests/functional/testplan/testing/test_filtering.py index 5675ad40d..9551d54fc 100644 --- a/tests/functional/testplan/testing/test_filtering.py +++ b/tests/functional/testplan/testing/test_filtering.py @@ -194,6 +194,56 @@ def test_programmatic_filtering(filter_obj, report_ctx): check_report_context(test_report, report_ctx) +@pytest.mark.parametrize( + "filter_obj, report_ctx", + ( + # Case 1, part not specified + ( + filtering.Pattern("XXX:Alpha:test_one") + | filtering.Pattern("XXX:Alpha:test_two"), + [ + ("XXX - part(0/3)", [("Alpha", ["test_one"])]), + ("XXX - part(1/3)", [("Alpha", ["test_two"])]), + ], + ), + # Case 2, part specified + ( + filtering.Pattern("XXX - part(0/3):Alpha:test_one") + | filtering.Pattern("XXX - part(0/3):Beta:test_three"), + [ + ("XXX - part(0/3)", [("Alpha", ["test_one"])]), + ], + ), + # Case 3, unix filename pattern in part + ( + filtering.Pattern("XXX - part([012]/*):Alpha") + | filtering.Pattern("XXX:Beta:test_three"), + [ + ("XXX - part(0/3)", [("Alpha", ["test_one"])]), + ("XXX - part(1/3)", [("Alpha", ["test_two"])]), + ( + "XXX - part(2/3)", + [("Alpha", ["test_three"]), ("Beta", ["test_three"])], + ), + ], + ), + # Case 4, ill-formed part + ( + filtering.Pattern("XXX - part*"), + [], + ), + ), +) +def test_programmatic_filtering_with_parts(filter_obj, report_ctx): + plan = TestplanMock(name="plan", test_filter=filter_obj) + for i in range(0, 3): + plan.add(MultiTest(name="XXX", suites=[Alpha(), Beta()], part=(i, 3))) + plan.run() + + test_report = plan.report + check_report_context(test_report, report_ctx) + + @pytest.mark.parametrize( "cmdline_args, report_ctx", ( @@ -226,7 +276,7 @@ def test_programmatic_filtering(filter_obj, report_ctx): ), # Case 4, tag filtering ( - ("--tags", "bar color=red"), + ("--tags", "bar", "color=red"), [ ( "XXX", @@ -244,9 +294,11 @@ def test_programmatic_filtering(filter_obj, report_ctx): # as they belong to the same category (tags) ( "--tags", - "bar color=blue", # bar OR color=blue + "bar", + "color=blue", # bar OR color=blue "--tags-all", - "baz color=red", # baz AND color=red + "baz", + "color=red", # baz AND color=red ), [ ( @@ -268,9 +320,11 @@ def test_programmatic_filtering(filter_obj, report_ctx): "*:*:test_one", "*:*:test_three", "--tags", - "bar color=blue", # bar OR color=blue + "bar", + "color=blue", # bar OR color=blue "--tags-all", - "baz color=red", # baz AND color=red + "baz", + "color=red", # baz AND color=red ), [ ("XXX", [("Beta", ["test_one", "test_three"])]), @@ -305,14 +359,12 @@ def test_command_line_filtering(cmdline_args, report_ctx): @pytest.mark.parametrize( "lines, report_ctx", ( - # Case 1, empty file, nothing matched - ([], []), - # Case 2, single line of pattern + # Case 1, single line of pattern ( ["XXX:*:test_two"], [("XXX", [("Alpha", ["test_two"]), ("Beta", ["test_two"])])], ), - # Case 3, multiple lines of pattern + # Case 2, multiple lines of pattern ( ["XXX:*:test_two", "YYY:*:test_three"], [ diff --git a/tests/functional/testplan/testing/test_listing.py b/tests/functional/testplan/testing/test_listing.py index 251dfd25d..bda1de2e2 100644 --- a/tests/functional/testplan/testing/test_listing.py +++ b/tests/functional/testplan/testing/test_listing.py @@ -1,20 +1,20 @@ import json +from functools import partial from pathlib import Path import boltons.iterutils import pytest -from testplan.testing.listing import SimpleJsonLister -from testplan.testing.multitest import MultiTest, testsuite, testcase - from testplan import TestplanMock +from testplan.common.utils.logger import TESTPLAN_LOGGER from testplan.common.utils.testing import ( - captured_logging, argv_overridden, + captured_logging, to_stdout, ) -from testplan.common.utils.logger import TESTPLAN_LOGGER -from testplan.testing import listing, filtering, ordering +from testplan.testing import filtering, listing, ordering +from testplan.testing.listing import SimpleJsonLister +from testplan.testing.multitest import MultiTest, testcase, testsuite @testsuite @@ -64,19 +64,39 @@ def test_a(self, env, result): DEFAULT_PATTERN_OUTPUT = to_stdout( "Primary", - " Primary::Beta --tags color=yellow", - " Primary::Beta::test_c", - " Primary::Beta::test_b --tags foo", - " Primary::Beta::test_a --tags color=red", - " Primary::Alpha", - " Primary::Alpha::test_c", - " Primary::Alpha::test_b --tags bar foo", - " Primary::Alpha::test_a --tags color=green", + " Primary:Beta --tags color=yellow", + " Primary:Beta:test_c", + " Primary:Beta:test_b --tags foo", + " Primary:Beta:test_a --tags color=red", + " Primary:Alpha", + " Primary:Alpha:test_c", + " Primary:Alpha:test_b --tags bar foo", + " Primary:Alpha:test_a --tags color=green", "Secondary", - " Secondary::Gamma", - " Secondary::Gamma::test_c", - " Secondary::Gamma::test_b --tags bar", - " Secondary::Gamma::test_a --tags color=blue", + " Secondary:Gamma", + " Secondary:Gamma:test_c", + " Secondary:Gamma:test_b --tags bar", + " Secondary:Gamma:test_a --tags color=blue", +) + +PATTERN_OUTPUT_WITH_PARTS = to_stdout( + "Primary - part(0/2)", + " Primary - part(0/2):Beta --tags color=yellow", + " Primary - part(0/2):Beta:test_c", + " Primary - part(0/2):Beta:test_a --tags color=red", + " Primary - part(0/2):Alpha", + " Primary - part(0/2):Alpha:test_c", + " Primary - part(0/2):Alpha:test_a --tags color=green", + "Primary - part(1/2)", + " Primary - part(1/2):Beta --tags color=yellow", + " Primary - part(1/2):Beta:test_b --tags foo", + " Primary - part(1/2):Alpha", + " Primary - part(1/2):Alpha:test_b --tags bar foo", + "Secondary", + " Secondary:Gamma", + " Secondary:Gamma:test_c", + " Secondary:Gamma:test_b --tags bar", + " Secondary:Gamma:test_a --tags color=blue", ) @@ -98,14 +118,30 @@ def test_a(self, env, result): ) +def prepare_plan(plan, prim_parts): + if prim_parts == 1: + plan.add(MultiTest(name="Primary", suites=[Beta(), Alpha()])) + else: + for i in range(prim_parts): + plan.add( + MultiTest( + name="Primary", + suites=[Beta(), Alpha()], + part=(i, prim_parts), + ) + ) + plan.add(MultiTest(name="Secondary", suites=[Gamma()])) + + @pytest.mark.parametrize( - "listing_obj,filter_obj,sorter_obj,expected_output", + "listing_obj,filter_obj,sorter_obj,prim_parts,expected_output", [ # Basic name listing ( listing.ExpandedNameLister(), filtering.Filter(), ordering.NoopSorter(), + 1, DEFAULT_NAME_OUTPUT, ), # Basic pattern listing @@ -113,6 +149,7 @@ def test_a(self, env, result): listing.ExpandedPatternLister(), filtering.Filter(), ordering.NoopSorter(), + 1, DEFAULT_PATTERN_OUTPUT, ), # Basic count listing @@ -120,16 +157,26 @@ def test_a(self, env, result): listing.CountLister(), filtering.Filter(), ordering.NoopSorter(), + 1, to_stdout( "Primary: (2 suites, 6 testcases)", "Secondary: (1 suite, 3 testcases)", ), ), + # Basic pattern listing with MultiTest parts + ( + listing.ExpandedPatternLister(), + filtering.Filter(), + ordering.NoopSorter(), + 2, + PATTERN_OUTPUT_WITH_PARTS, + ), # Custom sort & name listing ( listing.ExpandedNameLister(), filtering.Filter(), ordering.AlphanumericSorter(), + 1, to_stdout( "Primary", " Alpha", @@ -152,6 +199,7 @@ def test_a(self, env, result): listing.ExpandedNameLister(), filtering.Pattern("*:Alpha") | filtering.Pattern("*:Beta"), ordering.AlphanumericSorter(), + 1, to_stdout( "Primary", " Alpha", @@ -167,11 +215,8 @@ def test_a(self, env, result): ], ) def test_programmatic_listing( - runpath, listing_obj, filter_obj, sorter_obj, expected_output + runpath, listing_obj, filter_obj, sorter_obj, prim_parts, expected_output ): - multitest_x = MultiTest(name="Primary", suites=[Beta(), Alpha()]) - multitest_y = MultiTest(name="Secondary", suites=[Gamma()]) - plan = TestplanMock( name="plan", test_lister=listing_obj, @@ -181,9 +226,7 @@ def test_programmatic_listing( ) with captured_logging(TESTPLAN_LOGGER) as log_capture: - plan.add(multitest_x) - plan.add(multitest_y) - + prepare_plan(plan, prim_parts) assert log_capture.output == expected_output result = plan.run() @@ -191,39 +234,41 @@ def test_programmatic_listing( @pytest.mark.parametrize( - "cmdline_args,expected_output", + "cmdline_args,prim_parts,expected_output", [ - (["--list"], DEFAULT_NAME_OUTPUT), - (["--info", "pattern"], DEFAULT_PATTERN_OUTPUT), - (["--info", "name"], DEFAULT_NAME_OUTPUT), + (["--list"], 1, DEFAULT_NAME_OUTPUT), + (["--info", "pattern"], 1, DEFAULT_PATTERN_OUTPUT), + (["--info", "pattern"], 2, PATTERN_OUTPUT_WITH_PARTS), + (["--info", "name"], 1, DEFAULT_NAME_OUTPUT), ( ["--info", "name", "--patterns", "*:Alpha", "*:Beta:test_c"], + 2, to_stdout( - "Primary", + "Primary - part(0/2)", " Beta", " test_c", " Alpha", " test_c", - " test_b", " test_a", + "Primary - part(1/2)", + " Alpha", + " test_b", ), ), ], ) -def test_command_line_listing(runpath, cmdline_args, expected_output): - multitest_x = MultiTest(name="Primary", suites=[Beta(), Alpha()]) - multitest_y = MultiTest(name="Secondary", suites=[Gamma()]) +def test_command_line_listing( + runpath, cmdline_args, prim_parts, expected_output +): with argv_overridden(*cmdline_args): plan = TestplanMock(name="plan", parse_cmdline=True, runpath=runpath) with captured_logging(TESTPLAN_LOGGER) as log_capture: - plan.add(multitest_x) - plan.add(multitest_y) + prepare_plan(plan, prim_parts) + assert log_capture.output == expected_output result = plan.run() - - assert log_capture.output == expected_output assert len(result.test_report) == 0, "No tests should be run." @@ -271,10 +316,10 @@ def test_method(self, env, result, val): ( listing.PatternLister(), to_stdout( - *["Primary", " Primary::ParametrizedSuite"] + *["Primary", " Primary:ParametrizedSuite"] + [ - " Primary::ParametrizedSuite" - "::test_method ".format(idx) + " Primary:ParametrizedSuite" + ":test_method ".format(idx) for idx in range(listing.MAX_TESTCASES) ] + [ @@ -343,13 +388,6 @@ def validate_json_result(result_json): ) -def prepare_plan(plan): - multitest_x = MultiTest(name="Primary", suites=[Beta(), Alpha()]) - multitest_y = MultiTest(name="Secondary", suites=[Gamma()]) - plan.add(multitest_x) - plan.add(multitest_y) - - def test_json_listing(runpath): main = TestplanMock.main_wrapper( @@ -357,7 +395,7 @@ def test_json_listing(runpath): test_lister=SimpleJsonLister(), runpath=runpath, parse_cmdline=False, - )(prepare_plan) + )(partial(prepare_plan, prim_parts=1)) with captured_logging(TESTPLAN_LOGGER) as log_capture: main() @@ -374,7 +412,7 @@ def test_json_listing_to_file(runpath): test_lister_output=str(result_path), runpath=runpath, parse_cmdline=False, - )(prepare_plan) + )(partial(prepare_plan, prim_parts=1)) main() diff --git a/tests/unit/testplan/common/report/test_base.py b/tests/unit/testplan/common/report/test_base.py index d74d99a4c..a10e8240c 100644 --- a/tests/unit/testplan/common/report/test_base.py +++ b/tests/unit/testplan/common/report/test_base.py @@ -1,14 +1,13 @@ -import logging import functools +import logging import re -from unittest import mock import uuid +from unittest import mock import pytest from testplan.common import report from testplan.common.report.log import LOGGER - from testplan.common.utils.testing import disable_log_propagation DummyReport = functools.partial(report.Report, name="dummy") @@ -86,12 +85,12 @@ def test_equality_fail(self, override): assert rep_1 != rep_2 - def test_check_report_id_mismatch(self): + def test_check_report_definition_name_mismatch(self): """Should raise ValueError on failure""" - # These will have different ids - rep_1 = DummyReport(uid=1) - rep_2 = DummyReport(uid=2) + # These correspond to different tests + rep_1 = DummyReport(definition_name=1) + rep_2 = DummyReport(definition_name=2) with pytest.raises(AttributeError): rep_1._check_report(rep_2) @@ -102,8 +101,8 @@ def test_check_report_type_mismatch(self): class OtherReport(report.Report): pass - rep_1 = DummyReport(uid=5) - rep_2 = OtherReport(name="foo", uid=5) + rep_1 = DummyReport(name="foo") + rep_2 = OtherReport(name="foo") with pytest.raises(TypeError): rep_1._check_report(rep_2) diff --git a/tests/unit/testplan/testing/test_filtering.py b/tests/unit/testplan/testing/test_filtering.py index ee97f8955..0a8e8e7b7 100644 --- a/tests/unit/testplan/testing/test_filtering.py +++ b/tests/unit/testplan/testing/test_filtering.py @@ -70,6 +70,9 @@ def parametrized(self, env, result, val): multitest_D = MultiTest(name="DDD", suites=[Alpha(), Beta()]) multitest_E = MultiTest(name="EEE", suites=[Beta(), Gamma()]) multitest_F = MultiTest(name="FFF", suites=[Alpha(), Beta(), Gamma()]) +multitest_G0 = MultiTest(name="GGG", suites=[Alpha(), Beta()], part=(0, 3)) +multitest_G1 = MultiTest(name="GGG", suites=[Alpha(), Beta()], part=(1, 3)) +multitest_G2 = MultiTest(name="GGG", suites=[Alpha(), Beta()], part=(2, 3)) class TestTags: @@ -260,6 +263,10 @@ class TestPattern: ("A*:*", multitest_A, True), ("A*:*:*", multitest_A, True), ("AAA:foo:bar", multitest_A, True), + ("AAA - part(0/1):*:*", multitest_A, True), + ("GGG:Alpha:test_one", multitest_G0, True), + ("GGG - part(0/3):*", multitest_G0, True), + ("GGG - part(2/3):*", multitest_G1, False), ), ) def test_filter_test(self, pattern, multitest, expected):