diff --git a/docs/index.rst b/docs/index.rst index 0d6811cc5c..1961c2e07c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -21,28 +21,30 @@ Publications Presentations & Talks --------------------- -* Slides [`pdf `__][`talk `__] @ `9th EasyBuild User Meeting 2024 `__. -* Slides [`part 1 `__][`part 2 `__][`talk `__] @ `8th EasyBuild User Meeting 2023 `__. -* Slides [`pdf `__] @ `7th EasyBuild User Meeting 2022 `__. -* Slides [`pdf `__] @ `6th EasyBuild User Meeting 2021 `__. -* Slides [`pdf `__] @ `5th EasyBuild User Meeting 2020 `__. -* Slides [`pdf `__] @ `HPC System Testing BoF `__, SC'19. -* Slides [`pdf `__] @ `HUST 2019 `__, SC'19. -* Slides [`pdf `__] @ `HPC Knowledge Meeting '19 `__. -* Slides [`pdf `__] & `Talk `__ @ `FOSDEM'19 `__. -* Slides [`pdf `__] @ `4th EasyBuild User Meeting `__. -* Slides [`pdf `__] @ `HUST 2018 `__, SC'18. -* Slides [`pdf `__] @ `CSCS User Lab Day 2018 `__. -* Slides [`pdf `__] @ `HPC Advisory Council 2018 `__. -* Slides [`pdf `__] @ `SC17 `__. -* Slides [`pdf `__] @ `CUG 2017 `__. +* [`slides `__] "Introduction to ReFrame," CINECA visit, Jun 2024. +* [`slides `__][`recording `__] "Recent Advances in ReFrame," `9th EasyBuild User Meeting 2024 `__. +* [`slides `__][`recording `__] "Recent Advances in ReFrame," `8th EasyBuild User Meeting 2023 `__. +* [`slides `__][`recording `__] "Embracing ReFrame Programmable Configurations," `8th EasyBuild User Meeting 2023 `__. +* [`slides `__] "ReFrame Update," `7th EasyBuild User Meeting 2022 `__. +* [`slides `__] "Writing powerful HPC regression tests with ReFrame," `6th EasyBuild User Meeting 2021 `__ +* [`slides `__] "ReFrame: A Framework for Writing Regression Tests for HPC Systems," `5th EasyBuild User Meeting 2020 `__. +* [`slides `__] "Enabling Continuous Testing of HPC Systems using ReFrame," `HPC System Testing BoF `__, SC'19. +* [`slides `__] "Enabling Continuous Testing of HPC Systems using ReFrame," `HUST 2019 `__, SC'19. +* [`slides `__] "ReFrame: A Tool for Enabling Regression Testing and Continuous Integration for HPC Systems," `HPC Knowledge Meeting '19 `__. +* [`slides `__][`recording `__] "ReFrame: A Regression Testing and Continuous Integration Framework for HPC systems," `FOSDEM'19 `__. +* [`slides `__] "ReFrame: A Regression Testing and Continuous Integration Framework for HPC systems," `4th EasyBuild User Meeting `__. +* [`slides `__] "ReFrame: A Regression Testing and Continuous Integration Framework for HPC systems," `HUST 2018 `__, SC'18. +* [`slides `__] "Regression Testing and Continuous Integration with ReFrame," `CSCS User Lab Day 2018 `__. +* [`slides `__] "ReFrame: A Regression Testing Framework Enabling Continuous Integration of Large HPC Systems," `HPC Advisory Council 2018 `__. +* [`slides `__] "ReFrame: A Regression Testing Tool for HPC Systems," Regression testing BoF, `SC17 `__. +* [`slides `__] "ReFrame: A regression framework for checking the health of large HPC systems" `CUG 2017 `__. Webinars & Tutorials -------------------- -* "ReFrame – Efficient System and Application Performance Testing," CSCS Webinar, Aug. 29, 2022 [`slides `__] [`recording `__] [`demo run `__]. -* Tutorial at 6th EasyBuild User Meeting 2021 [`YouTube `__] +* [`slides `__][`recording `__][`demo run `__] "ReFrame – Efficient System and Application Performance Testing," CSCS Webinar, Aug. 29, 2022. +* [`recording `__] "ReFrame Tutorial," 6th EasyBuild User Meeting 2021. Papers diff --git a/docs/manpage.rst b/docs/manpage.rst index 156cf59955..d464c1e1e5 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -1043,7 +1043,7 @@ The way the tests are generated and how they interact with the test filtering op Parameterize a test on an existing variable. - This option will create a new test with a parameter named ``$VAR`` with the values given in the comma-separated list ``VAL0,VAL1,...``. + The test will behave as if the variable ``VAR`` was a parameter taking the values ``VAL0,VAL1,...``. The values will be converted based on the type of the target variable ``VAR``. The ``TEST.`` prefix will only parameterize the variable ``VAR`` of test ``TEST``. diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 50a4b558cb..326e14e73c 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -2135,8 +2135,3 @@ Finally, a stored session can be deleted using the :option:`--delete-stored-sess reframe --delete-stored-sessions=47e8d98f-e2b9-4019-9a41-1c44d8a53d1b Deleting a session will also delete all its test cases from the database. - - -.. tip:: - - You can disable results storage by either setting ``RFM_ENABLE_RESULTS_STORAGE=0`` or by setting the :attr:`storage.enable ` configuration parameter to ``False``. diff --git a/reframe/__init__.py b/reframe/__init__.py index 03e8efde2c..d2694b828b 100644 --- a/reframe/__init__.py +++ b/reframe/__init__.py @@ -6,7 +6,11 @@ import os import sys +<<<<<<< HEAD VERSION = '4.8.0-dev.0' +======= +VERSION = '4.7.1' +>>>>>>> master INSTALL_PREFIX = os.path.normpath( os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) ) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 5663f6e111..c7878e4272 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -1646,7 +1646,9 @@ def _resolve_fixtures(self): # registered under the same fixture class. So the loop below must # also inspect the fixture data the instance was registered with. for fixt_name, fixt_data in registry[f.cls].items(): - if f.scope != fixt_data.scope: + if fixt_data.variables != f.variables: + continue + elif f.scope != fixt_data.scope: continue elif fixt_data.variant_num not in target_variants: continue diff --git a/reframe/core/schedulers/slurm.py b/reframe/core/schedulers/slurm.py index 7b4a4bff07..235759ab00 100644 --- a/reframe/core/schedulers/slurm.py +++ b/reframe/core/schedulers/slurm.py @@ -351,6 +351,7 @@ def filternodes(self, job, nodes): option_parser.add_argument('-w', '--nodelist') option_parser.add_argument('-C', '--constraint') option_parser.add_argument('-x', '--exclude') + self.log(f'Filtering by Slurm options: {" ".join(options)}') parsed_args, _ = option_parser.parse_known_args(options) reservation = parsed_args.reservation partitions = parsed_args.partition @@ -363,7 +364,7 @@ def filternodes(self, job, nodes): else: nodes = {node for node in nodes if not node.in_state('RESERVED')} - self.log(f'[F] Filtering nodes by reservation={reservation}: ' + self.log(f'Filtering nodes by reservation={reservation}: ' f'available nodes now: {len(nodes)}') if partitions: @@ -377,27 +378,27 @@ def filternodes(self, job, nodes): ) partitions = {default_partition} if default_partition else set() self.log( - f'[F] No partition specified; using {default_partition!r}' + f'No partition specified; using {default_partition!r}' ) nodes = {n for n in nodes if n.partitions >= partitions} - self.log(f'[F] Filtering nodes by partition(s) {partitions}: ' + self.log(f'Filtering nodes by partition(s) {partitions}: ' f'available nodes now: {len(nodes)}') if constraints: nodes = {n for n in nodes if n.satisfies(constraints)} - self.log(f'[F] Filtering nodes by constraint(s) {constraints}: ' + self.log(f'Filtering nodes by constraint(s) {constraints}: ' f'available nodes now: {len(nodes)}') if nodelist: nodelist = nodelist.strip() nodes &= self._get_nodes_by_name(nodelist) - self.log(f'[F] Filtering nodes by nodelist: {nodelist}: ' + self.log(f'Filtering nodes by nodelist: {nodelist}: ' f'available nodes now: {len(nodes)}') if exclude_nodes: exclude_nodes = exclude_nodes.strip() nodes -= self._get_nodes_by_name(exclude_nodes) - self.log(f'[F] Excluding node(s): {exclude_nodes}: ' + self.log(f'Excluding node(s): {exclude_nodes}: ' f'available nodes now: {len(nodes)}') return nodes @@ -711,17 +712,29 @@ def is_down(self): return not self.is_avail() def satisfies(self, slurm_constraint): + def _replacemany(s, replacements): + for src, dst in replacements: + s = s.replace(src, dst) + + return s + # Convert the Slurm constraint to a Python expression and evaluate it, # but restrict our syntax to accept only AND or OR constraints and - # their combinations - if not re.match(r'^[\w\d\(\)\|\&]*$', slurm_constraint): + # their combinations; to properly treat `-` in constraints we need to + # convert them to valid Python identifiers before evaluating the + # constraint. + if not re.match(r'^[\-\w\d\(\)\|\&]*$', slurm_constraint): return False - names = {grp[0] - for grp in re.finditer(r'(\w(\w|\d)*)', slurm_constraint)} - expr = slurm_constraint.replace('|', ' or ').replace('&', ' and ') - vars = {n: True for n in self.active_features} - vars.update({n: False for n in names - self.active_features}) + names = { + grp[0] for grp in re.finditer(r'[\-\w][\-\w\d]*', slurm_constraint) + } + expr = _replacemany(slurm_constraint, + [('-', '_'), ('|', ' or '), ('&', ' and ')]) + vars = {n.replace('-', '_'): True for n in self.active_features} + vars.update({ + n.replace('-', '_'): False for n in names - self.active_features + }) try: return eval(expr, {}, vars) except BaseException: diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 5c699b452b..2337546bdd 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -1041,7 +1041,8 @@ def restrict_logging(): with exit_gracefully_on_error('failed to retrieve test case data', printer): printer.info(jsonext.dumps(reporting.testcase_info( - options.describe_stored_testcases, namepatt + options.describe_stored_testcases, + namepatt, options.filter_expr ), indent=2)) sys.exit(0) @@ -1637,9 +1638,12 @@ def module_unuse(*paths): if options.max_retries and runner.stats.failed(run=0): printer.retry_report(report) - # Print a failure report if we had failures in the last run + # Print a failure report in case of failures. + # If `--duration` or `--reruns` is used then take into account + # all runs, else (i.e., `--max-retries`) only the last run. success = True - if runner.stats.failed(): + runid = None if options.duration or options.reruns else -1 + if runner.stats.failed(run=runid): success = False printer.failure_report( report, @@ -1733,6 +1737,12 @@ def module_unuse(*paths): sys.exit(1) sys.exit(0) + except errors.RunSessionTimeout as err: + printer.warning(f'run session stopped: {err}') + if not success: + sys.exit(1) + else: + sys.exit(0) except (Exception, KeyboardInterrupt, errors.ReframeFatalError): exc_info = sys.exc_info() tb = ''.join(traceback.format_exception(*exc_info)) diff --git a/reframe/frontend/loader.py b/reframe/frontend/loader.py index 6486ebc77d..c05f7f69fe 100644 --- a/reframe/frontend/loader.py +++ b/reframe/frontend/loader.py @@ -193,17 +193,31 @@ def load_from_file(self, filename, force=False): try: dirname = os.path.dirname(filename) - with osext.change_dir(dirname): - with util.temp_sys_path(dirname): - if os.path.exists(os.path.join(dirname, '__init__.py')): - # If the containing directory is a package, - # import it, too. - parent = util.import_module_from_file(dirname).__name__ - else: - parent = None + # Load all parent modules of test file + parents = [] + while os.path.exists(os.path.join(dirname, '__init__.py')): + parents.append(os.path.join(dirname)) + dirname = os.path.split(dirname)[0] + + parent_module = None + for pdir in reversed(parents): + with osext.change_dir(pdir): + with util.temp_sys_path(pdir): + package_path = os.path.join(pdir, '__init__.py') + parent_module = util.import_module_from_file( + package_path, parent=parent_module + ).__name__ + + # Now load the actual test file + if not parents: + pdir = dirname + + with osext.change_dir(pdir): + with util.temp_sys_path(pdir): return self.load_from_module( - util.import_module_from_file(filename, force, parent) + util.import_module_from_file(filename, force, + parent_module) ) except Exception: exc_info = sys.exc_info() diff --git a/reframe/frontend/testgenerators.py b/reframe/frontend/testgenerators.py index b5f597d48a..041e4580cf 100644 --- a/reframe/frontend/testgenerators.py +++ b/reframe/frontend/testgenerators.py @@ -77,12 +77,12 @@ def _generate_tests(testcases, gen_fn): @time_function def distribute_tests(testcases, node_map): def _rfm_pin_run_nodes(obj): - nodelist = getattr(obj, '$nid') + nodelist = getattr(obj, '.nid') if not obj.local: obj.job.pin_nodes = nodelist def _rfm_pin_build_nodes(obj): - pin_nodes = getattr(obj, '$nid') + pin_nodes = getattr(obj, '.nid') if obj.build_job and not obj.local and not obj.build_locally: obj.build_job.pin_nodes = pin_nodes @@ -99,9 +99,9 @@ def _rfm_set_valid_systems(obj): 'valid_systems': [partition.fullname], # We add a partition parameter so as to differentiate the test # in case another test has the same nodes in another partition - '$part': builtins.parameter([partition.fullname], + '.part': builtins.parameter([partition.fullname], loggable=False), - '$nid': builtins.parameter( + '.nid': builtins.parameter( [[n] for n in node_map[partition.fullname]], fmt=util.nodelist_abbrev, loggable=False ) @@ -113,7 +113,7 @@ def _rfm_set_valid_systems(obj): # will not be overwritten by a parent post-init hook builtins.run_after('init')(_rfm_set_valid_systems), ] - ), ['$part', '$nid'] + ), ['.part', '.nid'] return _generate_tests(testcases, _make_dist_test) @@ -127,10 +127,10 @@ def _make_repeat_test(testcase): return make_test( cls.__name__, (cls,), { - '$repeat_no': builtins.parameter(range(num_repeats), + '.repeat_no': builtins.parameter(range(num_repeats), loggable=False) } - ), ['$repeat_no'] + ), ['.repeat_no'] return _generate_tests(testcases, _make_repeat_test) @@ -164,7 +164,7 @@ def _make_parameterized_test(testcase): ) continue - body[f'${var}'] = builtins.parameter(values, loggable=False) + body[f'.{var}'] = builtins.parameter(values, loggable=False) def _set_vars(self): for var in body.keys(): diff --git a/reframe/utility/__init__.py b/reframe/utility/__init__.py index 75c4008e3c..cfa38610ec 100644 --- a/reframe/utility/__init__.py +++ b/reframe/utility/__init__.py @@ -866,7 +866,7 @@ def count_digits(n): ''' num_digits = 1 - while n > 10: + while n >= 10: n /= 10 num_digits += 1 diff --git a/unittests/resources/checks_unlisted/fixtures_complex.py b/unittests/resources/checks_unlisted/fixtures_complex.py index 53be46f92f..d200502b82 100644 --- a/unittests/resources/checks_unlisted/fixtures_complex.py +++ b/unittests/resources/checks_unlisted/fixtures_complex.py @@ -102,3 +102,17 @@ def validate_fixture_resolution(self): ParamFixture.num_variants ) ]) + + +@rfm.simple_test +class TestC(rfm.RunOnlyRegressionTest): + valid_systems = ['*'] + valid_prog_environs = ['*'] + executable = 'echo' + f0 = fixture(SimpleFixture, scope='environment', variables={'data': 10}) + f1 = fixture(SimpleFixture, scope='environment', variables={'data': 20}) + + @sanity_function + def validate_vars(self): + return sn.all([sn.assert_eq(self.f0.data, 10), + sn.assert_eq(self.f1.data, 20)]) diff --git a/unittests/resources/checks_unlisted/testlib/nested/__init__.py b/unittests/resources/checks_unlisted/testlib/nested/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/unittests/resources/checks_unlisted/testlib/nested/dummy.py b/unittests/resources/checks_unlisted/testlib/nested/dummy.py new file mode 100644 index 0000000000..41e54fa7cd --- /dev/null +++ b/unittests/resources/checks_unlisted/testlib/nested/dummy.py @@ -0,0 +1,17 @@ +# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import reframe as rfm +import reframe.utility.sanity as sn +from ..utility import dummy_fixture + + +@rfm.simple_test +class dummy_test(rfm.RunOnlyRegressionTest): + valid_systems = ['*'] + valid_prog_environs = ['*'] + executable = 'true' + sanity_patterns = sn.assert_true(1) + dummy = fixture(dummy_fixture) diff --git a/unittests/resources/checks_unlisted/testlib/simple.py b/unittests/resources/checks_unlisted/testlib/simple.py index 1316168bd5..10218115a6 100644 --- a/unittests/resources/checks_unlisted/testlib/simple.py +++ b/unittests/resources/checks_unlisted/testlib/simple.py @@ -16,7 +16,7 @@ class simple_echo_check(rfm.RunOnlyRegressionTest, pin_prefix=True): executable = 'echo' executable_opts = ['Hello'] message = variable(str, value='World') - dummy = fixture(dummy_fixture, scope='environment') + dummy = fixture(dummy_fixture) @run_before('run') def set_executable_opts(self): diff --git a/unittests/test_cli.py b/unittests/test_cli.py index 2879966dc2..799b45f9ee 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -1076,6 +1076,13 @@ def test_reruns_with_duration(run_reframe): assert returncode == 1 +def test_exitcode_timeout(run_reframe): + assert_no_crash(*run_reframe( + more_options=['--duration=5s', '-n^HelloTest'], + checkpath=['unittests/resources/checks/hellocheck.py'] + )) + + @pytest.fixture(params=['name', 'rname', 'uid', 'ruid', 'random']) def exec_order(request): return request.param @@ -1263,7 +1270,7 @@ def test_testlib_inherit_fixture_in_different_files(run_reframe): action='run', ) assert returncode == 0 - assert 'Ran 3/3 test case(s)' in stdout + assert 'Ran 4/4 test case(s)' in stdout assert 'FAILED' not in stdout diff --git a/unittests/test_loader.py b/unittests/test_loader.py index d64fec6ba9..729708bde7 100644 --- a/unittests/test_loader.py +++ b/unittests/test_loader.py @@ -154,3 +154,9 @@ def test_relative_import_outside_rfm_prefix(loader, tmp_path): ) tests = loader.load_from_file(str(tmp_path / 'testlib' / 'simple.py')) assert len(tests) == 2 + + # Test nested library tests + tests = loader.load_from_file( + str(tmp_path / 'testlib' / 'nested' / 'dummy.py') + ) + assert len(tests) == 2 diff --git a/unittests/test_testgenerators.py b/unittests/test_testgenerators.py index 0d9b876d53..12cc7b44ab 100644 --- a/unittests/test_testgenerators.py +++ b/unittests/test_testgenerators.py @@ -51,7 +51,7 @@ def sys0p0_nodes(): nodelist_iter = sys0p0_nodes() for tc in new_cases: - nodes = getattr(tc.check, '$nid') + nodes = getattr(tc.check, '.nid') if tc.partition.fullname == 'sys0:p0': assert nodes == next(nodelist_iter) else: diff --git a/unittests/test_utility.py b/unittests/test_utility.py index 97d32c5810..2cb5ac462e 100644 --- a/unittests/test_utility.py +++ b/unittests/test_utility.py @@ -2070,6 +2070,14 @@ def test_nodelist_utilities(): assert nodelist(nodes) == 'nid0[00-99]-x,nid100-y' assert expand('nid0[00-99]-x,nid100-y') == nodes + # Test edge condition when node lists jump from N to N+1 digits + # See GH issue #3338 + nodes = ['vs-std-0009', 'vs-std-0010', 'vs-std-0099', 'vs-std-0100'] + assert nodelist(nodes) == 'vs-std-00[09-10],vs-std-0[099-100]' + assert expand('vs-std-00[09-10],vs-std-0[099-100]') == [ + 'vs-std-0009', 'vs-std-0010', 'vs-std-0099', 'vs-std-0100' + ] + # Test node duplicates assert nodelist(['nid001', 'nid001', 'nid002']) == 'nid001,nid00[1-2]' assert expand('nid001,nid00[1-2]') == ['nid001', 'nid001', 'nid002']