From 58fe85d56e0d76f310563dd14fe897a017cbc51a Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 29 Nov 2024 13:55:40 +0100 Subject: [PATCH 1/5] Do not ignore `--filter-expr` when using `--describe-stored-testcases` --- reframe/frontend/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 5c699b452..f55bf9894 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -1041,7 +1041,8 @@ def restrict_logging(): with exit_gracefully_on_error('failed to retrieve test case data', printer): printer.info(jsonext.dumps(reporting.testcase_info( - options.describe_stored_testcases, namepatt + options.describe_stored_testcases, + namepatt, options.filter_expr ), indent=2)) sys.exit(0) From a186646587c1998fc6ebb88af1f9c7d1ee1e8645 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 2 Dec 2024 17:37:21 +0100 Subject: [PATCH 2/5] Support relative imports in deeply nested tests --- reframe/frontend/loader.py | 32 +++++++++++++------ .../testlib/nested/__init__.py | 0 .../checks_unlisted/testlib/nested/dummy.py | 17 ++++++++++ .../checks_unlisted/testlib/simple.py | 2 +- unittests/test_cli.py | 2 +- unittests/test_loader.py | 6 ++++ 6 files changed, 48 insertions(+), 11 deletions(-) create mode 100644 unittests/resources/checks_unlisted/testlib/nested/__init__.py create mode 100644 unittests/resources/checks_unlisted/testlib/nested/dummy.py diff --git a/reframe/frontend/loader.py b/reframe/frontend/loader.py index 6486ebc77..c05f7f69f 100644 --- a/reframe/frontend/loader.py +++ b/reframe/frontend/loader.py @@ -193,17 +193,31 @@ def load_from_file(self, filename, force=False): try: dirname = os.path.dirname(filename) - with osext.change_dir(dirname): - with util.temp_sys_path(dirname): - if os.path.exists(os.path.join(dirname, '__init__.py')): - # If the containing directory is a package, - # import it, too. - parent = util.import_module_from_file(dirname).__name__ - else: - parent = None + # Load all parent modules of test file + parents = [] + while os.path.exists(os.path.join(dirname, '__init__.py')): + parents.append(os.path.join(dirname)) + dirname = os.path.split(dirname)[0] + + parent_module = None + for pdir in reversed(parents): + with osext.change_dir(pdir): + with util.temp_sys_path(pdir): + package_path = os.path.join(pdir, '__init__.py') + parent_module = util.import_module_from_file( + package_path, parent=parent_module + ).__name__ + + # Now load the actual test file + if not parents: + pdir = dirname + + with osext.change_dir(pdir): + with util.temp_sys_path(pdir): return self.load_from_module( - util.import_module_from_file(filename, force, parent) + util.import_module_from_file(filename, force, + parent_module) ) except Exception: exc_info = sys.exc_info() diff --git a/unittests/resources/checks_unlisted/testlib/nested/__init__.py b/unittests/resources/checks_unlisted/testlib/nested/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/unittests/resources/checks_unlisted/testlib/nested/dummy.py b/unittests/resources/checks_unlisted/testlib/nested/dummy.py new file mode 100644 index 000000000..41e54fa7c --- /dev/null +++ b/unittests/resources/checks_unlisted/testlib/nested/dummy.py @@ -0,0 +1,17 @@ +# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import reframe as rfm +import reframe.utility.sanity as sn +from ..utility import dummy_fixture + + +@rfm.simple_test +class dummy_test(rfm.RunOnlyRegressionTest): + valid_systems = ['*'] + valid_prog_environs = ['*'] + executable = 'true' + sanity_patterns = sn.assert_true(1) + dummy = fixture(dummy_fixture) diff --git a/unittests/resources/checks_unlisted/testlib/simple.py b/unittests/resources/checks_unlisted/testlib/simple.py index 1316168bd..10218115a 100644 --- a/unittests/resources/checks_unlisted/testlib/simple.py +++ b/unittests/resources/checks_unlisted/testlib/simple.py @@ -16,7 +16,7 @@ class simple_echo_check(rfm.RunOnlyRegressionTest, pin_prefix=True): executable = 'echo' executable_opts = ['Hello'] message = variable(str, value='World') - dummy = fixture(dummy_fixture, scope='environment') + dummy = fixture(dummy_fixture) @run_before('run') def set_executable_opts(self): diff --git a/unittests/test_cli.py b/unittests/test_cli.py index 2879966dc..1e1763592 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -1263,7 +1263,7 @@ def test_testlib_inherit_fixture_in_different_files(run_reframe): action='run', ) assert returncode == 0 - assert 'Ran 3/3 test case(s)' in stdout + assert 'Ran 4/4 test case(s)' in stdout assert 'FAILED' not in stdout diff --git a/unittests/test_loader.py b/unittests/test_loader.py index d64fec6ba..729708bde 100644 --- a/unittests/test_loader.py +++ b/unittests/test_loader.py @@ -154,3 +154,9 @@ def test_relative_import_outside_rfm_prefix(loader, tmp_path): ) tests = loader.load_from_file(str(tmp_path / 'testlib' / 'simple.py')) assert len(tests) == 2 + + # Test nested library tests + tests = loader.load_from_file( + str(tmp_path / 'testlib' / 'nested' / 'dummy.py') + ) + assert len(tests) == 2 From 887b0c7ead0ebaa221b937491e8d70c87261d677 Mon Sep 17 00:00:00 2001 From: Tom Lin Date: Tue, 26 Nov 2024 09:25:35 +0000 Subject: [PATCH 3/5] [bugfix] Fix error when more than one instance of the same fixture class are specified --- reframe/core/pipeline.py | 4 ++- .../checks_unlisted/fixtures_same_class.py | 33 +++++++++++++++++++ unittests/test_cli.py | 8 +++++ 3 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 unittests/resources/checks_unlisted/fixtures_same_class.py diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 5663f6e11..c7878e427 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -1646,7 +1646,9 @@ def _resolve_fixtures(self): # registered under the same fixture class. So the loop below must # also inspect the fixture data the instance was registered with. for fixt_name, fixt_data in registry[f.cls].items(): - if f.scope != fixt_data.scope: + if fixt_data.variables != f.variables: + continue + elif f.scope != fixt_data.scope: continue elif fixt_data.variant_num not in target_variants: continue diff --git a/unittests/resources/checks_unlisted/fixtures_same_class.py b/unittests/resources/checks_unlisted/fixtures_same_class.py new file mode 100644 index 000000000..4e2f2a84c --- /dev/null +++ b/unittests/resources/checks_unlisted/fixtures_same_class.py @@ -0,0 +1,33 @@ +# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import reframe as rfm +import reframe.utility.sanity as sn + + +class HelloFixture(rfm.RunOnlyRegressionTest): + executable = 'echo hello from fixture' + myvar = variable(str) + + @sanity_function + def assert_output(self): + return sn.assert_found(r'hello from fixture', self.stdout) + + +@rfm.simple_test +class TestA(rfm.RunOnlyRegressionTest): + valid_systems = ['*'] + valid_prog_environs = ['*'] + + dep1 = fixture(HelloFixture, scope='environment', variables={'myvar': 'a'}) + dep2 = fixture(HelloFixture, scope='environment', variables={'myvar': 'b'}) + + @run_after('setup') + def after_setup(self): + self.executable = f"echo {self.dep1.myvar} {self.dep2.myvar}" + + @sanity_function + def validate(self): + return sn.assert_found(r'a b', self.stdout) diff --git a/unittests/test_cli.py b/unittests/test_cli.py index 1e1763592..71dce245f 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -1227,6 +1227,14 @@ def test_fixture_resolution(run_reframe, run_action): ) assert returncode == 0 +def test_fixture_resolution_same_class(run_reframe, run_action): + returncode, stdout, stderr = run_reframe( + system='sys1', + environs=[], + checkpath=['unittests/resources/checks_unlisted/fixtures_same_class.py'], + action=run_action + ) + assert returncode == 0 def test_dynamic_tests(run_reframe, run_action): returncode, stdout, _ = run_reframe( From 8e5ab3c589713ac9981e1825b74a53103244bc24 Mon Sep 17 00:00:00 2001 From: Tom Lin Date: Tue, 26 Nov 2024 09:35:44 +0000 Subject: [PATCH 4/5] Fix formatting --- unittests/resources/checks_unlisted/fixtures_same_class.py | 3 +-- unittests/test_cli.py | 6 +++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/unittests/resources/checks_unlisted/fixtures_same_class.py b/unittests/resources/checks_unlisted/fixtures_same_class.py index 4e2f2a84c..329066c4c 100644 --- a/unittests/resources/checks_unlisted/fixtures_same_class.py +++ b/unittests/resources/checks_unlisted/fixtures_same_class.py @@ -10,11 +10,10 @@ class HelloFixture(rfm.RunOnlyRegressionTest): executable = 'echo hello from fixture' myvar = variable(str) - + @sanity_function def assert_output(self): return sn.assert_found(r'hello from fixture', self.stdout) - @rfm.simple_test class TestA(rfm.RunOnlyRegressionTest): diff --git a/unittests/test_cli.py b/unittests/test_cli.py index 71dce245f..5b4774de6 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -1227,15 +1227,19 @@ def test_fixture_resolution(run_reframe, run_action): ) assert returncode == 0 + def test_fixture_resolution_same_class(run_reframe, run_action): returncode, stdout, stderr = run_reframe( system='sys1', environs=[], - checkpath=['unittests/resources/checks_unlisted/fixtures_same_class.py'], + checkpath=[ + 'unittests/resources/checks_unlisted/fixtures_same_class.py' + ], action=run_action ) assert returncode == 0 + def test_dynamic_tests(run_reframe, run_action): returncode, stdout, _ = run_reframe( system='sys0', From 7a5d3daa2f7148afd228c379a8d6d41961c00558 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 5 Dec 2024 20:54:40 +0100 Subject: [PATCH 5/5] Simplify unit tests --- .../checks_unlisted/fixtures_complex.py | 14 ++++++++ .../checks_unlisted/fixtures_same_class.py | 32 ------------------- unittests/test_cli.py | 12 ------- 3 files changed, 14 insertions(+), 44 deletions(-) delete mode 100644 unittests/resources/checks_unlisted/fixtures_same_class.py diff --git a/unittests/resources/checks_unlisted/fixtures_complex.py b/unittests/resources/checks_unlisted/fixtures_complex.py index 53be46f92..d200502b8 100644 --- a/unittests/resources/checks_unlisted/fixtures_complex.py +++ b/unittests/resources/checks_unlisted/fixtures_complex.py @@ -102,3 +102,17 @@ def validate_fixture_resolution(self): ParamFixture.num_variants ) ]) + + +@rfm.simple_test +class TestC(rfm.RunOnlyRegressionTest): + valid_systems = ['*'] + valid_prog_environs = ['*'] + executable = 'echo' + f0 = fixture(SimpleFixture, scope='environment', variables={'data': 10}) + f1 = fixture(SimpleFixture, scope='environment', variables={'data': 20}) + + @sanity_function + def validate_vars(self): + return sn.all([sn.assert_eq(self.f0.data, 10), + sn.assert_eq(self.f1.data, 20)]) diff --git a/unittests/resources/checks_unlisted/fixtures_same_class.py b/unittests/resources/checks_unlisted/fixtures_same_class.py deleted file mode 100644 index 329066c4c..000000000 --- a/unittests/resources/checks_unlisted/fixtures_same_class.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -class HelloFixture(rfm.RunOnlyRegressionTest): - executable = 'echo hello from fixture' - myvar = variable(str) - - @sanity_function - def assert_output(self): - return sn.assert_found(r'hello from fixture', self.stdout) - -@rfm.simple_test -class TestA(rfm.RunOnlyRegressionTest): - valid_systems = ['*'] - valid_prog_environs = ['*'] - - dep1 = fixture(HelloFixture, scope='environment', variables={'myvar': 'a'}) - dep2 = fixture(HelloFixture, scope='environment', variables={'myvar': 'b'}) - - @run_after('setup') - def after_setup(self): - self.executable = f"echo {self.dep1.myvar} {self.dep2.myvar}" - - @sanity_function - def validate(self): - return sn.assert_found(r'a b', self.stdout) diff --git a/unittests/test_cli.py b/unittests/test_cli.py index 5b4774de6..1e1763592 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -1228,18 +1228,6 @@ def test_fixture_resolution(run_reframe, run_action): assert returncode == 0 -def test_fixture_resolution_same_class(run_reframe, run_action): - returncode, stdout, stderr = run_reframe( - system='sys1', - environs=[], - checkpath=[ - 'unittests/resources/checks_unlisted/fixtures_same_class.py' - ], - action=run_action - ) - assert returncode == 0 - - def test_dynamic_tests(run_reframe, run_action): returncode, stdout, _ = run_reframe( system='sys0',