From b18b428148b7ba841c36db4976b6c89e9d5ee1a5 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 17 Dec 2024 14:47:53 +0100 Subject: [PATCH 1/5] Update GH actions for Python 3.7 after the move of runners to Ubuntu 24.04 --- .github/workflows/main.yml | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 62447430d..0e941f242 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -46,6 +46,24 @@ jobs: run: | ./test_reframe.py + unittest-py37: + runs-on: ubuntu-22.04 + strategy: + matrix: + python-version: ['3.7'] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies and docs + run: | + ./bootstrap.sh +docs + - name: Generic Unittests + run: | + ./test_reframe.py + unittest-macos: runs-on: macos-latest strategy: @@ -120,7 +138,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v4 - name: Setup up Python ${{ matrix.python-version }} @@ -143,7 +161,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} From 58730e17a82200dbcfc41f489ffa692f1e1fed3f Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 11 Dec 2024 13:26:35 +0100 Subject: [PATCH 2/5] Optimize JSON decoding for sessions when using filters --- reframe/frontend/reporting/storage.py | 37 ++++++++++++++++++--------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py index 42a591a11..e4e370eb2 100644 --- a/reframe/frontend/reporting/storage.py +++ b/reframe/frontend/reporting/storage.py @@ -233,26 +233,39 @@ def _decode_sessions(self, results, sess_filter): Return a map of session uuids to decoded session data ''' + sess_info_patt = re.compile( + r'\"session_info\":\s+(?P\{.*?\})' + ) + + def _extract_sess_info(s): + return sess_info_patt.search(s).group('sess_info') + + @time_function + def _mass_json_decode(json_objs): + data = '[' + ','.join(json_objs) + ']' + getlogger().debug(f'decoding {len(data)} bytes') + return json.loads(data) + + session_infos = {} sessions = {} for uuid, json_blob in results: sessions.setdefault(uuid, json_blob) + session_infos.setdefault(uuid, _extract_sess_info(json_blob)) - # Join all sessions and decode them at once - reports_blob = '[' + ','.join(sessions.values()) + ']' - getprofiler().enter_region('json decode') - reports = json.loads(reports_blob) - getprofiler().exit_region() - - # Reindex and filter sessions based on their decoded data - sessions.clear() - for rpt in reports: + # Find the UUIDs to decode fully by inspecting only the session info + uuids = [] + for info in _mass_json_decode(session_infos.values()): try: - if self._db_filter_json(sess_filter, rpt['session_info']): - sessions[rpt['session_info']['uuid']] = rpt + if self._db_filter_json(sess_filter, info): + uuids.append(info['uuid']) except Exception: continue - return sessions + # Decode selected sessions + reports = _mass_json_decode(sessions[uuid] for uuid in uuids) + + # Return only the selected sessions + return {rpt['session_info']['uuid']: rpt for rpt in reports} @time_function def _fetch_testcases_raw(self, condition): From cc624d35c0c45b8d1986dceed074a93b64535236 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 17 Dec 2024 15:53:56 +0100 Subject: [PATCH 3/5] Address PR comments --- reframe/frontend/reporting/storage.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py index e4e370eb2..f8bd8584b 100644 --- a/reframe/frontend/reporting/storage.py +++ b/reframe/frontend/reporting/storage.py @@ -243,7 +243,9 @@ def _extract_sess_info(s): @time_function def _mass_json_decode(json_objs): data = '[' + ','.join(json_objs) + ']' - getlogger().debug(f'decoding {len(data)} bytes') + getlogger().debug( + f'decoding JSON raw data of length {len(data)}' + ) return json.loads(data) session_infos = {} @@ -254,10 +256,10 @@ def _mass_json_decode(json_objs): # Find the UUIDs to decode fully by inspecting only the session info uuids = [] - for info in _mass_json_decode(session_infos.values()): + for sess_info in _mass_json_decode(session_infos.values()): try: - if self._db_filter_json(sess_filter, info): - uuids.append(info['uuid']) + if self._db_filter_json(sess_filter, sess_info): + uuids.append(sess_info['uuid']) except Exception: continue From 727f46a823158d3bf5a1bda4d4367d6733f72d24 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 17 Dec 2024 14:17:52 +0100 Subject: [PATCH 4/5] Fix treatment of Slurm constraints in `job.options` --- reframe/core/schedulers/slurm.py | 8 +++++--- unittests/test_schedulers.py | 24 ++++++++++++++++++------ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/reframe/core/schedulers/slurm.py b/reframe/core/schedulers/slurm.py index 235759ab0..e07dc42b0 100644 --- a/reframe/core/schedulers/slurm.py +++ b/reframe/core/schedulers/slurm.py @@ -234,7 +234,7 @@ def emit_preamble(self, job): job._cli_options = other_cli_opts arg = '&'.join(f'({c.strip()})' for c in constraints) - job._sched_access = [f'--constraint={arg}'] + job._sched_access = [f'--constraint={arg}'] + access_other if not self._sched_access_in_submit: for opt in job.sched_access: @@ -244,8 +244,10 @@ def emit_preamble(self, job): prefix_patt = re.compile(r'(#\w+)') for opt in job.options + job.cli_options: if opt.strip().startswith(('-C', '--constraint')): - # Constraints are already processed - continue + if access.constraint: + # Constraints are already combined with the `sched_access` + # and processed + continue if not prefix_patt.match(opt): preamble.append('%s %s' % (self._prefix, opt)) diff --git a/unittests/test_schedulers.py b/unittests/test_schedulers.py index a11223121..c34c05af6 100644 --- a/unittests/test_schedulers.py +++ b/unittests/test_schedulers.py @@ -452,6 +452,13 @@ def test_prepare_without_smt(fake_job, slurm_only): assert re.search(r'--hint=nomultithread', fp.read()) is not None +def test_prepare_with_constraints(fake_job, slurm_only): + fake_job.options = ['--constraint=foo'] + prepare_job(fake_job) + with open(fake_job.script_filename) as fp: + assert re.search(r'#SBATCH --constraint=foo', fp.read()) is not None + + def test_prepare_nodes_option(make_exec_ctx, make_job, slurm_only): make_exec_ctx(test_util.TEST_CONFIG_FILE, 'testsys') job = make_job(sched_opts={'part_name': 'gpu'}) @@ -625,42 +632,47 @@ def test_no_empty_lines_in_preamble(minimal_job): def test_combined_access_constraint(make_job, slurm_only): - job = make_job(sched_access=['--constraint=c1']) + job = make_job(sched_access=['--constraint=c1', '-A acct', '-p part']) job.options = ['-C c2&c3'] prepare_job(job) with open(job.script_filename) as fp: script_content = fp.read() - print(script_content) + assert re.search('-A acct', script_content) + assert re.search('-p part', script_content) assert re.search(r'(?m)--constraint=\(c1\)&\(c2&c3\)$', script_content) assert re.search(r'(?m)--constraint=(c1|c2&c3)$', script_content) is None def test_combined_access_multiple_constraints(make_job, slurm_only): - job = make_job(sched_access=['--constraint=c1']) + job = make_job(sched_access=['--constraint=c1', '-A acct', '-p part']) job.options = ['--constraint=c2', '-C c3'] prepare_job(job) with open(job.script_filename) as fp: script_content = fp.read() + assert re.search('-A acct', script_content) + assert re.search('-p part', script_content) assert re.search(r'(?m)--constraint=\(c1\)&\(c3\)$', script_content) assert re.search(r'(?m)--constraint=(c1|c2|c3)$', script_content) is None def test_combined_access_verbatim_constraint(make_job, slurm_only): - job = make_job(sched_access=['--constraint=c1']) + job = make_job(sched_access=['--constraint=c1', '-A acct', '-p part']) job.options = ['#SBATCH --constraint=c2', '#SBATCH -C c3'] prepare_job(job) with open(job.script_filename) as fp: script_content = fp.read() + assert re.search('-A acct', script_content) + assert re.search('-p part', script_content) assert re.search(r'(?m)--constraint=c1$', script_content) assert re.search(r'(?m)^#SBATCH --constraint=c2$', script_content) assert re.search(r'(?m)^#SBATCH -C c3$', script_content) def test_sched_access_in_submit(make_job): - job = make_job(sched_access=['--constraint=c1', '--foo=bar']) + job = make_job(sched_access=['--constraint=c1', '-A acct']) job.options = ['--constraint=c2', '--xyz'] job.scheduler._sched_access_in_submit = True @@ -673,7 +685,7 @@ def test_sched_access_in_submit(make_job): print(script_content) assert '--xyz' in script_content - assert '--foo=bar' not in script_content + assert '-A acct' not in script_content if job.scheduler.registered_name in ('slurm', 'squeue'): # Constraints are combined in `sched_access` for Slurm backends assert '--constraint' not in script_content From 3924c8788051d1404390618e3b17ad4d902c70b0 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 19 Dec 2024 13:51:00 +0100 Subject: [PATCH 5/5] Bump patch level --- reframe/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reframe/__init__.py b/reframe/__init__.py index 2e98ac59d..73e581029 100644 --- a/reframe/__init__.py +++ b/reframe/__init__.py @@ -6,7 +6,7 @@ import os import sys -VERSION = '4.7.1' +VERSION = '4.7.2' INSTALL_PREFIX = os.path.normpath( os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) )