diff --git a/docs/config_reference.rst b/docs/config_reference.rst index ed994951a..26367b898 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -618,7 +618,7 @@ System Partition Configuration #. If the corresponding metadata files are not found, the processor information will be auto-detected. If the system partition is local (i.e., ``local`` scheduler + ``local`` launcher), the processor information is auto-detected unconditionally and stored in the corresponding metadata file for this partition. If the partition is remote, ReFrame will not try to auto-detect it unless the :envvar:`RFM_REMOTE_DETECT` or the :attr:`general.remote_detect` configuration option is set. - The commands for the remote auto-detection of processor topology can be configured through the :attr:`general.remote_command` configuration option. + The installation of ReFrame in the remote partition for the auto-detection of processor topology can be configured through the :attr:`general.remote_install` configuration option. The steps to auto-detect the remote processor information are the following: @@ -628,7 +628,7 @@ System Partition Configuration b. ReFrame changes to that directory and launches a job for the topology auto-detection ``reframe --detect-host-topology=topo.json``. The :option:`--detect-host-topology` option causes ReFrame to detect the topology of the current host, which in this case would be one of the remote compute nodes. - - If a custom set of commands for topology detection is specified through :attr:`general.remote_command`, ReFrame launches a job with the passed list of commands. If the ``reframe --detect-host-topology=topo.json`` command is missing in the list of commands it is added at the end of the list (last command). + - If a custom set of commands for the remote installation of ReFrame is specified through :attr:`general.remote_install`, ReFrame launches a job with the passed list of commands. The command ``reframe --detect-host-topology=topo.json`` is always added as a last command. Thus, the installation commands must make sure that the reframe installation is found. - Otherwise, ReFrame creates a fresh clone of itself in the temporary directory and launches a job that will first bootstrap the fresh clone and then run that clone with ``{launcher} ./bin/reframe --detect-host-topology=topo.json``. In case of errors during auto-detection, ReFrame will simply issue a warning and continue. @@ -1788,24 +1788,24 @@ General Configuration .. versionadded:: 3.7.0 -.. py:attribute:: general.remote_workdir +.. py:attribute:: general.remote_install :required: No - :default: ``"."`` + :default: ``[]`` - The temporary directory prefix that will be used to create a fresh ReFrame clone, in order to auto-detect the processor information of a remote partition. + List of commands to install reframe in the remote partition in order to auto-detect processor information. - .. versionadded:: 3.7.0 + .. versionadded:: 4.7.0 -.. py:attribute:: general.remote_command + +.. py:attribute:: general.remote_workdir :required: No - :default: ``[]`` + :default: ``"."`` - List of commands to be run in order to auto-detect processor information of remote partitions. - The commands must run reframe with ``--detect-host-topology=topo.json``. If this command is not specified it is added at the end of the list of commands. + The temporary directory prefix that will be used to create a fresh ReFrame clone, in order to auto-detect the processor information of a remote partition. - .. versionadded:: 4.7.0 + .. versionadded:: 3.7.0 .. py:attribute:: general.ignore_check_conflicts diff --git a/reframe/frontend/autodetect.py b/reframe/frontend/autodetect.py index c0e61531b..cdc16d0e7 100644 --- a/reframe/frontend/autodetect.py +++ b/reframe/frontend/autodetect.py @@ -152,6 +152,7 @@ def _emit_script_for_pip(job, env): job.prepare(commands, env, trap_errors=True, login=use_login_shell) def _emit_custom_script(job, env, commands): + commands.append('reframe --detect-host-topology=topo.json') job.prepare(commands, env, trap_errors=True, login=use_login_shell) getlogger().info( @@ -162,7 +163,7 @@ def _emit_custom_script(job, env, commands): try: prefix = runtime.runtime().get_option('general/0/remote_workdir') custom_command = runtime.runtime().get_option( - 'general/0/remote_command' + 'general/0/remote_install' ) with _copy_reframe(prefix) as (dirname, use_pip): with osext.change_dir(dirname): @@ -172,11 +173,10 @@ def _emit_custom_script(job, env, commands): sched_access=part.access) if custom_command: _emit_custom_script(job, [part.local_env], custom_command) + elif use_pip: + _emit_script_for_pip(job, [part.local_env]) else: - if use_pip: - _emit_script_for_pip(job, [part.local_env]) - else: - _emit_script_for_source(job, [part.local_env]) + _emit_script_for_source(job, [part.local_env]) getlogger().debug('submitting detection script') _log_contents(job.script_filename) job.submit() diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index f778b01c0..8e8c72dea 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -1037,20 +1037,6 @@ def restrict_logging(): ) sys.exit(0) - if site_config.get('general/0/remote_detect') and \ - site_config.get('general/0/remote_command'): - remote_commands = site_config.get('general/0/remote_command') - detect_topology_cmd = "reframe --detect-host-topology=topo.json" - detect_topology_found = [ - s for s in remote_commands if detect_topology_cmd in s - ] - if not detect_topology_found: - remote_cmd_args = site_config.get('general/0/remote_command') - printer.debug( - "Adding ReFrame topology detection to the curstom command" - ) - remote_cmd_args.append(detect_topology_cmd) - # Show configuration after everything is set up if options.show_config: # Restore logging level diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index 44f591f17..6676f2c46 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -520,11 +520,11 @@ "pipeline_timeout": {"type": ["number", "null"]}, "purge_environment": {"type": "boolean"}, "remote_detect": {"type": "boolean"}, - "remote_workdir": {"type": "string"}, - "remote_command": { + "remote_install": { "type": "array", "items": {"type": "string"} }, + "remote_workdir": {"type": "string"}, "report_file": {"type": "string"}, "report_junit": {"type": ["string", "null"]}, "resolve_module_conflicts": {"type": "boolean"}, @@ -590,8 +590,8 @@ "general/perf_report_spec": "now:now/last:/+job_nodelist+result", "general/purge_environment": false, "general/remote_detect": false, + "general/remote_install": [], "general/remote_workdir": ".", - "general/remote_command": [], "general/report_file": "${HOME}/.reframe/reports/run-report-{sessionid}.json", "general/report_junit": null, "general/resolve_module_conflicts": true, diff --git a/unittests/test_autodetect.py b/unittests/test_autodetect.py index b051704dc..56fc2d5e0 100644 --- a/unittests/test_autodetect.py +++ b/unittests/test_autodetect.py @@ -55,27 +55,23 @@ def default_exec_ctx(make_exec_ctx_g, temp_topo): @pytest.fixture -def remote_exec_ctx(make_exec_ctx, temp_topo): +def remote_exec_ctx(request, make_exec_ctx, temp_topo): if test_util.USER_CONFIG_FILE is None: pytest.skip('no user configuration file supplied') - ctx = make_exec_ctx(test_util.USER_CONFIG_FILE, - test_util.USER_SYSTEM, - {'general/remote_detect': True}) - yield ctx - - -@pytest.fixture -def remote_custom_exec_ctx(make_exec_ctx, temp_topo): - if test_util.USER_CONFIG_FILE is None: - pytest.skip('no user configuration file supplied') - - ctx = make_exec_ctx(test_util.USER_CONFIG_FILE, - test_util.USER_SYSTEM, - {'general/remote_detect': True, - 'general/remote_command': [ - 'echo \'{"dummy": "value"}\' > topo.json', - ]}) + custom_install = getattr(request, 'param', False) + + if not custom_install: + ctx = make_exec_ctx(test_util.USER_CONFIG_FILE, + test_util.USER_SYSTEM, + {'general/remote_detect': True}) + else: + ctx = make_exec_ctx(test_util.USER_CONFIG_FILE, + test_util.USER_SYSTEM, + {'general/remote_detect': True, + 'general/remote_install': [ + 'echo \'{"dummy": "value"}\' > topo.json', + ]}) yield ctx @@ -124,20 +120,17 @@ def test_autotect_with_invalid_files(invalid_topo_exec_ctx): assert part.devices == [] +@pytest.mark.parametrize('remote_exec_ctx', [False, True], indirect=True) def test_remote_autodetect(remote_exec_ctx): # All we can do with this test is to trigger the remote auto-detection # path; since we don't know what the remote user system is, we cannot test # if the topology is right. - partition = test_util.partition_by_scheduler() - if not partition: - pytest.skip('job submission not supported') - - autodetect.detect_topology() - - -def test_remote_custom_autodetect(remote_custom_exec_ctx): - # Test that a dummy topo.json file is created if a command to do - # so is specified as custom command for remote detection + # When the parameter in remote_exec_ctx is set to True, custom remote + # installation commands are tested creating a dummy json like topo.json. + # This is done to avoid a warning (which would be raised as an error) + # when ReFrame tries to search for the topo.json after + # reframe --detect-host-topology=topo.json, thus checking that the custom + # command was executed. partition = test_util.partition_by_scheduler() if not partition: pytest.skip('job submission not supported')