diff --git a/.github/workflows/spelling.yaml b/.github/workflows/spelling.yaml index e56205b44..f3bea3685 100644 --- a/.github/workflows/spelling.yaml +++ b/.github/workflows/spelling.yaml @@ -11,9 +11,9 @@ jobs: formatting: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Check Spelling - uses: crate-ci/typos@548ac37a5de9ce84871bf4db3c9b8c462896d480 # v1.16.24 + uses: crate-ci/typos@v1.20.4 with: files: ./lib/ramble/ramble ./lib/ramble/docs ./examples ./share ./bin ./etc ./var ./README.md config: ./.typos.toml diff --git a/.typos.toml b/.typos.toml index 18ce75dc8..8c2599f60 100644 --- a/.typos.toml +++ b/.typos.toml @@ -17,6 +17,8 @@ extend-ignore-re = [ "fom" = "fom" "namd" = "namd" "reord" = "reord" +"PN" = "PN" # fixing tPN in IOR +"repositor" = "repositor" # Fixing partial name without singular / plural suffix [default.extend-identifiers] "ATPase" = "ATPase" diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 000000000..e1f0b88e0 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,7 @@ +# This is the list of Ramble's significant contributors. +# +# This does not necessarily list everyone who has contributed code, +# especially since many employees of one corporation may be contributing. +# To see the full list of contributors, see the revision history in +# source control. +Google LLC diff --git a/LICENSE-APACHE b/LICENSE-APACHE index e94f2f3a6..7929d5ca8 100644 --- a/LICENSE-APACHE +++ b/LICENSE-APACHE @@ -187,7 +187,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2022-2024 Google LLC. + Copyright 2022-2024 The Ramble Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index 21585dfeb..c026d59ec 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ It works on Linux, macOS, and many supercomputers. Ramble can be used to configure a variety of experiments for applications. These can include anything from: - Scientific parameter sweeps - - Performance focused scalaing studies + - Performance focused scaling studies - Compiler flag sweeps To install ramble and configure your experiment workspace, make sure you have diff --git a/bin/ramble b/bin/ramble index d366c3dc9..911031236 100755 --- a/bin/ramble +++ b/bin/ramble @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/bin/ramble-python b/bin/ramble-python index 617142d91..7e608380d 100755 --- a/bin/ramble-python +++ b/bin/ramble-python @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/etc/ramble/defaults/spack.yaml b/etc/ramble/defaults/spack.yaml index 76560517c..9e1d214e7 100644 --- a/etc/ramble/defaults/spack.yaml +++ b/etc/ramble/defaults/spack.yaml @@ -1,4 +1,3 @@ spack: - concretized: false packages: {} environments: {} diff --git a/examples/basic_expansion_config.yaml b/examples/basic_expansion_config.yaml index 25e262691..c6543d70a 100644 --- a/examples/basic_expansion_config.yaml +++ b/examples/basic_expansion_config.yaml @@ -29,7 +29,6 @@ ramble: n_ranks: '1' n_nodes: '1' spack: - concretized: true packages: gcc9: spack_spec: gcc@9.3.0 target=x86_64 @@ -37,8 +36,8 @@ ramble: ompi412: spack_spec: openmpi@4.1.2 +legacylaunchers +pmi +thread_multiple +cxx target=x86_64 compiler: gcc9 - impi2018: - spack_spec: intel-mpi@2018.4.274 + impi2021: + spack_spec: intel-oneapi-mpi@2021.11.0 compiler: gcc9 openfoam: spack_spec: openfoam-org@7 @@ -57,5 +56,5 @@ ramble: - openfoam wrfv4: packages: - - impi2018 + - impi2021 - wrfv4 diff --git a/examples/basic_gromacs_config.yaml b/examples/basic_gromacs_config.yaml index cfd61d6b8..552f6a82d 100644 --- a/examples/basic_gromacs_config.yaml +++ b/examples/basic_gromacs_config.yaml @@ -38,13 +38,12 @@ ramble: size: '0003' type: 'rf' spack: - concretized: true packages: gcc9: spack_spec: gcc@9.4.0 target=x86_64 compiler_spec: gcc@9.4.0 - impi2018: - spack_spec: intel-mpi@2018.4.274 target=x86_64 + impi2021: + spack_spec: intel-oneapi-mpi@2021.11.0 target=x86_64 compiler: gcc9 gromacs: spack_spec: gromacs@2021.6 @@ -53,4 +52,4 @@ ramble: gromacs: packages: - gromacs - - impi2018 + - impi2021 diff --git a/examples/basic_hostname_config.yaml b/examples/basic_hostname_config.yaml index 7455e585a..4a901f279 100644 --- a/examples/basic_hostname_config.yaml +++ b/examples/basic_hostname_config.yaml @@ -44,6 +44,5 @@ ramble: n_nodes: '1' processes_per_node: '16' spack: - concretized: true packages: {} environments: {} diff --git a/examples/full_expansion_config.yaml b/examples/full_expansion_config.yaml index 58ef3c98e..942ff75e8 100644 --- a/examples/full_expansion_config.yaml +++ b/examples/full_expansion_config.yaml @@ -29,7 +29,6 @@ ramble: #(part1, 16, openfoam-skx, 2), (part1, 16, openfoam-skx, 4) #(part2, 32, openfoam-zen2, 2), (part2, 32, openfoam-zen2, 4) spack: - concretized: true packages: gcc9: spack_spec: gcc@9.3.0 target=x86_64 diff --git a/examples/slurm_execute_experiment.tpl b/examples/slurm_execute_experiment.tpl index f49841d7e..a3491481a 100644 --- a/examples/slurm_execute_experiment.tpl +++ b/examples/slurm_execute_experiment.tpl @@ -1,6 +1,7 @@ #!/bin/bash #SBATCH -N {n_nodes} #SBATCH --ntasks-per-node {processes_per_node} +#SBATCH -p {partition_name} #SBATCH -J {application_name}_{workload_name}_{experiment_name} # This is a template execution script for @@ -20,5 +21,6 @@ cd {experiment_run_dir} -{command} +scontrol show hostnames > {experiment_run_dir}/hostfile +{command} diff --git a/examples/vector_gromacs_software_config.yaml b/examples/vector_gromacs_software_config.yaml index 2099c7053..a8f33f164 100644 --- a/examples/vector_gromacs_software_config.yaml +++ b/examples/vector_gromacs_software_config.yaml @@ -26,13 +26,12 @@ ramble: - n_ranks - gromacs_version spack: - concretized: true packages: gcc9: spack_spec: gcc@9.4.0 target=x86_64 compiler_spec: gcc@9.4.0 - impi2018: - spack_spec: intel-mpi@2018.4.274 target=x86_64 + impi2021: + spack_spec: intel-oneapi-mpi@2021.11.0 target=x86_64 compiler: gcc9 gromacs-{gromacs_version}: spack_spec: gromacs@{gromacs_version} @@ -41,4 +40,4 @@ ramble: gromacs-{gromacs_version}: packages: - gromacs-{gromacs_version} - - impi2018 + - impi2021 diff --git a/examples/vector_matrix_gromacs_config.yaml b/examples/vector_matrix_gromacs_config.yaml index bfa883aec..e95dfe813 100644 --- a/examples/vector_matrix_gromacs_config.yaml +++ b/examples/vector_matrix_gromacs_config.yaml @@ -23,13 +23,12 @@ ramble: - type - n_ranks spack: - concretized: true packages: gcc9: spack_spec: gcc@9.4.0 target=x86_64 compiler_spec: gcc@9.4.0 - impi2018: - spack_spec: intel-mpi@2018.4.274 target=x86_64 + impi2021: + spack_spec: intel-oneapi-mpi@2021.11.0 target=x86_64 compiler: gcc9 gromacs: spack_spec: gromacs@2021.6 @@ -38,4 +37,4 @@ ramble: gromacs: packages: - gromacs - - impi2018 + - impi2021 diff --git a/lib/ramble/docs/conf.py b/lib/ramble/docs/conf.py index 8153e2422..94ff037f0 100644 --- a/lib/ramble/docs/conf.py +++ b/lib/ramble/docs/conf.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -130,6 +130,7 @@ def setup(sphinx): "sphinx.ext.viewcode", "sphinxcontrib.programoutput", "sphinxcontrib.jquery", + "sphinx_copybutton", ] # Set default graphviz options @@ -396,3 +397,9 @@ class RambleStyle(DefaultStyle): intersphinx_mapping = { "python": ("https://docs.python.org/3", None), } + +# sphinx_copybutton +# Do not copy the prompt, or any console outputs. +copybutton_exclude = '.gp, .go' +# Escape hatch for turning off the copy button. +copybutton_selector = "div:not(.hide-copy) > div.highlight > pre" diff --git a/lib/ramble/docs/configuration_files.rst b/lib/ramble/docs/configuration_files.rst index a2803ec93..9ff5e5fbd 100644 --- a/lib/ramble/docs/configuration_files.rst +++ b/lib/ramble/docs/configuration_files.rst @@ -1,4 +1,4 @@ -.. Copyright 2022-2024 Google LLC +.. Copyright 2022-2024 The Ramble Authors Licensed under the Apache License, Version 2.0 or the MIT license @@ -54,31 +54,31 @@ Ramble provides several configuration scopes, which are used to denote precedence of configuration options. In precedence order (from lowest to highest) Ramble contains the following scopes: -(1) **default**: Stored in ``$(prefix)/etc/ramble/defaults/``. These are the -default settings provided with Ramble. Users should generally not modify these -settings, and instead use a higher precedence configuration scope. These -defaults will change from version to version of Ramble. -(2) **system**: Store in ``/etc/ramble/``. These are Ramble settings for an -entire machine. These settings are typically managed by a systems -administrator, or something with root access on the machine. Settings defined -in this scope override settings in the **default** scope. -(3) **site**: Stored in ``$(prefix)/etc/ramble/``. Settings here only affect -*this instance* of Ramble, and they override both the **default** and -**system** scopes. -(4) **user**: Stored in ``~/.ramble/``. Settings here only affect a specific -user, and override **default**, **system**, and **site** scopes. -(5) **custom**: Stored in a custom directory, specified by ``--config-scope``. -If multiple scopes are listed on the command line, they are ordered from lowest -to highest precedence. Settings here override all previously defined scoped. -(6) **workspace configs dir**: Stored in ``$(workspace_root)/configs`` -generally as a ``.yaml`` file (i.e. ``variables.yaml``). These -settings apply to a specific workspace, and override all previous configuration -scopes. -(7) **workspace configuration file**: Stored in -``$(workspace_root)/configs/ramble.yaml``. Configuration scopes defined within -this config file override all previously defined configuration scopes. -(8) **command line**: Configuration options defined on the command line take -precedence over all other scopes. +1. **default**: Stored in ``$(prefix)/etc/ramble/defaults/``. These are the + default settings provided with Ramble. Users should generally not modify these + settings, and instead use a higher precedence configuration scope. These + defaults will change from version to version of Ramble. +2. **system**: Store in ``/etc/ramble/``. These are Ramble settings for an + entire machine. These settings are typically managed by a systems + administrator, or something with root access on the machine. Settings defined + in this scope override settings in the **default** scope. +3. **site**: Stored in ``$(prefix)/etc/ramble/``. Settings here only affect + *this instance* of Ramble, and they override both the **default** and + **system** scopes. +4. **user**: Stored in ``~/.ramble/``. Settings here only affect a specific + user, and override **default**, **system**, and **site** scopes. +5. **custom**: Stored in a custom directory, specified by ``--config-scope``. + If multiple scopes are listed on the command line, they are ordered from lowest + to highest precedence. Settings here override all previously defined scoped. +6. **workspace configs dir**: Stored in ``$(workspace_root)/configs`` + generally as a ``.yaml`` file (i.e. ``variables.yaml``). These + settings apply to a specific workspace, and override all previous configuration + scopes. +7. **workspace configuration file**: Stored in + ``$(workspace_root)/configs/ramble.yaml``. Configuration scopes defined within + this config file override all previously defined configuration scopes. +8. **command line**: Configuration options defined on the command line take + precedence over all other scopes. Each configuration directory may contain several configuration files, such as ``config.yaml``, ``variables.yaml``, or ``modifiers.yaml``. When configurations @@ -262,9 +262,14 @@ The format of this config section is as follows: The above example is general, and intended to show the available functionality of configuring environment variables. Below the ``env_vars`` level, one of four actions is available. These actions are: -* ``set`` - Define a variable equal to a given value. Overwrites previously configured values -* ``append`` - Append the given value to the end of a previous variable definition. Delimited for vars is defined by ``var_separator``, ``paths`` uses ``:`` -* ``prepend`` - Prepent the given value to the beginning of a previous variable definition. Only supports paths, delimiter is ``:`` + +* ``set`` - Define a variable equal to a given value. Overwrites previously + configured values +* ``append`` - Append the given value to the end of a previous variable + definition. Delimited for vars is defined by ``var_separator``, ``paths`` + uses ``:`` +* ``prepend`` - Prepent the given value to the beginning of a previous variable + definition. Only supports paths, delimiter is ``:`` * ``unset`` - Remove a variable definition, if it is set. .. _formatted-execs-config: @@ -471,7 +476,6 @@ environments created from those packages. Its format is as follows: .. code-block:: yaml spack: - concretized: [True/False] # Should be false unless defined in a concretized workspace [variables: {}] packages: : @@ -517,8 +521,8 @@ Below is an annotated example of the spack dictionary. gcc9: # Abstract name to refer to this package spack_spec: gcc@9.3.0 target=x86_64 # Spack spec for this package compiler_spec: gcc@9.3.0 # Spack compiler spec for this package - impi2018: - spack_spec: intel-mpi@2018.4.274 target=x86_64 + impi2021: + spack_spec: intel-oneapi-mpi@2021.11.0 target=x86_64 compiler: gcc9 # Other package name to use as compiler for this package gromacs: spack_spec: gromacs@2022.4 @@ -526,61 +530,12 @@ Below is an annotated example of the spack dictionary. environments: gromacs: packages: # List of packages to include in this environment - - impi2018 + - impi2021 - gromacs -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Vector and Matrix Packages and Environments: -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Package and environment definitions can generate many packages and environments -following Ramble's -:ref:`vector` / :ref:`matrix` logic. - -Below is an example of using this logic within the spack dictionary: - -.. code-block:: yaml - - spack: - packages: - gcc-{ver}: - variables: - ver: ['9.3.0', '10.3.0', '12.2.0'] - spack_spec: gcc@{ver} target=x86_64 - compiler_spec: gcc@{ver} - intel-mpi-{comp}: - variables: - comp: gcc-{ver} - ver: ['9.3.0', '10.3.0', '12.2.0'] - spack_spec: intel-mpi@2018.4.274 - compiler: {comp} - openmpi-{comp}: - variables: - comp: gcc-{ver} - ver: ['9.3.0', '10.3.0', '12.2.0'] - spack_spec: openmpi@4.1.4 - compiler: {comp} - wrf-{comp}: - variables: - comp: gcc-{ver} - ver: ['9.3.0', '10.3.0', '12.2.0'] - spack_spec: wrf@4.2 - compiler: {comp} - environments: - wrf-{comp}-{mpi}: - variables: - comp: gcc-{ver} - ver: ['9.3.0', '10.3.0', '12.2.0'] - mpi: [intel-mpi-{comp}, openmpi-{comp}'] - matrix: - - mpi - packages: - - {mpi} - - wrf-{comp} - -The above file will generate 3 versions of ``gcc``, 3 versions each of ``wrf``, -``intel-mpi`` and ``openmpi`` built with each ``gcc`` version, and 6 spack -environments, with each combination of the 2 ``mpi`` libraries and 3 compilers. +Packages and environments defined inside the ``spack`` config section are +merely templates. They will be rendered into explicit environments and packages +by each individual experiment. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ External Spack Environment Support: diff --git a/lib/ramble/docs/dev_guides.rst b/lib/ramble/docs/dev_guides.rst index 1e5f662cd..663d8d9a4 100644 --- a/lib/ramble/docs/dev_guides.rst +++ b/lib/ramble/docs/dev_guides.rst @@ -1,4 +1,4 @@ -.. Copyright 2022-2024 Google LLC +.. Copyright 2022-2024 The Ramble Authors Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/docs/dev_guides/application_dev_guide.rst b/lib/ramble/docs/dev_guides/application_dev_guide.rst index 14b4ce773..f234a0d20 100644 --- a/lib/ramble/docs/dev_guides/application_dev_guide.rst +++ b/lib/ramble/docs/dev_guides/application_dev_guide.rst @@ -1,4 +1,4 @@ -.. Copyright 2022-2024 Google LLC +.. Copyright 2022-2024 The Ramble Authors Licensed under the Apache License, Version 2.0 or the MIT license @@ -265,7 +265,7 @@ directives that are intended to be package manager specific. As an example, there are directives for Spack defined by: * :meth:`ramble.language.shared_language.software_spec` -* :meth:`ramble.language.shared_language.default_compiler` +* :meth:`ramble.language.shared_language.define_compiler` * :meth:`ramble.language.shared_language.required_package` These provide Ramble with information about how Spack could install and require diff --git a/lib/ramble/docs/getting_started.rst b/lib/ramble/docs/getting_started.rst index c4371d83d..7e2639b7e 100644 --- a/lib/ramble/docs/getting_started.rst +++ b/lib/ramble/docs/getting_started.rst @@ -1,4 +1,4 @@ -.. Copyright 2022-2024 Google LLC +.. Copyright 2022-2024 The Ramble Authors Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/docs/index.rst b/lib/ramble/docs/index.rst index 282e09d44..e41fe1100 100644 --- a/lib/ramble/docs/index.rst +++ b/lib/ramble/docs/index.rst @@ -1,4 +1,4 @@ -.. Copyright 2022-2024 Google LLC +.. Copyright 2022-2024 The Ramble Authors Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/docs/mirror-config.rst b/lib/ramble/docs/mirror-config.rst index 8c10da9a2..abcdc821e 100644 --- a/lib/ramble/docs/mirror-config.rst +++ b/lib/ramble/docs/mirror-config.rst @@ -1,4 +1,4 @@ -.. Copyright 2022-2024 Google LLC +.. Copyright 2022-2024 The Ramble Authors Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/docs/requirements.txt b/lib/ramble/docs/requirements.txt index e0248e2a5..975163043 100644 --- a/lib/ramble/docs/requirements.txt +++ b/lib/ramble/docs/requirements.txt @@ -3,3 +3,4 @@ docutils sphinx_rtd_theme sphinxcontrib-programoutput sphinxcontrib-jquery +sphinx-copybutton diff --git a/lib/ramble/docs/success_criteria.rst b/lib/ramble/docs/success_criteria.rst index a1fec0116..4638f1180 100644 --- a/lib/ramble/docs/success_criteria.rst +++ b/lib/ramble/docs/success_criteria.rst @@ -1,4 +1,4 @@ -.. Copyright 2022-2024 Google LLC +.. Copyright 2022-2024 The Ramble Authors Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/docs/tutorials.rst b/lib/ramble/docs/tutorials.rst index 622d91061..e5e04463b 100644 --- a/lib/ramble/docs/tutorials.rst +++ b/lib/ramble/docs/tutorials.rst @@ -1,4 +1,4 @@ -.. Copyright 2022-2024 Google LLC +.. Copyright 2022-2024 The Ramble Authors Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/docs/tutorials/10_using_modifiers.rst b/lib/ramble/docs/tutorials/10_using_modifiers.rst index 68d85ae1b..196600588 100644 --- a/lib/ramble/docs/tutorials/10_using_modifiers.rst +++ b/lib/ramble/docs/tutorials/10_using_modifiers.rst @@ -1,4 +1,4 @@ -.. Copyright 2022-2024 Google LLC +.. Copyright 2022-2024 The Ramble Authors Licensed under the Apache License, Version 2.0 or the MIT license @@ -144,12 +144,11 @@ The resulting configuration file should look like the following. variables: n_nodes: [1, 2] spack: - concretized: true packages: gcc9: spack_spec: gcc@9.4.0 intel-mpi: - spack_spec: intel-mpi@2018.4.274 + spack_spec: intel-oneapi-mpi@2021.11.0 compiler: gcc9 wrfv4: spack_spec: wrf@4.2 build_type=dm+sm compile_type=em_real nesting=basic ~chem @@ -182,10 +181,10 @@ modifier. Advanced Modifiers ----------------- -Some modifiers have additionally functionality, which can include requiring +Some modifiers have additional functionality, which can include requiring specific software packages to be present. An example of this is the ``intel-aps`` modifier, which applies Intel's -`Application Performan Snapshot`_ +`Application Performance Snapshot `_ to a workspace's experiments. To get information about the ``intel-aps`` modifier, execute: @@ -239,12 +238,11 @@ look like the following: variables: n_nodes: [1, 2] spack: - concretized: true packages: gcc9: spack_spec: gcc@9.4.0 intel-mpi: - spack_spec: intel-mpi@2018.4.274 + spack_spec: intel-oneapi-mpi@2021.11.0 compiler: gcc9 wrfv4: spack_spec: wrf@4.2 build_type=dm+sm compile_type=em_real nesting=basic ~chem @@ -309,12 +307,11 @@ configuration file should look like the following: variables: n_nodes: [1, 2] spack: - concretized: true packages: gcc9: spack_spec: gcc@9.4.0 intel-mpi: - spack_spec: intel-mpi@2018.4.274 + spack_spec: intel-oneapi-mpi@2021.11.0 compiler: gcc9 aps: spack_spec: intel-oneapi-vtune @@ -342,3 +339,18 @@ following: * Socket(s) - From ``lscpu`` * MPI Time - From ``intel-aps`` * Disk I/O Time - From ``intel-aps`` + +Clean the Workspace +------------------- + +Once you are finished with the tutorial content, make sure you deactivate your workspace: + +.. code-block:: console + + $ ramble workspace deactivate + +Additionally, you can remove the workspace and all of its content with: + +.. code-block:: console + + $ ramble workspace remove modifiers_wrf diff --git a/lib/ramble/docs/tutorials/11_using_internals.rst b/lib/ramble/docs/tutorials/11_using_internals.rst index ddf857bb5..1fd04199c 100644 --- a/lib/ramble/docs/tutorials/11_using_internals.rst +++ b/lib/ramble/docs/tutorials/11_using_internals.rst @@ -1,4 +1,4 @@ -.. Copyright 2022-2024 Google LLC +.. Copyright 2022-2024 The Ramble Authors Licensed under the Apache License, Version 2.0 or the MIT license @@ -138,12 +138,11 @@ the following: variables: n_nodes: [1, 2] spack: - concretized: true packages: gcc9: spack_spec: gcc@9.4.0 intel-mpi: - spack_spec: intel-mpi@2018.4.274 + spack_spec: intel-oneapi-mpi@2021.11.0 compiler: gcc9 wrfv4: spack_spec: wrf@4.2 build_type=dm+sm compile_type=em_real nesting=basic ~chem @@ -245,12 +244,11 @@ configuration file should look like the following: variables: n_nodes: [1, 2] spack: - concretized: true packages: gcc9: spack_spec: gcc@9.4.0 intel-mpi: - spack_spec: intel-mpi@2018.4.274 + spack_spec: intel-oneapi-mpi@2021.11.0 compiler: gcc9 wrfv4: spack_spec: wrf@4.2 build_type=dm+sm compile_type=em_real nesting=basic ~chem @@ -337,12 +335,11 @@ following: variables: n_nodes: [1, 2, 4] spack: - concretized: true packages: gcc9: spack_spec: gcc@9.4.0 intel-mpi: - spack_spec: intel-mpi@2018.4.274 + spack_spec: intel-oneapi-mpi@2021.11.0 compiler: gcc9 wrfv4: spack_spec: wrf@4.2 build_type=dm+sm compile_type=em_real nesting=basic ~chem @@ -360,3 +357,18 @@ following: Examining the experiment run directories, you should see ``start_time`` and ``end_time`` in the same places as they were when you ran the explicitly defined order experiments. + +Clean the Workspace +------------------- + +Once you are finished with the tutorial content, make sure you deactivate your workspace: + +.. code-block:: console + + $ ramble workspace deactivate + +Additionally, you can remove the workspace and all of its content with: + +.. code-block:: console + + $ ramble workspace remove internals_wrf diff --git a/lib/ramble/docs/tutorials/1_hello_world.rst b/lib/ramble/docs/tutorials/1_hello_world.rst index d90f2ccb0..2e8f3fdc4 100644 --- a/lib/ramble/docs/tutorials/1_hello_world.rst +++ b/lib/ramble/docs/tutorials/1_hello_world.rst @@ -1,4 +1,4 @@ -.. Copyright 2022-2024 Google LLC +.. Copyright 2022-2024 The Ramble Authors Licensed under the Apache License, Version 2.0 or the MIT license @@ -48,6 +48,7 @@ filter available application definitions. For example: might output the following: +.. rst-class:: hide-copy .. code-block:: console ==> 10 applications @@ -61,6 +62,7 @@ The ``ramble list`` command also accepts regular expressions. For example: might output the following: +.. rst-class:: hide-copy .. code-block:: console ==> 5 applications @@ -174,7 +176,6 @@ following contents: variables: n_ranks: '1' spack: - concretized: true packages: {} environments: {} diff --git a/lib/ramble/docs/tutorials/2_running_a_simple_gromacs_experiment.rst b/lib/ramble/docs/tutorials/2_running_a_simple_gromacs_experiment.rst index c01cec3fd..2c8e90a76 100644 --- a/lib/ramble/docs/tutorials/2_running_a_simple_gromacs_experiment.rst +++ b/lib/ramble/docs/tutorials/2_running_a_simple_gromacs_experiment.rst @@ -1,4 +1,4 @@ -.. Copyright 2022-2024 Google LLC +.. Copyright 2022-2024 The Ramble Authors Licensed under the Apache License, Version 2.0 or the MIT license @@ -86,8 +86,8 @@ software configuration: spack_spec = gcc@9.3.0 Software Specs: - impi2018: - spack_spec = intel-mpi@2018.4.274 + impi2021: + spack_spec = intel-oneapi-mpi@2021.11.0 gromacs: spack_spec = gromacs@2020.5 compiler = gcc9 diff --git a/lib/ramble/docs/tutorials/3_modifying_a_gromacs_experiment.rst b/lib/ramble/docs/tutorials/3_modifying_a_gromacs_experiment.rst index 7947eba8c..20681d059 100644 --- a/lib/ramble/docs/tutorials/3_modifying_a_gromacs_experiment.rst +++ b/lib/ramble/docs/tutorials/3_modifying_a_gromacs_experiment.rst @@ -1,4 +1,4 @@ -.. Copyright 2022-2024 Google LLC +.. Copyright 2022-2024 The Ramble Authors Licensed under the Apache License, Version 2.0 or the MIT license @@ -56,7 +56,7 @@ To get detailed information about where variable definitions come from, you can .. code-block:: console - $ ramble workspace info -v + $ ramble workspace info --expansions The experiments section of this command's output might contain the following: @@ -214,7 +214,7 @@ These changes should now be reflected in the output of: .. code-block:: console - $ ramble workspace info -v + $ ramble workspace info --expansions .. include:: shared/gromacs_execute.rst diff --git a/lib/ramble/docs/tutorials/4_using_vectors_and_matrices.rst b/lib/ramble/docs/tutorials/4_using_vectors_and_matrices.rst index 11b0ee053..224759c18 100644 --- a/lib/ramble/docs/tutorials/4_using_vectors_and_matrices.rst +++ b/lib/ramble/docs/tutorials/4_using_vectors_and_matrices.rst @@ -1,4 +1,4 @@ -.. Copyright 2022-2024 Google LLC +.. Copyright 2022-2024 The Ramble Authors Licensed under the Apache License, Version 2.0 or the MIT license @@ -25,17 +25,6 @@ trying to use Ramble on your own, and illustrate how you might fix them. .. include:: shared/gromacs_workspace.rst -Activate the Workspace ----------------------- - -As you are using a pre-existing workspace, ensure it is activated (NOTE: you -only need to run this if you do not currently have the workspace active). - -.. code-block:: console - - $ ramble workspace activate basic_gromacs - - Experiment Descriptions ----------------------- @@ -70,7 +59,7 @@ To get detailed information about where variable definitions come from, you can .. code-block:: console - $ ramble workspace info -v + $ ramble workspace info --expansions The experiments section of this command's output might contain the following: @@ -154,7 +143,7 @@ These changes should now be reflected in the output of: .. code-block:: console - $ ramble workspace info -v + $ ramble workspace info -vvv Using Vector Variables ---------------------- @@ -377,7 +366,7 @@ cross product of the ``app_workload`` and ``type`` variable definitions. Since each has a length of two, the result would be a matrix with four elements in it. -After saving an exiting this file, the resulting experiments can be seen using the: +After saving and exiting this file, the resulting experiments can be seen using the: .. code-block:: console @@ -445,3 +434,20 @@ Which should contain the following output: Experiment: gromacs.water_gmx50.rf_4ranks .. include:: shared/gromacs_execute.rst + +Cleaning the Workspace +---------------------- + +After you are finished with the content of this tutorial, make sure you +deactivate your workspace using: + +.. code-block:: console + + $ ramble workspace deactivate + +If you no longer need the workspace materials, remove the entire workspace +with: + +.. code-block:: console + + $ ramble workspace remove basic_gromacs diff --git a/lib/ramble/docs/tutorials/5_changing_your_software_stack.rst b/lib/ramble/docs/tutorials/5_changing_your_software_stack.rst index a2f65c390..d4691ff99 100644 --- a/lib/ramble/docs/tutorials/5_changing_your_software_stack.rst +++ b/lib/ramble/docs/tutorials/5_changing_your_software_stack.rst @@ -1,4 +1,4 @@ -.. Copyright 2022-2024 Google LLC +.. Copyright 2022-2024 The Ramble Authors Licensed under the Apache License, Version 2.0 or the MIT license @@ -48,10 +48,10 @@ environments: gcc9: Spack spec: gcc@9.4.0 target=x86_64 Compiler spec: gcc@9.4.0 - impi2018: + impi2021: Rendered Packages: - impi2018: - Spack spec: intel-mpi@2018.4.274 target=x86_64 + impi2021: + Spack spec: intel-oneapi-mpi@2021.11.0 target=x86_64 Compiler: gcc9 gromacs: Rendered Packages: @@ -63,7 +63,7 @@ environments: Rendered Environments: gromacs Packages: - gromacs - - impi2018 + - impi2021 Currently, this command outputs every package and software environment @@ -80,13 +80,12 @@ The relevant portion of the workspace configuration file is: .. code-block:: YAML spack: - concretized: true packages: gcc9: spack_spec: gcc@9.4.0 target=x86_64 compiler_spec: gcc@9.4.0 - impi2018: - spack_spec: intel-mpi@2018.4.274 target=x86_64 + impi2021: + spack_spec: intel-oneapi-mpi@2021.11.0 target=x86_64 compiler: gcc9 gromacs: spack_spec: gromacs@2021.6 @@ -95,7 +94,7 @@ The relevant portion of the workspace configuration file is: gromacs: packages: - gromacs - - impi2018 + - impi2021 In this configuration, the ``packages`` block defines software packages that can be used to build experiment environments out of. The ``environments`` block diff --git a/lib/ramble/docs/tutorials/6_configuring_a_scaling_study.rst b/lib/ramble/docs/tutorials/6_configuring_a_scaling_study.rst index a5f184b06..1ea471ae1 100644 --- a/lib/ramble/docs/tutorials/6_configuring_a_scaling_study.rst +++ b/lib/ramble/docs/tutorials/6_configuring_a_scaling_study.rst @@ -1,4 +1,4 @@ -.. Copyright 2022-2024 Google LLC +.. Copyright 2022-2024 The Ramble Authors Licensed under the Apache License, Version 2.0 or the MIT license @@ -144,7 +144,7 @@ define additional important details such as how many MPI ranks you want in each experiment (defined by the ``n_ranks`` variable), or how many MPI ranks should execute on each node (defined by the ``processes_per_node`` variable). You should add these details as workspace variables within your configuration file. -For the purposes of this tutorial, we will assume 4 MPI ranks per node and that +For the purposes of this tutorial, we will assume 16 MPI ranks per node and that the number of MPI ranks total will be the number of MPI ranks per node multiplied by the number of nodes. @@ -223,12 +223,11 @@ look like the following: variables: n_nodes: [1, 2] spack: - concretized: true packages: gcc9: spack_spec: gcc@9.4.0 intel-mpi: - spack_spec: intel-mpi@2018.4.274 + spack_spec: intel-oneapi-mpi@2021.11.0 compiler: gcc9 wrfv4: spack_spec: wrf@4.2 build_type=dm+sm compile_type=em_real nesting=basic ~chem @@ -253,3 +252,20 @@ you see fit, and make sure the ``gcc9`` references under ``intel-mpi`` and Ramble also supports uploading the analyzed data to online databases, using ``ramble workspace analyze --upload``. We will not cover this functionality in detail here, but it is very useful for production experiments. + +Cleaning the Workspace +---------------------- + +After you are finished with the content of this tutorial, make sure you +deactivate your workspace using: + +.. code-block:: console + + $ ramble workspace deactivate + +If you no longer need the workspace materials, remove the entire workspace +with: + +.. code-block:: console + + $ ramble workspace remove scaling_wrf diff --git a/lib/ramble/docs/tutorials/7_using_zips_and_matrices.rst b/lib/ramble/docs/tutorials/7_using_zips_and_matrices.rst index 41bca9877..8f716616f 100644 --- a/lib/ramble/docs/tutorials/7_using_zips_and_matrices.rst +++ b/lib/ramble/docs/tutorials/7_using_zips_and_matrices.rst @@ -1,4 +1,4 @@ -.. Copyright 2022-2024 Google LLC +.. Copyright 2022-2024 The Ramble Authors Licensed under the Apache License, Version 2.0 or the MIT license @@ -112,12 +112,11 @@ the above section. The result should look like the following: variables: n_nodes: [1, 2] spack: - concretized: true packages: gcc9: spack_spec: gcc@9.4.0 intel-mpi: - spack_spec: intel-mpi@2018.4.274 + spack_spec: intel-oneapi-mpi@2021.11.0 compiler: gcc9 wrfv4: spack_spec: wrf@4.2 build_type=dm+sm compile_type=em_real nesting=basic ~chem @@ -172,7 +171,7 @@ example: - platform_config - n_nodes -Would result in 6 experiments. Adding this to you workspace configuration, you +Would result in 6 experiments. Adding this to your workspace configuration, you should have the following in your ``ramble.yaml``: .. code-block:: YAML @@ -203,12 +202,11 @@ should have the following in your ``ramble.yaml``: - platform_config - n_nodes spack: - concretized: true packages: gcc9: spack_spec: gcc@9.4.0 intel-mpi: - spack_spec: intel-mpi@2018.4.274 + spack_spec: intel-oneapi-mpi@2021.11.0 compiler: gcc9 wrfv4: spack_spec: wrf@4.2 build_type=dm+sm compile_type=em_real nesting=basic ~chem @@ -266,12 +264,11 @@ Your final configuration file should look something like: - platform_config - n_nodes spack: - concretized: true packages: gcc9: spack_spec: gcc@9.4.0 intel-mpi: - spack_spec: intel-mpi@2018.4.274 + spack_spec: intel-oneapi-mpi@2021.11.0 compiler: gcc9 wrfv4: spack_spec: wrf@4.2 build_type=dm+sm compile_type=em_real nesting=basic ~chem diff --git a/lib/ramble/docs/tutorials/8_var_expansion_indirection_and_stack_parameterization.rst b/lib/ramble/docs/tutorials/8_var_expansion_indirection_and_stack_parameterization.rst index 96eccdae3..b34eb01dc 100644 --- a/lib/ramble/docs/tutorials/8_var_expansion_indirection_and_stack_parameterization.rst +++ b/lib/ramble/docs/tutorials/8_var_expansion_indirection_and_stack_parameterization.rst @@ -1,4 +1,4 @@ -.. Copyright 2022-2024 Google LLC +.. Copyright 2022-2024 The Ramble Authors Licensed under the Apache License, Version 2.0 or the MIT license @@ -95,12 +95,11 @@ final configuration from the previous tutorial. - platform_config - n_nodes spack: - concretized: true packages: gcc9: spack_spec: gcc@9.4.0 intel-mpi: - spack_spec: intel-mpi@2018.4.274 + spack_spec: intel-oneapi-mpi@2021.11.0 compiler: gcc9 wrfv4: spack_spec: wrf@4.2 build_type=dm+sm compile_type=em_real nesting=basic ~chem @@ -112,9 +111,8 @@ final configuration from the previous tutorial. - intel-mpi - wrfv4 -The above configuration will execute 6 experiments, comprising a basic scaling -study on three different sets of nodes across two different platforms. This -configuration was the final result of the :ref:`zips_and_matrices` tutorial. +The above configuration will execute 4 experiments, comprising a basic scaling +study on three different sets of nodes across two different platforms. You will expand this definition to perform the same sweep over multiple MPI implementations. Over the course of this tutorial, you will learn how to use @@ -193,12 +191,11 @@ generation as well. The result might look like the following: - platform_config - n_nodes spack: - concretized: true packages: gcc9: spack_spec: gcc@9.4.0 intel-mpi: - spack_spec: intel-mpi@2018.4.274 + spack_spec: intel-oneapi-mpi@2021.11.0 compiler: gcc9 openmpi: spack_spec: openmpi@3.1.6 +orterunprefix @@ -228,7 +225,7 @@ Should result in the following error: ==> Error: Experiment wrfv4.CONUS_12km.scaling_1_platform1 is not unique. -As you have implicitly defined 12 experiments (3 from ``n_nodes``, times 2 from +As you have implicitly defined 8 experiments (2 from ``n_nodes``, times 2 from ``platform_config``, times another 2 from ``mpi_name``), but you haven't updated the experiment name template. To resolve this, add ``{mpi_name}`` into the experiment name template. Additionally, you may explicitly add ``mpi_name`` @@ -264,12 +261,11 @@ into the matrix. The result might look like the following: - n_nodes - mpi_name spack: - concretized: true packages: gcc9: spack_spec: gcc@9.4.0 intel-mpi: - spack_spec: intel-mpi@2018.4.274 + spack_spec: intel-oneapi-mpi@2021.11.0 compiler: gcc9 openmpi: spack_spec: openmpi@3.1.6 +orterunprefix @@ -286,7 +282,7 @@ into the matrix. The result might look like the following: Variable Expansion and Indirection ---------------------------------- -At this stage, you have defined a workspace that will execute 12 experiments. +At this stage, you have defined a workspace that will execute 8 experiments. It is important to point out that different MPI implementations have different command line flags for controlling their behavior. The existing ``mpi_command`` should work fine with both Intel MPI, and OpenMPI but to illustrate how @@ -374,12 +370,11 @@ resulting configuration might look like the following: - n_nodes - mpi_name spack: - concretized: true packages: gcc9: spack_spec: gcc@9.4.0 intel-mpi: - spack_spec: intel-mpi@2018.4.274 + spack_spec: intel-oneapi-mpi@2021.11.0 compiler: gcc9 openmpi: spack_spec: openmpi@3.1.6 +orterunprefix @@ -464,12 +459,11 @@ The resulting configuration file might look like the following: - n_nodes - mpi_name spack: - concretized: true packages: gcc9: spack_spec: gcc@9.4.0 intel-mpi: - spack_spec: intel-mpi@2018.4.274 + spack_spec: intel-oneapi-mpi@2021.11.0 compiler: gcc9 openmpi: spack_spec: openmpi@3.1.6 +orterunprefix diff --git a/lib/ramble/docs/tutorials/9_success_criteria.rst b/lib/ramble/docs/tutorials/9_success_criteria.rst index 4d57b5d68..0bcf15b4d 100644 --- a/lib/ramble/docs/tutorials/9_success_criteria.rst +++ b/lib/ramble/docs/tutorials/9_success_criteria.rst @@ -1,4 +1,4 @@ -.. Copyright 2022-2024 Google LLC +.. Copyright 2022-2024 The Ramble Authors Licensed under the Apache License, Version 2.0 or the MIT license @@ -75,7 +75,7 @@ users to be sure their experiments completed successfully. When WRF runs, it outputs the time each timestep takes. To begin with, you will define a new success criteria within your workspace configuration file that -validates some timing data is present in the output an experiment. In the +validates some timing data is present in the output of an experiment. In the experiment output, the timing data is prefixed with the string ``Timing for main``. @@ -120,12 +120,11 @@ configuration file might look like the following: variables: n_nodes: [1, 2] spack: - concretized: true packages: gcc9: spack_spec: gcc@9.4.0 intel-mpi: - spack_spec: intel-mpi@2018.4.274 + spack_spec: intel-oneapi-mpi@2021.11.0 compiler: gcc9 wrfv4: spack_spec: wrf@4.2 build_type=dm+sm compile_type=em_real nesting=basic ~chem @@ -195,12 +194,11 @@ resulting configuration file might look like the following: variables: n_nodes: [1, 2] spack: - concretized: true packages: gcc9: spack_spec: gcc@9.4.0 intel-mpi: - spack_spec: intel-mpi@2018.4.274 + spack_spec: intel-oneapi-mpi@2021.11.0 compiler: gcc9 wrfv4: spack_spec: wrf@4.2 build_type=dm+sm compile_type=em_real nesting=basic ~chem @@ -258,3 +256,18 @@ Also, running analyze in debug mode as: Will print significnatly more output, but you should see where Ramble tests the ``timing-present`` and ``correct-timesteps`` success criteria in the output. + +Clean the Workspace +------------------- + +Once you are finished with the tutorial content, make sure you deactivate your workspace: + +.. code-block:: console + + $ ramble workspace deactivate + +Additionally, you can remove the workspace and all of its content with: + +.. code-block:: console + + $ ramble workspace remove success_wrf diff --git a/lib/ramble/docs/tutorials/mirrors.rst b/lib/ramble/docs/tutorials/mirrors.rst index c4fcd8858..7597f55bc 100644 --- a/lib/ramble/docs/tutorials/mirrors.rst +++ b/lib/ramble/docs/tutorials/mirrors.rst @@ -1,4 +1,4 @@ -.. Copyright 2022-2024 Google LLC +.. Copyright 2022-2024 The Ramble Authors Licensed under the Apache License, Version 2.0 or the MIT license @@ -65,7 +65,6 @@ Write the following configuration into the file, save, and exit: n_nodes: 1 processes_per_node: 30 spack: - concretized: false packages: {} environments: {} @@ -94,12 +93,11 @@ will look something like this: n_nodes: 1 processes_per_node: 30 spack: - concretized: true packages: gcc9: spack_spec: gcc@9.3.0 intel-mpi: - spack_spec: intel-mpi@2018.4.274 + spack_spec: intel-oneapi-mpi@2021.11.0 compiler: gcc9 wrfv4: spack_spec: wrf@4.2 build_type=dm+sm compile_type=em_real nesting=basic ~chem @@ -127,8 +125,8 @@ this mirror in the first place. ==> Executing phase mirror_inputs ==> Executing phase create_spack_env - ==> Concretized intel-mpi@2018.4.274%gcc@ - - intel-mpi@2018.4.274%gcc@_etc. + ==> Concretized intel-oneapi-mpi@2021.11.0%gcc@ + - intel-oneapi-mpi@2021.11.0%gcc@_etc. - ^(short list of software prerequisistes for intel-mpi) ==> Concretized wrf@4.2%gcc@ @@ -143,8 +141,8 @@ this mirror in the first place. ==> Concretized wrf@4.2%gcc@ - (long list of software prerequisites for wrf@4.2) - ==> Concretized intel-mpi@2018.4.274%gcc@ - - intel-mpi@2018.4.274%gcc@_etc. + ==> Concretized intel-oneapi-mpi@2021.11.0%gcc@ + - intel-oneapi-mpi@2021.11.0%gcc@_etc. - ^(short list of software prerequisistes for intel-mpi) ==> Executing phase mirror_software diff --git a/lib/ramble/docs/tutorials/shared/gromacs_execute.rst b/lib/ramble/docs/tutorials/shared/gromacs_execute.rst index f7b0a0550..08a7a0fad 100644 --- a/lib/ramble/docs/tutorials/shared/gromacs_execute.rst +++ b/lib/ramble/docs/tutorials/shared/gromacs_execute.rst @@ -1,4 +1,4 @@ -.. Copyright 2022-2024 Google LLC +.. Copyright 2022-2024 The Ramble Authors Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/docs/tutorials/shared/gromacs_vector_workspace.rst b/lib/ramble/docs/tutorials/shared/gromacs_vector_workspace.rst index c304f3a40..f7feb30b9 100644 --- a/lib/ramble/docs/tutorials/shared/gromacs_vector_workspace.rst +++ b/lib/ramble/docs/tutorials/shared/gromacs_vector_workspace.rst @@ -1,4 +1,4 @@ -.. Copyright 2022-2024 Google LLC +.. Copyright 2022-2024 The Ramble Authors Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/docs/tutorials/shared/gromacs_workspace.rst b/lib/ramble/docs/tutorials/shared/gromacs_workspace.rst index b1d8b52bb..1bc895a8f 100644 --- a/lib/ramble/docs/tutorials/shared/gromacs_workspace.rst +++ b/lib/ramble/docs/tutorials/shared/gromacs_workspace.rst @@ -1,4 +1,4 @@ -.. Copyright 2022-2024 Google LLC +.. Copyright 2022-2024 The Ramble Authors Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/docs/tutorials/shared/wrf_execute.rst b/lib/ramble/docs/tutorials/shared/wrf_execute.rst index 8b9351c38..39513d6b2 100644 --- a/lib/ramble/docs/tutorials/shared/wrf_execute.rst +++ b/lib/ramble/docs/tutorials/shared/wrf_execute.rst @@ -1,4 +1,4 @@ -.. Copyright 2022-2024 Google LLC +.. Copyright 2022-2024 The Ramble Authors Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/docs/tutorials/shared/wrf_scaling_workspace.rst b/lib/ramble/docs/tutorials/shared/wrf_scaling_workspace.rst index 4695ecacb..af6c8b270 100644 --- a/lib/ramble/docs/tutorials/shared/wrf_scaling_workspace.rst +++ b/lib/ramble/docs/tutorials/shared/wrf_scaling_workspace.rst @@ -1,4 +1,4 @@ -.. Copyright 2022-2024 Google LLC +.. Copyright 2022-2024 The Ramble Authors Licensed under the Apache License, Version 2.0 or the MIT license @@ -22,7 +22,7 @@ Alternatively, the files can be edited directly with: $ ramble workspace edit -Within the ``ramble.yaml`` file, write the following contents, which the +Within the ``ramble.yaml`` file, write the following contents, which is the final configuration from a previous tutorial. .. code-block:: YAML @@ -45,12 +45,11 @@ final configuration from a previous tutorial. variables: n_nodes: [1, 2] spack: - concretized: true packages: gcc9: spack_spec: gcc@9.4.0 intel-mpi: - spack_spec: intel-mpi@2018.4.274 + spack_spec: intel-oneapi-mpi@2021.11.0 compiler: gcc9 wrfv4: spack_spec: wrf@4.2 build_type=dm+sm compile_type=em_real nesting=basic ~chem @@ -66,6 +65,6 @@ The above configuration will execute 2 experiments, comprising a basic scaling study on 2 different sets of nodes. This is primarily defined by the use of vector experiments, which are documented in the :ref:`vector logic` portion of the workspace configuration file -documentation.. Vector experiments were also introduced in the :ref:`vector and +documentation. Vector experiments were also introduced in the :ref:`vector and matrix tutorial `. diff --git a/lib/ramble/docs/workspace.rst b/lib/ramble/docs/workspace.rst index a6fc6dfc0..d63f156f0 100644 --- a/lib/ramble/docs/workspace.rst +++ b/lib/ramble/docs/workspace.rst @@ -1,4 +1,4 @@ -.. Copyright 2022-2024 Google LLC +.. Copyright 2022-2024 The Ramble Authors Licensed under the Apache License, Version 2.0 or the MIT license @@ -167,7 +167,7 @@ To get basic information, and: .. code-block:: console - $ ramble workspace info -v + $ ramble workspace info -vvv To get more detailed information, including which variables are defined and where they come from. @@ -184,6 +184,80 @@ application definition files, one can use: $ ramble workspace concretize +To remove any unused software definitions from the workspace configuration, +as well as unused experiment templates, one can use: + +.. code-block:: console + + $ ramble workspace concretize --simplify + +Note: This command will also remove comments within the edited section +of the workspace config file. + +--------------------- +Workspace Deployments +--------------------- + +A deployment is one mechanism of transferring a configured workspace from one +location to another. Ramble provides commands to handle creating (and pushing) +a deployment from a local workspace to a remote location, or pulling a +deployment from a remote location into a local workspace. + +A deployment is a directory that contains the necessary artifacts required to +recreate the experiments in the workspace on a separate machine. Deployments +copy the workspace configuration file, along with creating an object +repository, containing the application, modifier, and any package manager files +needed for the experiments (that might not be upstreamed). This section +describes the commands that can be used to use deployments. + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Preparing a Workspace Deployment +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Once a workspace is configured, it can be used to create a deployment. To prepare a +deployment, one can use: + +.. code-block:: console + + $ ramble deployment push + +This will populate a directory named ``deployments``, where the default is the +name of the workspace. + +The name of the created deployment can be controlled using: + +.. code-block:: console + + $ ramble deployment push -d + +Additionally, Ramble can create a tar of the deployment using: + +.. code-block:: console + + $ ramble deployment push -t + +And upload the deployment to a remote URL using: + +.. code-block:: console + + $ ramble deployment push -u + + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Pulling a Workspace Deployment +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To apply a deployment to an existing workspace, the ``pull`` sub-command can be used. For example: + +.. code-block:: console + + ramble workspace pull -p file://path/to/deployment + +Will overwrite the contents of the currently active workspace with the contents +from the deployment contained in ``file://path/to/deployment``. + +It is important to note that this command is destructive, and there is no way +to revert a workspace back to its state prior to the pull action. .. _workspace-setup: diff --git a/lib/ramble/docs/workspace_config.rst b/lib/ramble/docs/workspace_config.rst index d11e16d0d..5ab72ff08 100644 --- a/lib/ramble/docs/workspace_config.rst +++ b/lib/ramble/docs/workspace_config.rst @@ -1,4 +1,4 @@ -.. Copyright 2022-2024 Google LLC +.. Copyright 2022-2024 The Ramble Authors Licensed under the Apache License, Version 2.0 or the MIT license @@ -1069,8 +1069,8 @@ variable can be used to submit the same experiment to multiple batch systems. packages: gcc9: spack_spec: gcc@9.3.0 target=x86_64 - impi2018: - spack_spec: intel-mpi@2018.4.274 target=x86_64 + impi2021: + spack_spec: intel-oneapi-mpi@2021.11.0 target=x86_64 compiler: gcc9 gromacs: spack_spec: gromacs@2022.4 @@ -1078,7 +1078,7 @@ variable can be used to submit the same experiment to multiple batch systems. environments: gromacs: packages: - - impi2018 + - impi2021 - gromacs The above example overrides the generated ``batch_submit`` variable to change @@ -1182,7 +1182,7 @@ The ``variables`` keyword is optional. It can be used to override the definition of variables from the chained experiment if needed. Once the experiments are defined, the final order of the chain can be viewed using -``ramble workspace info -v``. +``ramble workspace info -vvv``. **NOTE** When using the ``experiment_index`` variable, all experiments in a chain share the same value. This ensures the resulting experiment will be diff --git a/lib/ramble/external/__init__.py b/lib/ramble/external/__init__.py index ce7468d5d..62a5aac65 100644 --- a/lib/ramble/external/__init__.py +++ b/lib/ramble/external/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/external/ordereddict_backport.py b/lib/ramble/external/ordereddict_backport.py index d96208d20..d4b5be2a2 100644 --- a/lib/ramble/external/ordereddict_backport.py +++ b/lib/ramble/external/ordereddict_backport.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/llnl/__init__.py b/lib/ramble/llnl/__init__.py index 43ed09c43..b27bb20ee 100644 --- a/lib/ramble/llnl/__init__.py +++ b/lib/ramble/llnl/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/llnl/util/__init__.py b/lib/ramble/llnl/util/__init__.py index 60cd9ddff..8fa4e1588 100644 --- a/lib/ramble/llnl/util/__init__.py +++ b/lib/ramble/llnl/util/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/llnl/util/argparsewriter.py b/lib/ramble/llnl/util/argparsewriter.py index 2a53409eb..39313c38d 100644 --- a/lib/ramble/llnl/util/argparsewriter.py +++ b/lib/ramble/llnl/util/argparsewriter.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/llnl/util/compat.py b/lib/ramble/llnl/util/compat.py index 071b9fa03..8edbb8cdf 100644 --- a/lib/ramble/llnl/util/compat.py +++ b/lib/ramble/llnl/util/compat.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/llnl/util/filesystem.py b/lib/ramble/llnl/util/filesystem.py index 09d07fb87..c57a5d3e9 100644 --- a/lib/ramble/llnl/util/filesystem.py +++ b/lib/ramble/llnl/util/filesystem.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/llnl/util/lang.py b/lib/ramble/llnl/util/lang.py index b11ac773d..5fadc5d85 100644 --- a/lib/ramble/llnl/util/lang.py +++ b/lib/ramble/llnl/util/lang.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/llnl/util/link_tree.py b/lib/ramble/llnl/util/link_tree.py index 87fa9e9b7..133a3d452 100644 --- a/lib/ramble/llnl/util/link_tree.py +++ b/lib/ramble/llnl/util/link_tree.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/llnl/util/lock.py b/lib/ramble/llnl/util/lock.py index 8144910c9..eff70c70b 100644 --- a/lib/ramble/llnl/util/lock.py +++ b/lib/ramble/llnl/util/lock.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/llnl/util/multiproc.py b/lib/ramble/llnl/util/multiproc.py index 6ad89d0fd..94757ff1d 100644 --- a/lib/ramble/llnl/util/multiproc.py +++ b/lib/ramble/llnl/util/multiproc.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/llnl/util/symlink.py b/lib/ramble/llnl/util/symlink.py index 52f33f50d..1e35b5747 100644 --- a/lib/ramble/llnl/util/symlink.py +++ b/lib/ramble/llnl/util/symlink.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/llnl/util/tty/__init__.py b/lib/ramble/llnl/util/tty/__init__.py index cef50f412..893819699 100644 --- a/lib/ramble/llnl/util/tty/__init__.py +++ b/lib/ramble/llnl/util/tty/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/llnl/util/tty/colify.py b/lib/ramble/llnl/util/tty/colify.py index 93ffbc53f..b2cbecf1a 100644 --- a/lib/ramble/llnl/util/tty/colify.py +++ b/lib/ramble/llnl/util/tty/colify.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/llnl/util/tty/color.py b/lib/ramble/llnl/util/tty/color.py index 4539b83d7..c7a167611 100644 --- a/lib/ramble/llnl/util/tty/color.py +++ b/lib/ramble/llnl/util/tty/color.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/llnl/util/tty/log.py b/lib/ramble/llnl/util/tty/log.py index a4729d3d6..7a5bb8089 100644 --- a/lib/ramble/llnl/util/tty/log.py +++ b/lib/ramble/llnl/util/tty/log.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/llnl/util/tty/pty.py b/lib/ramble/llnl/util/tty/pty.py index 225112383..7d66a4a6e 100644 --- a/lib/ramble/llnl/util/tty/pty.py +++ b/lib/ramble/llnl/util/tty/pty.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/__init__.py b/lib/ramble/ramble/__init__.py index 631deb5b3..4bb6539a9 100644 --- a/lib/ramble/ramble/__init__.py +++ b/lib/ramble/ramble/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -8,7 +8,7 @@ #: major, minor, patch version for Ramble, in a tuple -ramble_version_info = (0, 4, 0) +ramble_version_info = (0, 5, 0) #: String containing Ramble version joined with .'s ramble_version = '.'.join(str(v) for v in ramble_version_info) diff --git a/lib/ramble/ramble/appkit.py b/lib/ramble/ramble/appkit.py index d59e82b6e..762a67d5e 100644 --- a/lib/ramble/ramble/appkit.py +++ b/lib/ramble/ramble/appkit.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/application.py b/lib/ramble/ramble/application.py index 16b968bd9..65296fe30 100644 --- a/lib/ramble/ramble/application.py +++ b/lib/ramble/ramble/application.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -15,7 +15,6 @@ import string import shutil import fnmatch -import enum import time from typing import List @@ -29,6 +28,7 @@ import spack.util.compression import ramble.config +import ramble.graphs import ramble.stage import ramble.mirror import ramble.fetch_strategy @@ -44,29 +44,40 @@ import ramble.util.env import ramble.util.directives import ramble.util.stats +import ramble.util.graph +import ramble.util.class_attributes from ramble.util.logger import logger from ramble.workspace import namespace -from ramble.language.application_language import ApplicationMeta, register_phase -from ramble.language.shared_language import SharedMeta, register_builtin +from ramble.language.application_language import ApplicationMeta +from ramble.language.shared_language import SharedMeta, register_builtin, register_phase from ramble.error import RambleError from enum import Enum -experiment_status = Enum('experiment_status', ['UNKNOWN', 'SETUP', 'SUCCESS', 'FAILED']) +experiment_status = Enum('experiment_status', ['UNKNOWN', 'SETUP', 'RUNNING', + 'COMPLETE', 'SUCCESS', 'FAILED']) + +_NULL_CONTEXT = 'null' + + +def _get_context_display_name(context): + return ( + f'default ({_NULL_CONTEXT}) context' + if context == _NULL_CONTEXT + else f'{context} context' + ) class ApplicationBase(object, metaclass=ApplicationMeta): name = None uses_spack = False _builtin_name = 'builtin::{name}' - _exec_prefix_builtin = 'builtin::' - _mod_prefix_builtin = 'modifier_builtin::' _builtin_required_key = 'required' - _workload_exec_key = 'executables' _inventory_file_name = 'ramble_inventory.json' _status_file_name = 'ramble_status.json' - _pipelines = ['analyze', 'archive', 'mirror', 'setup', 'pushtocache', 'execute'] + _pipelines = ['analyze', 'archive', 'mirror', 'setup', + 'pushdeployment', 'pushtocache', 'execute'] _language_classes = [ApplicationMeta, SharedMeta] #: Lists of strings which contains GitHub usernames of attributes. @@ -79,6 +90,8 @@ class ApplicationBase(object, metaclass=ApplicationMeta): def __init__(self, file_path): super().__init__() + ramble.util.class_attributes.convert_class_attributes(self) + self.keywords = ramble.keywords.keywords self._vars_are_expanded = False @@ -87,8 +100,9 @@ def __init__(self, file_path): self.variables = None self.no_expand_vars = None self.experiment_set = None - self.internals = None + self.internals = {} self.is_template = False + self.generated_experiments = [] self.repeats = ramble.repeats.Repeats() self._command_list = [] self.chained_experiments = None @@ -96,7 +110,7 @@ def __init__(self, file_path): self.chain_prepend = [] self.chain_append = [] self.chain_commands = {} - self._env_variable_sets = None + self._env_variable_sets = [] self.modifiers = [] self.experiment_tags = [] self._modifier_instances = [] @@ -104,6 +118,8 @@ def __init__(self, file_path): self._input_fetchers = None self.results = {} self._phase_times = {} + self._pipeline_graphs = None + self.custom_executables = {} self.hash_inventory = { 'application_definition': None, @@ -120,23 +136,26 @@ def __init__(self, file_path): self.application_class = 'ApplicationBase' self._verbosity = 'short' - self._inject_required_builtins() self.license_path = '' self.license_file = '' - self.build_phase_order() - ramble.util.directives.define_directive_methods(self) def copy(self): """Deep copy an application instance""" new_copy = type(self)(self._file_path) + self.generated_experiments.append(new_copy) + + if self._env_variable_sets: + new_copy.set_env_variable_sets(self._env_variable_sets.copy()) + if self.variables: + new_copy.set_variables(self.variables.copy(), self.experiment_set) + if self.internals: + new_copy.set_internals(self.internals.copy()) + if self._formatted_executables: + new_copy.set_formatted_executables(self._formatted_executables.copy()) - new_copy.set_env_variable_sets(self._env_variable_sets.copy()) - new_copy.set_variables(self.variables.copy(), self.experiment_set) - new_copy.set_internals(self.internals.copy()) - new_copy.set_formatted_executables(self._formatted_executables.copy()) new_copy.set_template(False) new_copy.repeats.set_repeats(False, 0) new_copy.set_chained_experiments(None) @@ -156,120 +175,31 @@ def is_actionable(self): return True def build_phase_order(self): - for pipeline in self._pipelines: - pipeline_phases = [] + if self._pipeline_graphs is not None: + return + self._pipeline_graphs = {} + for pipeline in self._pipelines: if pipeline not in self.phase_definitions: self.phase_definitions[pipeline] = {} - # Detect cycles - for phase in self.phase_definitions[pipeline].keys(): - - phase_stack = [phase] - phases_touched = set() - while phase_stack: - cur_phase = phase_stack.pop() - - if cur_phase in phases_touched: - raise PhaseCycleDetectedError( - 'Cycle detected when ordering phases in ' - f'application {self.name}\n' - f'Phase {phase} ultimately depends on itself.' - ) - - for dep_phase in self.phase_definitions[pipeline][cur_phase]: - if dep_phase not in self.phase_definitions[pipeline].keys(): - raise InvalidPhaseError(f'In application {self.name}, phase ' - f'{dep_phase} is a dependency of ' - f'phase {phase} but is not defined.') - phase_stack.append(dep_phase) - - phases_to_add = [phase for phase in self.phase_definitions[pipeline].keys()] - - while phases_to_add: - cur_phase = phases_to_add.pop(0) - - earliest_idx = 0 - for dep_phase in self.phase_definitions[pipeline][cur_phase]: - if dep_phase not in pipeline_phases: - earliest_idx = None - break - else: - earliest_idx = max(earliest_idx, pipeline_phases.index(dep_phase) + 1) - - if earliest_idx is None: - phases_to_add.append(cur_phase) - elif earliest_idx == 0 or earliest_idx == len(pipeline_phases): - pipeline_phases.append(cur_phase) - else: - pipeline_phases.insert(earliest_idx, cur_phase) - - setattr(self, f'_{pipeline}_phases', pipeline_phases) - - def _inject_required_builtins(self): - required_builtins = [] - for builtin, blt_conf in self.builtins.items(): - if blt_conf[self._builtin_required_key]: - required_builtins.append(builtin) - - for workload, wl_conf in self.workloads.items(): - if self._workload_exec_key in wl_conf: - # Insert in reverse order, to make sure they are correctly ordered. - for builtin in reversed(required_builtins): - blt_conf = self.builtins[builtin] - if builtin not in wl_conf[self._workload_exec_key]: - if blt_conf['injection_method'] == 'prepend': - wl_conf[self._workload_exec_key].insert(0, builtin) - else: - wl_conf[self._workload_exec_key].append(builtin) - - def _inject_required_modifier_builtins(self): - """Inject builtins defined as required from each modifier into this - application instance.""" - if not self.modifiers or len(self._modifier_instances) == 0: - return - - required_prepend_builtins = [] - required_append_builtins = [] - - mod_regex = re.compile(ramble.modifier.ModifierBase._mod_builtin_regex + - r'(?P.*)') - for mod_inst in self._modifier_instances: - for builtin, blt_conf in mod_inst.builtins.items(): - if blt_conf[self._builtin_required_key]: - blt_match = mod_regex.match(builtin) - - # Each builtin should only be added once. - added = False - - if blt_conf['injection_method'] == 'prepend': - if builtin not in required_prepend_builtins: - required_prepend_builtins.append(builtin) - added = True - else: # Append - if builtin not in required_append_builtins: - required_append_builtins.append(builtin) - added = True - - # Only update if the builtin was added to a list. - if added: - self._modifier_builtins[builtin] = { - 'func': getattr(mod_inst, - blt_match.group("func")) - } + self._pipeline_graphs[pipeline] = ramble.graphs.PhaseGraph( + self.phase_definitions[pipeline], + self + ) - for workload, wl_conf in self.workloads.items(): - if self._workload_exec_key in wl_conf: - # Insert prepend builtins in reverse order, to make sure they - # are correctly ordered. - for builtin in reversed(required_prepend_builtins): - if builtin not in wl_conf[self._workload_exec_key]: - wl_conf[self._workload_exec_key].insert(0, builtin) + for mod_inst in self._modifier_instances: + # Define phase nodes + for phase, phase_node in mod_inst.all_pipeline_phases(pipeline): + self._pipeline_graphs[pipeline].add_node( + phase_node, obj_inst=mod_inst + ) - # Append builtins can be inserted in their correct order. - for builtin in required_append_builtins: - if builtin not in wl_conf[self._workload_exec_key]: - wl_conf[self._workload_exec_key].append(builtin) + # Define phase edges + for phase, phase_node in mod_inst.all_pipeline_phases(pipeline): + self._pipeline_graphs[pipeline].define_edges( + phase_node, internal_order=True + ) def _long_print(self): out_str = [] @@ -294,21 +224,16 @@ def _long_print(self): out_str.append(colified(self.tags, tty=True)) out_str.append('\n') - if hasattr(self, '_setup_phases'): - out_str.append('\n') - out_str.append(rucolor.section_title('Setup Pipeline Phases:\n')) - out_str.append(colified(self._setup_phases, tty=True)) - - if hasattr(self, '_analyze_phases'): + for pipeline in self._pipelines: out_str.append('\n') - out_str.append(rucolor.section_title('Analyze Pipeline Phases:\n')) - out_str.append(colified(self._analyze_phases, tty=True)) + out_str.append(rucolor.section_title(f'Pipeline "{pipeline}" Phases:\n')) + out_str.append(colified(self.get_pipeline_phases(pipeline), tty=True)) # Print all FOMs without a context if hasattr(self, 'figures_of_merit'): out_str.append('\n') out_str.append(rucolor.section_title('Figure of merit contexts:\n')) - out_str.append(rucolor.nested_1('\t(null) context (default):\n')) + out_str.append(rucolor.nested_1(f'\t({_NULL_CONTEXT}) context (default):\n')) for name, conf in self.figures_of_merit.items(): if len(conf['contexts']) == 0: out_str.append(rucolor.nested_2(f'\t\t{name}\n')) @@ -326,36 +251,8 @@ def _long_print(self): if hasattr(self, 'workloads'): out_str.append('\n') - for wl_name, wl_conf in self.workloads.items(): - out_str.append(rucolor.section_title('Workload:') + f' {wl_name}\n') - out_str.append('\t' + rucolor.nested_1('Executables: ') + - f'{wl_conf["executables"]}\n') - out_str.append('\t' + rucolor.nested_1('Inputs: ') + - f'{wl_conf["inputs"]}\n') - out_str.append('\t' + rucolor.nested_1('Workload Tags: \n')) - if 'tags' in wl_conf and wl_conf['tags']: - out_str.append(colified(wl_conf['tags'], indent=8) + '\n') - - if wl_name in self.environment_variables: - out_str.append(rucolor.nested_1('\tEnvironment Variables:\n')) - for var, conf in self.environment_variables[wl_name].items(): - indent = '\t\t' - - out_str.append(rucolor.nested_2(f'{indent}{var}:\n')) - out_str.append(f'{indent}\tDescription: {conf["description"]}\n') - out_str.append(f'{indent}\tValue: {conf["value"]}\n') - - if wl_name in self.workload_variables: - out_str.append(rucolor.nested_1('\tVariables:\n')) - for var, conf in self.workload_variables[wl_name].items(): - indent = '\t\t' - - out_str.append(rucolor.nested_2(f'{indent}{var}:\n')) - out_str.append(f'{indent}\tDescription: {conf["description"]}\n') - out_str.append(f'{indent}\tDefault: {conf["default"]}\n') - if 'values' in conf: - out_str.append(f'{indent}\tSuggested Values: {conf["values"]}\n') - + for workload in self.workloads.values(): + out_str.append(workload.as_str()) out_str.append('\n') if hasattr(self, 'builtins'): @@ -380,10 +277,11 @@ def set_variables(self, variables, experiment_set): self.no_expand_vars = set() workload_name = self.expander.workload_name - if workload_name in self.workload_variables: - for var, conf in self.workload_variables[workload_name].items(): - if 'expandable' in conf and not conf['expandable']: - self.no_expand_vars.add(var) + if workload_name in self.workloads: + for name, var in self.workloads[workload_name].variables.items(): + if not var.expandable: + self.no_expand_vars.add(var.name) + self.expander.set_no_expand_vars(self.no_expand_vars) def set_internals(self, internals): @@ -413,7 +311,8 @@ def set_tags(self, tags): self.experiment_tags = self.tags.copy() workload_name = self.expander.workload_name - self.experiment_tags.extend(self.workloads[workload_name]['tags']) + if workload_name in self.workloads: + self.experiment_tags.extend(self.workloads[workload_name].tags) if tags: self.experiment_tags.extend(tags) @@ -426,7 +325,7 @@ def has_tags(self, tags): """Check if this instance has provided tags. Args: - tags (list): List of strings, where each string is an indivudal tag + tags (list): List of strings, where each string is an individual tag Returns: (bool): True if all tags are in this instance, False otherwise """ @@ -450,31 +349,42 @@ def experiment_log_file(self, logs_dir): '.out' def get_pipeline_phases(self, pipeline, phase_filters=['*']): + self.build_modifier_instances() + self.build_phase_order() + if pipeline not in self._pipelines: logger.die(f'Requested pipeline {pipeline} is not valid.\n', f'\tAvailable pipelinese are {self._pipelines}') phases = set() - if hasattr(self, f'_{pipeline}_phases'): - for phase in getattr(self, f'_{pipeline}_phases'): + if pipeline in self._pipeline_graphs: + for phase in self._pipeline_graphs[pipeline].walk(): for phase_filter in phase_filters: - if fnmatch.fnmatch(phase, phase_filter): + if fnmatch.fnmatch(phase.key, phase_filter): phases.add(phase) - else: - logger.die(f'Pipeline {pipeline} is not defined in application {self.name}') include_phase_deps = ramble.config.get('config:include_phase_dependencies') if include_phase_deps: phases_for_deps = list(phases) while phases_for_deps: cur_phase = phases_for_deps.pop(0) - for dep_phase in self.phase_definitions[pipeline][cur_phase]: - if dep_phase not in phases: - phases_for_deps.append(dep_phase) - phases.add(dep_phase) - - return [phase for phase in getattr(self, f'_{pipeline}_phases') - if phase in phases] + for phase in phases: + if phase is not cur_phase and phase not in phases: + if cur_phase.key in phase._order_before: + phases_for_deps.append(phase) + phases.add(phase) + + for dep_phase_name in cur_phase._order_after: + dep_node = self._pipeline_graphs[pipeline].get_node(dep_phase_name) + if dep_node not in phases: + phases_for_deps.append(dep_node) + phases.add(dep_node) + + phase_order = [] + for node in self._pipeline_graphs[pipeline].walk(): + if node in phases: + phase_order.append(node.key) + return phase_order def _short_print(self): return [self.name] @@ -497,6 +407,58 @@ def print_vars(self, header='', vars_to_print=None, indent=''): expanded = self.expander.expand_var(expansion_var) color.cprint(f'{indent} {var} = {val} ==> {expanded}'.replace('@', '@@')) + def build_used_variables(self, workspace): + """Build a set of all used variables + + By expanding all necessary portions of this experiment (required / + reserved keywords, templates, commands, etc...), determine which + variables are used throughout the experiment definition. + + Variables can have list definitions. These are iterated over to ensure + variables referenced by any of them are tracked properly. + + Args: + workspace (Workspace): Workspace to extract templates from + + Returns: + (set): All variable names used by this experiment. + """ + self.build_modifier_instances() + self.add_expand_vars(workspace) + + # Add all known keywords + for key in self.keywords.keys: + self.expander._used_variables.add(key) + self.expander.expand_var_name(key) + + # Add modifier mode variables: + for mod_inst in self._modifier_instances: + for var in mod_inst.mode_variables().keys(): + self.expander._used_variables.add(var) + self.expander.expand_var_name(var) + + if self.chained_experiments: + for chained_exp in self.chained_experiments: + if namespace.inherit_variables in chained_exp: + for var in chained_exp[namespace.inherit_variables]: + self.expander._used_variables.add(var) + + # Add variables from success criteria + criteria_list = workspace.success_list + for criteria in criteria_list.all_criteria(): + if criteria.mode == 'fom_comparison': + self.expander.expand_var(criteria.formula) + self.expander.expand_var(criteria.fom_name) + self.expander.expand_var(criteria.fom_context) + elif criteria.mode == 'application_function': + self.evaluate_success() + + for template_name, template_conf in workspace.all_templates(): + self.expander._used_variables.add(template_name) + self.expander.expand_var(template_conf['contents']) + + return self.expander._used_variables + def print_internals(self, indent=''): if not self.internals: return @@ -542,10 +504,8 @@ def format_doc(self, **kwargs): return results.getvalue() # Phase execution helpers - def run_phase(self, phase, workspace): + def run_phase(self, pipeline, phase, workspace): """Run a phase, by getting its function pointer""" - self.build_modifier_instances() - self._inject_required_modifier_builtins() self.add_expand_vars(workspace) if self.is_template: logger.debug(f'{self.name} is a template. Skipping phases') @@ -554,14 +514,18 @@ def run_phase(self, phase, workspace): logger.debug(f'{self.name} is a repeat base. Skipping phases') return - if hasattr(self, f'_{phase}'): - logger.msg(f' Executing phase {phase}') - start_time = time.time() - for mod_inst in self._modifier_instances: - mod_inst.run_phase_hook(workspace, phase) - phase_func = getattr(self, f'_{phase}') - phase_func(workspace) - self._phase_times[phase] = time.time() - start_time + phase_node = self._pipeline_graphs[pipeline].get_node(phase) + + if phase_node is None: + logger.die(f'Phase {phase} is not defined in pipeline {pipeline}') + + logger.msg(f' Executing phase {phase}') + start_time = time.time() + for mod_inst in self._modifier_instances: + mod_inst.run_phase_hook(workspace, pipeline, phase) + phase_func = phase_node.attribute + phase_func(workspace, app_inst=self) + self._phase_times[phase] = time.time() - start_time def print_phase_times(self, pipeline, phase_filters=['*']): """Print phase execution times by pipeline phase order @@ -604,7 +568,7 @@ def create_experiment_chain(self, workspace): f' Primary experiment {parent_namespace}\n' + f' Chained expeirment name: {exp_name}\n' + f' Chain definition: {str(exp)}') - chain_stack.append((exp_name, exp)) + chain_stack.append((exp_name, exp.copy())) parent_run_dir = self.expander.expand_var( self.expander.expansion_str(self.keywords.experiment_run_dir) @@ -744,6 +708,11 @@ def create_experiment_chain(self, workspace): if exp_inst: exp_inst.chain_order = self.chain_order.copy() + def define_variable(self, var_name, var_value): + self.expander._variables[var_name] = var_value + for mod_inst in self._modifier_instances: + mod_inst.expander._variables[var_name] = var_value + def build_modifier_instances(self): """Built a map of modifier names to modifier instances needed for this application instance @@ -752,8 +721,7 @@ def build_modifier_instances(self): if not self.modifiers: return - if len(self._modifier_instances) > 0: - return + self._modifier_instances = [] mod_type = ramble.repository.ObjectTypes.modifiers @@ -781,98 +749,72 @@ def build_modifier_instances(self): # (note: the base ramble variables are checked earlier too) self.keywords.check_required_keys(self.variables) + # Ensure no expand vars are set correctly for modifiers + for mod_inst in self._modifier_instances: + for var in mod_inst.no_expand_vars(): + self.expander.add_no_expand_var(var) + mod_inst.expander.add_no_expand_var(var) + def define_modifier_variables(self): """Extract default variable definitions from modifier instances""" - def _get_executables(self): - """Return executables for add_expand_vars""" - - executables = self.workloads[self.expander.workload_name]['executables'] - - # Use yaml defined executable order, if defined - if namespace.executables in self.internals: - executables = self.internals[namespace.executables] - + def _define_custom_executables(self): # Define custom executables if namespace.custom_executables in self.internals: for name, conf in self.internals[namespace.custom_executables].items(): - self.executables[name] = ramble.util.executable.CommandExecutable( + if name in self.executables or name in self.custom_executables: + experiment_namespace = self.expander.expand_var_name('experiment_namespace') + raise ExecutableNameError(f'In experiment {experiment_namespace} ' + f'a custom executable "{name}" is defined.\n' + f'However, an executable "{name}" is already ' + 'defined') + + self.custom_executables[name] = ramble.util.executable.CommandExecutable( name=name, **conf ) - # Perform executable injection - if namespace.executable_injection in self.internals: + def _get_exec_order(self, workload_name): + graph = self._get_executable_graph(workload_name) + order = [] + for node in graph.walk(): + order.append(node.key) + return order - supported_orders = enum.Enum('supported_orders', ['before', 'after']) + def _get_executable_graph(self, workload_name): + """Return executables for add_expand_vars""" + self._define_custom_executables() + exec_order = self.workloads[workload_name].executables + # Use yaml defined executable order, if defined + if namespace.executables in self.internals: + exec_order = self.internals[namespace.executables] - # Order can be 'before' or 'after. - # If `relative_to` is not set, then before adds to be the beginning of the list - # and after (default) adds to the end of the list - # If `relative_to` IS set, then before adds before the first instance of - # the executable in the list - # and after (default) adds after the last instance of the - # executable in the list - # If `relative_to` is set, and the executable name is not found, raise a fatal error. - for exec_injection in self.internals[namespace.executable_injection]: - exec_name = exec_injection['name'] + builtin_objects = [self] + all_builtins = [self.builtins] + for mod_inst in self._modifier_instances: + builtin_objects.append(mod_inst) + all_builtins.append(mod_inst.builtins) - injection_order = supported_orders.after - if 'order' in exec_injection: - if not hasattr(supported_orders, exec_injection['order']): - logger.die('In experiment ' - f'"{self.expander.experiment_namespace}" ' - f'injection order of executable "{exec_name}" is set to an ' - f'invalid value of "{injection_order}".\n' - f'Valid values are {supported_orders}.') + all_executables = self.executables.copy() + all_executables.update(self.custom_executables) - injection_order = getattr(supported_orders, exec_injection['order']) + executable_graph = ramble.graphs.ExecutableGraph(exec_order, all_executables, + builtin_objects, all_builtins, + self) - relative = None + # Perform executable injection + if namespace.executable_injection in self.internals: + for exec_injection in self.internals[namespace.executable_injection]: + exec_name = exec_injection['name'] + order = 'before' + if 'order' in exec_injection: + order = exec_injection['order'] + relative_to = None if 'relative_to' in exec_injection: - relative = exec_injection['relative_to'] - - if exec_name not in self.executables: - logger.die('In experiment ' - f'"{self.expander.experiment_namespace}" ' - f'attempting to inject a non existing executable "{exec_name}".') - - if relative is not None and relative not in executables: - logger.die('In experiment ' - f'"{self.expander.experiment_namespace}" ' - f'attempting to inject executable "{exec_name}" ' - f'relative to a non existing executable "{relative}".') - - if relative is None: - if injection_order == supported_orders.before: - executables.insert(0, exec_name) - elif injection_order == supported_orders.after: - executables.append(exec_name) - else: - - found = False - if injection_order == supported_orders.before: - relative_index = 0 - increment = 1 - elif injection_order == supported_orders.after: - relative_index = len(executables) - 1 - increment = -1 - - while not found and relative_index <= len(executables) \ - and relative_index >= 0: - if executables[relative_index] == relative: - found = True - else: - relative_index += increment - - if injection_order == supported_orders.before: - injection_index = relative_index - elif injection_order == supported_orders.after: - injection_index = relative_index + 1 + relative_to = exec_injection['relative_to'] + executable_graph.inject_executable(exec_name, order, relative_to) - executables.insert(injection_index, exec_name) - - return executables + return executable_graph def _set_input_path(self): """Put input_path into self.variables[input_file] for add_expand_vars""" @@ -892,31 +834,42 @@ def _set_default_experiment_variables(self): if they haven't been set already""" # Set default experiment variables, if they haven't been set already var_sets = [] - if self.expander.workload_name in self.workload_variables: - var_sets.append(self.workload_variables[self.expander.workload_name]) + if self.expander.workload_name in self.workloads: + var_sets.append(self.workloads[self.expander.workload_name].variables) for mod_inst in self._modifier_instances: var_sets.append(mod_inst.mode_variables()) for var_set in var_sets: - for var, conf in var_set.items(): + for var, val in var_set.items(): if var not in self.variables.keys(): - self.variables[var] = conf['default'] + self.variables[var] = val.default + + if self.expander.workload_name in self.workloads: + workload = self.workloads[self.expander.workload_name] - if self.expander.workload_name in self.environment_variables: - wl_env_vars = self.environment_variables[self.expander.workload_name] + for name, env_var in workload.environment_variables.items(): + action = 'set' + value = env_var.value - for name, vals in wl_env_vars.items(): + # Since the type coming from the schema can either be None, or + # a complex polymorphic type we need to ensure it has a + # sensible base structure when it is not given + if not self._env_variable_sets: + self._env_variable_sets.append({'set': {}}) - action = vals['action'] - value = vals['value'] + # Since the type coming from the schema can either be None, or + # a complex polymorphic type we need to ensure it has a + # sensible base structure when it is not given + if not self._env_variable_sets: + self._env_variable_sets.append({'set': {}}) for env_var_set in self._env_variable_sets: if action in env_var_set: - if name not in env_var_set[action].keys(): - env_var_set[action][name] = value + if env_var.name not in env_var_set[action].keys(): + env_var_set[action][env_var.name] = value - def _define_commands(self, executables): + def _define_commands(self, exec_graph): """Populate the internal list of commands based on executables Populates self._command_list with a list of the executable commands that @@ -933,55 +886,36 @@ def _define_commands(self, executables): # ensure all log files are purged and set up logs = set() - builtin_regex = re.compile(r'%s(?P.*)' % self._exec_prefix_builtin) - modifier_regex = re.compile(ramble.modifier.ModifierBase._mod_prefix_builtin + - r'(?P.*)') - for executable in executables: - if not builtin_regex.search(executable) and \ - not modifier_regex.search(executable): - command_config = self.executables[executable] - if command_config.redirect: - logs.add(command_config.redirect) + + for exec_node in exec_graph.walk(): + if isinstance(exec_node.attribute, ramble.util.executable.CommandExecutable): + exec_cmd = exec_node.attribute + if exec_cmd.redirect: + logs.add(exec_cmd.redirect) for log in logs: self._command_list.append('rm -f "%s"' % log) self._command_list.append('touch "%s"' % log) - for executable in executables: - builtin_match = builtin_regex.match(executable) + for exec_node in exec_graph.walk(): + exec_vars = {'executable_name': exec_node.key} - exec_vars = {'executable_name': executable} - if executable in self.executables: - exec_vars.update(self.executables[executable].variables) + if isinstance(exec_node.attribute, ramble.util.executable.CommandExecutable): + exec_vars.update(exec_node.attribute.variables) for mod in self._modifier_instances: - if mod.applies_to_executable(executable): + if mod.applies_to_executable(exec_node.key): exec_vars.update(mod.modded_variables(self)) - if builtin_match: - # Process builtin executables - - # Get internal method: - func_name = f'{builtin_match.group("func")}' - func = getattr(self, func_name) - func_cmds = func() - for cmd in func_cmds: - self._command_list.append(self.expander.expand_var(cmd, exec_vars)) - elif executable in self._modifier_builtins.keys(): - builtin_def = self._modifier_builtins[executable] - func = builtin_def['func'] - func_cmds = func() - for cmd in func_cmds: - self._command_list.append(self.expander.expand_var(cmd, exec_vars)) - else: + if isinstance(exec_node.attribute, ramble.util.executable.CommandExecutable): # Process directive defined executables - base_command = self.executables[executable].copy() + base_command = exec_node.attribute.copy() pre_commands = [] post_commands = [] for mod in self._modifier_instances: - if mod.applies_to_executable(executable): - pre_cmd, post_cmd = mod.apply_executable_modifiers(executable, + if mod.applies_to_executable(exec_node.key): + pre_cmd, post_cmd = mod.apply_executable_modifiers(exec_node.key, base_command, app_inst=self) pre_commands.extend(pre_cmd) @@ -1001,12 +935,20 @@ def _define_commands(self, executables): out_log = self.expander.expand_var(cmd_conf.redirect, exec_vars) output_operator = cmd_conf.output_capture redirect = f' {output_operator} "{out_log}"' + if ramble.config.get('config:shell') in ['bash', 'sh']: + redirect += ' 2>&1' for part in cmd_conf.template: command_part = f'{mpi_cmd}{part}{redirect}' self._command_list.append(self.expander.expand_var(command_part, exec_vars)) + else: # All Builtins + func = exec_node.attribute + func_cmds = func() + for cmd in func_cmds: + self._command_list.append(self.expander.expand_var(cmd, exec_vars)) + # Inject all appended chained experiments for chained_exp in self.chain_append: self._command_list.append(self.chain_commands[chained_exp]) @@ -1030,7 +972,7 @@ def _define_formatted_executables(self): n_indentation = 0 if namespace.indentation in formatted_conf: - n_indentation = int(formatted_conf[namespace.indentation]) + 1 + n_indentation = int(formatted_conf[namespace.indentation]) prefix = '' if namespace.prefix in formatted_conf: @@ -1040,9 +982,7 @@ def _define_formatted_executables(self): if namespace.join_separator in formatted_conf: join_separator = formatted_conf[namespace.join_separator].replace(r'\n', '\n') - indentation = '' - for _ in range(0, n_indentation + 1): - indentation += ' ' + indentation = ' ' * n_indentation formatted_str = '' for cmd in self._command_list: @@ -1081,10 +1021,10 @@ def add_expand_vars(self, workspace): """ if not self._vars_are_expanded: self._validate_experiment() - executables = self._get_executables() + exec_graph = self._get_executable_graph(self.expander.workload_name) self._set_default_experiment_variables() self._set_input_path() - self._define_commands(executables) + self._define_commands(exec_graph) self._define_formatted_executables() self._derive_variables_for_template_path(workspace) @@ -1100,13 +1040,15 @@ def _inputs_and_fetchers(self, workload=None): if self._input_fetchers is not None: return + self._input_fetchers = {} + workload_names = [workload] if workload else self.workloads.keys() inputs = {} for workload_name in workload_names: workload = self.workloads[workload_name] - for input_file in workload['inputs']: + for input_file in workload.inputs: if input_file not in self.inputs: logger.die( f'Workload {workload_name} references a non-existent input file ' @@ -1141,7 +1083,7 @@ def _inputs_and_fetchers(self, workload=None): register_phase('mirror_inputs', pipeline='mirror') - def _mirror_inputs(self, workspace): + def _mirror_inputs(self, workspace, app_inst=None): """Mirror application inputs Perform mirroring of inputs within this application class. @@ -1160,7 +1102,7 @@ def _mirror_inputs(self, workspace): register_phase('get_inputs', pipeline='setup') - def _get_inputs(self, workspace): + def _get_inputs(self, workspace, app_inst=None): """Download application inputs Download application inputs into the proper directory within the workspace. @@ -1212,7 +1154,7 @@ def _prepare_license_path(self, workspace): register_phase('license_includes', pipeline='setup') - def _license_includes(self, workspace): + def _license_includes(self, workspace, app_inst=None): logger.debug("Writing License Includes") self._prepare_license_path(workspace) @@ -1237,9 +1179,9 @@ def _license_includes(self, workspace): if cmd: f.write(cmd + '\n') - register_phase('make_experiments', pipeline='setup', depends_on=['get_inputs']) + register_phase('make_experiments', pipeline='setup', run_after=['get_inputs']) - def _make_experiments(self, workspace): + def _make_experiments(self, workspace, app_inst=None): """Create experiment directories Create the experiment this application encapsulates. This includes @@ -1373,9 +1315,9 @@ def populate_inventory(self, workspace, force_compute=False, require_exist=False self.experiment_hash = ramble.util.hashing.hash_json(self.hash_inventory) - register_phase('write_inventory', pipeline='setup', depends_on=['make_experiments']) + register_phase('write_inventory', pipeline='setup', run_after=['make_experiments']) - def _write_inventory(self, workspace): + def _write_inventory(self, workspace, app_inst=None): """Build and write an inventory of an experiment Write an inventory file describing all of the contents of this @@ -1390,7 +1332,7 @@ def _write_inventory(self, workspace): register_phase('archive_experiments', pipeline='archive') - def _archive_experiments(self, workspace): + def _archive_experiments(self, workspace, app_inst=None): """Archive an experiment directory Perform the archiving action on an experiment. @@ -1439,7 +1381,7 @@ def _archive_experiments(self, workspace): register_phase('prepare_analysis', pipeline='analyze') - def _prepare_analysis(self, workspace): + def _prepare_analysis(self, workspace, app_inst=None): """Prepapre experiment for analysis extraction This function performs any actions that are necessary before the @@ -1450,9 +1392,9 @@ def _prepare_analysis(self, workspace): """ pass - register_phase('analyze_experiments', pipeline='analyze', depends_on=['prepare_analysis']) + register_phase('analyze_experiments', pipeline='analyze', run_after=['prepare_analysis']) - def _analyze_experiments(self, workspace): + def _analyze_experiments(self, workspace, app_inst=None): """Perform experiment analysis. This method will build up the fom_values dictionary. Its structure is: @@ -1473,11 +1415,10 @@ def _analyze_experiments(self, workspace): """ if self.get_status() == experiment_status.UNKNOWN.name and not workspace.dry_run: - logger.die( - f'Workspace status is {self.get_status()}\n' - 'Make sure your workspace is fully setup with\n' - ' ramble workspace setup' + logger.warn( + f'Experiment has status is {self.get_status()}. Skipping analysis..\n' ) + return def format_context(context_match, context_format): @@ -1487,8 +1428,7 @@ def format_context(context_match, context_format): if group[1]: context_val[group[1]] = context_match[group[1]] - context_string = context_format.replace('{', '').replace('}', '') \ - + ' = ' + context_format.format(**context_val) + context_string = context_format.format(**context_val) return context_string fom_values = {} @@ -1548,10 +1488,10 @@ def format_context(context_match, context_format): for context in fom_conf['contexts']: context_name = active_contexts[context] \ if context in active_contexts \ - else 'null' + else _NULL_CONTEXT fom_contexts.append(context_name) else: - fom_contexts.append('null') + fom_contexts.append(_NULL_CONTEXT) for context in fom_contexts: if context not in fom_values: @@ -1609,14 +1549,18 @@ def format_context(context_match, context_format): self.results['CONTEXTS'] = [] for context, fom_map in fom_values.items(): - context_map = {'name': context, 'foms': []} + context_map = { + 'name': context, + 'foms': [], + 'display_name': _get_context_display_name(context), + } for fom_name, fom in fom_map.items(): fom_copy = fom.copy() fom_copy['name'] = fom_name context_map['foms'].append(fom_copy) - if context == 'null': + if context == _NULL_CONTEXT: self.results['CONTEXTS'].insert(0, context_map) else: self.results['CONTEXTS'].append(context_map) @@ -1715,6 +1659,12 @@ def is_numeric(value): if repeat_success or workspace.always_print_foms: logger.debug(f'Calculating statistics for {self.repeats.n_repeats} repeats of ' f'{base_exp_name}') + + # Add defined keywords as top level keys + for key in self.keywords.keys: + if self.keywords.is_key_level(key): + self.results[key] = self.expander.expand_var_name(key) + self.results['RAMBLE_VARIABLES'] = {} self.results['RAMBLE_RAW_VARIABLES'] = {} for var, val in self.variables.items(): @@ -1757,7 +1707,11 @@ def is_numeric(value): # Iterate through the aggregated foms, calculate stats, and insert into results for context, fom_dict in repeat_foms.items(): - context_map = {'name': context, 'foms': []} + context_map = { + 'name': context, + 'foms': [], + 'display_name': _get_context_display_name(context), + } for fom_key, fom_values in fom_dict.items(): fom_name = fom_key[0] @@ -1925,10 +1879,10 @@ def get_status(self): """Get the status of this experiment""" return self.variables[self.keywords.experiment_status] - register_phase('write_status', pipeline='analyze', depends_on=['analyze_experiments']) - register_phase('write_status', pipeline='setup', depends_on=['make_experiments']) + register_phase('write_status', pipeline='analyze', run_after=['analyze_experiments']) + register_phase('write_status', pipeline='setup', run_after=['make_experiments']) - def _write_status(self, workspace): + def _write_status(self, workspace, app_inst=None): """Phase to write an experiment's ramble_status.json file""" status_data = {} @@ -1946,6 +1900,22 @@ def _write_status(self, workspace): with open(status_path, 'w+') as f: spack.util.spack_json.dump(status_data, f) + register_phase('deploy_artifacts', pipeline='pushdeployment') + + def _deploy_artifacts(self, workspace, app_inst=None): + repo_path = os.path.join(workspace.named_deployment, 'object_repo') + + app_dir_name = os.path.basename(os.path.dirname(self._file_path)) + app_dir = os.path.join(repo_path, 'applications', app_dir_name) + fs.mkdirp(app_dir) + shutil.copyfile(self._file_path, os.path.join(app_dir, 'application.py')) + + for mod_inst in self._modifier_instances: + mod_dir_name = os.path.basename(os.path.dirname(mod_inst._file_path)) + mod_dir = os.path.join(repo_path, 'modifiers', mod_dir_name) + fs.mkdirp(mod_dir) + shutil.copyfile(mod_inst._file_path, os.path.join(mod_dir, 'modifier.py')) + register_builtin('env_vars', required=True) def env_vars(self): @@ -2006,6 +1976,12 @@ class ApplicationError(RambleError): """ +class ExecutableNameError(RambleError): + """ + Exception raised when a name collision in executables happens + """ + + class FormattedExecutableError(ApplicationError): """ Exception raise when there are issues defining formatted executables diff --git a/lib/ramble/ramble/application_types/__init__.py b/lib/ramble/ramble/application_types/__init__.py index 60cd9ddff..8fa4e1588 100644 --- a/lib/ramble/ramble/application_types/__init__.py +++ b/lib/ramble/ramble/application_types/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/application_types/executable.py b/lib/ramble/ramble/application_types/executable.py index 6d5417e78..416ddfdeb 100644 --- a/lib/ramble/ramble/application_types/executable.py +++ b/lib/ramble/ramble/application_types/executable.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/application_types/spack.py b/lib/ramble/ramble/application_types/spack.py index a4b8d95bd..75240d02f 100644 --- a/lib/ramble/ramble/application_types/spack.py +++ b/lib/ramble/ramble/application_types/spack.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -8,10 +8,13 @@ import os import six +import shutil -from ramble.language.application_language import register_phase -from ramble.language.shared_language import register_builtin +import llnl.util.filesystem as fs + +from ramble.language.shared_language import register_builtin, register_phase from ramble.application import ApplicationBase, ApplicationError +from ramble.software_environments import ExternalEnvironment import ramble.spack_runner import ramble.keywords from ramble.util.logger import logger @@ -39,13 +42,13 @@ class SpackApplication(ApplicationBase): """ uses_spack = True - _spec_groups = [('default_compilers', 'Default Compilers'), + _spec_groups = [('compilers', 'Compilers'), ('mpi_libraries', 'MPI Libraries'), ('software_specs', 'Software Specs')] _spec_keys = ['spack_spec', 'compiler_spec', 'compiler'] register_phase('make_experiments', pipeline='setup', - depends_on=['define_package_paths']) + run_after=['define_package_paths']) def __init__(self, file_path): super().__init__(file_path) @@ -77,10 +80,37 @@ def _long_print(self): return ''.join(out_str) + def build_used_variables(self, workspace): + """Build a set of all used variables + + By expanding all necessary portions of this experiment (required / + reserved keywords, templates, commands, etc...), determine which + variables are used throughout the experiment definition. + + Variables can have list definitions. These are iterated over to ensure + variables referenced by any of them are tracked properly. + + Args: + workspace (Workspace): Workspace to extract templates from + + Returns: + (set): All variable names used by this experiment. + """ + _ = super().build_used_variables(workspace) + + app_context = self.expander.expand_var_name(self.keywords.env_name) + + software_environments = workspace.software_environments + software_environments.render_environment(app_context, + self.expander, + require=False) + + return self.expander._used_variables + register_phase('software_install_requested_compilers', pipeline='setup', - depends_on=['software_create_env']) + run_after=['software_create_env']) - def _software_install_requested_compilers(self, workspace): + def _software_install_requested_compilers(self, workspace, app_inst=None): """Install compilers an application uses""" # See if we cached this already, and if so return env_path = self.expander.env_path @@ -101,38 +131,21 @@ def _software_install_requested_compilers(self, workspace): app_context = self.expander.expand_var_name(self.keywords.env_name) - compilers_to_install = set() - root_compilers = [] - for pkg_name in workspace.software_environments.get_env_packages(app_context): - pkg_spec = workspace.software_environments.get_spec(pkg_name) - if 'compiler' in pkg_spec: - if pkg_spec['compiler'] not in compilers_to_install: - logger.debug(f' Adding root compiler: {pkg_spec["compiler"]}') - compilers_to_install.add(pkg_spec['compiler']) - root_compilers.append(pkg_spec['compiler']) - - dep_compilers = [] - for comp_name in root_compilers: - cur_spec = workspace.software_environments.get_spec(comp_name) - while 'compiler' in cur_spec and cur_spec['compiler'] not in compilers_to_install: - compilers_to_install.add(cur_spec['compiler']) - dep_compilers.append(cur_spec['compiler']) - logger.debug(f' Adding dependency compiler: {cur_spec["compiler"]}') - cur_spec = workspace.software_environments.get_spec(cur_spec['compiler']) - - # Install all compilers, starting with deps: - for comp_pkg in reversed(root_compilers + dep_compilers): - spec_str = workspace.software_environments.get_spec_string(comp_pkg) - logger.debug(f'Installing compiler: {comp_pkg}') - self.spack_runner.install_compiler(spec_str) + software_envs = workspace.software_environments + software_env = software_envs.render_environment(app_context, self.expander) + + for compiler_spec in software_envs.compiler_specs_for_environment(software_env): + logger.debug(f'Installing compiler: {compiler_spec}') + self.spack_runner.install_compiler(compiler_spec) except ramble.spack_runner.RunnerError as e: logger.die(e) register_phase('software_create_env', pipeline='mirror') register_phase('software_create_env', pipeline='setup') + register_phase('software_create_env', pipeline='pushdeployment') - def _software_create_env(self, workspace): + def _software_create_env(self, workspace, app_inst=None): """Create the spack environment for this experiment Extract all specs this experiment uses, and write the spack environment @@ -179,13 +192,13 @@ def _software_create_env(self, workspace): f.write(self.expander.expand_var(contents)) env_context = self.expander.expand_var_name(self.keywords.env_name) - external_spack_env = workspace.external_spack_env(env_context) - if external_spack_env: - self.spack_runner.copy_from_external_env(external_spack_env) + software_envs = workspace.software_environments + software_env = software_envs.render_environment(env_context, self.expander) + if isinstance(software_env, ExternalEnvironment): + self.spack_runner.copy_from_external_env(software_env.external_env) else: - for pkg_name in workspace.software_environments.get_env_packages(env_context): - spec_str = workspace.software_environments.get_spec_string(pkg_name) - self.spack_runner.add_spec(spec_str) + for pkg_spec in software_envs.package_specs_for_environment(software_env): + self.spack_runner.add_spec(pkg_spec) self.spack_runner.generate_env_file() @@ -211,9 +224,9 @@ def _software_create_env(self, workspace): logger.die(e) register_phase('software_configure', pipeline='setup', - depends_on=['software_create_env', 'software_install_requested_compilers']) + run_after=['software_create_env', 'software_install_requested_compilers']) - def _software_configure(self, workspace): + def _software_configure(self, workspace, app_inst=None): """Concretize the spack environment for this experiment Perform spack's concretize step on the software environment generated @@ -246,9 +259,9 @@ def _software_configure(self, workspace): logger.die(e) register_phase('software_install', pipeline='setup', - depends_on=['software_configure']) + run_after=['software_configure']) - def _software_install(self, workspace): + def _software_install(self, workspace, app_inst=None): """Install application's software using spack""" # See if we cached this already, and if so return @@ -273,9 +286,9 @@ def _software_install(self, workspace): logger.die(e) register_phase('evaluate_requirements', pipeline='setup', - depends_on=['software_install']) + run_after=['software_install']) - def _evaluate_requirements(self, workspace): + def _evaluate_requirements(self, workspace, app_inst=None): """Evaluate all requirements for this experiment""" for mod_inst in self._modifier_instances: @@ -286,9 +299,9 @@ def _evaluate_requirements(self, workspace): self.spack_runner.validate_command(**expanded_req) register_phase('define_package_paths', pipeline='setup', - depends_on=['software_install', 'evaluate_requirements']) + run_after=['software_install', 'evaluate_requirements']) - def _define_package_paths(self, workspace): + def _define_package_paths(self, workspace, app_inst=None): """Define variables containing the path to all spack packages For every spack package defined within an application context, define @@ -309,26 +322,43 @@ def _define_package_paths(self, workspace): logger.msg('Defining Spack variables') + cache = workspace.pkg_path_cache + app_context = self.expander.expand_var_name(self.keywords.env_name) + software_environments = workspace.software_environments + software_environment = software_environments.render_environment( + app_context, self.expander + ) + # Try to resolve using local cache first + unresolved_specs = [] + for pkg_spec in software_environments.package_specs_for_environment( + software_environment + ): + if pkg_spec in cache: + spack_pkg_name, pkg_path = cache.get(pkg_spec) + self.variables[spack_pkg_name] = pkg_path + else: + unresolved_specs.append(pkg_spec) + if not unresolved_specs: + return + try: + logger.debug('Resolving package paths using Spack') self.spack_runner.set_dry_run(workspace.dry_run) self.spack_runner.set_env(self.expander.env_path) self.spack_runner.activate() - app_context = self.expander.expand_var_name(self.keywords.env_name) - - for pkg_name in \ - workspace.software_environments.get_env_packages(app_context): - spec_str = workspace.software_environments.get_spec_string(pkg_name) - spack_pkg_name, package_path = self.spack_runner.get_package_path(spec_str) - self.variables[spack_pkg_name] = package_path + for pkg_spec in unresolved_specs: + spack_pkg_name, pkg_path = self.spack_runner.get_package_path(pkg_spec) + self.variables[spack_pkg_name] = pkg_path + cache[pkg_spec] = (spack_pkg_name, pkg_path) except ramble.spack_runner.RunnerError as e: logger.die(e) - register_phase('mirror_software', pipeline='mirror', depends_on=['software_create_env']) + register_phase('mirror_software', pipeline='mirror', run_after=['software_create_env']) - def _mirror_software(self, workspace): + def _mirror_software(self, workspace, app_inst=None): """Mirror software source for this experiment using spack""" import re @@ -388,9 +418,9 @@ def _mirror_software(self, workspace): except ramble.spack_runner.RunnerError as e: logger.die(e) - register_phase('push_to_spack_cache', pipeline='pushtocache', depends_on=[]) + register_phase('push_to_spack_cache', pipeline='pushtocache', run_after=[]) - def _push_to_spack_cache(self, workspace): + def _push_to_spack_cache(self, workspace, app_inst=None): env_path = self.expander.env_path cache_tupl = ('push-to-cache', env_path) @@ -449,9 +479,34 @@ def _clean_hash_variables(self, workspace, variables): super()._clean_hash_variables(workspace, variables) - register_builtin('spack_source', required=True) - register_builtin('spack_activate', required=True) - register_builtin('spack_deactivate', required=False) + register_phase('deploy_artifacts', pipeline='pushdeployment', + run_after=['software_create_env']) + + def _deploy_artifacts(self, workspace, app_inst=None): + super()._deploy_artifacts(workspace, app_inst=app_inst) + env_path = self.expander.env_path + + try: + self.spack_runner.set_dry_run(workspace.dry_run) + self.spack_runner.set_env(env_path) + self.spack_runner.activate() + + repo_path = os.path.join(workspace.named_deployment, 'object_repo') + + for pkg, pkg_def in self.spack_runner.package_definitions(): + pkg_dir_name = os.path.basename(os.path.dirname(pkg_def)) + pkg_dir = os.path.join(repo_path, 'packages', pkg_dir_name) + fs.mkdirp(pkg_dir) + shutil.copyfile(pkg_def, os.path.join(pkg_dir, 'package.py')) + + self.spack_runner.deactivate() + + except ramble.spack_runner.RunnerError as e: + logger.die(e) + + register_builtin('spack_source', required=True, depends_on=['builtin::env_vars']) + register_builtin('spack_activate', required=True, depends_on=['builtin::spack_source']) + register_builtin('spack_deactivate', required=False, depends_on=['builtin::spack_source']) def spack_source(self): return self.spack_runner.generate_source_command() diff --git a/lib/ramble/ramble/caches.py b/lib/ramble/ramble/caches.py index f745812c5..66e1b51fb 100644 --- a/lib/ramble/ramble/caches.py +++ b/lib/ramble/ramble/caches.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/cmd/__init__.py b/lib/ramble/ramble/cmd/__init__.py index c63dc9b80..96ca9e231 100644 --- a/lib/ramble/ramble/cmd/__init__.py +++ b/lib/ramble/ramble/cmd/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -273,13 +273,20 @@ def find_workspace(args): # nothing was set; there's no active environment if not ws: return None - - # if we get here, env isn't the name of a spack environment; it has - # to be a path to an environment, or there is something wrong. + elif not ramble.workspace.is_workspace_dir(ws): + env_var = ramble.workspace.ramble_workspace_var + raise ramble.workspace.RambleActiveWorkspaceError( + f'The environment variable {env_var} refers to an invalid ramble workspace.' + ) + + # if we get here, ws isn't the name of a ramble workspace; it has + # to be a path to a workspace, or there is something wrong. if ramble.workspace.is_workspace_dir(ws): return ramble.workspace.Workspace(ws) - raise ramble.workspace.RambleWorkspaceError('no workspace in %s' % ws) + raise ramble.workspace.RambleWorkspaceError( + f'No workspace in {ws}' + ) def find_workspace_path(args): diff --git a/lib/ramble/ramble/cmd/attributes.py b/lib/ramble/ramble/cmd/attributes.py index a543cbe27..e80c8cda1 100644 --- a/lib/ramble/ramble/cmd/attributes.py +++ b/lib/ramble/ramble/cmd/attributes.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/cmd/clean.py b/lib/ramble/ramble/cmd/clean.py index 8c12973be..36d023765 100644 --- a/lib/ramble/ramble/cmd/clean.py +++ b/lib/ramble/ramble/cmd/clean.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -61,15 +61,20 @@ def clean(parser, args): if args.python_cache: logger.msg('Removing python cache files') - for directory in [lib_path, var_path]: - for root, dirs, files in os.walk(directory): - for f in files: - if f.endswith('.pyc') or f.endswith('.pyo'): - fname = os.path.join(root, f) - logger.debug(f'Removing {fname}') - os.remove(fname) - for d in dirs: - if d == '__pycache__': - dname = os.path.join(root, d) - logger.debug(f'Removing {dname}') - shutil.rmtree(dname) + remove_python_caches() + + +def remove_python_caches(): + logger.msg('Removing python cache files') + for directory in [lib_path, var_path]: + for root, dirs, files in os.walk(directory): + for f in files: + if f.endswith('.pyc') or f.endswith('.pyo'): + fname = os.path.join(root, f) + logger.debug(f'Removing {fname}') + os.remove(fname) + for d in dirs: + if d == '__pycache__': + dname = os.path.join(root, d) + logger.debug(f'Removing {dname}') + shutil.rmtree(dname) diff --git a/lib/ramble/ramble/cmd/commands.py b/lib/ramble/ramble/cmd/commands.py index 610e987b4..aad278ed2 100644 --- a/lib/ramble/ramble/cmd/commands.py +++ b/lib/ramble/ramble/cmd/commands.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/cmd/common/__init__.py b/lib/ramble/ramble/cmd/common/__init__.py index c4db969c7..f412f78bd 100644 --- a/lib/ramble/ramble/cmd/common/__init__.py +++ b/lib/ramble/ramble/cmd/common/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/cmd/common/arguments.py b/lib/ramble/ramble/cmd/common/arguments.py index e21f94925..07562be5f 100644 --- a/lib/ramble/ramble/cmd/common/arguments.py +++ b/lib/ramble/ramble/cmd/common/arguments.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -79,9 +79,9 @@ def specs(): def repo_type(): from ramble.repository import default_type, OBJECT_NAMES return Args( - '-t', '--type', default=default_type.name, + '-t', '--type', default='any', help=f"type of repositories to manage. Defaults to '{default_type.name}'. " - f"Allowed types are {str(OBJECT_NAMES)}", + f"Allowed types are {', '.join(OBJECT_NAMES)}, or any", ) diff --git a/lib/ramble/ramble/cmd/common/info.py b/lib/ramble/ramble/cmd/common/info.py index 23c415d7a..b0db7ccb7 100644 --- a/lib/ramble/ramble/cmd/common/info.py +++ b/lib/ramble/ramble/cmd/common/info.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/cmd/common/list.py b/lib/ramble/ramble/cmd/common/list.py index adde463ff..b5cab6fec 100644 --- a/lib/ramble/ramble/cmd/common/list.py +++ b/lib/ramble/ramble/cmd/common/list.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/cmd/config.py b/lib/ramble/ramble/cmd/config.py index f6bf87288..fcf7a5567 100644 --- a/lib/ramble/ramble/cmd/config.py +++ b/lib/ramble/ramble/cmd/config.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/cmd/debug.py b/lib/ramble/ramble/cmd/debug.py index f4df1151b..4a1f2676f 100644 --- a/lib/ramble/ramble/cmd/debug.py +++ b/lib/ramble/ramble/cmd/debug.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/cmd/deployment.py b/lib/ramble/ramble/cmd/deployment.py new file mode 100644 index 000000000..bedcbab0f --- /dev/null +++ b/lib/ramble/ramble/cmd/deployment.py @@ -0,0 +1,180 @@ +# Copyright 2022-2024 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import os +import sys + +import llnl.util.filesystem as fs + +import spack.util.spack_json as sjson +import spack.util.url as surl + +import ramble.cmd +import ramble.cmd.common.arguments +import ramble.cmd.common.arguments as arguments + +import ramble.fetch_strategy +import ramble.config +import ramble.stage +import ramble.workspace +import ramble.workspace.shell +import ramble.pipeline +import ramble.filters + +if sys.version_info >= (3, 3): + from collections.abc import Sequence # novm noqa: F401 +else: + from collections import Sequence # noqa: F401 + + +description = '(experimental) manage workspace deployments' +section = 'workspaces' +level = 'short' + +subcommands = [ + 'push', + 'pull', +] + + +def deployment_push_setup_parser(subparser): + """Push a workspace deployment""" + subparser.add_argument( + '--tar-archive', '-t', action='store_true', + dest='tar_archive', + help='create a tar.gz of the deployment directory.') + + subparser.add_argument( + '--deployment-name', '-d', dest='deployment_name', + default=None, + help='Name for deployment. Uses workspace name if not set.') + + subparser.add_argument( + '--upload-url', '-u', dest='upload_url', + default=None, + help='URL to upload deployment into. Upload tar if `-t` is specified..') + + arguments.add_common_arguments(subparser, ['phases', 'include_phase_dependencies', + 'where', 'exclude_where', 'filter_tags']) + + +def deployment_push(args): + current_pipeline = ramble.pipeline.pipelines.pushdeployment + ws = ramble.cmd.require_active_workspace(cmd_name='deployment push') + + filters = ramble.filters.Filters( + phase_filters=args.phases, + include_where_filters=args.where, + exclude_where_filters=args.exclude_where, + tags=args.filter_tags + ) + + pipeline_cls = ramble.pipeline.pipeline_class(current_pipeline) + + pipeline = pipeline_cls(ws, + filters, + create_tar=args.tar_archive, + upload_url=args.upload_url, + deployment_name=args.deployment_name) + + with ws.write_transaction(): + deployment_run_pipeline(args, pipeline) + + +def deployment_pull_setup_parser(subparser): + """Pull a workspace deployment into current workspace""" + subparser.add_argument( + '--deployment-path', '-p', dest='deployment_path', + help='Path to deployment that should be pulled') + + +def deployment_pull(args): + def pull_file(src, dest): + fetcher = ramble.fetch_strategy.URLFetchStrategy(url=src) + stage_dir = os.path.dirname(dest) + fs.mkdirp(stage_dir) + with ramble.stage.InputStage(fetcher, path=stage_dir, + name=os.path.basename(src)) as stage: + stage.fetch() + + ws = ramble.cmd.require_active_workspace(cmd_name='deployment pull') + + with ws.write_transaction(): + # Fetch deployment index first: + push_cls = ramble.pipeline.PushDeploymentPipeline + + remote_index_path = surl.join( + args.deployment_path, ramble.pipeline.PushDeploymentPipeline.index_filename + ) + local_index_path = \ + os.path.join(ws.root, push_cls.index_filename) + + pull_file(remote_index_path, local_index_path) + + with open(local_index_path, 'r') as f: + index_data = sjson.load(f) + + for file in index_data[push_cls.index_namespace]: + src = surl.join(args.deployment_path, file) + dest = os.path.join(ws.root, file) + if os.path.exists(dest): + fs.force_remove(dest) + + pull_file(src, dest) + + +def deployment_run_pipeline(args, pipeline): + include_phase_dependencies = getattr(args, 'include_phase_dependencies', None) + if include_phase_dependencies: + with ramble.config.override('config:include_phase_dependencies', True): + pipeline.run() + else: + pipeline.run() + + +#: Dictionary mapping subcommand names and aliases to functions +subcommand_functions = {} + + +def sanitize_arg_name(base_name): + """Allow function names to be remapped (eg `-` to `_`) """ + formatted_name = base_name.replace('-', '_') + return formatted_name + + +def setup_parser(subparser): + sp = subparser.add_subparsers(metavar='SUBCOMMAND', + dest='deployment_command') + + for name in subcommands: + if isinstance(name, (list, tuple)): + name, aliases = name[0], name[1:] + else: + aliases = [] + + # add commands to subcommands dict + function_name = sanitize_arg_name('deployment_%s' % name) + + function = globals()[function_name] + for alias in [name] + aliases: + subcommand_functions[alias] = function + + # make a subparser and run the command's setup function on it + setup_parser_cmd_name = sanitize_arg_name('deployment_%s_setup_parser' % name) + setup_parser_cmd = globals()[setup_parser_cmd_name] + + subsubparser = sp.add_parser( + name, aliases=aliases, help=setup_parser_cmd.__doc__, + description=setup_parser_cmd.__doc__) + setup_parser_cmd(subsubparser) + + +def deployment(parser, args): + """Look for a function called deployment_ and call it.""" + action = subcommand_functions[args.deployment_command] + action(args) diff --git a/lib/ramble/ramble/cmd/edit.py b/lib/ramble/ramble/cmd/edit.py index 7d2ad7918..826bc8c94 100644 --- a/lib/ramble/ramble/cmd/edit.py +++ b/lib/ramble/ramble/cmd/edit.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/cmd/flake8.py b/lib/ramble/ramble/cmd/flake8.py index 05987f20a..e06927ee5 100644 --- a/lib/ramble/ramble/cmd/flake8.py +++ b/lib/ramble/ramble/cmd/flake8.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/cmd/help.py b/lib/ramble/ramble/cmd/help.py index f16ea4cbd..41327aaf1 100644 --- a/lib/ramble/ramble/cmd/help.py +++ b/lib/ramble/ramble/cmd/help.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/cmd/info.py b/lib/ramble/ramble/cmd/info.py index 78fb6ebcd..b24e9070d 100644 --- a/lib/ramble/ramble/cmd/info.py +++ b/lib/ramble/ramble/cmd/info.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/cmd/license.py b/lib/ramble/ramble/cmd/license.py index 8e0403d97..b88fda2ae 100644 --- a/lib/ramble/ramble/cmd/license.py +++ b/lib/ramble/ramble/cmd/license.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -65,20 +65,30 @@ ] -def _all_ramble_files(root=ramble.paths.prefix): +def _get_modified_files(root): + """Get a list of modified files in the current repository.""" + diff_args = ['-C', root, 'diff', 'HEAD', '--name-only'] + files = git(*diff_args, output=str).split() + return files + + +def _all_ramble_files(root=ramble.paths.prefix, modified_only=False): """Generates root-relative paths of all files in the ramble repository.""" - visited = set() - for cur_root, folders, files in os.walk(root): - for filename in files: - path = os.path.realpath(os.path.join(cur_root, filename)) + if modified_only: + yield from _get_modified_files(root) + else: + visited = set() + for cur_root, folders, files in os.walk(root): + for filename in files: + path = os.path.realpath(os.path.join(cur_root, filename)) - if path not in visited: - yield os.path.relpath(path, root) - visited.add(path) + if path not in visited: + yield os.path.relpath(path, root) + visited.add(path) -def _licensed_files(root=ramble.paths.prefix): - for relpath in _all_ramble_files(root): +def _licensed_files(root=ramble.paths.prefix, modified_only=False): + for relpath in _all_ramble_files(root, modified_only=modified_only): if any(regex.match(relpath) for regex in licensed_files): yield relpath @@ -118,7 +128,7 @@ def error_messages(self): def _check_license(lines, path): license_lines = [ - r'Copyright 2022-2024 Google LLC', # noqa: E501 + r'Copyright 2022-2024 The Ramble Authors', # noqa: E501 r'Licensed under the Apache License, Version 2.0 or the MIT license', # noqa: E501 r', at your', # noqa: E501 @@ -178,7 +188,7 @@ def verify(args): license_errors = LicenseError() - for relpath in _licensed_files(args.root): + for relpath in _licensed_files(args.root, modified_only=args.modified): path = os.path.join(args.root, relpath) with open(path) as f: lines = [line for line in f][:license_lines] @@ -203,6 +213,13 @@ def setup_parser(subparser): verify_parser.add_argument( '--root', action='store', default=ramble.paths.prefix, help='scan a different prefix for license issues') + verify_parser.add_argument( + '--modified', + '-m', + action='store_true', + default=False, + help='verify only the modified files as outputted by `git ls-files --modified`', + ) def license(parser, args): diff --git a/lib/ramble/ramble/cmd/list.py b/lib/ramble/ramble/cmd/list.py index 598f16949..0191af0f3 100644 --- a/lib/ramble/ramble/cmd/list.py +++ b/lib/ramble/ramble/cmd/list.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/cmd/mirror.py b/lib/ramble/ramble/cmd/mirror.py index e493ad2e1..9b2147c68 100644 --- a/lib/ramble/ramble/cmd/mirror.py +++ b/lib/ramble/ramble/cmd/mirror.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/cmd/mods.py b/lib/ramble/ramble/cmd/mods.py index 892e8a80c..0cd128074 100644 --- a/lib/ramble/ramble/cmd/mods.py +++ b/lib/ramble/ramble/cmd/mods.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/cmd/on.py b/lib/ramble/ramble/cmd/on.py index 421fef4da..8b818db03 100644 --- a/lib/ramble/ramble/cmd/on.py +++ b/lib/ramble/ramble/cmd/on.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -32,6 +32,16 @@ def setup_parser(subparser): help='execution template for each experiment', required=False) + subparser.add_argument( + '--enable-per-experiment-prints', action='store_true', + dest='per_experiment_prints_on', + help='Enable per experiment prints (phases and log paths).') + + subparser.add_argument( + '--suppress-run-header', action='store_true', + dest='run_header_off', + help='Disable the logger header.') + arguments.add_common_arguments(subparser, ['where', 'exclude_where', 'filter_tags']) @@ -42,14 +52,19 @@ def ramble_on(args): executor = args.executor if args.executor else '{batch_submit}' filters = ramble.filters.Filters( - phase_filters=[], + phase_filters=['*'], include_where_filters=args.where, exclude_where_filters=args.exclude_where, tags=args.filter_tags ) + suppress_per_experiment_prints = not args.per_experiment_prints_on + suppress_run_header = args.run_header_off + pipeline_cls = ramble.pipeline.pipeline_class(current_pipeline) - pipeline = pipeline_cls(ws, filters, executor=executor) + pipeline = pipeline_cls(ws, filters, executor=executor, + suppress_per_experiment_prints=suppress_per_experiment_prints, + suppress_run_header=suppress_run_header) with ws.write_transaction(): pipeline.run() diff --git a/lib/ramble/ramble/cmd/python.py b/lib/ramble/ramble/cmd/python.py index f4ec3dd15..b1191f217 100644 --- a/lib/ramble/ramble/cmd/python.py +++ b/lib/ramble/ramble/cmd/python.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/cmd/repo.py b/lib/ramble/ramble/cmd/repo.py index 47ddf5f65..8c8afbe0c 100644 --- a/lib/ramble/ramble/cmd/repo.py +++ b/lib/ramble/ramble/cmd/repo.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -40,8 +40,12 @@ def setup_parser(subparser): create_parser.add_argument( 'directory', help="directory to create the repo in") create_parser.add_argument( - 'namespace', help="namespace to identify objects " - "in the repository. defaults to the directory name", nargs='?') + 'namespace', + metavar='new_namespace', + help="namespace to identify objects " + "in the repository. defaults to the directory name", + nargs='?', + ) create_parser.add_argument( '-d', '--subdirectory', action='store', @@ -89,113 +93,149 @@ def setup_parser(subparser): def repo_create(args): """Create a new repository.""" - obj_type = ramble.repository.ObjectTypes[args.type] + if args.type == 'any': + unified_repo = True + obj_type = ramble.repository.default_type + repo_type = 'applications and modifiers' + register_type = '' + else: + unified_repo = False + obj_type = ramble.repository.ObjectTypes[args.type] + repo_type = ramble.repository.ObjectTypes[args.type].name + register_type = f' -t {repo_type}' + subdir = args.subdirectory - if subdir is None: - subdir = ramble.repository.type_definitions[obj_type]['dir_name'] full_path, namespace = ramble.repository.create_repo( - args.directory, args.namespace, subdir, object_type=obj_type + args.directory, args.namespace, subdir, object_type=obj_type, + unified_repo=unified_repo ) - logger.msg(f"Created {obj_type.name} repo with namespace '{namespace}'.") + logger.msg(f"Created {repo_type} repo with namespace '{namespace}'.") logger.msg("To register it with ramble, run this command:", - f'ramble repo -t {obj_type.name} add {full_path}') + f'ramble repo{register_type} add {full_path}') def repo_add(args): """Add a repository to Ramble's configuration.""" path = args.path - obj_type = ramble.repository.ObjectTypes[args.type] - type_def = ramble.repository.type_definitions[obj_type] + if args.type == 'any': + obj_types = ramble.repository.ObjectTypes + else: + obj_types = [ramble.repository.ObjectTypes[args.type]] - # real_path is absolute and handles substitution. - canon_path = ramble.util.path.canonicalize_path(path) + for obj_type in obj_types: + type_def = ramble.repository.type_definitions[obj_type] - # check if the path exists - if not os.path.exists(canon_path): - logger.die(f"No such file or directory: {path}") + # real_path is absolute and handles substitution. + canon_path = ramble.util.path.canonicalize_path(path) - # Make sure the path is a directory. - if not os.path.isdir(canon_path): - logger.die(f"Not a Ramble repository: {path}") + # check if the path exists + if not os.path.exists(canon_path): + logger.die(f"No such file or directory: {path}") - # Make sure it's actually a ramble repository by constructing it. - repo = ramble.repository.Repo(canon_path, obj_type) + # Make sure the path is a directory. + if not os.path.isdir(canon_path): + logger.die(f"Not a Ramble repository: {path}") - # If that succeeds, finally add it to the configuration. - repos = ramble.config.get(type_def['config_section'], scope=args.scope) - if not repos: - repos = [] + # Make sure it's actually a ramble repository by constructing it. + repo = ramble.repository.Repo(canon_path, obj_type) - if repo.root in repos or path in repos: - logger.die(f"Repository is already registered with Ramble: {path}") + # If that succeeds, finally add it to the configuration. + repos = ramble.config.get(type_def['config_section'], scope=args.scope) + if not repos: + repos = [] - repos.insert(0, canon_path) - ramble.config.set(type_def['config_section'], repos, args.scope) - logger.msg(f"Added {obj_type.name} repo with namespace '{repo.namespace}'.") + if repo.root in repos or path in repos: + logger.warn(f"{obj_type.name} repository is already registered with Ramble: {path}") + else: + repos.insert(0, canon_path) + ramble.config.set(type_def['config_section'], repos, args.scope) + logger.msg(f"Added {obj_type.name} repo with namespace '{repo.namespace}'.") def repo_remove(args): """Remove a repository from Ramble's configuration.""" - obj_type = ramble.repository.ObjectTypes[args.type] - type_def = ramble.repository.type_definitions[obj_type] - - repos = ramble.config.get(type_def['config_section'], scope=args.scope) - namespace_or_path = args.namespace_or_path - - # If the argument is a path, remove that repository from config. - canon_path = ramble.util.path.canonicalize_path(namespace_or_path) - for repo_path in repos: - repo_canon_path = ramble.util.path.canonicalize_path(repo_path) - if canon_path == repo_canon_path: - repos.remove(repo_path) - ramble.config.set(type_def['config_section'], repos, args.scope) - logger.msg(f"Removed {obj_type.name} repository {repo_path}") - return - - # If it is a namespace, remove corresponding repo - for path in repos: - try: - repo = ramble.repository.Repo(path, obj_type) - if repo.namespace == namespace_or_path: - repos.remove(path) + if args.type == 'any': + obj_types = ramble.repository.ObjectTypes + else: + obj_types = [ramble.repository.ObjectTypes[args.type]] + + repo_removed = [False] * len(obj_types) + + for obj_idx, obj_type in enumerate(obj_types): + type_def = ramble.repository.type_definitions[obj_type] + + repos = ramble.config.get(type_def['config_section'], scope=args.scope) + namespace_or_path = args.namespace_or_path + + obj_complete = False + # If the argument is a path, remove that repository from config. + canon_path = ramble.util.path.canonicalize_path(namespace_or_path) + for repo_path in repos: + repo_canon_path = ramble.util.path.canonicalize_path(repo_path) + if canon_path == repo_canon_path: + repos.remove(repo_path) ramble.config.set(type_def['config_section'], repos, args.scope) - logger.msg(f"Removed {obj_type.name} repository {repo.root} " - f"with namespace '{repo.namespace}'.") - return - except ramble.repository.RepoError: - continue - - logger.die( - f"No {obj_type.name} repository with path or namespace: {namespace_or_path}" - ) + logger.msg(f"Removed {obj_type.name} repository {repo_path}") + obj_complete = True + repo_removed[obj_idx] = True + break + + if obj_complete: + break + + # If it is a namespace, remove corresponding repo + for path in repos: + try: + repo = ramble.repository.Repo(path, obj_type) + if repo.namespace == namespace_or_path: + repos.remove(path) + ramble.config.set(type_def['config_section'], repos, args.scope) + logger.msg(f"Removed {obj_type.name} repository {repo.root} " + f"with namespace '{repo.namespace}'.") + repo_removed[obj_idx] = True + obj_complete = True + break + except ramble.repository.RepoError: + continue + + if not any(repo_removed): + all_types = [str(obj_type.name) for obj_type in obj_types] + logger.die( + f"No repository for {all_types} with path or namespace: {namespace_or_path}" + ) def repo_list(args): """Show registered repositories and their namespaces.""" - obj_type = ramble.repository.ObjectTypes[args.type] - type_def = ramble.repository.type_definitions[obj_type] - - roots = ramble.config.get(type_def['config_section'], scope=args.scope) - repos = [] - for r in roots: - try: - repos.append(ramble.repository.Repo(r, obj_type)) - except ramble.repository.RepoError: - continue + if args.type == 'any': + obj_types = ramble.repository.ObjectTypes + else: + obj_types = [ramble.repository.ObjectTypes[args.type]] - if sys.stdout.isatty(): - msg = f"{len(repos)} {obj_type.name} repository" - msg += "y." if len(repos) == 1 else "ies." - logger.msg(msg) + for obj_type in obj_types: + type_def = ramble.repository.type_definitions[obj_type] - if not repos: - return + roots = ramble.config.get(type_def['config_section'], scope=args.scope) + repos = [] + for r in roots: + try: + repos.append(ramble.repository.Repo(r, obj_type)) + except ramble.repository.RepoError: + continue + + if sys.stdout.isatty(): + msg = f"{len(repos)} {obj_type.name} repositor" + msg += "y." if len(repos) == 1 else "ies." + logger.msg(msg) + + if not repos: + return - max_ns_len = max(len(r.namespace) for r in repos) - for repo in repos: - fmt = "%%-%ds%%s" % (max_ns_len + 4) - print(fmt % (repo.namespace, repo.root)) + max_ns_len = max(len(r.namespace) for r in repos) + for repo in repos: + fmt = "%%-%ds%%s" % (max_ns_len + 4) + print(fmt % (repo.namespace, repo.root)) def repo(parser, args): @@ -205,7 +245,7 @@ def repo(parser, args): 'remove': repo_remove, 'rm': repo_remove} - if args.type not in ramble.repository.OBJECT_NAMES: + if args.type != 'any' and args.type not in ramble.repository.OBJECT_NAMES: logger.die(f"Repository type '{args.type}' is not valid.") action[args.repo_command](args) diff --git a/lib/ramble/ramble/cmd/results.py b/lib/ramble/ramble/cmd/results.py index 4ede4d38e..10807cab2 100644 --- a/lib/ramble/ramble/cmd/results.py +++ b/lib/ramble/ramble/cmd/results.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/cmd/software_definitions.py b/lib/ramble/ramble/cmd/software_definitions.py index 4d7795a7e..592479c69 100644 --- a/lib/ramble/ramble/cmd/software_definitions.py +++ b/lib/ramble/ramble/cmd/software_definitions.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -60,7 +60,7 @@ def collect_definitions(): The maps are global to this module, and reused in other internal methods. """ - top_level_attrs = ['default_compilers', 'software_specs'] + top_level_attrs = ['compilers', 'software_specs'] types_to_print = [ ramble.repository.ObjectTypes.applications diff --git a/lib/ramble/ramble/cmd/unit_test.py b/lib/ramble/ramble/cmd/unit_test.py index 9cfac3b8e..6e4deeb93 100644 --- a/lib/ramble/ramble/cmd/unit_test.py +++ b/lib/ramble/ramble/cmd/unit_test.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/cmd/workspace.py b/lib/ramble/ramble/cmd/workspace.py index ff6cffa65..456c709d9 100644 --- a/lib/ramble/ramble/cmd/workspace.py +++ b/lib/ramble/ramble/cmd/workspace.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -193,7 +193,8 @@ def workspace_deactivate(args): ) if ramble.workspace.active_workspace() is None: - logger.die('No workspace is currently active.') + if ramble.workspace.ramble_workspace_var not in os.environ: + logger.die('No workspace is currently active.') cmds = ramble.workspace.shell.deactivate_header(args.shell) env_mods = ramble.workspace.shell.deactivate() @@ -301,7 +302,7 @@ def _workspace_create(name_or_path, dir=False, def workspace_remove_setup_parser(subparser): """remove an existing workspace""" subparser.add_argument( - 'rm_wrkspc', metavar='wrkspc', nargs='+', + 'rm_wrkspc', metavar='workspace', nargs='+', help='workspace(s) to remove') arguments.add_common_arguments(subparser, ['yes_to_all']) @@ -340,14 +341,33 @@ def workspace_remove(args): def workspace_concretize_setup_parser(subparser): """Concretize a workspace""" - pass + subparser.add_argument( + '-f', '--force-concretize', + dest='force_concretize', + action='store_true', + help='Overwrite software environment configuration with defaults defined in application ' + + 'definition', + required=False) + subparser.add_argument( + '--simplify', + dest='simplify', + action='store_true', + help='Remove unused software and experiment templates from workspace config', + required=False) def workspace_concretize(args): ws = ramble.cmd.require_active_workspace(cmd_name='workspace concretize') - logger.debug('Concretizing workspace') - ws.concretize() + if args.simplify: + logger.debug('Simplifying workspace config') + ws.simplify() + else: + if args.force_concretize: + ws.force_concretize = True + + logger.debug('Concretizing workspace') + ws.concretize() def workspace_run_pipeline(args, pipeline): @@ -539,6 +559,9 @@ def workspace_info(args): # Print workspace variables information workspace_vars = ws.get_workspace_vars() + ws.software_environments = ramble.software_environments.SoftwareEnvironments(ws) + software_environments = ws.software_environments + # Build experiment set experiment_set = ws.build_experiment_set() @@ -587,6 +610,10 @@ def workspace_info(args): for exp_name, _, _ in print_experiment_set.filtered_experiments(filters): app_inst = experiment_set.get_experiment(exp_name) + if app_inst.uses_spack: + software_environments.render_environment( + app_inst.expander.expand_var('{env_name}'), app_inst.expander + ) if print_header: color.cprint(rucolor.nested_1(' Application: ') + @@ -650,8 +677,9 @@ def workspace_info(args): # Print software stack information if args.software: color.cprint('') - software_environments = ramble.software_environments.SoftwareEnvironments(ws) - software_environments.print_environments(verbosity=args.verbose) + # software_environments.print_environments(verbosity=args.verbose) + color.cprint(rucolor.section_title('Software Stack:')) + color.cprint(software_environments.info(verbosity=args.verbose, indent=4, color_level=1)) # # workspace list @@ -850,6 +878,6 @@ def setup_parser(subparser): def workspace(parser, args): - """Look for a function called environment_ and call it.""" + """Look for a function called workspace_ and call it.""" action = subcommand_functions[args.workspace_command] action(args) diff --git a/lib/ramble/ramble/config.py b/lib/ramble/ramble/config.py index fed1f0520..8d14fa97e 100644 --- a/lib/ramble/ramble/config.py +++ b/lib/ramble/ramble/config.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -478,7 +478,7 @@ def remove_scope(self, scope_name): def file_scopes(self): """List of writable scopes with an associated file.""" return [s for s in self.scopes.values() - if (type(s) == ConfigScope + if (type(s) == ConfigScope # noqa: E721 or type(s) == SingleFileScope)] def highest_precedence_scope(self): diff --git a/lib/ramble/ramble/context.py b/lib/ramble/ramble/context.py index d90f9ae07..d4dce1a53 100644 --- a/lib/ramble/ramble/context.py +++ b/lib/ramble/ramble/context.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/error.py b/lib/ramble/ramble/error.py index e74da2c3c..9b3f8e8ae 100644 --- a/lib/ramble/ramble/error.py +++ b/lib/ramble/ramble/error.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/expander.py b/lib/ramble/ramble/expander.py index be9ee54ca..f574a8122 100644 --- a/lib/ramble/ramble/expander.py +++ b/lib/ramble/ramble/expander.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -13,6 +13,8 @@ import math import random +from typing import Dict + import ramble.error import ramble.keywords from ramble.util.logger import logger @@ -20,12 +22,23 @@ import spack.util.naming supported_math_operators = { - ast.Add: operator.add, ast.Sub: operator.sub, - ast.Mult: operator.mul, ast.Div: operator.truediv, ast.Pow: - operator.pow, ast.BitXor: operator.xor, ast.USub: operator.neg, - ast.Eq: operator.eq, ast.NotEq: operator.ne, ast.Gt: operator.gt, - ast.GtE: operator.ge, ast.Lt: operator.lt, ast.LtE: operator.le, - ast.And: operator.and_, ast.Or: operator.or_, ast.Mod: operator.mod + ast.Add: operator.add, + ast.Sub: operator.sub, + ast.Mult: operator.mul, + ast.Div: operator.truediv, + ast.FloorDiv: operator.floordiv, + ast.Pow: operator.pow, + ast.BitXor: operator.xor, + ast.USub: operator.neg, + ast.Eq: operator.eq, + ast.NotEq: operator.ne, + ast.Gt: operator.gt, + ast.GtE: operator.ge, + ast.Lt: operator.lt, + ast.LtE: operator.le, + ast.And: operator.and_, + ast.Or: operator.or_, + ast.Mod: operator.mod, } supported_scalar_function_pointers = { @@ -108,7 +121,7 @@ def add_children(self, children): def define_value(self, expansion_dict, allow_passthrough=True, expansion_func=str, evaluation_func=eval, - no_expand_vars=set()): + no_expand_vars=set(), used_vars=set()): """Define the value for this node. Construct the value of self. This builds up a string representation of @@ -153,6 +166,7 @@ def define_value(self, expansion_dict, allow_passthrough=True, required_passthrough = False if kw_parts[0] in expansion_dict: + used_vars.add(kw_parts[0]) # Exit expansion for variables defined as no_expand if kw_parts[0] in no_expand_vars: self.value = expansion_dict[kw_parts[0]] @@ -293,6 +307,7 @@ def __init__(self, variables, experiment_set, no_expand_vars=set()): self._variables = variables self._no_expand_vars = no_expand_vars + self._used_variables = set() self._experiment_set = experiment_set @@ -303,7 +318,6 @@ def __init__(self, variables, experiment_set, no_expand_vars=set()): self._application_namespace = None self._workload_namespace = None self._experiment_namespace = None - self._env_namespace = None self._env_path = None self._application_input_dir = None @@ -314,6 +328,14 @@ def __init__(self, variables, experiment_set, no_expand_vars=set()): self._workload_run_dir = None self._experiment_run_dir = None + def add_no_expand_var(self, var: str): + """Add a new variable to the no expand set + + Args: + var (str): Variable that should not expand + """ + self._no_expand_vars.add(var) + def set_no_expand_vars(self, no_expand_vars): self._no_expand_vars = no_expand_vars.copy() @@ -365,15 +387,6 @@ def experiment_namespace(self): return self._experiment_namespace - @property - def env_namespace(self): - if not self._env_namespace: - var = self.expansion_str(self._keywords.env_name) + \ - '.' + self.expansion_str(self._keywords.workload_name) - self._env_namespace = self.expand_var(var) - - return self._env_namespace - @property def env_path(self): if not self._env_path: @@ -451,7 +464,8 @@ def expand_lists(self, var): except SyntaxError: return var - def expand_var_name(self, var_name, extra_vars=None, allow_passthrough=True): + def expand_var_name(self, var_name: str, extra_vars: Dict = None, + allow_passthrough: bool = True, typed: bool = False): """Convert a variable name to an expansion string, and expand it Take a variable name (var) and convert it to an expansion string by @@ -459,26 +473,30 @@ def expand_var_name(self, var_name, extra_vars=None, allow_passthrough=True): expand_var, and return the result. Args: - var_name: String name of variable to expand - extra_vars: Variable definitions to use with highest precedence - allow_passthrough: Whether the string is allowed to have keywords - after expansion + var_name (str): String name of variable to expand + extra_vars (dict): Variable definitions to use with highest precedence + allow_passthrough (bool): Whether the string is allowed to have keywords + after expansion + typed (bool): Whether the return type should be typed or not """ return self.expand_var(self.expansion_str(var_name), extra_vars=extra_vars, - allow_passthrough=allow_passthrough) + allow_passthrough=allow_passthrough, + typed=typed) - def expand_var(self, var, extra_vars=None, allow_passthrough=True): + def expand_var(self, var: str, extra_vars: Dict = None, + allow_passthrough: bool = True, typed: bool = False): """Perform expansion of a string Expand a string by building up a dict of all expansion variables. Args: - var: String variable to expand - extra_vars: Variable definitions to use with highest precedence - allow_passthrough: Whether the string is allowed to have keywords - after expansion + var (str): String variable to expand + extra_vars (dict): Variable definitions to use with highest precedence + allow_passthrough (bool): Whether the string is allowed to have keywords + after expansion + typed (bool): Whether the return type should be typed or not """ passthrough_setting = allow_passthrough @@ -503,6 +521,15 @@ def expand_var(self, var, extra_vars=None, allow_passthrough=True): f'{e}') logger.debug(f'END OF EXPAND_VAR STACK {value}') + if typed: + logger.debug(f'BEGINNING OF TYPING ON {value}') + try: + value = ast.literal_eval(value) + logger.debug(f'END OF TYPING {value}') + except ValueError: + logger.debug('END OF TYPING Failed with ValueError') + except SyntaxError: + logger.debug('END OF TYPING Failed with SyntaxError') return value def evaluate_predicate(self, in_str, extra_vars=None): @@ -553,7 +580,8 @@ def _partial_expand(self, expansion_vars, in_str, allow_passthrough=True): allow_passthrough=allow_passthrough, expansion_func=self._partial_expand, evaluation_func=self.perform_math_eval, - no_expand_vars=self._no_expand_vars) + no_expand_vars=self._no_expand_vars, + used_vars=self._used_variables) return str(str_graph.root.value) @@ -577,6 +605,8 @@ def perform_math_eval(self, in_str): except MathEvaluationError as e: logger.debug(f' Math input is: "{in_str}"') logger.debug(e) + except RambleSyntaxError as e: + raise RambleSyntaxError(f'{str(e)} in "{in_str}"') return in_str @@ -691,6 +721,9 @@ def _eval_comparisons(self, node): if len(node.ops) == 1 and isinstance(node.ops[0], ast.In): return self._eval_comp_in(node) + if len(node.ops) == 1 and isinstance(node.ops[0], ast.Is): + raise RambleSyntaxError('Encountered unsupported operator `is`') + # Try to evaluate the comparison logic, if not return the node as is. try: cur_left = self.eval_math(node.left) @@ -719,8 +752,9 @@ def _eval_comp_in(self, node): """Handle in nodes in the ast Perform extraction of ` in ` syntax. - Raises an exception if the experiment does not exist. + + Also, evaluated ` in [list, of, values]` syntax. """ if isinstance(node.left, ast.Name): var_name = self._ast_name(node.left) @@ -733,6 +767,21 @@ def _eval_comp_in(self, node): f'"{var_name} in {namespace}"') self.__raise_syntax_error(node) return val + # ast.Str was deprecated. short-circuit the test for it to avoid issues with newer python. + # TODO: Remove `or` logic after 3.6 & 3.7 series python are unsupported + elif isinstance(node.left, ast.Constant) or \ + (hasattr(ast, 'Str') and isinstance(node.left, ast.Str)): + lhs_value = self.eval_math(node.left) + + found = False + for comp in node.comparators: + if isinstance(comp, ast.List): + for elt in comp.elts: + rhs_value = self.eval_math(elt) + if lhs_value == rhs_value: + found = True + return found + self.__raise_syntax_error(node) def _eval_binary_ops(self, node): diff --git a/lib/ramble/ramble/experiment_set.py b/lib/ramble/ramble/experiment_set.py index a16e2c29b..3e692d050 100644 --- a/lib/ramble/ramble/experiment_set.py +++ b/lib/ramble/ramble/experiment_set.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -247,6 +247,85 @@ def _compute_mpi_vars(self, expander, variables): if not n_threads: variables[self.keywords.n_threads] = 1 + def _prepare_experiment(self, exp_template_name, variables, context, repeats): + """Prepare an experiment instance + + Create an experiment instance based on the input variables, context, + repeats, and template name. + + Args: + exp_template_name (str): Template name for experiments + variables (dict): Dictionary of variables for this experiment + context (Context): Context object for this experiment + repeats (Repeats): Repeats object for this experiment + + Returns: + (Application): Instance of an application class for this experiment + """ + variables[self.keywords.env_path] = os.path.join( + self._workspace.software_dir, Expander.expansion_str(self.keywords.env_name) + ) + + experiment_suffix = '' + # After generating the base experiment, append the index to repeat experiments + if repeats.repeat_index: + experiment_suffix = f'.{repeats.repeat_index}' + + expander = ramble.expander.Expander(variables, self) + self._compute_mpi_vars(expander, variables) + final_app_name = expander.expand_var_name(self.keywords.application_name, + allow_passthrough=False) + final_wl_name = expander.expand_var_name(self.keywords.workload_name, + allow_passthrough=False) + final_exp_name = expander.expand_var(exp_template_name + experiment_suffix, + allow_passthrough=False) + + variables[self.keywords.experiment_template_name] = (exp_template_name + + experiment_suffix) + variables[self.keywords.application_name] = final_app_name + variables[self.keywords.workload_name] = final_wl_name + variables[self.keywords.experiment_name] = final_exp_name + variables[self.keywords.experiment_index] = len(self.experiments) + 1 + + experiment_namespace = expander.experiment_namespace + variables[self.keywords.experiment_namespace] = experiment_namespace + + variables[self.keywords.log_file] = os.path.join('{experiment_run_dir}', + '{experiment_name}.out') + + variables[self.keywords.simplified_application_namespace] = \ + spack.util.naming.simplify_name( + expander.expand_var_name( + self.keywords.application_namespace + ) + ) + variables[self.keywords.simplified_workload_namespace] = \ + spack.util.naming.simplify_name( + expander.expand_var_name( + self.keywords.workload_namespace + ) + ) + variables[self.keywords.simplified_experiment_namespace] = \ + spack.util.naming.simplify_name( + expander.expand_var_name( + self.keywords.experiment_namespace + ) + ) + + app_inst = ramble.repository.get(final_app_name).copy() + app_inst.set_variables(variables, self) + app_inst.set_env_variable_sets(context.env_variables) + app_inst.set_internals(context.internals) + app_inst.set_template(context.is_template) + app_inst.repeats = repeats + app_inst.set_chained_experiments(context.chained_experiments) + app_inst.set_modifiers(context.modifiers) + app_inst.set_tags(context.tags) + app_inst.set_formatted_executables(context.formatted_executables) + app_inst.read_status() + + return app_inst + def _ingest_experiments(self): """Ingest experiments based on the current context. @@ -310,6 +389,7 @@ def _ingest_experiments(self): render_group.zips = final_context.zips render_group.matrices = final_context.matrices render_group.n_repeats = final_context.n_repeats + render_group.used_variables = set() excluded_experiments = set() if final_context.exclude: @@ -320,7 +400,8 @@ def _ingest_experiments(self): final_context.exclude) if perform_explicit_exclude: - for exclude_exp_vars, _ in renderer.render_objects(exclude_group): + for exclude_exp_vars, _ in renderer.render_objects(exclude_group, + ignore_used=False): expander = ramble.expander.Expander(exclude_exp_vars, self) self._compute_mpi_vars(expander, exclude_exp_vars) exclude_exp_name = expander.expand_var(experiment_template_name, @@ -332,89 +413,53 @@ def _ingest_experiments(self): if namespace.where in final_context.exclude: exclude_where = final_context.exclude[namespace.where] - rendered_experiments = set() - for experiment_vars, repeats in \ - renderer.render_objects(render_group, exclude_where=exclude_where): - experiment_vars[self.keywords.env_path] = \ - os.path.join(self._workspace.software_dir, - Expander.expansion_str(self.keywords.env_name) + '.' + - Expander.expansion_str(self.keywords.workload_name)) - - experiment_suffix = '' - # After generating the base experiment, append the index to repeat experiments - if repeats.repeat_index: - experiment_suffix = f'.{repeats.repeat_index}' - - expander = ramble.expander.Expander(experiment_vars, self) - self._compute_mpi_vars(expander, experiment_vars) - final_app_name = expander.expand_var_name(self.keywords.application_name, - allow_passthrough=False) - final_wl_name = expander.expand_var_name(self.keywords.workload_name, - allow_passthrough=False) - final_exp_name = expander.expand_var(experiment_template_name + experiment_suffix, - allow_passthrough=False) + tracking_group = ramble.renderer.RenderGroup('experiment', 'create') + tracking_group.variables = final_context.variables + tracking_group.zips = final_context.zips + tracking_group.matrices = final_context.matrices + tracking_group.n_repeats = final_context.n_repeats + tracking_group.used_variables = set() - # Skip explicitly excluded experiments - if final_exp_name in excluded_experiments: - continue + used_variables = set() + for tracking_vars, repeats in \ + renderer.render_objects(tracking_group, exclude_where=exclude_where, + ignore_used=False, fatal=False): + app_inst = self._prepare_experiment(experiment_template_name, + tracking_vars, final_context, repeats) - experiment_vars[self.keywords.experiment_template_name] = (experiment_template_name - + experiment_suffix) - experiment_vars[self.keywords.application_name] = final_app_name - experiment_vars[self.keywords.workload_name] = final_wl_name - experiment_vars[self.keywords.experiment_name] = final_exp_name - experiment_vars[self.keywords.experiment_index] = len(self.experiments) + 1 + final_exp_name = app_inst.expander.expand_var_name(self.keywords.experiment_namespace) - experiment_namespace = expander.experiment_namespace + exp_used_variables = app_inst.build_used_variables(self._workspace) + used_variables = used_variables.union(exp_used_variables) + render_group.used_variables = used_variables.copy() - experiment_vars[self.keywords.log_file] = os.path.join('{experiment_run_dir}', - '{experiment_name}.out') + rendered_experiments = set() + for experiment_vars, repeats in \ + renderer.render_objects(render_group, exclude_where=exclude_where): + app_inst = self._prepare_experiment(experiment_template_name, experiment_vars, + final_context, repeats) - experiment_vars[self.keywords.simplified_application_namespace] = \ - spack.util.naming.simplify_name( - expander.expand_var_name( - self.keywords.application_namespace - ) - ) - experiment_vars[self.keywords.simplified_workload_namespace] = \ - spack.util.naming.simplify_name( - expander.expand_var_name( - self.keywords.workload_namespace - ) - ) - experiment_vars[self.keywords.simplified_experiment_namespace] = \ - spack.util.naming.simplify_name( - expander.expand_var_name( - self.keywords.experiment_namespace - ) - ) + final_exp_name = app_inst.expander.expand_var_name(self.keywords.experiment_name) + final_exp_namespace = \ + app_inst.expander.expand_var_name(self.keywords.experiment_namespace) - logger.debug(' Final name: %s' % final_exp_name) + # Skip explicitly excluded experiments + if final_exp_name not in excluded_experiments: + logger.debug(f' Final name: {final_exp_namespace}') - if experiment_namespace in rendered_experiments: - logger.die(f'Experiment {experiment_namespace} is not unique.') - rendered_experiments.add(experiment_namespace) + if final_exp_namespace in rendered_experiments: + logger.die(f'Experiment {final_exp_namespace} is not unique.') - try: - self.keywords.check_required_keys(experiment_vars) - except ramble.keywords.RambleKeywordError as e: - raise RambleVariableDefinitionError( - f'In experiment {self.experiment_namespace}: {e}' - ) + try: + self.keywords.check_required_keys(experiment_vars) + except ramble.keywords.RambleKeywordError as e: + raise RambleVariableDefinitionError( + f'In experiment {final_exp_namespace}: {e}' + ) - app_inst = ramble.repository.get(final_app_name) - app_inst.set_variables(experiment_vars, self) - app_inst.set_env_variable_sets(final_context.env_variables) - app_inst.set_internals(final_context.internals) - app_inst.set_template(final_context.is_template) - app_inst.repeats = repeats - app_inst.set_chained_experiments(final_context.chained_experiments) - app_inst.set_modifiers(final_context.modifiers) - app_inst.set_tags(final_context.tags) - app_inst.set_formatted_executables(final_context.formatted_executables) - app_inst.read_status() - self.experiments[experiment_namespace] = app_inst - self.experiment_order.append(experiment_namespace) + rendered_experiments.add(final_exp_namespace) + self.experiments[final_exp_namespace] = app_inst + self.experiment_order.append(final_exp_namespace) def build_experiment_chains(self): base_experiments = self.experiment_order.copy() @@ -450,6 +495,17 @@ def all_experiments(self): yield exp, inst, count count += 1 + def template_experiments(self): + """Iterator over template experiments in this set""" + + for exp, inst in self.experiments.items(): + if inst.is_template: + yield exp, inst + + for exp, inst in self.chained_experiments.items(): + if inst.is_template: + yield exp, inst + def num_experiments(self): """Return the number of total experiments in this set""" count = 0 diff --git a/lib/ramble/ramble/experimental/uploader.py b/lib/ramble/ramble/experimental/uploader.py index a9e9f2064..48d57cd86 100644 --- a/lib/ramble/ramble/experimental/uploader.py +++ b/lib/ramble/ramble/experimental/uploader.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/fetch_strategy.py b/lib/ramble/ramble/fetch_strategy.py index a8b17a372..1c0303812 100644 --- a/lib/ramble/ramble/fetch_strategy.py +++ b/lib/ramble/ramble/fetch_strategy.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/filters.py b/lib/ramble/ramble/filters.py index b63d01455..340d53bf6 100644 --- a/lib/ramble/ramble/filters.py +++ b/lib/ramble/ramble/filters.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/graphs.py b/lib/ramble/ramble/graphs.py new file mode 100644 index 000000000..b96a65d72 --- /dev/null +++ b/lib/ramble/ramble/graphs.py @@ -0,0 +1,418 @@ +# Copyright 2022-2024 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import enum +import graphlib + +import ramble.error +import ramble.util.graph + +from ramble.util.logger import logger + + +class AttributeGraph(object): + + node_type = 'object' + + def __init__(self, obj_inst): + self._obj_inst = obj_inst + self.node_definitions = {} + self.adj_list = {} + self._prepared = False + self._sorted = None + + def _make_editable(self): + """Make this graph editable, and remove any defined ordering""" + if self._prepared: + self._sorted = None + self._prepared = False + + def update_graph(self, node, dep_nodes=[], internal_order=False): + """Update the graph with a new node and / or new dependencies. + + Given a node, and list of dependencies, define new edges in the + graph. If the node is new, also construct a new phase node. + + Args: + node (GraphNode): Node to inject or modify + dep_nodes (list(GraphNode)): List of nodes that are dependencies + internal_order (Boolean): True to process internal dependencies, + False to skip + + """ + + self._make_editable() + self.add_node(node) + self.define_edges(node, dep_nodes, internal_order=internal_order) + + def add_node(self, node): + """Add a node to the graph + + Args: + node (GraphNode): Node to add into graph + """ + + self._make_editable() + + if node.key not in self.node_definitions: + self.node_definitions[node.key] = node + + if node not in self.adj_list: + self.adj_list[node] = set() + + def define_edges(self, node, dep_nodes=[], internal_order=False): + """Define graph edges + + Process dependencies, and internal orderings (inside the node object) + to define new graph edges. + + Args: + node (GraphNode): Node to inject or modify + dep_nodes (list(GraphNode)): List of nodes that are dependencies + internal_order (Boolean): True to process internal dependencies, + False to skip + """ + + for dep in dep_nodes: + if dep.key not in self.node_definitions: + self.node_definitions[dep.key] = dep + self.adj_list[dep] = set() + self.adj_list[node].add(dep) + + if internal_order: + for dep in node._order_after: + dep_node = self.node_definitions[dep] + self.adj_list[node].add(dep_node) + + for dep in node._order_before: + dep_node = self.node_definitions[dep] + self.adj_list[dep_node].add(node) + + def walk(self): + """Walk the graph in topological ordering and yield each node. + + Construct a topological ordering of the current graph, walk it, and + yield each node one by one. + + Yields: + node (GraphNode): Each node in the graph + """ + if not self._prepared: + sorter = graphlib.TopologicalSorter(self.adj_list) + try: + self._sorted = tuple(sorter.static_order()) + except graphlib.CycleError as e: + try: + exp_name = self._obj_inst.expander.experiment_namespace + except AttributeError: + exp_name = self._obj_inst.name + raise GraphCycleError(f'In experiment {exp_name} a cycle was detected ' + f'when processing the {self.node_type} graph.\n' + + str(e)) + self._prepared = True + + for node in self._sorted: + yield node + + def get_node(self, key): + """Given a key, return the node containing this key + + Args: + key (str): Name of key to find in the graph + + Returns: + (GraphNode): Node representing the key requested. Returns None if + the key isn't found. + """ + for node in self.walk(): + if node.key == key: + return node + return None + + +class PhaseGraph(AttributeGraph): + + node_type = 'phase' + + def __init__(self, phase_definitions, obj_inst): + """Construct a phase graph for a pipeline + + Parse a single pipeline's phase definitions, and build an adjacency + list from this. Using the graph utiltites, construct a topological + sorting of the graph. + + Args: + phase_definitions (dict): Definitions of phases. Should be of the + format {'phase_name': GraphNode} + obj_inst (obj): Object instance to extract phase functions from + """ + + super().__init__(obj_inst) + + # Define all graph nodes + for phase_node in phase_definitions.values(): + if phase_node.obj_inst is None: + phase_node.obj_inst = obj_inst + + if phase_node.attribute is None: + phase_func = getattr(obj_inst, f'_{phase_node.key}') + phase_node.set_attribute(phase_func) + + self.add_node(phase_node) + + # Define graph edges + for phase_node in phase_definitions.values(): + self.define_edges(phase_node, internal_order=True) + + def add_node(self, node, obj_inst=None): + """Add a new phase node to the graph + + Extract the phase function from the object instance, and inject a new node into the graph. + + Args: + node (GraphNode): Phase node to add into graph + obj_inst (Object): Object that owns the phase + """ + + func_obj = obj_inst + if func_obj is None: + func_obj = self._obj_inst + + phase_func = getattr(func_obj, f'_{node.key}') + node.set_attribute(phase_func) + + super().add_node(node) + + def update_graph(self, phase_name, dependencies=[], + internal_order=False, obj_inst=None): + """Update the graph with a new phase and / or new dependencies. + + Given a phase name, and list of dependencies, define new edges in the + graph. If the phase is new, also construct a new phase node. + + Args: + phase_name (str): Name of the phase to inject or modify + dependencies (list(str)): List of phase names to inject dependencies on + internal_order (Boolean): True to process internal dependencies, + False to skip + obj_inst (object): Application or modifier instance to extract phase function from + """ + if self._prepared: + del self._sorted + self._sorted = None + self._prepared = False + + if phase_name not in self.node_definitions: + phase_node = ramble.util.graph.GraphNode(phase_name) + self.add_node(phase_node, obj_inst) + + phase_node = self.node_definitions[phase_name] + + dep_nodes = [] + for dep in dependencies: + if dep not in self.node_definitions: + dep_node = ramble.util.graph.GraphNode(dep) + self.add_node(dep_node, obj_inst) + + dep_node = self.node_definitions[dep] + dep_nodes.append(dep_node) + + super().define_edges(phase_node, dep_nodes) + + +class ExecutableGraph(AttributeGraph): + """Graph that handles command executables and builtins""" + + node_type = 'command executable' + supported_injection_orders = enum.Enum('supported_injection_orders', ['before', 'after']) + + def __init__(self, exec_order, executables, builtin_objects, builtin_groups, obj_inst): + """Construct a new ExecutableGraph + + Executable graphs have node attributes that are either of type + CommandExecutable, or are a function pointer to a builtin. + + Args: + exec_order (list(str)): List of executable names in execution order + executables (dict): Dictionary of executable definitions. + Keys are executable names, values are CommandExecutables + builtin_objects (list(object)): List of objects to associate with each builtin + group (in order) + builtins (list(dict)): List of dictionaries containing definitions of builtins. + Keys are names values are configurations of builtins. + modifier_builtins (dict): Dictionary containing definitions of modifier builtins. + Keys are names values are configurations of builtins. + Modifier builtins are inserted between application builtins + and executables. + obj_inst (object): Object instance to extract attributes from (when necessary) + """ + super().__init__(obj_inst) + self._builtin_dependencies = {} + + # Define nodes for executable + for exec_name, cmd_exec in executables.items(): + exec_node = ramble.util.graph.GraphNode(exec_name, cmd_exec, obj_inst=obj_inst) + self.node_definitions[exec_name] = exec_node + if exec_name in exec_order: + super().update_graph(exec_node) + + # Define nodes for builtins + for builtin_obj, builtins in zip(builtin_objects, builtin_groups): + for builtin, blt_conf in builtins.items(): + self._builtin_dependencies[builtin] = blt_conf['depends_on'].copy() + blt_func = getattr(builtin_obj, blt_conf['name']) + exec_node = ramble.util.graph.GraphNode(builtin, + attribute=blt_func, + obj_inst=builtin_obj) + self.node_definitions[builtin] = exec_node + + dep_exec = None + for exec_name in exec_order: + if dep_exec is not None: + exec_node = self.node_definitions[exec_name] + dep_node = self.node_definitions[dep_exec] + super().update_graph(exec_node, [dep_node]) + dep_exec = exec_name + + head_node = None + tail_node = None + for node in self.walk(): + if head_node is None: + head_node = node + tail_node = node + + tail_prepend_builtin = None + tail_append_builtin = None + + # Add (missing) required builtins + for builtins in builtin_groups: + for builtin, blt_conf in builtins.items(): + if blt_conf['required'] and self.get_node(builtin) is None: + blt_node = self.node_definitions[builtin] + super().update_graph(blt_node) + + if blt_conf['injection_method'] == 'prepend': + if head_node is not None: + super().update_graph(head_node, [blt_node]) + + if tail_prepend_builtin is not None: + super().update_graph(blt_node, [tail_prepend_builtin]) + tail_prepend_builtin = blt_node + elif blt_conf['injection_method'] == 'append': + if tail_node is not None: + super().update_graph(blt_node, [tail_node]) + + if tail_append_builtin is not None: + super().update_graph(blt_node, [tail_append_builtin]) + tail_append_builtin = blt_node + + if blt_conf['depends_on']: + deps = [] + for dep in blt_conf['depends_on']: + dep_node = self.node_definitions[dep] + super().update_graph(dep_node) + deps.append(dep_node) + + exec_node = self.node_definitions[builtin] + super().update_graph(exec_node, deps) + + def inject_executable(self, exec_name, injection_order, relative): + """Inject an executable into the graph + + Args: + exec_name (str): Name of executable to inject + injection_order (str): Order for injection. Can be 'before' or 'after' + relative (str): Name of executable to inject relative to. Can be + None to inject relative to the whole set of executables. + """ + # Order can be 'before' or 'after. + # If `relative_to` is not set, then before adds to be the beginning of the list + # and after (default) adds to the end of the list + # If `relative_to` IS set, then before adds before the first instance of + # the executable in the list + # and after (default) adds after the last instance of the + # executable in the list + # If `relative_to` is set, and the executable name is not found, raise a fatal error. + + exec_node = self.node_definitions[exec_name] + cur_exec_order = [] + for node in self.walk(): + cur_exec_order.append(node) + + exp_name = self._obj_inst.expander.experiment_namespace + order = self.supported_injection_orders.after + if injection_order is not None: + if not hasattr(self.supported_injection_orders, injection_order): + logger.die('In experiment ' + f'"{exp_name}" ' + f'injection order of executable "{exec_name}" is set to an ' + f'invalid value of "{injection_order}".\n' + f'Valid values are {self.supported_injection_orders}.') + order = getattr(self.supported_injection_orders, injection_order) + + if exec_name not in self.node_definitions: + logger.die('In experiment ' + f'"{exp_name}" ' + f'attempting to inject a non existing executable "{exec_name}".') + + if relative is not None: + relative_error = False + if relative not in self.node_definitions: + relative_error = True + + relative_node = self.node_definitions[relative] + if relative_node not in cur_exec_order: + relative_error = True + + if relative_error: + logger.die('In experiment ' + f'"{exp_name}" ' + f'attempting to inject executable "{exec_name}" ' + f'relative to a non existing executable "{relative}".') + + relative_node = self.node_definitions[relative] + order_index = cur_exec_order.index(relative_node) + + if order == self.supported_injection_orders.before: + super().update_graph(relative_node, [exec_node]) + if order_index > 0: + super().update_graph(exec_node, [cur_exec_order[order_index - 1]]) + elif order == self.supported_injection_orders.after: + super().update_graph(exec_node, [relative_node]) + if order_index < len(cur_exec_order) - 1: + super().update_graph(cur_exec_order[order_index + 1], [exec_node]) + else: + # If relative is none, determine head and tail nodes to inject properly + head_node = cur_exec_order[0] + tail_node = cur_exec_order[-1] + + super().update_graph(exec_node) + if order == self.supported_injection_orders.before: + super().update_graph(head_node, [exec_node]) + elif order == self.supported_injection_orders.after: + super().update_graph(exec_node, [tail_node]) + + # If exec_name is a builtin, inject edges to it's dependencies + if exec_name in self._builtin_dependencies: + dep_nodes = [] + for dep in self._builtin_dependencies[exec_name]: + dep_node = self.node_definitions[dep] + dep_nodes.append(dep_node) + super().update_graph(exec_node, dep_nodes) + + +class GraphError(ramble.error.RambleError): + """ + Exception raised with errors in a graph type + """ + + +class GraphCycleError(GraphError): + """ + Exception raised when a cycle is detected in a graph + """ diff --git a/lib/ramble/ramble/keywords.py b/lib/ramble/ramble/keywords.py index bd0ae0b53..ed579d79d 100644 --- a/lib/ramble/ramble/keywords.py +++ b/lib/ramble/ramble/keywords.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -26,6 +26,7 @@ 'workload_namespace': {'type': key_type.reserved, 'level': output_level.key}, 'simplified_workload_namespace': {'type': key_type.reserved, 'level': output_level.key}, 'license_input_dir': {'type': key_type.reserved, 'level': output_level.variable}, + 'experiments_file': {'type': key_type.reserved, 'level': output_level.key}, 'experiment_name': {'type': key_type.reserved, 'level': output_level.key}, 'experiment_run_dir': {'type': key_type.reserved, 'level': output_level.variable}, 'experiment_status': {'type': key_type.reserved, 'level': output_level.key}, @@ -122,6 +123,26 @@ def is_variable_level(self, key): return False return self.keys[key]['level'] == output_level.variable + def all_required_keys(self): + """Yield all required keys + + Yields: + (str): Key name + """ + for key in self.keys.keys(): + if self.is_required(key): + yield key + + def all_reserved_keys(self): + """Yield all reserved keys + + Yields: + (str): Key name + """ + for key in self.keys.keys(): + if self.is_reserved(key): + yield key + def check_reserved_keys(self, definitions): """Check a dictionary of variable definitions for reserved keywords""" if not definitions: diff --git a/lib/ramble/ramble/language/application_language.py b/lib/ramble/ramble/language/application_language.py index 990f7a340..5e210b285 100644 --- a/lib/ramble/ramble/language/application_language.py +++ b/lib/ramble/ramble/language/application_language.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -6,10 +6,10 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. +import ramble.workload import ramble.language.language_base from ramble.language.language_base import DirectiveError import ramble.language.shared_language -from ramble.schema.types import OUTPUT_CAPTURE import ramble.language.language_helpers import ramble.success_criteria @@ -70,33 +70,24 @@ def workload(name, executables=None, executable=None, input=None, """ def _execute_workload(app): - app.workloads[name] = { - 'executables': [], - 'inputs': [], - 'tags': [], - } - all_execs = ramble.language.language_helpers.require_definition(executable, executables, + app.executables, 'executable', 'executables', 'workload') - app.workloads[name]['executables'] = all_execs.copy() - - all_inputs = ramble.language.language_helpers.merge_definitions(input, inputs) - - app.workloads[name]['inputs'] = all_inputs.copy() + all_inputs = ramble.language.language_helpers.merge_definitions(input, + inputs, + app.inputs) - if tags: - app.workloads[name]['tags'] = tags.copy() + app.workloads[name] = ramble.workload.Workload(name, all_execs, all_inputs, tags) return _execute_workload @application_directive('executables') -def executable(name, template, use_mpi=False, variables={}, redirect='{log_file}', - output_capture=OUTPUT_CAPTURE.DEFAULT, **kwargs): +def executable(name, template, **kwargs): """Adds an executable to this application Defines a new executable that can be used to configure workloads and @@ -104,10 +95,13 @@ def executable(name, template, use_mpi=False, variables={}, redirect='{log_file} Executables may or may not use MPI. - Args: - template: The template command this executable should generate from - use_mpi: (Boolean) determines if this executable should be - wrapped with an `mpirun` like command or not. + Required Args: + name (str): Name of the executable + template (list[str] or str): The template command this executable should generate from + + Optional Args: + use_mpi or mpi: (Boolean) determines if this executable should be + wrapped with an `mpirun` like command or not. variables (dict): dictionary of variable definitions to use for this executable only redirect (Optional): Sets the path for outputs to be written to. @@ -120,8 +114,7 @@ def executable(name, template, use_mpi=False, variables={}, redirect='{log_file} def _execute_executable(app): from ramble.util.executable import CommandExecutable app.executables[name] = CommandExecutable( - name=name, template=template, use_mpi=use_mpi, redirect=redirect, - output_capture=output_capture) + name=name, template=template, **kwargs) return _execute_executable @@ -159,7 +152,7 @@ def _execute_input_file(app): return _execute_input_file -@application_directive('workload_variables') +@application_directive(dicts=()) def workload_variable(name, default, description, values=None, workload=None, workloads=None, expandable=True, **kwargs): """Define a new variable to be used in experiments @@ -174,26 +167,23 @@ def workload_variable(name, default, description, values=None, workload=None, def _execute_workload_variable(app): all_workloads = ramble.language.language_helpers.require_definition(workload, workloads, + app.workloads, 'workload', 'workloads', 'workload_variable') for wl_name in all_workloads: - if wl_name not in app.workload_variables: - app.workload_variables[wl_name] = {} - - app.workload_variables[wl_name][name] = { - 'default': default, - 'description': description, - 'expandable': expandable - } - if values: - app.workload_variables[wl_name][name]['values'] = values + app.workloads[wl_name].add_variable( + ramble.workload.WorkloadVariable( + name, default=default, description=description, + values=values, expandable=expandable + ) + ) return _execute_workload_variable -@application_directive('environment_variables') +@application_directive(dicts=()) def environment_variable(name, value, description, workload=None, workloads=None, **kwargs): """Define an environment variable to be used in experiments @@ -204,25 +194,23 @@ def environment_variable(name, value, description, workload=None, def _execute_environment_variable(app): all_workloads = ramble.language.language_helpers.require_definition(workload, workloads, + app.workloads, 'workload', 'workloads', 'environment_variable') for wl_name in all_workloads: - if wl_name not in app.environment_variables: - app.environment_variables[wl_name] = {} - - app.environment_variables[wl_name][name] = { - 'value': value, - 'action': 'set', - 'description': description, - } + app.workloads[wl_name].add_environment_variable( + ramble.workload.WorkloadEnvironmentVariable( + name, value=value, description=description + ) + ) return _execute_environment_variable @application_directive('phase_definitions') -def register_phase(name, pipeline=None, depends_on=[]): +def register_phase(name, pipeline=None, run_before=[], run_after=[]): """Register a phase Phases are portions of a pipeline that will execute when @@ -237,30 +225,49 @@ def register_phase(name, pipeline=None, depends_on=[]): Args: - name: The name of the phase. Phases are functions named '_'. - pipeline: The name of the pipeline this phase should be registered into. - - depends_on: A list of phase names this phase depends on + - run_before: A list of phase names this phase should run before + - run_after: A list of phase names this phase should run after """ def _execute_register_phase(app): + import ramble.util.graph if pipeline not in app._pipelines: raise DirectiveError('Directive register_phase was ' f'given an invalid pipeline "{pipeline}"\n' 'Available pipelines are: ' f' {app._pipelines}') - if not isinstance(depends_on, list): + if not isinstance(run_before, list): raise DirectiveError('Directive register_phase was ' 'given an invalid type for ' - 'the depends_on attribute in application ' + 'the run_before attribute in application ' f'{app.name}') + if not isinstance(run_after, list): + raise DirectiveError('Directive register_phase was ' + 'given an invalid type for ' + 'the run_after attribute in application ' + f'{app.name}') + + if not hasattr(app, f'_{name}'): + raise DirectiveError('Directive register_phase was ' + f'given an undefined phase {name} ' + f'in application {app.name}') + if pipeline not in app.phase_definitions: app.phase_definitions[pipeline] = {} - if name not in app.phase_definitions[pipeline]: - app.phase_definitions[pipeline][name] = [] + if name in app.phase_definitions[pipeline]: + phase_node = app.phase_definitions[pipeline][name] + else: + phase_node = ramble.util.graph.GraphNode(name) + + for before in run_before: + phase_node.order_before(before) + + for after in run_after: + phase_node.order_after(after) - for dep in depends_on: - if dep not in app.phase_definitions[pipeline][name]: - app.phase_definitions[pipeline][name].append(dep) + app.phase_definitions[pipeline][name] = phase_node return _execute_register_phase diff --git a/lib/ramble/ramble/language/language_base.py b/lib/ramble/ramble/language/language_base.py index 0295d69a0..1dac93af7 100644 --- a/lib/ramble/ramble/language/language_base.py +++ b/lib/ramble/ramble/language/language_base.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -95,7 +95,8 @@ def __init__(cls, name, bases, attr_dict): directive_attrs = { '_directive_functions': {}, - '_directive_classes': {} + '_directive_classes': {}, + '_directive_names': DirectiveMeta._directive_names.copy() } for attr in directive_attrs.keys(): diff --git a/lib/ramble/ramble/language/language_helpers.py b/lib/ramble/ramble/language/language_helpers.py index 107496c81..246e75bc8 100644 --- a/lib/ramble/ramble/language/language_helpers.py +++ b/lib/ramble/ramble/language/language_helpers.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -6,12 +6,13 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. +import fnmatch import six from ramble.language.language_base import DirectiveError -def merge_definitions(single_type, multiple_type): +def merge_definitions(single_type, multiple_type, multiple_pattern_match): """Merge definitions of a type This method will merge two optional definitions of single_type and @@ -19,7 +20,8 @@ def merge_definitions(single_type, multiple_type): Args: single_type: Single string for type name - multiple_type: List of strings for type names + multiple_type: List of strings for type names, may contain wildcards + multiple_pattern_match: List of strings to match against patterns in multiple_type Returns: List of all type names (Merged if both single_type and multiple_type definitions are valid) @@ -31,12 +33,13 @@ def merge_definitions(single_type, multiple_type): all_types.append(single_type) if multiple_type: - all_types.extend(multiple_type) + expanded_multiple_type = expand_patterns(multiple_type, multiple_pattern_match) + all_types.extend(expanded_multiple_type) return all_types -def require_definition(single_type, multiple_type, +def require_definition(single_type, multiple_type, multiple_pattern_match, single_arg_name, multiple_arg_name, directive_name): """Require at least one definition for a type in a directive @@ -48,7 +51,8 @@ def require_definition(single_type, multiple_type, Args: single_type: Single string for type name - multiple_type: List of strings for type names + multiple_type: List of strings for type names, may contain wildcards + multiple_pattern_match: List of strings to match against patterns in multiple_type single_arg_name: String name of the single_type argument in the directive multiple_arg_name: String name of the multiple_type argument in the directive directive_name: Name of the directive requiring a type @@ -71,4 +75,32 @@ def require_definition(single_type, multiple_type, f'for the {multiple_arg_name} argument. ' f'Type was {type(multiple_type)}') - return merge_definitions(single_type, multiple_type) + return merge_definitions(single_type, multiple_type, multiple_pattern_match) + + +def expand_patterns(multiple_type: list, multiple_pattern_match: list): + """Expand wildcard patterns within a list of names + + This method takes an input list containing wildcard patterns and expands the + wildcard with values matching a list of names. Returns a list containing + matching names and any inputs with zero matches. + + Args: + multiple_types: List of strings for type names, may contain wildcards + multiple_pattern_match: List of strings to match against patterns in multiple_type + + Returns: + List of expanded patterns matching the names list plus patterns + not found in the names list. + """ + + expanded_patterns = [] + for input in multiple_type: + matched_inputs = fnmatch.filter(multiple_pattern_match, input) + if matched_inputs: + for matching_name in matched_inputs: + expanded_patterns.append(matching_name) + else: + expanded_patterns.append(input) + + return expanded_patterns diff --git a/lib/ramble/ramble/language/modifier_language.py b/lib/ramble/ramble/language/modifier_language.py index ce4803a19..97ee16c9d 100644 --- a/lib/ramble/ramble/language/modifier_language.py +++ b/lib/ramble/ramble/language/modifier_language.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -83,6 +83,7 @@ def _execute_variable_modification(mod): all_modes = ramble.language.language_helpers.require_definition(mode, modes, + mod.modes, 'mode', 'modes', 'variable_modification') @@ -180,6 +181,7 @@ def _env_var_modification(mod): all_modes = ramble.language.language_helpers.require_definition(mode, modes, + mod.modes, 'mode', 'modes', 'env_var_modification') @@ -261,6 +263,7 @@ def modifier_variable(name: str, default, description: str, values: Optional[lis def _define_modifier_variable(mod): all_modes = ramble.language.language_helpers.require_definition(mode, modes, + mod.modes, 'mode', 'modes', 'modifier_variable') @@ -269,14 +272,10 @@ def _define_modifier_variable(mod): if mode_name not in mod.modifier_variables: mod.modifier_variables[mode_name] = {} - mod.modifier_variables[mode_name][name] = { - 'default': default, - 'description': description, - 'expandable': expandable - } - - if values: - mod.modifier_variables[mode_name][name]['values'] = values + mod.modifier_variables[mode_name][name] = ramble.workload.WorkloadVariable( + name, default=default, description=description, + values=values, expandable=expandable + ) return _define_modifier_variable @@ -309,7 +308,12 @@ def _new_package_manager_requirement(mod): raise DirectiveError(f'package_manager_requirement validation type is ' f'{validation_type} but no regex is given') - for mode in modes: + exp_modes = modes + if isinstance(exp_modes, list): + exp_modes = ramble.language.language_helpers.expand_patterns(exp_modes, + mod.modes) + + for mode in exp_modes: if mode not in mod.package_manager_requirements: mod.package_manager_requirements[mode] = [] diff --git a/lib/ramble/ramble/language/shared_language.py b/lib/ramble/ramble/language/shared_language.py index cfd3e897b..89bb0d33c 100644 --- a/lib/ramble/ramble/language/shared_language.py +++ b/lib/ramble/ramble/language/shared_language.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -107,23 +107,23 @@ def _execute_figure_of_merit(obj): return _execute_figure_of_merit -@shared_directive('default_compilers') -def default_compiler(name, spack_spec, compiler_spec=None, compiler=None): - """Defines the default compiler that will be used with this object +@shared_directive('compilers') +def define_compiler(name, spack_spec, compiler_spec=None, compiler=None): + """Defines the compiler that will be used with this object Adds a new compiler spec to this object. Software specs should reference a compiler that has been added. """ - def _execute_default_compiler(obj): + def _execute_define_compiler(obj): if hasattr(obj, 'uses_spack') and getattr(obj, 'uses_spack'): - obj.default_compilers[name] = { + obj.compilers[name] = { 'spack_spec': spack_spec, 'compiler_spec': compiler_spec, 'compiler': compiler } - return _execute_default_compiler + return _execute_define_compiler @shared_directive('software_specs') @@ -222,7 +222,7 @@ def _execute_success_criteria(obj): @shared_directive('builtins') -def register_builtin(name, required=True, injection_method='prepend'): +def register_builtin(name, required=True, injection_method='prepend', depends_on=[]): """Register a builtin Builtins are methods that return lists of strings. These methods represent @@ -268,7 +268,7 @@ def example_builtin(self): def _store_builtin(obj): if injection_method not in supported_injection_methods: raise ramble.language.language_base.DirectiveError( - f'Object {obj.name} has an invalid ' + f'Object {obj.name} defines builtin {name} with an invalid ' f'injection method of {injection_method}.\n' f'Valid methods are {str(supported_injection_methods)}' ) @@ -277,10 +277,83 @@ def _store_builtin(obj): obj.builtins[builtin_name] = {'name': name, 'required': required, - 'injection_method': injection_method} + 'injection_method': injection_method, + 'depends_on': depends_on.copy()} return _store_builtin +@shared_directive('phase_definitions') +def register_phase(name, pipeline=None, run_before=[], run_after=[]): + """Register a phase + + Phases are portions of a pipeline that will execute when + executing a full pipeline. + + Registering a phase allows an object to know what the phases + dependencies are, to ensure the execution order is correct. + + If called multiple times, the dependencies are combined together. Only one + instance of a phase will show up in the resulting dependency list for a phase. + + Args: + - name: The name of the phase. Phases are functions named '_'. + - pipeline: The name of the pipeline this phase should be registered into. + - run_before: A list of phase names this phase should run before + - run_after: A list of phase names this phase should run after + """ + + def _execute_register_phase(obj): + import ramble.util.graph + if pipeline not in obj._pipelines: + raise ramble.language.language_base.DirectiveError( + 'Directive register_phase was ' + f'given an invalid pipeline "{pipeline}"\n' + 'Available pipelines are: ' + f' {obj._pipelines}' + ) + + if not isinstance(run_before, list): + raise ramble.language.language_base.DirectiveError( + 'Directive register_phase was ' + 'given an invalid type for ' + 'the run_before attribute in object ' + f'{obj.name}' + ) + + if not isinstance(run_after, list): + raise ramble.language.language_base.DirectiveError( + 'Directive register_phase was ' + 'given an invalid type for ' + 'the run_after attribute in object ' + f'{obj.name}' + ) + + if not hasattr(obj, f'_{name}'): + raise ramble.language.language_base.DirectiveError( + 'Directive register_phase was ' + f'given an undefined phase {name} ' + f'in object {obj.name}' + ) + + if pipeline not in obj.phase_definitions: + obj.phase_definitions[pipeline] = {} + + if name in obj.phase_definitions[pipeline]: + phase_node = obj.phase_definitions[pipeline][name] + else: + phase_node = ramble.util.graph.GraphNode(name) + + for before in run_before: + phase_node.order_before(before) + + for after in run_after: + phase_node.order_after(after) + + obj.phase_definitions[pipeline][name] = phase_node + + return _execute_register_phase + + @shared_directive(dicts=()) def maintainers(*names: str): """Add a new maintainer directive, to specify maintainers in a declarative way. diff --git a/lib/ramble/ramble/main.py b/lib/ramble/ramble/main.py index 66616daf6..822df5e85 100644 --- a/lib/ramble/ramble/main.py +++ b/lib/ramble/ramble/main.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -812,6 +812,8 @@ def _main(argv=None): ramble.workspace.shell.activate(ws) # print the context but delay this exception so that commands like # `ramble config edit` can still work with a bad workspace. + except ramble.workspace.RambleActiveWorkspaceError as e: + workspace_format_error = e except ramble.config.ConfigFormatError as e: e.print_context() workspace_format_error = e @@ -861,18 +863,22 @@ def finish_parse_and_run(parser, cmd_name, workspace_format_error): # Now that we know what command this is and what its args are, determine # whether we can continue with a bad workspace and raise if not. edit_cmds = ["workspace", "config"] - allowed_subcommands = ['edit', 'list'] + allowed_subcommands = ['edit', 'list', 'deactivate'] + if workspace_format_error: raise_error = False if cmd_name.strip() in edit_cmds: - logger.msg( - "Error while reading workspace config. In some cases this can be " + - "avoided by passing `-W` to ramble" - ) - raise_error = True subcommand = getattr(args, "%s_command" % cmd_name, None) - if subcommand in allowed_subcommands: - raise_error = False + + if subcommand != "deactivate": + raise_error = True + logger.msg( + "Error while reading workspace config. In some cases this can be " + + "avoided by passing `-W` to ramble or by running\n" + + "`ramble workspace deactivate`" + ) + if subcommand in allowed_subcommands: + raise_error = False if raise_error: raise workspace_format_error diff --git a/lib/ramble/ramble/mirror.py b/lib/ramble/ramble/mirror.py index a0c69a706..efd12d730 100644 --- a/lib/ramble/ramble/mirror.py +++ b/lib/ramble/ramble/mirror.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/modifier.py b/lib/ramble/ramble/modifier.py index 026b77076..24a130faf 100644 --- a/lib/ramble/ramble/modifier.py +++ b/lib/ramble/ramble/modifier.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -20,6 +20,7 @@ from ramble.error import RambleError import ramble.util.colors as rucolor import ramble.util.directives +import ramble.util.class_attributes from ramble.util.logger import logger @@ -28,10 +29,8 @@ class ModifierBase(object, metaclass=ModifierMeta): uses_spack = False _builtin_name = 'modifier_builtin::{obj_name}::{name}' _mod_prefix_builtin = r'modifier_builtin::' - _mod_builtin_regex = r'modifier_builtin::(?P[\w-]+)::' - _builtin_required_key = 'required' - builtin_group = 'modifier' _language_classes = [ModifierMeta, SharedMeta] + _pipelines = ['analyze', 'archive', 'mirror', 'setup', 'pushtocache', 'execute'] modifier_class = 'ModifierBase' @@ -43,6 +42,8 @@ class ModifierBase(object, metaclass=ModifierMeta): def __init__(self, file_path): super().__init__() + ramble.util.class_attributes.convert_class_attributes(self) + self._file_path = file_path self._on_executables = ['*'] self.expander = None @@ -72,14 +73,20 @@ def set_usage_mode(self, mode): self._usage_mode = mode elif hasattr(self, '_default_usage_mode'): self._usage_mode = self._default_usage_mode - logger.msg(f' Using default usage mode {self._usage_mode} on modifier {self.name}') + if len(logger.log_stack) >= 1: + logger.msg( + f' Using default usage mode {self._usage_mode} on modifier {self.name}' + ) else: if len(self.modes) > 1 or len(self.modes) == 0: raise InvalidModeError('Cannot auto determine usage ' f'mode for modifier {self.name}') self._usage_mode = list(self.modes.keys())[0] - logger.msg(f' Using default usage mode {self._usage_mode} on modifier {self.name}') + if len(logger.log_stack) >= 1: + logger.msg( + f' Using default usage mode {self._usage_mode} on modifier {self.name}' + ) def set_on_executables(self, on_executables): """Set the executables this modifier applies to. @@ -129,10 +136,10 @@ def _long_print(self): indent = '\t\t' for var, conf in self.modifier_variables[mode_name].items(): out_str.append(rucolor.nested_2(f'{indent}{var}:\n')) - out_str.append(f'{indent}\tDescription: {conf["description"]}\n') - out_str.append(f'{indent}\tDefault: {conf["default"]}\n') - if 'values' in conf: - out_str.append(f'{indent}\tSuggested Values: {conf["values"]}\n') + out_str.append(f'{indent}\tDescription: {conf.description}\n') + out_str.append(f'{indent}\tDefault: {conf.default}\n') + if conf.values: + out_str.append(f'{indent}\tSuggested Values: {conf.values}\n') if mode_name in self.variable_modifications: out_str.append(rucolor.nested_1('\tVariable Modifications:\n')) @@ -158,9 +165,9 @@ def _long_print(self): out_str.append(f'\t{name} = {config}\n') out_str.append('\n') - if hasattr(self, 'default_compilers'): + if hasattr(self, 'compilers'): out_str.append(rucolor.section_title('Default Compilers:\n')) - for comp_name, comp_def in self.default_compilers.items(): + for comp_name, comp_def in self.compilers.items(): out_str.append(rucolor.nested_2(f'\t{comp_name}:\n')) out_str.append(rucolor.nested_3('\t\tSpack Spec:') + f'{comp_def["spack_spec"].replace("@", "@@")}\n') @@ -275,27 +282,54 @@ def all_package_manager_requirements(self): for req in self.package_manager_requirements[self._usage_mode]: yield req + def all_pipeline_phases(self, pipeline): + if pipeline in self.phase_definitions: + for phase_name, phase_node in self.phase_definitions[pipeline].items(): + yield phase_name, phase_node + + def no_expand_vars(self): + """Iterator over non-expandable variables in current mode + + Yields: + (str): Variable name + """ + + if self._usage_mode in self.modifier_variables: + for var, var_conf in self.modifier_variables[self._usage_mode].items(): + if not var_conf.expandable: + yield var + def mode_variables(self): """Return a dict of variables that should be defined for the current mode""" if self._usage_mode in self.modifier_variables: return self.modifier_variables[self._usage_mode] - return {} + else: + return {} - def run_phase_hook(self, workspace, hook_name): + def run_phase_hook(self, workspace, pipeline, hook_name): """Run a modifier hook. Hooks are internal functions named _{hook_name}. This is a wrapper to extract the hook function, and execute it properly. + + Hooks are only executed if they are not defined as a phase from the + modifier. """ - hook_func_name = f'_{hook_name}' - if hasattr(self, hook_func_name): - phase_func = getattr(self, hook_func_name) + run_hook = True + if pipeline in self.phase_definitions: + if hook_name in self.phase_definitions[pipeline]: + run_hook = False + + if run_hook: + hook_func_name = f'_{hook_name}' + if hasattr(self, hook_func_name): + phase_func = getattr(self, hook_func_name) - phase_func(workspace) + phase_func(workspace) def _prepare_analysis(self, workspace): """Hook to perform analysis that a modifier defines. diff --git a/lib/ramble/ramble/modifier_types/basic.py b/lib/ramble/ramble/modifier_types/basic.py index 54aa22463..fddb77150 100644 --- a/lib/ramble/ramble/modifier_types/basic.py +++ b/lib/ramble/ramble/modifier_types/basic.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/modifier_types/spack.py b/lib/ramble/ramble/modifier_types/spack.py index 5f320a2ca..180c6cde3 100644 --- a/lib/ramble/ramble/modifier_types/spack.py +++ b/lib/ramble/ramble/modifier_types/spack.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/modkit.py b/lib/ramble/ramble/modkit.py index 36bceefee..eb39a3bf1 100644 --- a/lib/ramble/ramble/modkit.py +++ b/lib/ramble/ramble/modkit.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/namespace.py b/lib/ramble/ramble/namespace.py index bf273f552..d60d00827 100644 --- a/lib/ramble/ramble/namespace.py +++ b/lib/ramble/ramble/namespace.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/paths.py b/lib/ramble/ramble/paths.py index 9934c0c9e..243d2354f 100644 --- a/lib/ramble/ramble/paths.py +++ b/lib/ramble/ramble/paths.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/pipeline.py b/lib/ramble/ramble/pipeline.py index ff5989b20..303ef82aa 100644 --- a/lib/ramble/ramble/pipeline.py +++ b/lib/ramble/ramble/pipeline.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -52,6 +52,8 @@ def __init__(self, workspace, filters): self.force_inventory = False self.require_inventory = False self.action_string = 'Operating on' + self.suppress_per_experiment_prints = False + self.suppress_run_header = False dt = self.workspace.date_string() log_file = f'{self.name}.{dt}.out' @@ -64,15 +66,9 @@ def __init__(self, workspace, filters): self.log_path_latest = os.path.join(self.workspace.log_dir, f'{self.name}.latest.out') - fs.mkdirp(self.log_dir) - - # Create simlinks to give known paths - self.create_simlink(self.log_dir, self.log_dir_latest) - self.create_simlink(self.log_path, self.log_path_latest) - - self._experiment_set = workspace.build_experiment_set() self._software_environments = ramble.software_environments.SoftwareEnvironments(workspace) self.workspace.software_environments = self._software_environments + self._experiment_set = workspace.build_experiment_set() def _construct_hash(self): """Hash all of the experiments, construct workspace inventory""" @@ -116,24 +112,23 @@ def _construct_hash(self): self.workspace.hash_file_name), 'w+') as f: f.write(self.workspace.workspace_hash + '\n') - def _validate(self): - """Perform validation that this pipeline can be executed""" - if not self.workspace.is_concretized(): - error_message = 'Cannot run %s in a ' % self.name + \ - 'non-conretized workspace\n' + \ - 'Run `ramble workspace concretize` on this ' + \ - 'workspace first.\n' + \ - 'Then ensure its spack configuration is ' + \ - 'properly configured.' - logger.die(error_message) - def _prepare(self): """Perform preparation for pipeline execution""" pass def _execute(self): """Hook for executing the pipeline""" + num_exps = self._experiment_set.num_filtered_experiments(self.filters) + + if logger.enabled: + fs.mkdirp(self.log_dir) + # Also create simlink to give known paths + self.create_simlink(self.log_dir, self.log_dir_latest) + + if self.suppress_per_experiment_prints and not self.suppress_run_header: + logger.all_msg(f' Log files for experiments are stored in: {self.log_dir}') + count = 1 for exp, app_inst, idx in self._experiment_set.filtered_experiments(self.filters): exp_log_path = app_inst.experiment_log_file(self.log_dir) @@ -141,16 +136,18 @@ def _execute(self): experiment_index_value = \ app_inst.expander.expand_var_name(app_inst.keywords.experiment_index) - logger.all_msg(f'Experiment #{idx} ({count}/{num_exps}):') - logger.all_msg(f' name: {exp}') - logger.all_msg(f' root experiment_index: {experiment_index_value}') - logger.all_msg(f' log file: {exp_log_path}') + if not self.suppress_per_experiment_prints: + logger.all_msg(f'Experiment #{idx} ({count}/{num_exps}):') + logger.all_msg(f' name: {exp}') + logger.all_msg(f' root experiment_index: {experiment_index_value}') + logger.all_msg(f' log file: {exp_log_path}') logger.add_log(exp_log_path) phase_list = app_inst.get_pipeline_phases(self.name, self.filters.phases) - disable_progress = ramble.config.get('config:disable_progress_bar', False) + disable_progress = ramble.config.get('config:disable_progress_bar', False) \ + or self.suppress_per_experiment_prints if not disable_progress: progress = tqdm.tqdm(total=len(phase_list), leave=True, @@ -161,7 +158,7 @@ def _execute(self): progress.set_description( f'Processing phase {phase} ({phase_idx}/{len(phase_list)})' ) - app_inst.run_phase(phase, self.workspace) + app_inst.run_phase(self.name, phase, self.workspace) if not disable_progress: progress.update() app_inst.print_phase_times(self.name, self.filters.phases) @@ -170,7 +167,8 @@ def _execute(self): progress.close() logger.remove_log() - logger.all_msg(f' Returning to log file: {logger.active_log()}') + if not self.suppress_per_experiment_prints: + logger.all_msg(f' Returning to log file: {logger.active_log()}') count += 1 def _complete(self): @@ -179,20 +177,23 @@ def _complete(self): def run(self): """Run the full pipeline""" - logger.all_msg('Streaming details to log:') - logger.all_msg(f' {self.log_path}') - if self.workspace.dry_run: - cprint('@*g{ -- DRY-RUN -- DRY-RUN -- DRY-RUN -- DRY-RUN -- DRY-RUN --}') - - experiment_count = self._experiment_set.num_filtered_experiments(self.filters) - experiment_total = self._experiment_set.num_experiments() - logger.all_msg( - f' {self.action_string} {experiment_count} out of ' - f'{experiment_total} experiments:' - ) + if not self.suppress_run_header: + logger.all_msg('Streaming details to log:') + logger.all_msg(f' {self.log_path}') + if self.workspace.dry_run: + cprint('@*g{ -- DRY-RUN -- DRY-RUN -- DRY-RUN -- DRY-RUN -- DRY-RUN --}') + + experiment_count = self._experiment_set.num_filtered_experiments(self.filters) + experiment_total = self._experiment_set.num_experiments() + logger.all_msg( + f' {self.action_string} {experiment_count} out of ' + f'{experiment_total} experiments:' + ) logger.add_log(self.log_path) - self._validate() + if logger.enabled: + self.create_simlink(self.log_path, self.log_path_latest) + self._prepare() self._execute() self._complete() @@ -227,14 +228,24 @@ def __init__(self, workspace, filters, output_formats=['text'], upload=False): self.upload_results = upload def _prepare(self): - for _, app_inst, _ in self._experiment_set.filtered_experiments(self.filters): + + # We only want to let the user run analyze if one of the following is true: + # - At least one expeirment is set up + # - `--dry-run` is enabled + found_valid_experiment = False + for exp, app_inst, _ in self._experiment_set.filtered_experiments(self.filters): if not (app_inst.is_template or app_inst.repeats.is_repeat_base): - if app_inst.get_status() == ramble.application.experiment_status.UNKNOWN.name: - logger.die( - f'Workspace status is {app_inst.get_status()}\n' - 'Make sure your workspace is fully setup with\n' - ' ramble workspace setup' - ) + if app_inst.get_status() != ramble.application.experiment_status.UNKNOWN.name: + found_valid_experiment = True + + if not found_valid_experiment and self._experiment_set.num_experiments() \ + and not self.workspace.dry_run: + logger.die( + 'No analyzeable experiment detected.' + ' Make sure your workspace is setup with\n' + ' ramble workspace setup' + ) + super()._construct_hash() super()._prepare() @@ -267,6 +278,12 @@ def __init__(self, workspace, filters, create_tar=False, self.archive_prefix = archive_prefix self.archive_name = None + if self.upload_url and not self.create_tar: + logger.warn( + 'Upload URL is currently only supported when using tar format (-t)\n' + 'Archive will not be uploaded.' + ) + def _prepare(self): import glob super()._construct_hash() @@ -295,13 +312,7 @@ def _prepare(self): # Copy current configs archive_configs = os.path.join(self.workspace.latest_archive_path, ramble.workspace.workspace_config_path) - fs.mkdirp(archive_configs) - for root, dirs, files in os.walk(self.workspace.config_dir): - for name in files: - src = os.path.join(self.workspace.config_dir, root, name) - dest = src.replace(self.workspace.config_dir, archive_configs) - fs.mkdirp(os.path.dirname(dest)) - shutil.copyfile(src, dest) + _copy_tree(self.workspace.config_dir, archive_configs) # Copy current software spack files file_names = ['spack.yaml', 'spack.lock'] @@ -372,10 +383,8 @@ def _complete(self): if archive_url: tar_path = self.workspace.latest_archive_path + tar_extension remote_tar_path = archive_url + '/' + self.workspace.latest_archive + tar_extension - fetcher = ramble.fetch_strategy.URLFetchStrategy(tar_path) - fetcher.stage = ramble.stage.DIYStage(self.workspace.latest_archive_path) - fetcher.stage.archive_file = tar_path - fetcher.archive(remote_tar_path) + _upload_file(tar_path, remote_tar_path) + logger.all_msg(f"Archive Uploaded to {remote_tar_path}") class MirrorPipeline(Pipeline): @@ -477,13 +486,21 @@ class ExecutePipeline(Pipeline): name = 'execute' - def __init__(self, workspace, filters, executor='{batch_submit}'): + def __init__(self, workspace, filters, executor='{batch_submit}', + suppress_per_experiment_prints=True, suppress_run_header=False): super().__init__(workspace, filters) self.action_string = 'Executing' self.require_inventory = True self.executor = executor + self.suppress_per_experiment_prints = suppress_per_experiment_prints + self.suppress_run_header = suppress_run_header def _execute(self): + super()._execute() + + if not self.suppress_run_header: + logger.all_msg('Running executors...') + for exp, app_inst, idx in self._experiment_set.filtered_experiments(self.filters): if app_inst.is_template: logger.debug(f'{app_inst.name} is a template. Skipping execution.') @@ -502,9 +519,144 @@ def _execute(self): executor(*exec_args) +class PushDeploymentPipeline(Pipeline): + """class for the `prepare-deployment` pipeline""" + + name = 'pushdeployment' + index_filename = 'index.json' + index_namespace = 'deployment_files' + tar_extension = '.tar.gz' + + def __init__(self, workspace, filters, + create_tar=False, upload_url=None, + deployment_name=None): + super().__init__(workspace, filters) + self.action_string = 'Pushing deployment of' + self.require_inventory = True + self.create_tar = create_tar + self.upload_url = upload_url + self.object_repo_name = 'object_repo' + + if deployment_name: + workspace.deployment_name = deployment_name + self.deployment_name = deployment_name + else: + self.deployment_name = workspace.name + + def _execute(self): + from spack.util import spack_yaml as syaml + + configs_dir = os.path.join(self.workspace.named_deployment, + ramble.workspace.workspace_config_path) + fs.mkdirp(configs_dir) + + _copy_tree(self.workspace.config_dir, configs_dir) + + aux_software_dir = os.path.join(configs_dir, ramble.workspace.auxiliary_software_dir_name) + fs.mkdirp(aux_software_dir) + aux_repo_conf = os.path.join(aux_software_dir, 'repos.yaml') + + repo_conf_defs = [('repos', 'repos.yaml'), + ('modifier_repos', 'modifier_repos.yaml')] + + for repo_conf in repo_conf_defs: + aux_repo_conf = os.path.join(aux_software_dir, repo_conf[1]) + repo_data = syaml.syaml_dict() + if os.path.exists(aux_repo_conf): + with open(aux_repo_conf, 'r') as f: + repo_data = syaml.load_config(f.read()) + else: + repo_data[repo_conf[0]] = [] + + add_repo = True + for repo in repo_data[repo_conf[0]]: + if repo == f'../../{self.object_repo_name}': + add_repo = False + if add_repo: + repo_data[repo_conf[0]].append(f'../../{self.object_repo_name}') + + with open(aux_repo_conf, 'w+') as f: + f.write(syaml.dump_config(repo_data)) + + repo_path = os.path.join(self.workspace.named_deployment, self.object_repo_name) + object_types = ['applications', 'modifiers', 'packages'] + for object_type in object_types: + fs.mkdirp(os.path.join(repo_path, object_type)) + + for conf_file in ['repo.yaml', 'modifier_repo.yaml']: + with open(os.path.join(repo_path, conf_file), 'w+') as f: + f.write('repo:\n') + f.write(f' namespace: deployment_{self.deployment_name}\n') + + super()._execute() + + def _deployment_files(self): + """Yield the full path to each file in a deployment""" + for root, dirs, files in os.walk(self.workspace.named_deployment): + for name in files: + yield os.path.join(self.workspace.named_deployment, root, name) + + def _complete(self): + # Create an index.json of the deployment + deployment_index = {self.index_namespace: []} + for file in self._deployment_files(): + deployment_index[self.index_namespace].append( + file.replace(self.workspace.named_deployment + os.path.sep, '') + ) + index_file = os.path.join(self.workspace.named_deployment, self.index_filename) + with open(index_file, 'w+') as f: + f.write(sjson.dump(deployment_index)) + + tar_path = os.path.join(self.workspace.deployments_dir, + self.deployment_name + self.tar_extension) + if self.create_tar: + tar = which('tar', required=True) + with py.path.local(self.workspace.deployments_dir).as_cwd(): + tar('-czf', tar_path, self.deployment_name) + + if self.upload_url: + remote_base = self.upload_url + '/' + self.deployment_name + + for file in self._deployment_files(): + dest = file.replace(self.workspace.named_deployment, remote_base) + _upload_file(file, dest) + + if self.create_tar: + stage_dir = self.workspace.deployments_dir + tar_path = os.path.join(stage_dir, self.deployment_name + self.tar_extension) + remote_tar_path = self.upload_url + '/' + self.deployment_name + self.tar_extension + _upload_file(tar_path, remote_tar_path) + + logger.all_msg(f'Deployment created in: {self.workspace.named_deployment}') + if self.create_tar: + logger.all_msg(f' Tar of deployment created in: {tar_path}') + if self.upload_url: + remote_base = self.upload_url + '/' + self.deployment_name + logger.all_msg(f' Deployment uploaded to: {remote_base}') + + +def _copy_tree(src_dir, dest_dir): + """Copy all files in src_dir to dest_dir""" + for root, dirs, files in os.walk(src_dir): + for name in files: + src = os.path.join(src_dir, root, name) + dest = src.replace(src_dir, dest_dir) + fs.mkdirp(os.path.dirname(dest)) + shutil.copyfile(src, dest) + + +def _upload_file(src_file, dest_file): + stage_dir = os.path.dirname(src_file) + fetcher = ramble.fetch_strategy.URLFetchStrategy(src_file) + fetcher.stage = ramble.stage.DIYStage(stage_dir) + fetcher.stage.archive_file = src_file + fetcher.archive(dest_file) + + pipelines = Enum('pipelines', [AnalyzePipeline.name, ArchivePipeline.name, MirrorPipeline.name, - SetupPipeline.name, PushToCachePipeline.name, ExecutePipeline.name] + SetupPipeline.name, PushToCachePipeline.name, ExecutePipeline.name, + PushDeploymentPipeline.name] ) _pipeline_map = { @@ -513,7 +665,8 @@ def _execute(self): pipelines.mirror: MirrorPipeline, pipelines.setup: SetupPipeline, pipelines.pushtocache: PushToCachePipeline, - pipelines.execute: ExecutePipeline + pipelines.execute: ExecutePipeline, + pipelines.pushdeployment: PushDeploymentPipeline } diff --git a/lib/ramble/ramble/renderer.py b/lib/ramble/ramble/renderer.py index 56690c9f6..a89681234 100644 --- a/lib/ramble/ramble/renderer.py +++ b/lib/ramble/ramble/renderer.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -51,6 +51,7 @@ def __init__(self, obj_type=None, action='create'): self.variables = {} self.zips = {} self.matrices = [] + self.used_variables = set() self.n_repeats = 0 def copy_contents(self, in_group): @@ -65,6 +66,9 @@ def copy_contents(self, in_group): if in_group.matrices: self.matrices.extend(in_group.matrices) + if in_group.used_variables: + self.used_variables = in_group.used_variables.copy() + def from_dict(self, name_template, in_dict): """Extract RenderGroup definitions from a dictionary @@ -106,7 +110,7 @@ def from_dict(self, name_template, in_dict): class Renderer(object): - def render_objects(self, render_group, exclude_where=None): + def render_objects(self, render_group, exclude_where=None, ignore_used=True, fatal=True): """Render objects based on the input variables and matrices Internally collects all matrix and vector variables. @@ -142,9 +146,10 @@ def render_objects(self, render_group, exclude_where=None): """ variables = render_group.variables - zips = render_group.zips + zips = render_group.zips.copy() matrices = render_group.matrices n_repeats = render_group.n_repeats + used_variables = render_group.used_variables.copy() object_variables = {} expander = ramble.expander.Expander(variables, None) @@ -159,6 +164,35 @@ def render_objects(self, render_group, exclude_where=None): consumed_zips = set() matrix_objects = [] + if ignore_used: + # Add variables / zips in matrices to used variables + if matrices: + for matrix in matrices: + for mat_var in matrix: + used_variables.add(mat_var) + + # Update zip definitions based on variables that are used. + # If a zip has one variable that is used, the entire zip is + # considered used. + # If a zip contains no used variables, ignore the entire zip. + if zips: + remove_zips = set() + for zip_group in zips: + + keep_zip = zip_group in used_variables + for var_name in zips[zip_group]: + if var_name in used_variables: + keep_zip = True + + if keep_zip: + for var_name in zips[zip_group]: + used_variables.add(var_name) + else: + remove_zips.add(zip_group) + + for zip_name in remove_zips: + del zips[zip_name] + if zips: zipped_vars = set() @@ -332,7 +366,7 @@ def render_objects(self, render_group, exclude_where=None): # Extract vector variables max_vector_size = 0 for var, val in object_variables.items(): - if isinstance(val, list): + if isinstance(val, list) and (var in used_variables or not ignore_used): vector_vars[var] = val.copy() max_vector_size = max(len(val), max_vector_size) @@ -343,7 +377,7 @@ def render_objects(self, render_group, exclude_where=None): if len(val) != max_vector_size: length_mismatch = True - if length_mismatch: + if fatal and length_mismatch: err_context = object_variables[render_group.context] err_str = f'Length mismatch in vector variables in {render_group.object} ' \ f'{err_context}\n' @@ -356,7 +390,8 @@ def render_objects(self, render_group, exclude_where=None): for i in range(0, max_vector_size): obj_vars = {} for var, val in vector_vars.items(): - obj_vars[var] = val[i] + if len(val) > i: + obj_vars[var] = val[i] if matrix_objects: for matrix_object in matrix_objects: diff --git a/lib/ramble/ramble/repeats.py b/lib/ramble/ramble/repeats.py index fb4f11472..1aa4edf96 100644 --- a/lib/ramble/ramble/repeats.py +++ b/lib/ramble/ramble/repeats.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/repository.py b/lib/ramble/ramble/repository.py index 37722b4e7..1f2ea8114 100644 --- a/lib/ramble/ramble/repository.py +++ b/lib/ramble/ramble/repository.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -63,13 +63,15 @@ default_type = ObjectTypes.applications +unified_config = 'repo.yaml' + type_definitions = { ObjectTypes.applications: { 'file_name': 'application.py', 'dir_name': 'applications', 'abbrev': 'app', 'config_section': 'repos', - 'config': 'repo.yaml', + 'accepted_configs': ['application_repo.yaml', unified_config], 'singular': 'application', }, ObjectTypes.modifiers: { @@ -77,7 +79,7 @@ 'dir_name': 'modifiers', 'abbrev': 'mod', 'config_section': 'modifier_repos', - 'config': 'modifier_repo.yaml', + 'accepted_configs': ['modifier_repo.yaml', unified_config], 'singular': 'modifier', } } @@ -271,7 +273,8 @@ def _create_new_cache(self): # Warn about invalid names that look like objects. if not nm.valid_module_name(obj_name): if not obj_name.startswith('.') and not any( - obj_name == obj_info['config'] for obj_info in type_definitions.values() + obj_name in obj_info['accepted_configs'] for obj_info in + type_definitions.values() ): logger.warn( f'Skipping {self.object_type} ' @@ -807,7 +810,6 @@ def __init__(self, root, object_type=default_type): self.object_file_name = type_definitions[object_type]['file_name'] self.object_type = object_type self.object_abbrev = type_definitions[object_type]['abbrev'] - self.config_name = type_definitions[object_type]['config'] self.base_namespace = f'{global_namespace}.{self.object_abbrev}' # check and raise BadRepoError on fail. @@ -816,7 +818,13 @@ def check(condition, msg): raise BadRepoError(msg) # Validate repository layout. - self.config_file = os.path.join(self.root, self.config_name) + self.config_name = None + self.config_file = None + for config in type_definitions[object_type]['accepted_configs']: + config_file = os.path.join(self.root, config) + if os.path.exists(config_file): + self.config_name = config + self.config_file = config_file check(os.path.isfile(self.config_file), "No %s found in '%s'" % (self.config_name, root)) @@ -833,6 +841,7 @@ def check(condition, msg): objects_dir = config["subdirectory"] if "subdirectory" in config else \ type_definitions[object_type]['dir_name'] + self.objects_path = os.path.join(self.root, objects_dir) check(os.path.isdir(self.objects_path), "No directory '%s' found in '%s'" % (objects_dir, @@ -1199,7 +1208,8 @@ def __contains__(self, obj_name): def create_repo(root, namespace=None, subdir=type_definitions[default_type]['dir_name'], - object_type=default_type): + object_type=default_type, + unified_repo=True): """Create a new repository in root with the specified namespace. If the namespace is not provided, use basename of root. @@ -1236,21 +1246,44 @@ def create_repo(root, namespace=None, "Cannot create repository in %s: can't access parent!" % root) try: - config_path = os.path.join(root, type_definitions[object_type]['config']) - objects_path = os.path.join(root, subdir) + object_dirs = [] + if unified_repo: + # If unified, and no subdir, create obj dirs + # If unified and subdir, create subdir + # If not unified and no subdir, create obj dir + # If not unified and subdir, create subdir + config_name = unified_config + for obj_type in type_definitions.values(): + objects_path = os.path.join(root, obj_type['dir_name']) + object_dirs.append(objects_path) + else: + config_name = type_definitions[object_type]['accepted_configs'][0] + objects_path = os.path.join(root, type_definitions[object_type]['dir_name']) + object_dirs.append(objects_path) - fs.mkdirp(objects_path) + if subdir is not None: + object_dirs = [os.path.join(root, subdir)] + + for objects_path in object_dirs: + fs.mkdirp(objects_path) + + config_path = os.path.join(root, config_name) with open(config_path, 'w') as config: config.write("repo:\n") config.write(f" namespace: '{namespace}'\n") - if subdir != type_definitions[object_type]['dir_name']: + if subdir is not None: config.write(f" subdirectory: '{subdir}'\n") except (IOError, OSError) as e: # try to clean up. if existed: shutil.rmtree(config_path, ignore_errors=True) - shutil.rmtree(objects_path, ignore_errors=True) + if unified_repo: + for obj_type in type_definitions.values(): + objects_path = os.path.join(root, obj_type['dir_name']) + shutil.rmtree(objects_path, ignore_errors=True) + else: + shutil.rmtree(objects_path, ignore_errors=True) else: shutil.rmtree(root, ignore_errors=True) diff --git a/lib/ramble/ramble/schema/__init__.py b/lib/ramble/ramble/schema/__init__.py index b8c120352..50db788d9 100644 --- a/lib/ramble/ramble/schema/__init__.py +++ b/lib/ramble/ramble/schema/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/schema/applications.py b/lib/ramble/ramble/schema/applications.py index 9464da255..b8986250d 100644 --- a/lib/ramble/ramble/schema/applications.py +++ b/lib/ramble/ramble/schema/applications.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/schema/config.py b/lib/ramble/ramble/schema/config.py index 7c7ec4a67..9ee942827 100644 --- a/lib/ramble/ramble/schema/config.py +++ b/lib/ramble/ramble/schema/config.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/schema/env_vars.py b/lib/ramble/ramble/schema/env_vars.py index 9ec3463cb..535123b65 100644 --- a/lib/ramble/ramble/schema/env_vars.py +++ b/lib/ramble/ramble/schema/env_vars.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/schema/formatted_executables.py b/lib/ramble/ramble/schema/formatted_executables.py index 35eef4a54..b66c256bb 100644 --- a/lib/ramble/ramble/schema/formatted_executables.py +++ b/lib/ramble/ramble/schema/formatted_executables.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/schema/internals.py b/lib/ramble/ramble/schema/internals.py index 3176ddfb3..58226420d 100644 --- a/lib/ramble/ramble/schema/internals.py +++ b/lib/ramble/ramble/schema/internals.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/schema/licenses.py b/lib/ramble/ramble/schema/licenses.py index 70e7b2a60..91e287748 100644 --- a/lib/ramble/ramble/schema/licenses.py +++ b/lib/ramble/ramble/schema/licenses.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/schema/merged.py b/lib/ramble/ramble/schema/merged.py index 28b3c53a8..1e6414e16 100644 --- a/lib/ramble/ramble/schema/merged.py +++ b/lib/ramble/ramble/schema/merged.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/schema/mirrors.py b/lib/ramble/ramble/schema/mirrors.py index 00f50992f..f2d3d34bd 100644 --- a/lib/ramble/ramble/schema/mirrors.py +++ b/lib/ramble/ramble/schema/mirrors.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/schema/modifier_repos.py b/lib/ramble/ramble/schema/modifier_repos.py index fce4a17b7..98bd96132 100644 --- a/lib/ramble/ramble/schema/modifier_repos.py +++ b/lib/ramble/ramble/schema/modifier_repos.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/schema/modifiers.py b/lib/ramble/ramble/schema/modifiers.py index 1a979d745..fec7b31c8 100644 --- a/lib/ramble/ramble/schema/modifiers.py +++ b/lib/ramble/ramble/schema/modifiers.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/schema/repos.py b/lib/ramble/ramble/schema/repos.py index f8dd434be..637cc3353 100644 --- a/lib/ramble/ramble/schema/repos.py +++ b/lib/ramble/ramble/schema/repos.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/schema/spack.py b/lib/ramble/ramble/schema/spack.py index 2a7da3a40..4ea5d9582 100644 --- a/lib/ramble/ramble/schema/spack.py +++ b/lib/ramble/ramble/schema/spack.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -12,21 +12,12 @@ :lines: 12- """ # noqa E501 -import ramble.schema.variables -import ramble.schema.applications -import ramble.schema.zips #: Properties for inclusion in other schemas properties = { 'spack': { 'type': 'object', 'properties': { - 'concretized': { - 'type': 'boolean', - 'default': False - }, - 'variables': ramble.schema.variables.variables_def, - 'zips': ramble.schema.zips.zips_def, 'packages': { 'type': 'object', 'additionalProperties': { @@ -41,11 +32,6 @@ 'type': 'string', 'default': None, }, - 'variables': ramble.schema.variables.variables_def, - 'zips': ramble.schema.zips.zips_def, - 'matrix': ramble.schema.applications.matrix_def, - 'matrices': ramble.schema.applications.matrices_def, - 'exclude': ramble.schema.applications.exclude_def, }, 'additionalProperties': False, 'default': {} @@ -67,11 +53,6 @@ 'items': {'type': 'string'}, 'default': [] }, - 'variables': ramble.schema.variables.variables_def, - 'zips': ramble.schema.zips.zips_def, - 'matrix': ramble.schema.applications.matrix_def, - 'matrices': ramble.schema.applications.matrices_def, - 'exclude': ramble.schema.applications.exclude_def, }, 'additionalProperties': False, 'default': {} diff --git a/lib/ramble/ramble/schema/success_criteria.py b/lib/ramble/ramble/schema/success_criteria.py index bd42708f4..61ad341ae 100644 --- a/lib/ramble/ramble/schema/success_criteria.py +++ b/lib/ramble/ramble/schema/success_criteria.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/schema/types.py b/lib/ramble/ramble/schema/types.py index bf7a977d5..496790860 100644 --- a/lib/ramble/ramble/schema/types.py +++ b/lib/ramble/ramble/schema/types.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/schema/variables.py b/lib/ramble/ramble/schema/variables.py index 493766459..288fbedb5 100644 --- a/lib/ramble/ramble/schema/variables.py +++ b/lib/ramble/ramble/schema/variables.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -12,14 +12,11 @@ :lines: 12- """ # noqa E501 -import ramble.schema.types - - variables_def = { 'type': ['object', 'null'], 'default': {}, 'properties': {}, - 'additionalProperties': ramble.schema.types.array_or_scalar_of_strings_or_nums + 'additionalProperties': True } properties = { diff --git a/lib/ramble/ramble/schema/workspace.py b/lib/ramble/ramble/schema/workspace.py index e84b6855b..980de6a9b 100644 --- a/lib/ramble/ramble/schema/workspace.py +++ b/lib/ramble/ramble/schema/workspace.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/schema/zips.py b/lib/ramble/ramble/schema/zips.py index 85f166bc3..fa7c07f85 100644 --- a/lib/ramble/ramble/schema/zips.py +++ b/lib/ramble/ramble/schema/zips.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/software_environments.py b/lib/ramble/ramble/software_environments.py index d7298b64b..2d8bd7e03 100644 --- a/lib/ramble/ramble/software_environments.py +++ b/lib/ramble/ramble/software_environments.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -6,7 +6,7 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. -import llnl.util.tty.color as color +from enum import Enum import ramble.repository import ramble.workspace @@ -20,402 +20,665 @@ import ramble.util.matrices import ramble.util.colors as rucolor +package_managers = Enum("package_managers", ['spack']) -class SoftwareEnvironments(object): - """Class to represent a set of software environments - This class contains logic to take the dictionary representations of - software environments, and unify their format. - """ +class SoftwarePackage(object): + """Class to represent a single software package""" - supported_confs = ['v2'] + def __init__(self, name: str, spec: str, compiler: str = None, + compiler_spec: str = None, package_manager: str = 'spack'): + """Software package constructor - def __init__(self, workspace): - self.keywords = ramble.keywords.keywords - - self._raw_packages = {} - self._packages = {} - self._package_map = {} - self._raw_environments = {} - self._environments = {} - self._environment_map = {} - self._workspace = workspace - self.spack_dict = self._workspace.get_spack_dict().copy() + Args: + name (str): Name of package + spec (str): Package spec (used to install / load package) + compiler (optional str): Name of package definition to use as compiler + for this package + compiler_spec (optional str): Spec string to use when this package + is used as a compiler + package_manager (optional str): Name of package manager for this package + """ - conf_type = self._detect_conf_type() + self.name = name + self.spec = spec + self.compiler = compiler + self.compiler_spec = compiler_spec + self._package_type = 'Rendered' + self.package_manager = package_managers[package_manager] + + def spec_str(self, all_packages: dict = {}, compiler=False): + """Return a spec string for this software package + + Args: + all_packages (dict): Dictionary of all package definitions. + Used to look up compiler packages. + compiler (boolean): True of this package is used as a compiler for + another package. False if this is just a primary package. + Toggles returning compiler_spec vs. spec in case they are + different. + + Returns: + (str): String representation of the spec for this package definition + """ - if conf_type not in self.supported_confs: + out_str = '' + if self.package_manager == package_managers.spack: + if compiler and self.compiler_spec: + out_str = self.compiler_spec + else: + out_str = self.spec + + if compiler: + return out_str + + if self.compiler in all_packages: + out_str += ' %' + all_packages[self.compiler].spec_str(all_packages, compiler=True) + elif self.compiler: + out_str += f' (built with {self.compiler})' + else: raise RambleSoftwareEnvironmentError( - f'Software configuration type {conf_type} is not one of ' + - f'{str(self.supported_confs)}' + f'Package {self.name} uses an unknown ' + f'package manager {self.package_manager}' ) - setup_method = getattr(self, f'_{conf_type}_setup') - setup_method() + return out_str - def _detect_conf_type(self): - """Auto-detect the type of configuration provided. + def info(self, indent: int = 0, verbosity: int = 0, color_level: int = 0): + """String representation of package information - Default configuration type is 'invalid'. + Args: + indent (int): Number of spaces to indent lines with + verbosity (int): Verbosity level - v2 configurations follow the format: + Returns: + (str): String representation of this package + """ - spack: - concretized: [true/false] - packages: {} - environments: {} + indentation = ' ' * indent + color = rucolor.level_func(color_level) + out_str = color(f'{indentation}{self._package_type} package: {self.name}\n') + out_str += f'{indentation} Spec: {self.spec.replace("@", "@@")}\n' + if self.compiler: + out_str += f'{indentation} Compiler: {self.compiler}\n' + if self.compiler_spec: + out_str += f'{indentation} Compiler Spec: {self.compiler_spec.replace("@", "@@")}\n' + return out_str + + def __str__(self): + """String representation of software package + + Returns: + (str): String representation of this software package """ - conf_type = 'invalid' - - if namespace.packages in self.spack_dict and \ - namespace.environments in self.spack_dict: - conf_type = 'v2' - - logger.debug(f'Detected config type of: {conf_type}') - - return conf_type - - def _v2_setup(self): - """Process a v2 `spack:` dictionary in the workspace configuration.""" - logger.debug('Performing v2 software setup.') - - pkg_renderer = ramble.renderer.Renderer() - env_renderer = ramble.renderer.Renderer() - - expander = ramble.expander.Expander({}, None) - - workspace_vars = self._workspace.get_workspace_vars().copy() - workspace_zips = self._workspace.get_workspace_zips().copy() - - if namespace.variables in self.spack_dict and \ - self.spack_dict[namespace.variables] is not None: - workspace_vars.update(self.spack_dict[namespace.variables]) - - if namespace.zips in self.spack_dict and \ - self.spack_dict[namespace.zips] is not None: - workspace_zips.update(self.spack_dict[namespace.zips]) - - if namespace.packages in self.spack_dict: - for pkg_template, pkg_info in self.spack_dict[namespace.packages].items(): - self._raw_packages[pkg_template] = pkg_info - self._package_map[pkg_template] = {} - pkg_group = ramble.renderer.RenderGroup('package', 'create') - pkg_group.variables.update(workspace_vars) - pkg_group.zips.update(workspace_zips) - pkg_group.from_dict(pkg_template, pkg_info) - - pkg_group.variables['package_name'] = pkg_template - - exclude_pkgs = set() - exclude_where = [] - if namespace.exclude in pkg_info: - exclude_group = ramble.renderer.RenderGroup('package', 'exclude') - exclude_group.variables.update(workspace_vars) - exclude_group.variables['package_name'] = pkg_template - perform_explicit_exclude = \ - exclude_group.from_dict(pkg_template, pkg_info[namespace.exclude]) - - if namespace.where in pkg_info[namespace.exclude]: - exclude_where = pkg_info[namespace.exclude][namespace.where].copy() - - if perform_explicit_exclude: - for exclude_vars, _ in pkg_renderer.render_objects(exclude_group): - final_name = expander.expand_var_name('package_name', - extra_vars=exclude_vars) - exclude_pkgs.add(final_name) - - for rendered_vars, _ in pkg_renderer.render_objects(pkg_group, - exclude_where=exclude_where): - final_name = expander.expand_var_name('package_name', - extra_vars=rendered_vars) - if final_name in exclude_pkgs: - continue - - self._packages[final_name] = {} - self._package_map[pkg_template].update({final_name: None}) - - spack_spec = expander.expand_var(pkg_info['spack_spec'], - extra_vars=rendered_vars) - self._packages[final_name]['spack_spec'] = spack_spec - - if 'compiler_spec' in pkg_info: - comp_spec = expander.expand_var(pkg_info['compiler_spec'], - extra_vars=rendered_vars) - self._packages[final_name]['compiler_spec'] = comp_spec - - if 'compiler' in pkg_info: - comp = expander.expand_var(pkg_info['compiler'], - extra_vars=rendered_vars) - self._packages[final_name]['compiler'] = comp - - if namespace.environments in self.spack_dict: - for env_template, env_info in self.spack_dict[namespace.environments].items(): - env_group = ramble.renderer.RenderGroup('environment', 'create') - env_group.variables.update(workspace_vars) - env_group.zips.update(workspace_zips) - env_group.from_dict(env_template, env_info) - self._raw_environments[env_template] = env_info - self._environment_map[env_template] = {} - - env_group.variables['environment_name'] = env_template - - exclude_envs = set() - exclude_where = [] - if namespace.exclude in env_info: - exclude_group = ramble.renderer.RenderGroup('environment', 'exclude') - exclude_group.variables.update(workspace_vars) - exclude_group.variables['environment_name'] = env_template - perform_explicit_exclude = \ - exclude_group.from_dict(env_template, env_info[namespace.exclude]) - - if namespace.where in env_info[namespace.exclude]: - exclude_where = env_info[namespace.exclude][namespace.where].copy() - - if perform_explicit_exclude: - for exclude_vars, _ in env_renderer.render_objects(exclude_group): - final_name = expander.expand_var_name('environment_name', - extra_vars=exclude_vars) - exclude_envs.add(final_name) - - for rendered_vars, _ in env_renderer.render_objects(env_group, - exclude_where=exclude_where): - final_name = expander.expand_var_name('environment_name', - extra_vars=rendered_vars) - - if final_name in exclude_envs: - continue - - self._environment_map[env_template].update({final_name: None}) - - self._environments[final_name] = {} - - if namespace.external_env in env_info: - external_env = expander.expand_var(env_info[namespace.external_env], - extra_vars=rendered_vars) - self._environments[final_name][namespace.external_env] = \ - external_env + return self.info() - if namespace.packages in env_info: - self._environments[final_name][namespace.packages] = [] - env_packages = self._environments[final_name][namespace.packages] - - for pkg_name in env_info[namespace.packages]: - expanded_pkg = expander.expand_var(pkg_name, - extra_vars=rendered_vars) - if expanded_pkg: - env_packages.append(expanded_pkg) - - pkgs_with_compiler = [] - missing_pkgs = set() - for pkg_name in env_packages: - if pkg_name in self._packages: - pkg_info = self._packages[pkg_name] - - if 'compiler' in pkg_info and pkg_info['compiler'] in env_packages: - pkgs_with_compiler.append((pkg_name, pkg_info['compiler'])) - else: - missing_pkgs.add(pkg_name) - - if pkgs_with_compiler: - logger.warn( - f'Environment {final_name} contains packages and their ' - 'compilers in the package list. These include:' - ) - for pkg_name, comp_name in pkgs_with_compiler: - logger.warn( - f' Package: {pkg_name}, Compiler: {comp_name}' - ) - logger.warn( - 'This might cause problems when installing the packages.' - ) - if missing_pkgs: - err_msg = f'Environment {final_name} refers to the following ' \ - 'packages, which are not defined:\n' - for pkg_name in missing_pkgs: - err_msg += f'\t{pkg_name}\n' - err_msg += 'Please make sure all packages are defined ' \ - 'before using this environment.' - logger.die(err_msg) - - def print_environments(self, verbosity=0): - color.cprint(rucolor.section_title('Software Stack:')) - color.cprint(rucolor.nested_1(' Packages:')) - for raw_pkg in self.all_raw_packages(): - color.cprint(rucolor.nested_2(f' {raw_pkg}:')) - - pkg_info = self.raw_package_info(raw_pkg) + def __eq__(self, other): + """Equvialence test for two package definitions - if verbosity >= 1: - if namespace.variables in pkg_info and pkg_info[namespace.variables]: - color.cprint(rucolor.nested_3(' Variables:')) - for var, val in pkg_info[namespace.variables].items(): - color.cprint(f' {var} = {val}') - - if namespace.matrices in pkg_info and pkg_info[namespace.matrices]: - color.cprint(rucolor.nested_3(' Matrices:')) - for matrix in pkg_info[namespace.matrices]: - base_str = ' - ' - for var in matrix: - color.cprint(f'{base_str}- {var}') - base_str = ' ' - - if namespace.matrix in pkg_info and pkg_info[namespace.matrix]: - color.cprint(rucolor.nested_3(' Matrix:')) - for var in pkg_info[namespace.matrix]: - color.cprint(f' - {var}') - - color.cprint(rucolor.nested_3(' Rendered Packages:')) - for pkg in self.mapped_packages(raw_pkg): - color.cprint(rucolor.nested_4(f' {pkg}:')) - pkg_spec = self.get_spec(pkg) - spec_str = pkg_spec[namespace.spack_spec].replace('@', '@@') - color.cprint(f' Spack spec: {spec_str}') - if namespace.compiler_spec in pkg_spec and pkg_spec[namespace.compiler_spec]: - spec_str = pkg_spec[namespace.compiler_spec].replace('@', '@@') - color.cprint(f' Compiler spec: {spec_str}') - if namespace.compiler in pkg_spec and pkg_spec[namespace.compiler]: - color.cprint(f' Compiler: {pkg_spec[namespace.compiler]}') - - color.cprint(rucolor.nested_1(' Environments:')) - for raw_env in self.all_raw_environments(): - color.cprint(rucolor.nested_2(f' {raw_env}:')) - - env_info = self.raw_environment_info(raw_env) + Args: + other (SoftwarePackage): Package to compare with self. + + Returns: + (boolean): True if packages are the same, False otherwise + """ + + return self.name == other.name and self.spec == other.spec and \ + self.compiler == other.compiler and self.compiler_spec == other.compiler_spec + + +class TemplatePackage(SoftwarePackage): + """Class representing a template software package""" + + def __init__(self, name: str, spec: str, + compiler: str = None, compiler_spec: str = None, + package_manager: str = 'spack'): + """Template package constructor + + Args: + name (str): Name of package + spec (str): Package spec (used to install / load package) + compiler (optional str): Name of package definition to use as compiler + for this package + compiler_spec (optional str): Spec string to use when this package + is used as a compiler + package_manager (optional str): Name of package manager for this package + """ + super().__init__(name, spec, compiler=compiler, + compiler_spec=compiler_spec, + package_manager=package_manager) + self._rendered_packages = {} + self._package_type = 'Template' + + def info(self, indent: int = 0, verbosity: int = 0, color_level: int = 0): + """String representation of package information + + Args: + indent (int): Number of spaces to indent lines with + verbosity (int): Verbosity level + + Returns: + (str): String representation of this package + """ + + out_str = super().info(indent, verbosity, color_level) + new_indent = indent + 4 + for pkg in self._rendered_packages.values(): + out_str += pkg.info(indent=new_indent, verbosity=verbosity, + color_level=color_level + 1) + return out_str + + def render_package(self, expander: object): + """Render a SoftwarePackage from this TemplatePackage + + Args: + expander (Expander): Expander to use to render a package from this template + + Returns: + (SoftwarePackage): Rendered SoftwarePackage + """ + name = expander.expand_var(self.name) + spec = expander.expand_var(self.spec) + compiler = expander.expand_var(self.compiler) if self.compiler else None + compiler_spec = expander.expand_var(self.compiler_spec) if self.compiler_spec else None + + new_pkg = SoftwarePackage(name, spec, compiler, compiler_spec) + + if new_pkg.name in self._rendered_packages: + if new_pkg != self._rendered_packages[name]: + raise RambleSoftwareEnvironmentError( + f'Package {new_pkg.name} defined multiple times with ' + 'inconsistent definitions.\n' + 'New definition is:\n' + f'{new_pkg}' + 'Old definition is:\n' + f'{self._rendered_packages[name]}' + ) + return self._rendered_packages[name] + else: + return new_pkg + + def add_rendered_package(self, new_package: object, all_packages: dict): + """Add a rendered package to this template's list of rendered packages + + Args: + new_package (SoftwarePackage): New package definition to add + all_packages (dict): Dictionary of all package definitions + """ + + if new_package.name not in self._rendered_packages: + self._rendered_packages[new_package.name] = new_package + all_packages[new_package.name] = new_package + + +class SoftwareEnvironment(object): + """Class representing a single software environment""" + + def __init__(self, name: str): + """SoftwareEnvironment constructor + + Args: + name (str): Name of the environment + """ + + self.name = name + self._packages = [] + self._environment_type = 'Rendered' + + def info(self, indent: int = 0, verbosity: int = 0, color_level: int = 0): + """Software environment information + Args: + indent (int): Number of spaces to inject as indentation + verbosity (int): Verbosity level + + Returns: + (str): information of this environment + """ + + indentation = ' ' * indent + color = rucolor.level_func(color_level) + out_str = color(f'{indentation}{self._environment_type} environment: {self.name}\n') + out_str += f'{indentation} Packages:\n' + for pkg in self._packages: if verbosity >= 1: - if namespace.variables in env_info and env_info[namespace.variables]: - color.cprint(rucolor.nested_3(' Variables:')) - for var, val in env_info[namespace.variables].items(): - color.cprint(f' {var} = {val}') - - if namespace.matrices in env_info and env_info[namespace.matrices]: - color.cprint(rucolor.nested_3(' Matrices:')) - for matrix in env_info[namespace.matrices]: - base_str = ' - ' - for var in matrix: - color.cprint(f'{base_str}- {var}') - base_str = ' ' - - if namespace.matrix in env_info and env_info[namespace.matrix]: - color.cprint(rucolor.nested_3(' Matrix:')) - for var in env_info[namespace.matrix]: - color.cprint(f' - {var}') - - color.cprint(rucolor.nested_3(' Rendered Environments:')) - for env in self.mapped_environments(raw_env): - color.cprint(rucolor.nested_4(f' {env} Packages:')) - for pkg in self.get_env_packages(env): - color.cprint(f' - {pkg}') - - def get_env(self, environment_name): - """Return a reference to the environment definition""" - if environment_name not in self._environments: - raise RambleSoftwareEnvironmentError( - f'Environment {environment_name} is not defined.' - ) + out_str += f'{indentation} - {pkg.name} = {pkg.spec_str().replace("@", "@@")}\n' + else: + out_str += f'{indentation} - {pkg.name}\n' + return out_str - return self._environments[environment_name] + def __str__(self): + """String representation of this environment - def get_spec_string(self, package_name): - """Return the full spec string given a package name""" - if package_name not in self._packages: - raise RambleSoftwareEnvironmentError( - f'Package {package_name} is not defined.' - ) + Returns: + (str): Representation of this environment + """ + return self.info(indent=0) - spec_string = self._packages[package_name]['spack_spec'] - compiler_str = '' - if 'compiler' in self._packages[package_name]: - comp_name = self._packages[package_name]['compiler'] - comp_spec = self.get_spec(comp_name) - compiler_str = f' %{comp_spec["spack_spec"]}' - if 'compiler_spec' in comp_spec: - compiler_str = f' %{comp_spec["compiler_spec"]}' - return spec_string + compiler_str - - def get_spec(self, package_name): - """Return a single spec given its name""" - if package_name not in self._packages: - raise RambleSoftwareEnvironmentError( - f'Package {package_name} is not defined.' - ) + def add_package(self, package: object): + """Add a package definition to this environment - return self._packages[package_name] + Args: + package (SoftwarePackage): Package object + """ + self._packages.append(package) - def get_env_packages(self, environment_name): - """Return all of the packages used by an environment""" - if environment_name not in self._environments: - raise RambleSoftwareEnvironmentError( - f'Environment {environment_name} is not defined.' - ) + def __eq__(self, other): + """Equivalence test for environments - if namespace.packages in self._environments[environment_name]: - for name in self._environments[environment_name][namespace.packages]: - yield name + Args: + other (SoftwareEnvironment): Environment to compare with self - def _require_raw_package(self, pkg): - """Raise an error if the raw package is not defined""" - if pkg not in self._raw_packages.keys(): - raise RambleSoftwareEnvironmentError( - f'Package {pkg} is not defined.' - ) + Returns: + (boolean): True if environments are equivalent, False otherwise + """ + equal = self.name == other.name and len(self._packages) == len(other._packages) - def _require_raw_environment(self, env): - """Raise an error if the raw environment is not defined""" - if env not in self._raw_environments.keys(): - raise RambleSoftwareEnvironmentError( - f'Environment {env} is not defined.' - ) + if not equal: + return False + + for self_pkg, other_pkg in zip(self._packages, other._packages): + if self_pkg != other_pkg: + return False + + return True + + +class ExternalEnvironment(SoftwareEnvironment): + """Class representing an externally defined software environment""" + + def __init__(self, name: str, name_or_path: str): + """Constructor for external software environment + + """ + super().__init__(name) + self.external_env = name_or_path + + +class TemplateEnvironment(SoftwareEnvironment): + """Class representing a template software environment""" + + def __init__(self, name: str): + """TemplateEnvironment constructor + + Args: + name (str): Name of this environment + """ + super().__init__(name) + self._package_names = set() + self._rendered_environments = {} + self._environment_type = 'Template' + + def add_package_name(self, package: str = None): + self._package_names.add(package) + + def info(self, indent: int = 0, verbosity: int = 0, color_level: int = 0): + """Software environment information + + Args: + indent (int): Number of spaces to inject as indentation + verbosity (int): Verbosity level + + Returns: + (str): information of this environment + """ + out_str = super().info(indent, verbosity, color_level=color_level) + new_indent = indent + 4 + for env in self._rendered_environments.values(): + out_str += env.info(new_indent, verbosity, color_level=color_level + 1) + return out_str + + def __str__(self): + """String representation of this environment + + Returns: + (str): String representation of this environment (none of it's rendered environments) + """ + + return super().info() + + def render_environment(self, expander: object, all_package_templates: dict, + all_packages: dict): + """Render a SoftwareEnvironment from this TemplateEnvironment + + Args: + expander (Expander): Expander object to use when rendering + all_packages (dict): All package definitions + + Returns: + (SoftwareEnvironment) Reference to the rendered SoftwareEnvironment + """ + name = expander.expand_var(self.name) + + new_env = SoftwareEnvironment(name) + + for env_pkg_template in self._package_names: + rendered_env_pkg_name = expander.expand_var(env_pkg_template) + + if rendered_env_pkg_name: + added = False + for template_pkg in all_package_templates.values(): + rendered_pkg = template_pkg.render_package(expander) + + if rendered_env_pkg_name == rendered_pkg.name: + if rendered_pkg.name in all_packages: + if rendered_pkg != all_packages[rendered_pkg.name]: + raise RambleSoftwareEnvironmentError( + f'Environment {name} defined multiple times in inconsistent ' + f'ways.\nPackage with differences is {rendered_pkg.name}' + ) + rendered_pkg = all_packages[rendered_pkg.name] + else: + all_packages[rendered_pkg.name] = rendered_pkg + + added = True + template_pkg.add_rendered_package(rendered_pkg, all_packages) + new_env.add_package(rendered_pkg) + + if not added: + raise RambleSoftwareEnvironmentError( + f'Environment template {self.name} references ' + f'undefined package {env_pkg_template} rendered to {rendered_env_pkg_name}' + ) + + return new_env + + def add_rendered_environment(self, environment: object, all_environments: dict, + all_packages: dict): + """Add a rendered environment to this template + + Args: + environment (SoftwareEnvironment): Reference to rendered environment + all_environments (dict): Dictionary containing all environments + all_packages (dict): Dictionary containing all packages + """ + if environment.name not in self._rendered_environments: + self._rendered_environments[environment.name] = environment + all_environments[environment.name] = environment + for template_pkg, rendered_pkg in zip(self._packages, environment._packages): + template_pkg.add_rendered_package(rendered_pkg, all_packages) def all_packages(self): - """Yield each package name""" - for pkg in self._packages.keys(): - yield pkg + """Iterator over all packages in this environment - def all_raw_packages(self): - """Yield each raw package name""" - for pkg in self._raw_packages.keys(): - yield pkg + Yields: + (SoftwarePackage) Each package in this environment + """ + for _, pkg_obj in self._packages: + yield pkg_obj - def raw_package_info(self, raw_pkg): - """Return the information for a raw package""" - self._require_raw_package(raw_pkg) - return self._raw_packages[raw_pkg] +class SoftwareEnvironments(object): + """Class representing a group of software environments""" - def mapped_packages(self, raw_pkg): - """Yield each package rendered from a raw package""" - self._require_raw_package(raw_pkg) + def __init__(self, workspace): + """SoftwareEnvironments constructor - for pkg in self._package_map[raw_pkg].keys(): - yield pkg + Args: + workspace (Workspace): Reference to workspace owning the software descriptions + """ - def all_environments(self): - """Yield each environment name""" - for env in self._environments.keys(): - yield env + self._workspace = workspace + self._spack_dict = workspace.get_spack_dict().copy() + self._environment_templates = {} + self._package_templates = {} + self._rendered_packages = {} + self._rendered_environments = {} - def all_raw_environments(self): - """Yield raw environment names""" - for env in self._raw_environments.keys(): - yield env + self._define_templates() - def raw_environment_info(self, env): - """Return the information for a raw environment""" - if env not in self._raw_environments.keys(): - raise RambleSoftwareEnvironmentError( - f'Environment {env} is not defined.' + def info(self, indent: int = 0, verbosity: int = 0, color_level: int = 0): + """Information for all packages and environments + + Args: + indent (int): Number of spaces to indent lines with + verbosity (int): Verbosity level + + Returns: + (str): Representation of all packages and environments + """ + out_str = '' + for pkg in self._package_templates.values(): + out_str += pkg.info(indent, verbosity=verbosity, color_level=color_level) + for env in self._environment_templates.values(): + out_str += env.info(indent, verbosity=verbosity, color_level=color_level) + return out_str + + def unused_environments(self): + """Iterator over environment templates that do not have any rendered environments + + Yields: + (TemplateEnvironment) Each unused template environment in this group + """ + for env in self._environment_templates.values(): + if not env._rendered_environments: + yield env + + def unused_packages(self): + """Iterator over package templates that do not have any rendered packages + + Yields: + (TemplatePackage) Each unused template package in this group + """ + for pkg in self._package_templates.values(): + if not pkg._rendered_packages: + yield pkg + + def __str__(self): + """String representation of all packages and environments in this object + + Returns: + (str): Representation of all packages and environments + """ + return self.info(indent=0) + + def _define_templates(self): + """Process software dictionary to generate templates""" + + if namespace.packages in self._spack_dict: + for pkg_template, pkg_info in self._spack_dict[namespace.packages].items(): + spec = pkg_info['spack_spec'] if 'spack_spec' in pkg_info else pkg_info['spec'] + compiler = pkg_info['compiler'] \ + if 'compiler' in pkg_info and pkg_info['compiler'] else None + compiler_spec = pkg_info['compiler_spec'] \ + if 'compiler_spec' in pkg_info and pkg_info['compiler_spec'] else None + new_pkg = TemplatePackage( + pkg_template, spec, compiler=compiler, compiler_spec=compiler_spec + ) + self._package_templates[pkg_template] = new_pkg + + if namespace.environments in self._spack_dict: + for env_template, env_info in self._spack_dict[namespace.environments].items(): + if namespace.external_env in env_info and env_info[namespace.external_env]: + # External environments are considered rendered + new_env = ExternalEnvironment(env_template, env_info[namespace.external_env]) + self._rendered_environments[env_template] = new_env + else: + # Define a new template environment + new_env = TemplateEnvironment(env_template) + if namespace.packages in env_info: + for package in env_info[namespace.packages]: + new_env.add_package_name(package) + self._environment_templates[env_template] = new_env + + def define_compiler_packages(self, environment, expander): + """Define packages for compilers in this environment + + If compilers referenced by (environment) are not defined, create + definitions for them to properly create compiler specs. + + Args: + environment (SoftwareEnvironment): Environment to extract necessary + compilers from + expander (Expander): Expander object to use when constructing + compiler package names + """ + for pkg in environment._packages: + if pkg.compiler: + cur_compiler = pkg.compiler + while cur_compiler and cur_compiler not in self._rendered_packages: + added = False + for template_name, template_def in self._package_templates.items(): + rendered_name = expander.expand_var(template_name) + + if rendered_name == cur_compiler: + rendered_pkg = template_def.render_package(expander) + + if cur_compiler in self._rendered_packages and \ + rendered_pkg != self._rendered_packages[cur_compiler]: + raise RambleSoftwareEnvironmentError( + f'Package {rendered_pkg.name} defined ' + 'multiple times in inconsistent ways' + ) + added = True + template_def.add_rendered_package(rendered_pkg, + self._rendered_packages) + self._rendered_packages[rendered_pkg.name] = rendered_pkg + + if rendered_pkg.compiler: + cur_compiler = rendered_pkg.compiler + if not added: + raise RambleSoftwareEnvironmentError( + f'Compiler {pkg.compiler} used, but not ' + f'defined in environment {environment.name} ' + f'by package {pkg.name}' + ) + + def compiler_specs_for_environment(self, environment: object): + """Iterator over compiler specs for a given environment + + Assumes all compilers have been defined via + self.define_compiler_packages() + + Args: + environment (SoftwareEnvironment): Environment to extract compiler specs for + + Yields: + (str) Spec string for each compiler + """ + + root_compilers = [] + for pkg in environment._packages: + if pkg.compiler: + if pkg.compiler not in self._rendered_packages: + raise RambleSoftwareEnvironmentError( + f'Compiler {pkg.compiler} used, but not ' + f'defined in environment {environment.name} ' + f'by package {pkg.name}' + ) + + root_compilers.append(pkg.compiler) + + dep_compilers = [] + for comp in root_compilers: + comp_pkg = self._rendered_packages[comp] + + if comp_pkg.compiler: + cur_compiler = comp_pkg.compiler + + while cur_compiler and cur_compiler not in dep_compilers: + dep_compilers.append(cur_compiler) + if comp_pkg.compiler: + cur_compiler = self._rendered_packages[comp_pkg.compiler].name + + for comp in reversed(root_compilers + dep_compilers): + comp_pkg = self._rendered_packages[comp] + yield comp_pkg.spec_str(all_packages=self._rendered_packages, + compiler=False) + + def package_specs_for_environment(self, environment: object): + """Iterator over package specs for a given environment + + Assumes all compilers have been defined via + self.define_compiler_packages() + + Args: + environment (SoftwareEnvironment): Environment to extract package specs for + + Yields: + (str) Spec string for each package + """ + + for pkg in environment._packages: + yield pkg.spec_str(all_packages=self._rendered_packages, compiler=False) + + def _check_environment(self, environment): + """Check an environment for common issues + + Args: + environment (SoftwareEnvironment): Environment to check for issues in + """ + + pkg_names = set() + + for pkg in environment._packages: + pkg_names.add(pkg.name) + + used_compilers = set() + compiler_warnings = [] + for pkg in environment._packages: + if pkg.compiler and pkg.compiler in pkg_names: + compiler_warnings.append((pkg.name, pkg.compiler)) + + logger.debug(f' Used compilers: {used_compilers}') + logger.debug(f' Compiler warnings: {compiler_warnings}') + if compiler_warnings: + logger.warn( + f'Environment {environment.name} contains packages and their ' + 'compilers in the package list. These include:' ) + for pkg_name, comp_name in compiler_warnings: + logger.warn( + f' Package: {pkg_name}, Compiler: {comp_name}' + ) + logger.warn( + 'This might cause problems when installing the packages.' + ) + + def render_environment(self, env_name: str, expander: object, require=True): + """Render an environment needed by an experiment + + Args: + env_name (str): Name of environment needed by the experiment + expander (Expander): Expander object from the experiment + + Returns: + (SoftwareEnvironment): Reference to software environment for + the experiment + """ - return self._raw_environments[env] + # Check for an external environment before checking templates + if env_name in self._rendered_environments: + if isinstance(self._rendered_environments[env_name], ExternalEnvironment): + return self._rendered_environments[env_name] + + for template_name, template_def in self._environment_templates.items(): + rendered_name = expander.expand_var(template_name) + if rendered_name == env_name: + rendered_env = template_def.render_environment(expander, self._package_templates, + self._rendered_packages) + + if rendered_env.name == env_name: + if env_name in self._rendered_environments: + if rendered_env != self._rendered_environments[env_name]: + raise RambleSoftwareEnvironmentError( + f'Environment {env_name} defined multiple times ' + 'in inconsistent ways' + ) + rendered_env = self._rendered_environments[env_name] - def mapped_environments(self, raw_env): - """Yield each environment rendered from a raw environment""" - self._require_raw_environment(raw_env) + template_def.add_rendered_environment(rendered_env, + self._rendered_environments, + self._rendered_packages) + self.define_compiler_packages(rendered_env, expander) + self._check_environment(rendered_env) + return rendered_env - for env in self._environment_map[raw_env].keys(): - yield env + if require: + raise RambleSoftwareEnvironmentError( + f'No defined environment matches required name {env_name}' + ) class RambleSoftwareEnvironmentError(ramble.error.RambleError): diff --git a/lib/ramble/ramble/spack_runner.py b/lib/ramble/ramble/spack_runner.py index 2d06f473c..f1a7c4ba8 100644 --- a/lib/ramble/ramble/spack_runner.py +++ b/lib/ramble/ramble/spack_runner.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -29,7 +29,7 @@ spack_namespace = 'spack' -package_name_regex = re.compile(r"\s*(?P[\w][\w-]+).*") +package_name_regex = re.compile(r"[\s-]*(?P[\w][\w-]+).*") class SpackRunner(object): @@ -65,7 +65,8 @@ class SpackRunner(object): 'mirrors.yaml', 'repos.yaml', 'packages.yaml', 'modules.yaml', 'config.yaml', 'upstreams.yaml', - 'bootstrap.yaml', 'spack.yaml'] + 'bootstrap.yaml', 'spack.yaml', + 'spack_includes.yaml'] def __init__(self, shell='bash', dry_run=False): """ @@ -106,6 +107,7 @@ def __init__(self, shell='bash', dry_run=False): self.compiler_config_dir = None self.configs = [] self.configs_applied = False + self.env_contents = [] self.installer = self.spack.copy() self.installer.add_default_prefix(ramble.config.get(f'{self.install_config_name}:prefix')) @@ -365,8 +367,6 @@ def activate(self): self.installer.add_default_env(self.env_key, self.env_path) self.concretizer.add_default_env(self.env_key, self.env_path) - self.env_contents = [] - self.active = True def deactivate(self): @@ -519,7 +519,9 @@ def _env_file_dict(self): env_file[spack_namespace]['concretizer']['unify'] = True env_file[spack_namespace]['specs'] = syaml.syaml_list() - env_file[spack_namespace]['specs'].extend(self.env_contents) + # Ensure the specs content are consistently sorted. + # Otherwise the hash checking may artificially miss due to ordering. + env_file[spack_namespace]['specs'].extend(sorted(self.env_contents)) env_file[spack_namespace]['include'] = self.includes @@ -635,10 +637,10 @@ def install(self): def get_package_path(self, package_spec): """Return the installation directory for a package""" loc_args = ['location', '-i'] - loc_args.extend(package_spec.split()) + loc_args.extend(shlex.split(package_spec)) name_args = ['find', '--format={name}'] - name_args.extend(package_spec.split()) + name_args.extend(shlex.split(package_spec)) if not self.dry_run: name = self._run_command(self.spack, name_args, return_output=True).strip() @@ -648,8 +650,8 @@ def get_package_path(self, package_spec): self._dry_run_print(self.spack, name_args) self._dry_run_print(self.spack, loc_args) - name = os.path.join(package_spec.split()[0]) - location = os.path.join('dry-run', 'path', 'to', package_spec.split()[0]) + name = os.path.join(shlex.split(package_spec)[0]) + location = os.path.join('dry-run', 'path', 'to', shlex.split(package_spec)[0]) return (name, location) def mirror_environment(self, mirror_path): @@ -746,6 +748,22 @@ def validate_command(self, command='', validation_type='not_empty', regex=None): else: self._dry_run_print(self.spack, args) + def package_definitions(self): + """For each package in this environment, yield the path to its application.py file""" + package_def_name = 'package.py' + location_args = ['location', '-p'] + + self._check_active() + + if not self.dry_run: + for pkg in self.env_contents: + args = location_args.copy() + args.append(pkg) + path = self._run_command(self.spack, args, return_output=True).strip() + yield pkg, os.path.join(path, package_def_name) + else: + self._dry_run_print(self.spack, location_args) + def _raise_validation_error(self, command, validation_type): raise ValidationFailedError( f'Validation of: "spack {command}" failed ' diff --git a/lib/ramble/ramble/spec.py b/lib/ramble/ramble/spec.py index a81f8fd05..eca60d391 100644 --- a/lib/ramble/ramble/spec.py +++ b/lib/ramble/ramble/spec.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/stage.py b/lib/ramble/ramble/stage.py index 2f0c457fb..d9dd4d7fe 100644 --- a/lib/ramble/ramble/stage.py +++ b/lib/ramble/ramble/stage.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/success_criteria.py b/lib/ramble/ramble/success_criteria.py index 984a9658d..2b4bb96a8 100644 --- a/lib/ramble/ramble/success_criteria.py +++ b/lib/ramble/ramble/success_criteria.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/application_inheritance.py b/lib/ramble/ramble/test/application_inheritance.py index b60fa32af..b232754e6 100644 --- a/lib/ramble/ramble/test/application_inheritance.py +++ b/lib/ramble/ramble/test/application_inheritance.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -18,14 +18,28 @@ def test_basic_inheritance(mutable_mock_apps_repo): assert app_inst.executables['bar'].mpi assert 'test_wl' in app_inst.workloads - assert app_inst.workloads['test_wl']['executables'] == ['builtin::env_vars', 'foo'] - assert app_inst.workloads['test_wl']['inputs'] == ['input'] + assert app_inst.workloads['test_wl'].executables == ['foo'] + assert app_inst.workloads['test_wl'].inputs == ['input'] + + exec_graph = app_inst._get_executable_graph('test_wl') + assert exec_graph.get_node('foo') is not None + assert exec_graph.get_node('builtin::env_vars') is not None + assert 'test_wl2' in app_inst.workloads - assert app_inst.workloads['test_wl2']['executables'] == ['builtin::env_vars', 'bar'] - assert app_inst.workloads['test_wl2']['inputs'] == ['input'] + assert app_inst.workloads['test_wl2'].executables == ['bar'] + assert app_inst.workloads['test_wl2'].inputs == ['input'] + + exec_graph = app_inst._get_executable_graph('test_wl2') + assert exec_graph.get_node('bar') is not None + assert exec_graph.get_node('builtin::env_vars') is not None + assert 'test_wl3' in app_inst.workloads - assert app_inst.workloads['test_wl3']['executables'] == ['builtin::env_vars', 'foo'] - assert app_inst.workloads['test_wl3']['inputs'] == ['inherited_input'] + assert app_inst.workloads['test_wl3'].executables == ['foo'] + assert app_inst.workloads['test_wl3'].inputs == ['inherited_input'] + + exec_graph = app_inst._get_executable_graph('test_wl3') + assert exec_graph.get_node('foo') is not None + assert exec_graph.get_node('builtin::env_vars') is not None assert 'test_fom' in app_inst.figures_of_merit fom_conf = app_inst.figures_of_merit['test_fom'] @@ -46,10 +60,7 @@ def test_basic_inheritance(mutable_mock_apps_repo): assert app_inst.inputs['inherited_input']['description'] == \ 'Again, not a file' - assert 'test_wl' in app_inst.workload_variables - assert 'my_var' in app_inst.workload_variables['test_wl'] - assert app_inst.workload_variables['test_wl']['my_var']['default'] == \ - '1.0' - - assert app_inst.workload_variables['test_wl']['my_var']['description'] \ - == 'Example var' + assert 'my_base_var' in app_inst.workloads['test_wl'].variables + assert 'my_var' in app_inst.workloads['test_wl'].variables + assert 'Shadowed' in app_inst.workloads['test_wl'].variables['my_var'].description + assert app_inst.workloads['test_wl'].variables['my_var'].default == '1.0' diff --git a/lib/ramble/ramble/test/application_language.py b/lib/ramble/ramble/test/application_language.py index 2cfa7bd22..cacefc38e 100644 --- a/lib/ramble/ramble/test/application_language.py +++ b/lib/ramble/ramble/test/application_language.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -9,6 +9,7 @@ import pytest import enum +import deprecation from ramble.appkit import * # noqa @@ -30,9 +31,7 @@ def test_application_type_features(app_class): assert hasattr(test_app, 'executables') assert hasattr(test_app, 'figures_of_merit') assert hasattr(test_app, 'inputs') - assert hasattr(test_app, 'workload_variables') - assert hasattr(test_app, 'environment_variables') - assert hasattr(test_app, 'default_compilers') + assert hasattr(test_app, 'compilers') assert hasattr(test_app, 'software_specs') assert hasattr(test_app, 'required_packages') assert hasattr(test_app, 'maintainers') @@ -211,7 +210,9 @@ def add_input_file(app_inst, input_num=1, func_type=func_types.directive): return input_defs -def add_default_compiler(app_inst, spec_num=1, func_type=func_types.directive): +# TODO: can this be dried with the modifier language add_compiler? +@deprecation.fail_if_not_removed +def add_compiler(app_inst, spec_num=1, func_type=func_types.directive): spec_name = 'Compiler%spec_num' spec_spack_spec = f'compiler_base@{spec_num}.0 +var1 ~var2' spec_compiler_spec = 'compiler1_base@{spec_num}' @@ -223,11 +224,11 @@ def add_default_compiler(app_inst, spec_num=1, func_type=func_types.directive): } if func_type == func_types.directive: - default_compiler(spec_name, spack_spec=spec_spack_spec, # noqa: F405 - compiler_spec=spec_compiler_spec)(app_inst) + define_compiler(spec_name, spack_spec=spec_spack_spec, # noqa: F405 + compiler_spec=spec_compiler_spec)(app_inst) elif func_type == func_types.method: - app_inst.default_compiler(spec_name, spack_spec=spec_spack_spec, # noqa: F405 - compiler_spec=spec_compiler_spec) + app_inst.define_compiler(spec_name, spack_spec=spec_spack_spec, # noqa: F405 + compiler_spec=spec_compiler_spec) else: assert False @@ -241,11 +242,11 @@ def add_default_compiler(app_inst, spec_num=1, func_type=func_types.directive): } if func_type == func_types.directive: - default_compiler(spec_name, spack_spec=spec_spack_spec, # noqa: F405 - compiler_spec=spec_compiler_spec)(app_inst) + define_compiler(spec_name, spack_spec=spec_spack_spec, # noqa: f405 + compiler_spec=spec_compiler_spec)(app_inst) elif func_type == func_types.method: - app_inst.default_compiler(spec_name, spack_spec=spec_spack_spec, # noqa: F405 - compiler_spec=spec_compiler_spec) + app_inst.define_compiler(spec_name, spack_spec=spec_spack_spec, # noqa: F405 + compiler_spec=spec_compiler_spec) else: assert False @@ -308,13 +309,13 @@ def test_workload_directive(app_class, func_type): assert hasattr(app_inst, 'workloads') assert wl_name in app_inst.workloads - assert 'executables' in app_inst.workloads[wl_name] - assert 'inputs' in app_inst.workloads[wl_name] + assert app_inst.workloads[wl_name].executables is not None + assert app_inst.workloads[wl_name].inputs is not None for test in test_defs['executables']: - assert test in app_inst.workloads[wl_name]['executables'] + assert app_inst.workloads[wl_name].find_executable(test) is not None for test in test_defs['inputs']: - assert test in app_inst.workloads[wl_name]['inputs'] + assert app_inst.workloads[wl_name].find_input(test) is not None @pytest.mark.parametrize('func_type', func_types) @@ -371,23 +372,23 @@ def test_input_file_directive(app_class, func_type): @pytest.mark.parametrize('func_type', func_types) @pytest.mark.parametrize('app_class', app_types) -def test_default_compiler_directive(app_class, func_type): +def test_define_compiler_directive(app_class, func_type): app_inst = app_class('/not/a/path') test_defs = {} if app_inst.uses_spack: - test_defs.update(add_default_compiler(app_inst, 1, func_type=func_type)) - test_defs.update(add_default_compiler(app_inst, 2, func_type=func_type)) + test_defs.update(add_compiler(app_inst, 1, func_type=func_type)) + test_defs.update(add_compiler(app_inst, 2, func_type=func_type)) - assert hasattr(app_inst, 'default_compilers') + assert hasattr(app_inst, 'compilers') for name, info in test_defs.items(): - assert name in app_inst.default_compilers + assert name in app_inst.compilers for key, value in info.items(): - assert app_inst.default_compilers[name][key] == value + assert app_inst.compilers[name][key] == value else: - test_defs.update(add_default_compiler(app_inst, 1, func_type=func_type)) + test_defs.update(add_compiler(app_inst, 1, func_type=func_type)) - assert hasattr(app_inst, 'default_compilers') - assert not app_inst.default_compilers + assert hasattr(app_inst, 'compilers') + assert not app_inst.compilers @pytest.mark.parametrize('func_type', func_types) diff --git a/lib/ramble/ramble/test/application_tests.py b/lib/ramble/ramble/test/application_tests.py index b67745a70..700a3291f 100644 --- a/lib/ramble/ramble/test/application_tests.py +++ b/lib/ramble/ramble/test/application_tests.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -10,6 +10,7 @@ import pytest import ramble.workspace +import ramble.workload pytestmark = pytest.mark.usefixtures('mutable_config', 'mutable_mock_workspace_path', @@ -26,29 +27,52 @@ def test_app_features(mutable_mock_apps_repo, app): assert hasattr(app_inst, 'executables') assert hasattr(app_inst, 'figures_of_merit') assert hasattr(app_inst, 'inputs') - assert hasattr(app_inst, 'default_compilers') + assert hasattr(app_inst, 'compilers') assert hasattr(app_inst, 'software_specs') assert hasattr(app_inst, 'required_packages') - assert hasattr(app_inst, 'workload_variables') - assert hasattr(app_inst, 'environment_variables') assert hasattr(app_inst, 'builtins') def test_basic_app(mutable_mock_apps_repo): basic_inst = mutable_mock_apps_repo.get('basic') - assert 'foo' in basic_inst.executables - assert basic_inst.executables['foo'].template == ['bar'] - assert not basic_inst.executables['foo'].mpi - assert 'bar' in basic_inst.executables - assert basic_inst.executables['bar'].template == ['baz'] - assert basic_inst.executables['bar'].mpi assert 'test_wl' in basic_inst.workloads - assert basic_inst.workloads['test_wl']['executables'] == ['builtin::env_vars', 'foo'] - assert basic_inst.workloads['test_wl']['inputs'] == ['input'] + assert len(basic_inst.workloads['test_wl'].executables) == 1 + foo_exec = basic_inst.workloads['test_wl'].find_executable('foo') + assert foo_exec is not None + foo_exec = basic_inst.executables[foo_exec] + assert foo_exec.template == ['bar'] + assert not foo_exec.mpi + + assert len(basic_inst.workloads['test_wl'].inputs) == 1 + example_input = basic_inst.workloads['test_wl'].find_input('input') + assert example_input is not None + + assert len(basic_inst.workloads['test_wl'].variables) == 2 + my_var = basic_inst.workloads['test_wl'].find_variable('my_var') + assert my_var is not None + assert my_var.default == '1.0' + assert my_var.description == 'Example var' + assert 'test_wl2' in basic_inst.workloads - assert basic_inst.workloads['test_wl2']['executables'] == ['builtin::env_vars', 'bar'] - assert basic_inst.workloads['test_wl2']['inputs'] == ['input'] + assert len(basic_inst.workloads['test_wl2'].executables) == 1 + bar_exec = basic_inst.workloads['test_wl2'].find_executable('bar') + assert bar_exec is not None + bar_exec = basic_inst.executables[bar_exec] + assert bar_exec.template == ['baz'] + assert bar_exec.mpi + + assert len(basic_inst.workloads['test_wl2'].inputs) == 1 + example_input = basic_inst.workloads['test_wl2'].find_input('input') + assert example_input is not None + + exec_graph = basic_inst._get_executable_graph('test_wl') + assert exec_graph.get_node('foo') is not None + assert exec_graph.get_node('builtin::env_vars') is not None + + exec_graph = basic_inst._get_executable_graph('test_wl2') + assert exec_graph.get_node('bar') is not None + assert exec_graph.get_node('builtin::env_vars') is not None assert 'test_fom' in basic_inst.figures_of_merit fom_conf = basic_inst.figures_of_merit['test_fom'] @@ -64,14 +88,6 @@ def test_basic_app(mutable_mock_apps_repo): assert basic_inst.inputs['input']['description'] == \ 'Not a file' - assert 'test_wl' in basic_inst.workload_variables - assert 'my_var' in basic_inst.workload_variables['test_wl'] - assert basic_inst.workload_variables['test_wl']['my_var']['default'] == \ - '1.0' - - assert basic_inst.workload_variables['test_wl']['my_var']['description'] \ - == 'Example var' - @pytest.mark.parametrize('app_name', ['basic', 'zlib']) def test_application_copy_is_deep(mutable_mock_apps_repo, app_name): @@ -125,13 +141,6 @@ def test_application_copy_is_deep(mutable_mock_apps_repo, app_name): copy_inst = src_inst.copy() - test_attrs = ['_setup_phases', '_analyze_phases', '_archive_phases', - '_mirror_phases'] - - # Test Phases - for attr in test_attrs: - assert getattr(copy_inst, attr) == getattr(src_inst, attr) - # Test variables for var, val in src_inst.variables.items(): assert var in copy_inst.variables.keys() @@ -184,9 +193,9 @@ def test_required_builtins(mutable_mock_apps_repo, app): required_builtins.append(builtin) for workload, wl_conf in app_inst.workloads.items(): - if app_inst._workload_exec_key in wl_conf: - for builtin in required_builtins: - assert builtin in wl_conf[app_inst._workload_exec_key] + exec_graph = app_inst._get_executable_graph(workload) + for builtin in required_builtins: + assert exec_graph.get_node(builtin) is not None def test_register_builtin_app(mutable_mock_apps_repo): @@ -201,11 +210,12 @@ def test_register_builtin_app(mutable_mock_apps_repo): excluded_builtins.append(builtin) for workload, wl_conf in app_inst.workloads.items(): - if app_inst._workload_exec_key in wl_conf: - for builtin in required_builtins: - assert builtin in wl_conf[app_inst._workload_exec_key] - for builtin in excluded_builtins: - assert builtin not in wl_conf[app_inst._workload_exec_key] + exec_graph = app_inst._get_executable_graph(workload) + + for builtin in required_builtins: + assert exec_graph.get_node(builtin) is not None + for builtin in excluded_builtins: + assert exec_graph.get_node(builtin) is None @pytest.mark.parametrize('app', [ @@ -245,8 +255,8 @@ def basic_exp_dict(): } -def test_get_executables_initial(mutable_mock_apps_repo): - """_get_executables, test1, workload executables""" +def test_get_executable_graph_initial(mutable_mock_apps_repo): + """_get_executable_graph, test1, workload executables""" executable_application_instance = mutable_mock_apps_repo.get('basic') @@ -254,19 +264,20 @@ def test_get_executables_initial(mutable_mock_apps_repo): # Set up the instance to test just the initial part of the function executable_application_instance.expander = ramble.expander.Expander(expansion_vars, None) - executable_application_instance.workloads = {'test_wl': {'executables': ['foo'], - 'inputs': ['input']}, - 'test_wl2': {'executables': ['bar'], - 'inputs': ['input']}} + test_wl = ramble.workload.Workload('test_wl', executables=['foo'], inputs=['input']) + test_wl2 = ramble.workload.Workload('test_wl2', executables=['bar'], inputs=['input']) + executable_application_instance.workloads = {'test_wl': test_wl, + 'test_wl2': test_wl2} executable_application_instance.internals = {} - executables = executable_application_instance._get_executables() + executable_graph = executable_application_instance._get_executable_graph('test_wl2') + bar_node = executable_graph.get_node('bar') - assert 'bar' in executables + assert bar_node is not None -def test_get_executables_yaml_defined(mutable_mock_apps_repo): - """_get_executables, test2, yaml-defined order""" +def test_get_executable_graph_yaml_defined(mutable_mock_apps_repo): + """_get_executable_graph, test2, yaml-defined order""" executable_application_instance = mutable_mock_apps_repo.get('basic') @@ -274,15 +285,15 @@ def test_get_executables_yaml_defined(mutable_mock_apps_repo): # Set up the instance to pass the initial part of the function executable_application_instance.expander = ramble.expander.Expander(expansion_vars, None) - executable_application_instance.workloads = {'test_wl': {'executables': ['foo'], - 'inputs': ['input']}, - 'test_wl2': {'executables': ['bar'], - 'inputs': ['input']}} + test_wl = ramble.workload.Workload('test_wl', executables=['foo'], inputs=['input']) + test_wl2 = ramble.workload.Workload('test_wl2', executables=['bar'], inputs=['input']) + executable_application_instance.workloads = {'test_wl': test_wl, + 'test_wl2': test_wl2} # Insert namespace.executables into the instance's internals to pass the # second part of the function defined_internals = { - 'executables': { + 'custom_executables': { 'test_exec': { 'template': [ 'test_exec' @@ -290,17 +301,23 @@ def test_get_executables_yaml_defined(mutable_mock_apps_repo): 'use_mpi': False, 'redirect': '{log_file}' } - } + }, + 'executables': [ + 'bar', + 'test_exec' + ] } executable_application_instance.set_internals(defined_internals) - executables = executable_application_instance._get_executables() + executable_graph = executable_application_instance._get_executable_graph('test_wl') - assert 'test_exec' in executables + test_node = executable_graph.get_node('test_exec') + assert test_node is not None -def test_get_executables_custom_executables(mutable_mock_apps_repo): - """_get_executables, test3, custom executables""" + +def test_get_executable_graph_custom_executables(mutable_mock_apps_repo): + """_get_executable_graph, test3, custom executables""" executable_application_instance = mutable_mock_apps_repo.get('basic') @@ -308,10 +325,10 @@ def test_get_executables_custom_executables(mutable_mock_apps_repo): # Set up the instance to pass the initial part of the function executable_application_instance.expander = ramble.expander.Expander(expansion_vars, None) - executable_application_instance.workloads = {'test_wl': {'executables': ['foo'], - 'inputs': ['input']}, - 'test_wl2': {'executables': ['bar'], - 'inputs': ['input']}} + test_wl = ramble.workload.Workload('test_wl', executables=['foo'], inputs=['input']) + test_wl2 = ramble.workload.Workload('test_wl2', executables=['bar'], inputs=['input']) + executable_application_instance.workloads = {'test_wl': test_wl, + 'test_wl2': test_wl2} # Insert namespace.executables into the instance's internals to pass the # second part of the function @@ -324,13 +341,18 @@ def test_get_executables_custom_executables(mutable_mock_apps_repo): 'use_mpi': False, 'redirect': '{log_file}', } - } + }, + 'executables': [ + 'test_exec2', + 'bar' + ] } executable_application_instance.set_internals(defined_internals) - executable_application_instance._get_executables() + executable_graph = executable_application_instance._get_executable_graph('test_wl2') + test_node = executable_graph.get_node('test_exec2') - assert 'test_exec2' in executable_application_instance.executables + assert test_node is not None def test_set_input_path(mutable_mock_apps_repo): @@ -392,19 +414,17 @@ def test_set_default_experiment_variables(mutable_mock_apps_repo): # Set up the instance to pass the initial part of the function executable_application_instance.expander = ramble.expander.Expander(expansion_vars, None) - executable_application_instance.workloads = {'test_wl': {'executables': ['foo'], - 'inputs': ['input']}, - 'test_wl2': {'executables': ['bar'], - 'inputs': ['input']}} + test_wl = ramble.workload.Workload('test_wl', executables=['foo'], inputs=['input']) + test_wl2 = ramble.workload.Workload('test_wl2', executables=['bar'], inputs=['input']) + test_wl2.add_variable(ramble.workload.WorkloadVariable('n_ranks', default='1')) + executable_application_instance.workloads = {'test_wl': test_wl, + 'test_wl2': test_wl2} executable_application_instance.internals = {} executable_application_instance.inputs = {'input': {'target_dir': '.'}} executable_application_instance.variables = {} - executable_application_instance.workload_variables = {'test_wl2': {'n_ranks': - {'default': '1'}}} - executable_application_instance._set_default_experiment_variables() assert executable_application_instance.variables['n_ranks'] == '1' @@ -420,20 +440,18 @@ def test_define_commands(mutable_mock_apps_repo): # Set up the instance to pass the initial part of the function executable_application_instance.expander = ramble.expander.Expander(expansion_vars, None) - executable_application_instance.workloads = {'test_wl': {'executables': ['foo'], - 'inputs': ['input']}, - 'test_wl2': {'executables': ['bar'], - 'inputs': ['input']}} + test_wl = ramble.workload.Workload('test_wl', executables=['foo'], inputs=['input']) + test_wl2 = ramble.workload.Workload('test_wl2', executables=['bar'], inputs=['input']) + test_wl2.add_variable(ramble.workload.WorkloadVariable('n_ranks', default='1')) + executable_application_instance.workloads = {'test_wl': test_wl, + 'test_wl2': test_wl2} executable_application_instance.internals = {} executable_application_instance.inputs = {'input': {'target_dir': '.'}} executable_application_instance.variables = {} - executables = executable_application_instance._get_executables() - - executable_application_instance.workload_variables = {'test_wl2': {'n_ranks': - {'default': '1'}}} + exec_graph = executable_application_instance._get_executable_graph('test_wl2') executable_application_instance.set_formatted_executables( {'command': {'join_separator': '\n'}} @@ -441,7 +459,7 @@ def test_define_commands(mutable_mock_apps_repo): executable_application_instance._set_default_experiment_variables() executable_application_instance.chain_prepend = [] - executable_application_instance._define_commands(executables) + executable_application_instance._define_commands(exec_graph) executable_application_instance._define_formatted_executables() assert 'mpirun' in executable_application_instance.variables['command'] @@ -471,7 +489,6 @@ def test_derive_variables_for_template_path(mutable_mock_apps_repo): n_nodes: '2' spack: - concretized: true packages: {} environments: {} """ @@ -494,30 +511,40 @@ def test_derive_variables_for_template_path(mutable_mock_apps_repo): # Set up the instance to pass the initial part of the function executable_application_instance.expander = ramble.expander.Expander(expansion_vars, None) - executable_application_instance.workloads = {'test_wl': {'executables': ['foo'], - 'inputs': ['input']}, - 'test_wl2': {'executables': ['bar'], - 'inputs': ['input'], - 'template': ['input'], - }, - } + test_wl = ramble.workload.Workload('test_wl', executables=['foo'], inputs=['input']) + test_wl2 = ramble.workload.Workload('test_wl2', executables=['bar'], inputs=['input']) + test_wl2.add_variable(ramble.workload.WorkloadVariable('n_ranks', default='1')) + executable_application_instance.workloads = {'test_wl': test_wl, + 'test_wl2': test_wl2} executable_application_instance.internals = {} executable_application_instance.inputs = {'input': {'target_dir': '.'}} executable_application_instance.variables = {} - executables = executable_application_instance._get_executables() - - executable_application_instance.workload_variables = {'test_wl2': {'n_ranks': - {'default': '1'}}} + exec_graph = executable_application_instance._get_executable_graph('test_wl2') executable_application_instance._set_default_experiment_variables() executable_application_instance.chain_prepend = [] - executable_application_instance._define_commands(executables) + executable_application_instance._define_commands(exec_graph) executable_application_instance._define_formatted_executables() test_answer = "/workspace/experiments/bar/test_wl2/baz/execute_experiment" executable_application_instance._derive_variables_for_template_path(ws1) assert executable_application_instance.variables['execute_experiment'] == test_answer + + +def test_class_attributes(mutable_mock_apps_repo): + basic_inst = mutable_mock_apps_repo.get('basic') + basic_copy = basic_inst.copy() + + instances = [basic_inst, basic_copy] + for inst in instances: + assert hasattr(inst, 'workloads') + assert 'test_wl' in inst.workloads + + basic_copy.workload('added_workload', executables=['foo']) + + assert 'added_workload' in basic_copy.workloads + assert 'added_workload' not in basic_inst.workloads diff --git a/lib/ramble/ramble/test/cache_fetch.py b/lib/ramble/ramble/test/cache_fetch.py index 2e33ac497..1ff1b9e31 100644 --- a/lib/ramble/ramble/test/cache_fetch.py +++ b/lib/ramble/ramble/test/cache_fetch.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/cmd/attributes.py b/lib/ramble/ramble/test/cmd/attributes.py index f8faa88d7..412512100 100644 --- a/lib/ramble/ramble/test/cmd/attributes.py +++ b/lib/ramble/ramble/test/cmd/attributes.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/cmd/clean.py b/lib/ramble/ramble/test/cmd/clean.py index e8dd46f54..8c5348554 100644 --- a/lib/ramble/ramble/test/cmd/clean.py +++ b/lib/ramble/ramble/test/cmd/clean.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -15,8 +15,9 @@ clean = ramble.main.RambleCommand('clean') -pytestmark = pytest.mark.skipif(sys.platform == "win32", - reason="does not run on windows") +pytestmark = pytest.mark.skipif( + sys.platform == "win32", reason="does not run on windows" +) @pytest.fixture() @@ -33,25 +34,29 @@ def __call__(self, *args, **kwargs): counts[self.name] += 1 monkeypatch.setattr( - ramble.caches.fetch_cache, 'destroy', Counter('downloads'), - raising=False) + ramble.caches.fetch_cache, 'destroy', Counter('downloads'), raising=False + ) + monkeypatch.setattr(ramble.caches.misc_cache, 'destroy', Counter('caches')) monkeypatch.setattr( - ramble.caches.misc_cache, 'destroy', Counter('caches')) + ramble.cmd.clean, 'remove_python_caches', Counter('python_caches') + ) yield counts -all_effects = ['downloads', 'caches'] +all_effects = ['downloads', 'caches', 'python_caches'] -@pytest.mark.usefixtures( - 'config' +@pytest.mark.usefixtures('config') +@pytest.mark.parametrize( + 'command_line,effects', + [ + ('-d', ['downloads']), + ('-m', ['caches']), + ('-p', ['python_caches']), + ('-a', all_effects), + ], ) -@pytest.mark.parametrize('command_line,effects', [ - ('-d', ['downloads']), - ('-m', ['caches']), - ('-a', all_effects), -]) def test_function_calls(command_line, effects, mock_calls_for_clean): # Call the command with the supplied command line diff --git a/lib/ramble/ramble/test/cmd/config.py b/lib/ramble/ramble/test/cmd/config.py index 190f9cdf1..feadb968c 100644 --- a/lib/ramble/ramble/test/cmd/config.py +++ b/lib/ramble/ramble/test/cmd/config.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/cmd/debug.py b/lib/ramble/ramble/test/cmd/debug.py index 95f077c55..58e5ded00 100644 --- a/lib/ramble/ramble/test/cmd/debug.py +++ b/lib/ramble/ramble/test/cmd/debug.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/cmd/info.py b/lib/ramble/ramble/test/cmd/info.py index fa8f6ea66..0ee876f68 100644 --- a/lib/ramble/ramble/test/cmd/info.py +++ b/lib/ramble/ramble/test/cmd/info.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -54,8 +54,8 @@ def test_info_fields(app_query, parser, info_lines): expected_fields = ( 'Description:', - 'Setup Pipeline Phases:', - 'Analyze Pipeline Phases:', + 'Pipeline "setup" Phases:', + 'Pipeline "analyze" Phases:', 'Tags:' ) @@ -73,8 +73,8 @@ def test_info_fields(app_query, parser, info_lines): def test_spack_info_software(app_query): expected_fields = ( 'Description:', - 'Setup Pipeline Phases:', - 'Analyze Pipeline Phases:', + 'Pipeline "setup" Phases:', + 'Pipeline "analyze" Phases:', 'Tags:', 'spack_spec =', 'compiler =', @@ -92,8 +92,8 @@ def test_spack_info_software(app_query): def test_mock_spack_info_software(mock_applications, app_query): expected_fields = ( 'Description:', - 'Setup Pipeline Phases:', - 'Analyze Pipeline Phases:', + 'Pipeline "setup" Phases:', + 'Pipeline "analyze" Phases:', 'Tags:', 'Package Manager Configs:', 'spack_spec =', diff --git a/lib/ramble/ramble/test/cmd/list.py b/lib/ramble/ramble/test/cmd/list.py index 2e39a4cc9..fef9f8698 100644 --- a/lib/ramble/ramble/test/cmd/list.py +++ b/lib/ramble/ramble/test/cmd/list.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/cmd/mirror.py b/lib/ramble/ramble/test/cmd/mirror.py index 5a7a73b20..cad9a06eb 100644 --- a/lib/ramble/ramble/test/cmd/mirror.py +++ b/lib/ramble/ramble/test/cmd/mirror.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/cmd/mods.py b/lib/ramble/ramble/test/cmd/mods.py index 73dc32afd..080a29db8 100644 --- a/lib/ramble/ramble/test/cmd/mods.py +++ b/lib/ramble/ramble/test/cmd/mods.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/cmd/on.py b/lib/ramble/ramble/test/cmd/on.py index febd7226d..a32163fd8 100644 --- a/lib/ramble/ramble/test/cmd/on.py +++ b/lib/ramble/ramble/test/cmd/on.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -34,9 +34,6 @@ def test_on_command(mutable_mock_workspace_path): ramble.test.cmd.workspace.add_basic(ws) ramble.test.cmd.workspace.check_basic(ws) - workspace('concretize') - assert ws.is_concretized() - workspace('setup') assert os.path.exists(ws.root + '/all_experiments') @@ -58,9 +55,6 @@ def test_execute_pipeline(mutable_mock_workspace_path): ramble.test.cmd.workspace.add_basic(ws) ramble.test.cmd.workspace.check_basic(ws) - ws.concretize() - assert ws.is_concretized() - setup_pipeline = setup_pipeline_class(ws, filters) setup_pipeline.run() assert os.path.exists(ws.root + '/all_experiments') @@ -77,9 +71,6 @@ def test_on_where(mutable_mock_workspace_path): ramble.test.cmd.workspace.add_basic(ws) ramble.test.cmd.workspace.check_basic(ws) - workspace('concretize') - assert ws.is_concretized() - workspace('setup') assert os.path.exists(ws.root + '/all_experiments') @@ -94,9 +85,6 @@ def test_on_executor(mutable_mock_workspace_path): ramble.test.cmd.workspace.add_basic(ws) ramble.test.cmd.workspace.check_basic(ws) - workspace('concretize') - assert ws.is_concretized() - workspace('setup') assert os.path.exists(ws.root + '/all_experiments') diff --git a/lib/ramble/ramble/test/cmd/python.py b/lib/ramble/ramble/test/cmd/python.py index 0e120a9a7..047fbe39f 100644 --- a/lib/ramble/ramble/test/cmd/python.py +++ b/lib/ramble/ramble/test/cmd/python.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/cmd/repo.py b/lib/ramble/ramble/test/cmd/repo.py index 9f4a5db44..e9e0cce27 100644 --- a/lib/ramble/ramble/test/cmd/repo.py +++ b/lib/ramble/ramble/test/cmd/repo.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -28,7 +28,7 @@ def test_create_add_list_remove(mutable_config, tmpdir): assert os.path.exists(os.path.join(str(tmpdir), 'applications')) # Add the new repository and check it appears in the list output - repo('add', '--scope=site', str(tmpdir)) + repo('add', '-t', 'applications', '--scope=site', str(tmpdir)) output = repo('list', '--scope=site', output=str) assert 'mockrepo' in output @@ -47,7 +47,7 @@ def test_create_add_list_remove_flags(mutable_config, tmpdir, subdir): assert os.path.exists(os.path.join(str(tmpdir), subdir)) # Add the new repository and check it appears in the list output - repo('add', '--scope=site', str(tmpdir)) + repo('add', '-t', 'applications', '--scope=site', str(tmpdir)) output = repo('list', '--scope=site', output=str) assert 'mockrepo' in output diff --git a/lib/ramble/ramble/test/cmd/results.py b/lib/ramble/ramble/test/cmd/results.py index 7fd59da79..bd00f75ee 100644 --- a/lib/ramble/ramble/test/cmd/results.py +++ b/lib/ramble/ramble/test/cmd/results.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/cmd/software_definitions.py b/lib/ramble/ramble/test/cmd/software_definitions.py index 3536bf2b8..7f382c752 100644 --- a/lib/ramble/ramble/test/cmd/software_definitions.py +++ b/lib/ramble/ramble/test/cmd/software_definitions.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/cmd/workspace.py b/lib/ramble/ramble/test/cmd/workspace.py index 65025bda5..dc896ff0a 100644 --- a/lib/ramble/ramble/test/cmd/workspace.py +++ b/lib/ramble/ramble/test/cmd/workspace.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -15,9 +15,9 @@ import ramble.application import ramble.workspace -from ramble.software_environments import RambleSoftwareEnvironmentError from ramble.main import RambleCommand, RambleCommandError from ramble.test.dry_run_helpers import search_files_for_string +from ramble.namespace import namespace import ramble.config import ramble.filters import ramble.pipeline @@ -103,8 +103,18 @@ def check_info_basic(output): assert 'Workload' in output assert 'Experiment' in output assert 'Software Stack' in output - assert 'Packages' in output - assert 'Environments' in output + + +def check_info_zlib(output): + assert 'zlib' in output + assert 'ensure_installed' in output + + assert 'Application' in output + assert 'Workload' in output + assert 'Experiment' in output + assert 'Software Stack' in output + assert 'Template Package' in output + assert 'Template Environment' in output def check_results(ws): @@ -178,11 +188,21 @@ def test_workspace_info(): test_experiment: variables: n_nodes: '2' - + zlib: + workloads: + ensure_installed: + experiments: + test_experiment: + variables: + n_nodes: '2' spack: - concretized: true - packages: {} - environments: {} + packages: + zlib: + spack_spec: 'zlib' + environments: + zlib: + packages: + - zlib """ workspace_name = 'test_info' @@ -221,10 +241,21 @@ def test_workspace_info_prints_all_levels(): test_experiment: variables: n_nodes: '2' + zlib: + workloads: + ensure_installed: + experiments: + test_experiment: + variables: + n_nodes: '2' spack: - concretized: true - packages: {} - environments: {} + packages: + zlib: + spack_spec: 'zlib' + environments: + zlib: + packages: + - zlib """ config_file = """ @@ -284,7 +315,6 @@ def test_workspace_info_with_experiment_chain(): n_nodes: '2' spack: - concretized: true packages: {} environments: {} """ @@ -394,14 +424,56 @@ def test_remove_workspace(): def test_concretize_command(): - ws_name = 'test' - workspace('create', ws_name) + test_config = """ +ramble: + variables: + mpi_command: 'mpirun -n {n_ranks} -ppn {processes_per_node}' + batch_submit: 'batch_submit {execute_experiment}' + processes_per_node: '5' + n_ranks: '{processes_per_node}*{n_nodes}' + applications: + basic: + workloads: + test_wl: + experiments: + test_experiment: + variables: + n_nodes: '2' + test_wl2: + experiments: + test_experiment: + variables: + n_nodes: '2' + zlib: + workloads: + ensure_installed: + experiments: + test_experiment: + variables: + n_nodes: '2' + spack: + packages: {} + environments: {} +""" - with ramble.workspace.read('test') as ws: - add_basic(ws) - check_basic(ws) - workspace('concretize') - assert ws.is_concretized() + workspace_name = 'test_concretize_command' + ws1 = ramble.workspace.create(workspace_name) + ws1.write() + + config_path = os.path.join(ws1.config_dir, ramble.workspace.config_file_name) + + with open(config_path, 'w+') as f: + f.write(test_config) + + ws1._re_read() + + assert search_files_for_string([config_path], 'packages: {}') is True + assert search_files_for_string([config_path], 'spack_spec: zlib') is False + + workspace('concretize', global_args=['-w', workspace_name]) + + assert search_files_for_string([config_path], 'packages: {}') is False + assert search_files_for_string([config_path], 'spack_spec: zlib') is True def test_concretize_nothing(): @@ -413,8 +485,126 @@ def test_concretize_nothing(): add_basic(ws) check_basic(ws) + assert search_files_for_string([ws.config_file_path], 'packages: {}') is True + assert search_files_for_string([ws.config_file_path], 'spack_spec:') is False + ws.concretize() - assert ws.is_concretized() + + assert search_files_for_string([ws.config_file_path], 'packages: {}') is True + assert search_files_for_string([ws.config_file_path], 'spack_spec:') is False + + +def test_concretize_concrete_config(): + test_config = """ +ramble: + variables: + mpi_command: 'mpirun -n {n_ranks} -ppn {processes_per_node}' + batch_submit: 'batch_submit {execute_experiment}' + processes_per_node: '5' + n_ranks: '{processes_per_node}*{n_nodes}' + applications: + basic: + workloads: + test_wl: + experiments: + test_experiment: + variables: + n_nodes: '2' + test_wl2: + experiments: + test_experiment: + variables: + n_nodes: '2' + zlib: + workloads: + ensure_installed: + experiments: + test_experiment: + variables: + n_nodes: '2' + spack: + packages: + zlib: + spack_spec: 'zlib' + environments: + zlib: + packages: + - zlib +""" + + workspace_name = 'test_concretize_concrete_config' + ws1 = ramble.workspace.create(workspace_name) + ws1.write() + + config_path = os.path.join(ws1.config_dir, ramble.workspace.config_file_name) + + with open(config_path, 'w+') as f: + f.write(test_config) + + ws1._re_read() + + with pytest.raises(ramble.workspace.RambleWorkspaceError) as e: + workspace('concretize', global_args=['-w', workspace_name]) + assert 'Cannot concretize an already concretized workspace.' in e + + +def test_force_concretize(): + test_config = """ +ramble: + variables: + mpi_command: 'mpirun -n {n_ranks} -ppn {processes_per_node}' + batch_submit: 'batch_submit {execute_experiment}' + processes_per_node: '5' + n_ranks: '{processes_per_node}*{n_nodes}' + applications: + basic: + workloads: + test_wl: + experiments: + test_experiment: + variables: + n_nodes: '2' + test_wl2: + experiments: + test_experiment: + variables: + n_nodes: '2' + zlib: + workloads: + ensure_installed: + experiments: + test_experiment: + variables: + n_nodes: '2' + spack: + packages: + zlib-test: + spack_spec: 'zlib-test' + environments: + zlib-test: + packages: + - zlib-test +""" + + workspace_name = 'test_force_concretize' + ws1 = ramble.workspace.create(workspace_name) + ws1.write() + + config_path = os.path.join(ws1.config_dir, ramble.workspace.config_file_name) + + with open(config_path, 'w+') as f: + f.write(test_config) + + ws1._re_read() + + with pytest.raises(ramble.workspace.RambleWorkspaceError) as e: + workspace('concretize', global_args=['-w', workspace_name]) + assert 'Cannot concretize an already concretized workspace.' in e + + workspace('concretize', '-f', global_args=['-w', workspace_name]) + + assert search_files_for_string([config_path], 'zlib:') is True + assert search_files_for_string([config_path], 'zlib-test') is False def test_setup_command(): @@ -426,12 +616,29 @@ def test_setup_command(): check_basic(ws) workspace('concretize') - assert ws.is_concretized() workspace('setup') assert os.path.exists(ws.root + '/all_experiments') +def test_setup_command_with_missing_log_dir(): + ws_name = "test" + workspace("create", ws_name) + + with ramble.workspace.read("test") as ws: + add_basic(ws) + check_basic(ws) + # Missing log directory shouldn't prevent workspace + # setup, as long as the workspace is considered valid + # by the `is_workspace_dir` check. + os.rmdir(ws.log_dir) + + workspace("concretize") + + workspace("setup") + assert os.path.exists(ws.root + "/all_experiments") + + def test_setup_nothing(): ws_name = 'test' workspace('create', ws_name) @@ -445,7 +652,6 @@ def test_setup_nothing(): check_basic(ws) ws.concretize() - assert ws.is_concretized() setup_pipeline = pipeline_cls(ws, filters) setup_pipeline.run() @@ -461,7 +667,6 @@ def test_anlyze_command(): check_basic(ws) workspace('concretize') - assert ws.is_concretized() workspace('setup') assert os.path.exists(ws.root + '/all_experiments') @@ -485,7 +690,6 @@ def test_analyze_nothing(): check_basic(ws) ws.concretize() - assert ws.is_concretized() setup_pipeline = setup_cls(ws, filters) setup_pipeline.run() @@ -508,8 +712,6 @@ def test_workspace_flag_named(): check_basic(ws) workspace('concretize', global_args=flag_args) - with ramble.workspace.read(ws_name) as ws: - assert ws.is_concretized() workspace('setup', global_args=flag_args) with ramble.workspace.read(ws_name) as ws: @@ -536,8 +738,6 @@ def test_workspace_flag_anon(tmpdir): check_basic(ws) workspace('concretize', global_args=flag_args) - with ramble.workspace.Workspace(ws_path) as ws: - assert ws.is_concretized() workspace('setup', global_args=flag_args) with ramble.workspace.Workspace(ws_path) as ws: @@ -567,7 +767,6 @@ def test_no_workspace_flag(): workspace('concretize', global_args=flag_args) ramble.workspace.activate(ws) workspace('concretize') - assert ws.is_concretized() with pytest.raises(RambleCommandError): workspace('setup', global_args=flag_args) @@ -638,7 +837,6 @@ def test_dryrun_setup(): variables: n_nodes: '2' spack: - concretized: true packages: {} environments: {} """ @@ -689,7 +887,6 @@ def test_matrix_vector_workspace_full(): - n_nodes - - idx spack: - concretized: true packages: {} environments: {} """ @@ -758,7 +955,6 @@ def test_invalid_vector_workspace(): n_nodes: [1, 2, 4] idx: [1, 2, 3, 4, 5, 6] spack: - concretized: true packages: {} environments: {} """ @@ -807,7 +1003,6 @@ def test_invalid_size_matrices_workspace(): - - n_nodes - - idx spack: - concretized: true packages: {} environments: {} """ @@ -852,7 +1047,6 @@ def test_undefined_var_matrices_workspace(): matrices: - - foo spack: - concretized: true packages: {} environments: {} """ @@ -896,7 +1090,6 @@ def test_non_vector_var_matrices_workspace(): matrices: - - foo spack: - concretized: true packages: {} environments: {} """ @@ -941,7 +1134,6 @@ def test_multi_use_vector_var_matrices_workspace(): - - foo - - foo spack: - concretized: true packages: {} environments: {} """ @@ -989,7 +1181,6 @@ def test_reconcretize_in_configs_dir(tmpdir): variables: foo: 1 spack: - concretized: false packages: {} environments: {} """ @@ -1013,15 +1204,22 @@ def write_config(ws_path, config): with config_path.as_cwd(): write_config(ws_path, test_config) + with ramble.workspace.Workspace(ws_path) as ws: + spack_dict = ws.get_spack_dict() + print(f'spack_dict before = {spack_dict}') + workspace('concretize', global_args=workspace_flags) + with ramble.workspace.Workspace(ws_path) as ws: - assert ws.is_concretized() + spack_dict = ws.get_spack_dict() + assert spack_dict[namespace.environments] write_config(ws_path, test_config) workspace('concretize', global_args=workspace_flags) with ramble.workspace.Workspace(ws_path) as ws: - assert ws.is_concretized() + spack_dict = ws.get_spack_dict() + assert spack_dict[namespace.environments] def test_workspace_archive(): @@ -1041,7 +1239,6 @@ def test_workspace_archive(): variables: n_nodes: '2' spack: - concretized: true packages: {} environments: {} """ @@ -1131,7 +1328,6 @@ def test_workspace_archive_include_secrets(): variables: n_nodes: '2' spack: - concretized: true packages: {} environments: {} """ @@ -1191,7 +1387,6 @@ def test_workspace_tar_archive(): variables: n_nodes: '2' spack: - concretized: true packages: {} environments: {} """ @@ -1269,7 +1464,6 @@ def test_workspace_tar_upload_archive(): variables: n_nodes: '2' spack: - concretized: true packages: {} environments: {} """ @@ -1352,7 +1546,6 @@ def test_workspace_tar_upload_archive_config_url(): variables: n_nodes: '2' spack: - concretized: true packages: {} environments: {} """ @@ -1435,7 +1628,6 @@ def test_dryrun_noexpvars_setup(): experiments: test_experiment: {} spack: - concretized: true packages: {} environments: {} """ @@ -1500,7 +1692,6 @@ def test_workspace_include(): include: - '%s' spack: - concretized: true packages: {} environments: {} """ % inc_file @@ -1536,7 +1727,6 @@ def test_invalid_template_name_errors(tpl_name, capsys): experiments: test_experiment: {} spack: - concretized: true packages: {} environments: {} """ @@ -1599,7 +1789,6 @@ def test_custom_executables_info(): redirect: '{log_file}' output_capture: '&>' spack: - concretized: true packages: {} environments: {} """ @@ -1662,7 +1851,6 @@ def test_custom_executables_order_info(): - wl_level_cmd - app_level_cmd spack: - concretized: true packages: {} environments: {} """ @@ -1683,14 +1871,14 @@ def test_custom_executables_order_info(): assert "['exp_level_cmd', 'wl_level_cmd', 'app_level_cmd']" in output -def test_invalid_spack_config_errors(capsys): - test_config = """ +def test_workspace_simplify(): + test_ws_config = """ ramble: variables: mpi_command: 'mpirun -n {n_ranks} -ppn {processes_per_node}' batch_submit: 'batch_submit {execute_experiment}' processes_per_node: '5' - n_ranks: '{processes_per_node}' + n_ranks: '{processes_per_node}*{n_nodes}' applications: zlib: workloads: @@ -1698,20 +1886,87 @@ def test_invalid_spack_config_errors(capsys): experiments: test_experiment: variables: - n_ranks: '1' + n_nodes: '2' + zlib-configs: + workloads: + ensure_installed: + experiments: + unused_exp_template: + template: True + variables: + n_nodes: '1' + spack: + packages: + zlib: + spack_spec: zlib + zlib-configs: + spack_spec: zlib-configs + unused-pkg: + spack_spec: unused + environments: + zlib: + packages: + - zlib + zlib-configs: + packages: + - zlib-configs + unused-env: + packages: + - unused-pkg +""" + test_app_config = """ +applications: + basic: + workloads: + test_wl: + experiments: + app_not_in_ws_config: + variables: + n_ranks: 1 +""" + test_spack_config = """ +spack: + packages: + pkg_not_in_ws_config: + spack_spec: 'gcc@10.5.0' + compiler_spec: gcc@10.5.0 """ - workspace_name = 'test_invalid_spack_config_errors' + workspace_name = 'test_simplify' ws1 = ramble.workspace.create(workspace_name) ws1.write() - config_path = os.path.join(ws1.config_dir, ramble.workspace.config_file_name) + ws_config_path = os.path.join(ws1.config_dir, ramble.workspace.config_file_name) + app_config_path = os.path.join(ws1.config_dir, 'applications.yaml') + spack_config_path = os.path.join(ws1.config_dir, 'spack.yaml') - with open(config_path, 'w+') as f: - f.write(test_config) + with open(ws_config_path, 'w+') as f: + f.write(test_ws_config) + with open(app_config_path, 'w+') as f: + f.write(test_app_config) + with open(spack_config_path, 'w+') as f: + f.write(test_spack_config) ws1._re_read() - with pytest.raises(RambleSoftwareEnvironmentError): - ramble.software_environments.SoftwareEnvironments(ws1) - captured = capsys.readouterr() - assert "Software configuration type invalid is not one of ['v2']" in captured.err + + assert search_files_for_string([ws_config_path], 'spack_spec: zlib') is True + assert search_files_for_string([ws_config_path], 'unused-pkg') is True + assert search_files_for_string([ws_config_path], 'unused-env') is True + assert search_files_for_string([ws_config_path], 'unused_exp_template') is True + assert search_files_for_string([ws_config_path], 'spack_spec: zlib-configs') is True + assert search_files_for_string([ws_config_path], 'app_not_in_ws_config') is False + assert search_files_for_string([ws_config_path], 'pkg_not_in_ws_config') is False + + workspace('concretize', '--simplify', global_args=['-w', workspace_name]) + + print(ws_config_path) + + assert search_files_for_string([ws_config_path], 'spack_spec: zlib') is True # keep used pkg + assert search_files_for_string([ws_config_path], 'unused-pkg') is False # remove unused pkg + assert search_files_for_string([ws_config_path], 'unused-env') is False # remove unused env + # remove unused experiment template and associated pkgs/envs + assert search_files_for_string([ws_config_path], 'unused_exp_template') is False + assert search_files_for_string([ws_config_path], 'spack_spec: zlib-configs') is False + # ensure apps/pkgs/envs are not merged into workspace config from other config files + assert search_files_for_string([ws_config_path], 'app_not_in_ws_config') is False + assert search_files_for_string([ws_config_path], 'pkg_not_in_ws_config') is False diff --git a/lib/ramble/ramble/test/commands.py b/lib/ramble/ramble/test/commands.py index 793074d0d..df0592320 100644 --- a/lib/ramble/ramble/test/commands.py +++ b/lib/ramble/ramble/test/commands.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/concretize_builtin.py b/lib/ramble/ramble/test/concretize_builtin.py index f4ed4d4ea..a14ebe775 100644 --- a/lib/ramble/ramble/test/concretize_builtin.py +++ b/lib/ramble/ramble/test/concretize_builtin.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -68,7 +68,6 @@ def test_concretize_does_not_set_required(mutable_config, mutable_mock_workspace matrix: - n_nodes spack: - concretized: false packages: {} environments: {} """ diff --git a/lib/ramble/ramble/test/conftest.py b/lib/ramble/ramble/test/conftest.py index 6b8229c85..6e545389e 100644 --- a/lib/ramble/ramble/test/conftest.py +++ b/lib/ramble/ramble/test/conftest.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/dry_run_helpers.py b/lib/ramble/ramble/test/dry_run_helpers.py index 2b7f68a0f..07e4846f9 100644 --- a/lib/ramble/ramble/test/dry_run_helpers.py +++ b/lib/ramble/ramble/test/dry_run_helpers.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -37,7 +37,6 @@ def dry_run_config(section_name, injections, config_path, test_dict['spack'] = syaml.syaml_dict() spack_dict = test_dict['spack'] - spack_dict['concretized'] = False spack_dict['packages'] = syaml.syaml_dict() spack_dict['environments'] = syaml.syaml_dict() diff --git a/lib/ramble/ramble/test/end_to_end/analyze_fom_output.py b/lib/ramble/ramble/test/end_to_end/analyze_fom_output.py new file mode 100644 index 000000000..0f5385db9 --- /dev/null +++ b/lib/ramble/ramble/test/end_to_end/analyze_fom_output.py @@ -0,0 +1,69 @@ +# Copyright 2022-2024 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import os +import glob + +import pytest + +import ramble.workspace +import ramble.config +import ramble.software_environments +from ramble.main import RambleCommand + + +# everything here uses the mock_workspace_path +pytestmark = pytest.mark.usefixtures( + 'mutable_config', + 'mutable_mock_workspace_path', +) + +workspace = RambleCommand('workspace') + + +def test_analyze_fom_output(): + test_config = """ +ramble: + variables: + mpi_command: '' + batch_submit: 'batch_submit {execute_experiment}' + processes_per_node: '1' + applications: + hostname: + workloads: + local: + experiments: + test: + variables: + n_nodes: '1' + spack: + packages: {} + environments: {} +""" + workspace_name = 'test-fom-output' + ws = ramble.workspace.create(workspace_name) + ws.write() + + config_path = os.path.join(ws.config_dir, ramble.workspace.config_file_name) + + with open(config_path, 'w+') as f: + f.write(test_config) + + ws._re_read() + + workspace('setup', '--dry-run', global_args=['-w', workspace_name]) + exp_out = os.path.join(ws.experiment_dir, 'hostname', 'local', 'test', 'test.out') + with open(exp_out, 'w+') as f: + f.write('test-user.c.googlers.com\n') + workspace('analyze', global_args=['-w', workspace_name]) + result_file = glob.glob(os.path.join(ws.root, 'results.latest.txt'))[0] + + with open(result_file, 'r') as f: + content = f.read() + assert 'default (null) context figures of merit' in content + assert 'possible hostname = test-user.c.googlers.com' in content diff --git a/lib/ramble/ramble/test/end_to_end/chained_experiment_var_inheritance.py b/lib/ramble/ramble/test/end_to_end/chained_experiment_var_inheritance.py index 92bcf2134..ab7cc5a08 100644 --- a/lib/ramble/ramble/test/end_to_end/chained_experiment_var_inheritance.py +++ b/lib/ramble/ramble/test/end_to_end/chained_experiment_var_inheritance.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -94,7 +94,6 @@ def test_chained_experiment_variable_inheritance(mutable_config, variables: n_nodes: '2' spack: - concretized: true packages: gcc: spack_spec: gcc@9.3.0 target=x86_64 diff --git a/lib/ramble/ramble/test/end_to_end/config_section_env_vars.py b/lib/ramble/ramble/test/end_to_end/config_section_env_vars.py index 175a30919..47e8a53ac 100644 --- a/lib/ramble/ramble/test/end_to_end/config_section_env_vars.py +++ b/lib/ramble/ramble/test/end_to_end/config_section_env_vars.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -44,7 +44,6 @@ def test_config_section_env_vars(mutable_config, mutable_mock_workspace_path, mo variables: n_nodes: 1 spack: - concretized: true packages: {} environments: {} """ diff --git a/lib/ramble/ramble/test/end_to_end/custom_executables.py b/lib/ramble/ramble/test/end_to_end/custom_executables.py index f5c3bad10..6e6da73d4 100644 --- a/lib/ramble/ramble/test/end_to_end/custom_executables.py +++ b/lib/ramble/ramble/test/end_to_end/custom_executables.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -66,8 +66,8 @@ def test_custom_executables(mutable_config, mutable_mock_workspace_path, mock_ap - baz executable_injection: - name: before_all - order: before - name: after_all + order: after - name: before_env_vars order: before relative_to: builtin::env_vars @@ -82,7 +82,6 @@ def test_custom_executables(mutable_config, mutable_mock_workspace_path, mock_ap MY_VAR: 'TEST' OTHER_ENV_VAR: 'ANOTHER_TEST' spack: - concretized: true packages: {} environments: {} """ diff --git a/lib/ramble/ramble/test/end_to_end/define_package_paths.py b/lib/ramble/ramble/test/end_to_end/define_package_paths.py new file mode 100644 index 000000000..11ff47254 --- /dev/null +++ b/lib/ramble/ramble/test/end_to_end/define_package_paths.py @@ -0,0 +1,95 @@ +# Copyright 2022-2024 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import os + +import pytest + +import ramble.workspace +import ramble.config +import ramble.software_environments +from ramble.main import RambleCommand + + +# everything here uses the mock_workspace_path +pytestmark = pytest.mark.usefixtures( + 'mutable_config', + 'mutable_mock_workspace_path', +) + +workspace = RambleCommand('workspace') + + +def _spack_loc_log_line(pkg_spec): + return f'with args: [\'location\', \'-i\', \'{pkg_spec}\']' + + +def test_define_package_paths(): + test_config = """ +ramble: + variables: + mpi_command: 'mpirun -n {n_ranks} -ppn {processes_per_node}' + batch_submit: '{execute_experiment}' + processes_per_node: '1' + applications: + gromacs: + workloads: + water_bare: + experiments: + test1: + variables: + n_nodes: '1' + test2: + variables: + n_nodes: '2' + spack: + packages: + gromacs: + spack_spec: gromacs + intel-mpi: + spack_spec: intel-oneapi-mpi@2021.11.0 + environments: + gromacs: + packages: + - gromacs + - intel-mpi +""" + workspace_name = 'test-define-package-paths' + ws = ramble.workspace.create(workspace_name) + ws.write() + + config_path = os.path.join(ws.config_dir, ramble.workspace.config_file_name) + + with open(config_path, 'w+') as f: + f.write(test_config) + + ws._re_read() + + workspace( + 'setup', + '--dry-run', + global_args=['-w', workspace_name], + ) + + # test1 should attempt to invoke `spack location -i` on dep packages. + test1_log = os.path.join(ws.log_dir, 'setup.latest', 'gromacs.water_bare.test1.out') + gromacs_log_line = _spack_loc_log_line('gromacs') + impi_log_line = _spack_loc_log_line('intel-oneapi-mpi@2021.11.0') + with open(test1_log, 'r') as f: + content = f.read() + assert 'Executing phase define_package_paths' in content + assert content.count(gromacs_log_line) == 1 + assert content.count(impi_log_line) == 1 + + # test2 should use cached paths without invoking spack. + test2_log = os.path.join(ws.log_dir, 'setup.latest', 'gromacs.water_bare.test2.out') + with open(test2_log, 'r') as f: + content = f.read() + assert 'Executing phase define_package_paths' in content + assert gromacs_log_line not in content + assert impi_log_line not in content diff --git a/lib/ramble/ramble/test/end_to_end/dryrun_chained_experiments.py b/lib/ramble/ramble/test/end_to_end/dryrun_chained_experiments.py index f59f474bd..a27e35d86 100644 --- a/lib/ramble/ramble/test/end_to_end/dryrun_chained_experiments.py +++ b/lib/ramble/ramble/test/end_to_end/dryrun_chained_experiments.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -83,7 +83,6 @@ def test_dryrun_chained_experiments(mutable_config, variables: n_nodes: '2' spack: - concretized: true packages: gcc: spack_spec: gcc@9.3.0 target=x86_64 diff --git a/lib/ramble/ramble/test/end_to_end/dryrun_copies_external_env.py b/lib/ramble/ramble/test/end_to_end/dryrun_copies_external_env.py index cf05a53cd..19adea545 100644 --- a/lib/ramble/ramble/test/end_to_end/dryrun_copies_external_env.py +++ b/lib/ramble/ramble/test/end_to_end/dryrun_copies_external_env.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -52,7 +52,6 @@ def test_dryrun_copies_external_env(mutable_config, mutable_mock_workspace_path, variables: n_nodes: '1' spack: - concretized: true packages: {{}} environments: wrfv4: @@ -78,7 +77,7 @@ def test_dryrun_copies_external_env(mutable_config, mutable_mock_workspace_path, setup_pipeline = setup_cls(ws, filters) setup_pipeline.run() - env_file = os.path.join(ws.software_dir, 'wrfv4.CONUS_12km', 'spack.yaml') + env_file = os.path.join(ws.software_dir, 'wrfv4', 'spack.yaml') assert os.path.exists(env_file) diff --git a/lib/ramble/ramble/test/end_to_end/dryrun_series_contains_package_paths.py b/lib/ramble/ramble/test/end_to_end/dryrun_series_contains_package_paths.py index 5c6e50ae4..518637db7 100644 --- a/lib/ramble/ramble/test/end_to_end/dryrun_series_contains_package_paths.py +++ b/lib/ramble/ramble/test/end_to_end/dryrun_series_contains_package_paths.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -46,7 +46,6 @@ def test_dryrun_series_contains_package_paths(mutable_config, n_nodes: '1' test_id: [1, 2] spack: - concretized: true packages: zlib: spack_spec: zlib diff --git a/lib/ramble/ramble/test/end_to_end/env_var_builtin.py b/lib/ramble/ramble/test/end_to_end/env_var_builtin.py index b0060de15..7005399e8 100644 --- a/lib/ramble/ramble/test/end_to_end/env_var_builtin.py +++ b/lib/ramble/ramble/test/end_to_end/env_var_builtin.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -60,7 +60,6 @@ def test_env_var_builtin(mutable_config, mutable_mock_workspace_path, mock_appli set: MY_VAR: 'TEST' spack: - concretized: true packages: {} environments: {} """ @@ -125,3 +124,43 @@ def test_env_var_builtin(mutable_config, mutable_mock_workspace_path, mock_appli if export_found and cmd3_regex.search(line): cmd_found = True assert cmd_found and export_found + + +def test_env_var_from_app_only(mutable_config, mutable_mock_workspace_path, mock_applications): + test_config = """ +ramble: + variables: + mpi_command: 'mpirun -n {n_ranks} -ppn {processes_per_node}' + batch_submit: 'batch_submit {execute_experiment}' + partition: 'part1' + processes_per_node: '16' + n_threads: '1' + applications: + interleved-env-vars: + workloads: + test_wl: + experiments: + simple_test: + variables: + n_nodes: 1 + spack: + packages: {} + environments: {} +""" + workspace_name = 'test_env_var_from_app_only' + with ramble.workspace.create(workspace_name) as ws: + ws.write() + + config_path = os.path.join(ws.config_dir, ramble.workspace.config_file_name) + + with open(config_path, 'w+') as f: + f.write(test_config) + ws._re_read() + + workspace('setup', '--dry-run', global_args=['-w', workspace_name]) + + experiment_root = ws.experiment_dir + exp1_dir = os.path.join(experiment_root, 'interleved-env-vars', 'test_wl', 'simple_test') + + with open(os.path.join(exp1_dir, 'execute_experiment'), 'r') as f: + assert 'FROM_DIRECTIVE' in f.read() diff --git a/lib/ramble/ramble/test/end_to_end/exclusive_filtered_vector_workloads.py b/lib/ramble/ramble/test/end_to_end/exclusive_filtered_vector_workloads.py index 20c561df0..d0d27bf25 100644 --- a/lib/ramble/ramble/test/end_to_end/exclusive_filtered_vector_workloads.py +++ b/lib/ramble/ramble/test/end_to_end/exclusive_filtered_vector_workloads.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -42,7 +42,6 @@ def test_exclusive_filtered_vector_workloads(mutable_config, mutable_mock_worksp application_workload: ['parallel' ,'serial', 'local'] n_nodes: 1 spack: - concretized: true packages: {} environments: {} """ diff --git a/lib/ramble/ramble/test/end_to_end/expanded_fom_dry_run.py b/lib/ramble/ramble/test/end_to_end/expanded_fom_dry_run.py index 5dd26d2cb..530b7787d 100644 --- a/lib/ramble/ramble/test/end_to_end/expanded_fom_dry_run.py +++ b/lib/ramble/ramble/test/end_to_end/expanded_fom_dry_run.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -43,7 +43,6 @@ def test_expanded_foms_dry_run(mutable_config, n_nodes: 1 n_ranks: 1 spack: - concretized: false packages: {} environments: {} """ diff --git a/lib/ramble/ramble/test/end_to_end/experiment_excludes.py b/lib/ramble/ramble/test/end_to_end/experiment_excludes.py index 1c8759993..428cdccd3 100644 --- a/lib/ramble/ramble/test/end_to_end/experiment_excludes.py +++ b/lib/ramble/ramble/test/end_to_end/experiment_excludes.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -86,7 +86,6 @@ def test_wrfv4_exclusions(mutable_config, mutable_mock_workspace_path): where: - '{n_nodes} == 16' spack: - concretized: true packages: gcc: spack_spec: gcc@8.5.0 @@ -154,7 +153,7 @@ def test_wrfv4_exclusions(mutable_config, mutable_mock_workspace_path): out_files, 'Would download https://www2.mmm.ucar.edu/wrf/users/benchmark/v422/v42_bench_conus12km.tar.gz') # noqa # Test software directories - software_dirs = ['wrfv4.CONUS_12km', 'wrfv4-portable.CONUS_12km'] + software_dirs = ['wrfv4', 'wrfv4-portable'] software_base_dir = os.path.join(ws1.root, ramble.workspace.workspace_software_path) assert os.path.exists(software_base_dir) for software_dir in software_dirs: diff --git a/lib/ramble/ramble/test/end_to_end/experiment_repeats.py b/lib/ramble/ramble/test/end_to_end/experiment_repeats.py index cf41edf2f..df769b09c 100644 --- a/lib/ramble/ramble/test/end_to_end/experiment_repeats.py +++ b/lib/ramble/ramble/test/end_to_end/experiment_repeats.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -61,7 +61,6 @@ def test_gromacs_repeats(mutable_config, mutable_mock_workspace_path): size: '0003' type: 'pme' spack: - concretized: true packages: gcc: spack_spec: gcc@8.5.0 @@ -110,7 +109,7 @@ def test_gromacs_repeats(mutable_config, mutable_mock_workspace_path): out_files, 'Would download https://ftp.gromacs.org/pub/benchmarks/water_GMX50_bare.tar.gz') # noqa # Test software directories - software_dirs = ['gromacs.water_gmx50', 'gromacs.water_bare'] + software_dirs = ['gromacs'] software_base_dir = os.path.join(ws1.root, ramble.workspace.workspace_software_path) assert os.path.exists(software_base_dir) for software_dir in software_dirs: diff --git a/lib/ramble/ramble/test/end_to_end/experiment_templates.py b/lib/ramble/ramble/test/end_to_end/experiment_templates.py new file mode 100644 index 000000000..01735fff0 --- /dev/null +++ b/lib/ramble/ramble/test/end_to_end/experiment_templates.py @@ -0,0 +1,94 @@ +# Copyright 2022-2024 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import os + +import pytest + +import ramble.workspace +import ramble.config +import ramble.software_environments +from ramble.main import RambleCommand + +# everything here uses the mock_workspace_path +pytestmark = pytest.mark.usefixtures('mutable_config', + 'mutable_mock_workspace_path') + +workspace = RambleCommand('workspace') + + +def test_experiment_templates(mutable_config, mutable_mock_workspace_path): + test_config = r""" +ramble: + variables: + processes_per_node: 16 + mpi_command: 'mpirun -n {n_ranks} -ppn {processes_per_node}' + batch_submit: '{execute_experiment}' + n_ranks: 1 + applications: + hostname: + workloads: + serial: + experiments: + unused_template: + template: True + used_template: + template: True + template_false: + template: False + base_exp: + chained_experiments: + - name: 'hostname.serial.used_template' + command: '{execute_experiment}' + order: 'before_root' + - name: 'hostname.serial.template_false' + command: '{execute_experiment}' + order: 'after_root' +""" + workspace_name = 'test_experiment_templates' + with ramble.workspace.create(workspace_name) as ws1: + ws1.write() + + config_path = os.path.join(ws1.config_dir, ramble.workspace.config_file_name) + + with open(config_path, 'w+') as f: + f.write(test_config) + + ws1._re_read() + + workspace('setup', global_args=['-w', workspace_name]) + + expected_experiments = [ + 'template_false', + 'base_exp', + ] + + expected_chained_experiments = [ + '0.hostname.serial.used_template', + '1.hostname.serial.template_false' + ] + + unexpected_experiments = [ + 'unused_template', + 'used_template' + ] + + for exp in unexpected_experiments: + exp_dir = os.path.join(ws1.root, 'experiments', 'hostname', 'serial', exp) + assert not os.path.isdir(exp_dir) + + for exp in expected_experiments: + exp_dir = os.path.join(ws1.root, 'experiments', 'hostname', 'serial', exp) + assert os.path.isdir(exp_dir) + assert os.path.exists(os.path.join(exp_dir, 'execute_experiment')) + + for exp in expected_chained_experiments: + exp_dir = os.path.join(ws1.root, 'experiments', 'hostname', 'serial', 'base_exp', + 'chained_experiments', exp) + assert os.path.isdir(exp_dir) + assert os.path.exists(os.path.join(exp_dir, 'execute_experiment')) diff --git a/lib/ramble/ramble/test/end_to_end/explicit_zips.py b/lib/ramble/ramble/test/end_to_end/explicit_zips.py index 5100de6a0..92bf0f67d 100644 --- a/lib/ramble/ramble/test/end_to_end/explicit_zips.py +++ b/lib/ramble/ramble/test/end_to_end/explicit_zips.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -80,7 +80,6 @@ def test_wrfv4_explicit_zips(mutable_config, mutable_mock_workspace_path): - environments - partitions spack: - concretized: true packages: gcc: spack_spec: gcc@8.5.0 @@ -148,7 +147,7 @@ def test_wrfv4_explicit_zips(mutable_config, mutable_mock_workspace_path): out_files, 'Would download https://www2.mmm.ucar.edu/wrf/users/benchmark/v422/v42_bench_conus12km.tar.gz') # noqa # Test software directories - software_dirs = ['wrfv4.CONUS_12km', 'wrfv4-portable.CONUS_12km'] + software_dirs = ['wrfv4', 'wrfv4-portable'] software_base_dir = os.path.join(ws1.root, ramble.workspace.workspace_software_path) assert os.path.exists(software_base_dir) for software_dir in software_dirs: diff --git a/lib/ramble/ramble/test/end_to_end/formatted_executables.py b/lib/ramble/ramble/test/end_to_end/formatted_executables.py index aa177ec5a..e6f8e31eb 100644 --- a/lib/ramble/ramble/test/end_to_end/formatted_executables.py +++ b/lib/ramble/ramble/test/end_to_end/formatted_executables.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -58,7 +58,6 @@ def test_formatted_executables(mutable_config, mutable_mock_workspace_path, mock variables: n_nodes: 1 spack: - concretized: true packages: {} environments: {} """ @@ -86,10 +85,10 @@ def test_formatted_executables(mutable_config, mutable_mock_workspace_path, mock with open(exp_script, 'r') as f: data = f.read() - assert '; from_ws echo' in data assert 'from_app echo' in data - assert ' from_wl echo' in data - assert ' from_exp echo' in data + assert ';' + ' ' * 9 + 'from_ws echo' in data + assert '\n' + ' ' * 11 + 'from_wl echo' in data + assert '\n' + ' ' * 10 + 'from_exp echo' in data def test_redefined_executable_errors(mutable_config, mutable_mock_workspace_path, @@ -115,7 +114,6 @@ def test_redefined_executable_errors(mutable_config, mutable_mock_workspace_path var_exec_name: 'nothing' n_nodes: 1 spack: - concretized: true packages: {} environments: {} """ diff --git a/lib/ramble/ramble/test/end_to_end/globbing_patterns.py b/lib/ramble/ramble/test/end_to_end/globbing_patterns.py new file mode 100644 index 000000000..dc2479e29 --- /dev/null +++ b/lib/ramble/ramble/test/end_to_end/globbing_patterns.py @@ -0,0 +1,194 @@ +# Copyright 2022-2024 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import os + +import pytest + +import ramble.workspace +import ramble.config +import ramble.software_environments +from ramble.main import RambleCommand + + +# everything here uses the mock_workspace_path +pytestmark = pytest.mark.usefixtures('mutable_config', + 'mutable_mock_workspace_path') + +workspace = RambleCommand('workspace') + + +def test_globbing_patterns( + mutable_config, + mutable_mock_workspace_path, + mock_applications, + mock_modifiers): + test_config = """ +ramble: + variables: + mpi_command: 'mpirun -n {n_ranks} -ppn {processes_per_node}' + batch_submit: 'batch_submit {execute_experiment}' + partition: 'part1' + processes_per_node: '16' + n_threads: '1' + applications: + glob-patterns: + workloads: + test_one_exec: + experiments: + test_no_wildcards_one_exec: + variables: + n_nodes: 1 + test_three_exec: + experiments: + test_wildcard_3_execs: + variables: + n_nodes: 1 + modifiers: + - name: glob-patterns-mod + mode: test-glob + spack: + packages: {} + environments: {} +""" + workspace_name = 'test_globbing_patterns' + with ramble.workspace.create(workspace_name) as ws: + ws.write() + + config_path = os.path.join(ws.config_dir, ramble.workspace.config_file_name) + + with open(config_path, 'w+') as f: + f.write(test_config) + ws._re_read() + + workspace('setup', '--dry-run', global_args=['-w', workspace_name]) + + experiment_root = ws.experiment_dir + exp1_dir = os.path.join(experiment_root, 'glob-patterns', 'test_one_exec', + 'test_no_wildcards_one_exec') + exp1_script = os.path.join(exp1_dir, 'execute_experiment') + exp2_dir = os.path.join(experiment_root, 'glob-patterns', 'test_three_exec', + 'test_wildcard_3_execs') + exp2_script = os.path.join(exp2_dir, 'execute_experiment') + + import re + test_cmd_regex = re.compile('base test .*>>') + glob_cmd_regex = re.compile('test foo .*>>') + baz_regex = re.compile('baz .*>>') + + test_wl_var_regex = re.compile('wl_var_test') + glob_wl_var_regex = re.compile('wl_var_glob') + baz_wl_var_regex = re.compile('wl_var_baz') + + test_env_var_regex = re.compile('env_var_test') + glob_env_var_regex = re.compile('env_var_glob') + baz_env_var_regex = re.compile('env_var_baz') + + glob_var_mod_regex = re.compile('var_mod_modified') + glob_env_var_mod_regex = re.compile('env_var_mod=modded') + + with open(exp1_script, 'r') as f: + # Check for only 'test' executable command + test_cmd_found = False + glob_cmd_not_found = True + baz_cmd_not_found = True + + # Check for both test and glob workload vars + test_wl_var_found = False + glob_wl_var_found = False + baz_wl_var_not_found = True + + # Check for both test and glob env vars + test_env_var_found = False + glob_env_var_found = False + baz_env_var_not_found = True + + for line in f.readlines(): + # Executables + if test_cmd_regex.search(line): + test_cmd_found = True + if glob_cmd_regex.search(line): + glob_cmd_not_found = False + if baz_regex.search(line): + baz_cmd_not_found = False + + # Workload vars + if test_wl_var_regex.search(line): + test_wl_var_found = True + if glob_wl_var_regex.search(line): + glob_wl_var_found = True + if baz_wl_var_regex.search(line): + baz_wl_var_not_found = False + + # Env vars + if test_env_var_regex.search(line): + test_env_var_found = True + if glob_env_var_regex.search(line): + glob_env_var_found = True + if baz_env_var_regex.search(line): + baz_env_var_not_found = False + + assert test_cmd_found and glob_cmd_not_found and baz_cmd_not_found + assert test_wl_var_found and glob_wl_var_found and baz_wl_var_not_found + assert test_env_var_found and glob_env_var_found and baz_env_var_not_found + + with open(exp2_script, 'r') as f: + # Check for executables matching 'test*' glob pattern + test_cmd_found = False + glob_cmd_found = False + baz_cmd_not_found = True + + # Check for only glob workload var + test_wl_var_not_found = True + glob_wl_var_found = False + baz_wl_var_not_found = True + + # Check for only glob env var + test_env_var_not_found = True + glob_env_var_found = False + baz_env_var_not_found = True + + # Check for modifier globbing + glob_var_mod_found = False # checks both variable modifier and modifier variable + glob_env_var_mod_found = False + + for line in f.readlines(): + # Executables + if test_cmd_regex.search(line): + test_cmd_found = True + if glob_cmd_regex.search(line): + glob_cmd_found = True + if baz_regex.search(line): + baz_cmd_not_found = False + + # Workload vars + if test_wl_var_regex.search(line): + test_wl_var_not_found = False + if glob_wl_var_regex.search(line): + glob_wl_var_found = True + if baz_wl_var_regex.search(line): + baz_wl_var_not_found = False + + # Env vars + if test_env_var_regex.search(line): + test_env_var_not_found = False + if glob_env_var_regex.search(line): + glob_env_var_found = True + if baz_env_var_regex.search(line): + baz_env_var_not_found = False + + # Modifier + if glob_var_mod_regex.search(line): + glob_var_mod_found = True + if glob_env_var_mod_regex.search(line): + glob_env_var_mod_found = True + + assert test_cmd_found and glob_cmd_found and baz_cmd_not_found + assert test_wl_var_not_found and glob_wl_var_found and baz_wl_var_not_found + assert test_env_var_not_found and glob_env_var_found and baz_env_var_not_found + assert glob_var_mod_found and glob_env_var_mod_found diff --git a/lib/ramble/ramble/test/end_to_end/gromacs_size_expansion.py b/lib/ramble/ramble/test/end_to_end/gromacs_size_expansion.py index ba208ae29..387435383 100644 --- a/lib/ramble/ramble/test/end_to_end/gromacs_size_expansion.py +++ b/lib/ramble/ramble/test/end_to_end/gromacs_size_expansion.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -42,7 +42,6 @@ def test_gromacs_size_expansion(mutable_config, mutable_mock_workspace_path): n_nodes: '1' size: '0000.96' spack: - concretized: true packages: gcc: spack_spec: gcc@8.5.0 diff --git a/lib/ramble/ramble/test/end_to_end/inclusive_filtered_vector_workloads.py b/lib/ramble/ramble/test/end_to_end/inclusive_filtered_vector_workloads.py index 7f7ebf7c1..79bbec2b7 100644 --- a/lib/ramble/ramble/test/end_to_end/inclusive_filtered_vector_workloads.py +++ b/lib/ramble/ramble/test/end_to_end/inclusive_filtered_vector_workloads.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -42,7 +42,6 @@ def test_inclusive_filtered_vector_workloads(mutable_config, mutable_mock_worksp application_workload: ['parallel' ,'serial', 'local'] n_nodes: 1 spack: - concretized: true packages: {} environments: {} """ diff --git a/lib/ramble/ramble/test/end_to_end/known_applications.py b/lib/ramble/ramble/test/end_to_end/known_applications.py index 3b001ba5e..a42c56340 100644 --- a/lib/ramble/ramble/test/end_to_end/known_applications.py +++ b/lib/ramble/ramble/test/end_to_end/known_applications.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -71,7 +71,6 @@ def test_known_applications(application, capsys): n_nodes: '1' processes_per_node: '1'\n""") f.write(""" spack: - concretized: false packages: {} environments: {}\n""") diff --git a/lib/ramble/ramble/test/end_to_end/merge_config_files.py b/lib/ramble/ramble/test/end_to_end/merge_config_files.py index 935064cbf..f9b8c1cf1 100644 --- a/lib/ramble/ramble/test/end_to_end/merge_config_files.py +++ b/lib/ramble/ramble/test/end_to_end/merge_config_files.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -38,7 +38,6 @@ def test_merge_config_files(mutable_config, mutable_mock_workspace_path, mock_ap test_spack = """ spack: - concretized: true packages: zlib: spack_spec: zlib@1.2.12 @@ -56,7 +55,6 @@ def test_merge_config_files(mutable_config, mutable_mock_workspace_path, mock_ap n_threads: '1' applications: {} spack: - concretized: false packages: {} environments: {} """ diff --git a/lib/ramble/ramble/test/end_to_end/missing_required_dry_run.py b/lib/ramble/ramble/test/end_to_end/missing_required_dry_run.py index 3d697b201..d7adf4cd5 100644 --- a/lib/ramble/ramble/test/end_to_end/missing_required_dry_run.py +++ b/lib/ramble/ramble/test/end_to_end/missing_required_dry_run.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -42,7 +42,6 @@ def test_missing_required_dry_run(mutable_config, mutable_mock_workspace_path): variables: n_nodes: '8' spack: - concretized: true packages: gcc8: spack_spec: gcc@8.2.0 target=x86_64 diff --git a/lib/ramble/ramble/test/end_to_end/nested_compilers_are_installed.py b/lib/ramble/ramble/test/end_to_end/nested_compilers_are_installed.py index ad1cbba7f..01ab89567 100644 --- a/lib/ramble/ramble/test/end_to_end/nested_compilers_are_installed.py +++ b/lib/ramble/ramble/test/end_to_end/nested_compilers_are_installed.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -45,7 +45,6 @@ def test_nested_compilers_are_installed(mutable_config, mutable_mock_workspace_p variables: n_nodes: '1' spack: - concretized: true packages: gcc8: spack_spec: gcc@8.5.0 diff --git a/lib/ramble/ramble/test/end_to_end/package_manager_config.py b/lib/ramble/ramble/test/end_to_end/package_manager_config.py index 2aca09c59..a7ed0c910 100644 --- a/lib/ramble/ramble/test/end_to_end/package_manager_config.py +++ b/lib/ramble/ramble/test/end_to_end/package_manager_config.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -38,7 +38,6 @@ def test_package_manager_config_zlib(mock_applications): test: variables: {} spack: - concretized: true packages: zlib: spack_spec: 'zlib' @@ -60,7 +59,7 @@ def test_package_manager_config_zlib(mock_applications): workspace('setup', '--dry-run', global_args=['-w', workspace_name]) - spack_yaml = os.path.join(ws.software_dir, 'zlib-configs.ensure_installed', + spack_yaml = os.path.join(ws.software_dir, 'zlib-configs', 'spack.yaml') assert os.path.isfile(spack_yaml) diff --git a/lib/ramble/ramble/test/end_to_end/package_manager_requirements.py b/lib/ramble/ramble/test/end_to_end/package_manager_requirements.py index 8c0169ee9..2eb62e24a 100644 --- a/lib/ramble/ramble/test/end_to_end/package_manager_requirements.py +++ b/lib/ramble/ramble/test/end_to_end/package_manager_requirements.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -41,7 +41,6 @@ def test_package_manager_requirements_zlib(mock_applications, mock_modifiers): test: variables: {} spack: - concretized: true packages: {} environments: zlib-configs: @@ -65,7 +64,7 @@ def test_package_manager_requirements_zlib(mock_applications, mock_modifiers): workspace('setup', global_args=['-w', workspace_name]) - spack_yaml = os.path.join(ws.software_dir, 'zlib-configs.ensure_installed', + spack_yaml = os.path.join(ws.software_dir, 'zlib-configs', 'spack.yaml') assert os.path.isfile(spack_yaml) @@ -94,7 +93,6 @@ def test_package_manager_requirements_error(mock_applications, mock_modifiers): test: variables: {} spack: - concretized: true packages: {} environments: zlib-configs: diff --git a/lib/ramble/ramble/test/end_to_end/passthrough_variables.py b/lib/ramble/ramble/test/end_to_end/passthrough_variables.py index 6d970c311..8f848adda 100644 --- a/lib/ramble/ramble/test/end_to_end/passthrough_variables.py +++ b/lib/ramble/ramble/test/end_to_end/passthrough_variables.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -46,7 +46,6 @@ def test_passthrough_variables(mutable_config, mutable_mock_workspace_path, mock variables: n_nodes: 1 spack: - concretized: true packages: {} environments: {} """ @@ -94,7 +93,6 @@ def test_disable_passthrough(mutable_config, mutable_mock_workspace_path): mpi_command: '{undefined_var}' n_ranks: '1' spack: - concretized: true packages: {} environments: {} """ diff --git a/lib/ramble/ramble/test/end_to_end/phase_selection.py b/lib/ramble/ramble/test/end_to_end/phase_selection.py index 57cec6529..0ab088b23 100644 --- a/lib/ramble/ramble/test/end_to_end/phase_selection.py +++ b/lib/ramble/ramble/test/end_to_end/phase_selection.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -78,7 +78,6 @@ def test_workspace_phase_selection(mutable_config, mutable_mock_workspace_path, - n_nodes - env_name spack: - concretized: true packages: gcc: spack_spec: gcc@8.5.0 diff --git a/lib/ramble/ramble/test/end_to_end/phase_selection_with_dependencies.py b/lib/ramble/ramble/test/end_to_end/phase_selection_with_dependencies.py index b847d844f..6305ed054 100644 --- a/lib/ramble/ramble/test/end_to_end/phase_selection_with_dependencies.py +++ b/lib/ramble/ramble/test/end_to_end/phase_selection_with_dependencies.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -79,7 +79,6 @@ def test_workspace_phase_selection_with_dependencies(mutable_config, - n_nodes - env_name spack: - concretized: true packages: gcc: spack_spec: gcc@8.5.0 diff --git a/lib/ramble/ramble/test/end_to_end/shared_context.py b/lib/ramble/ramble/test/end_to_end/shared_context.py index 380d547e2..af2c7f0e2 100644 --- a/lib/ramble/ramble/test/end_to_end/shared_context.py +++ b/lib/ramble/ramble/test/end_to_end/shared_context.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -51,7 +51,6 @@ def test_shared_contexts( variables: n_nodes: 1 spack: - concretized: true packages: {} environments: {} """ diff --git a/lib/ramble/ramble/test/end_to_end/spack_env_cache.py b/lib/ramble/ramble/test/end_to_end/spack_env_cache.py new file mode 100644 index 000000000..7131cbf0d --- /dev/null +++ b/lib/ramble/ramble/test/end_to_end/spack_env_cache.py @@ -0,0 +1,119 @@ +# Copyright 2022-2024 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import os + +import pytest + +import ramble.workspace +import ramble.config +import ramble.software_environments +from ramble.main import RambleCommand + + +# everything here uses the mock_workspace_path +pytestmark = pytest.mark.usefixtures( + 'mutable_config', + 'mutable_mock_workspace_path', +) + +workspace = RambleCommand('workspace') + + +def test_spack_env_cache(): + test_config = """ +ramble: + variables: + mpi_command: 'mpirun -n {n_ranks} -ppn {processes_per_node}' + batch_submit: '{execute_experiment}' + processes_per_node: '1' + applications: + gromacs: + workloads: + water_bare: + experiments: + test1: + variables: + n_nodes: '1' + test2: + variables: + n_nodes: '2' + env_name: 'g2' + test3: + variables: + n_nodes: '3' + water_gmx50: + experiments: + test4: + variables: + n_nodes: '1' + spack: + packages: + intel-mpi: + spack_spec: intel-oneapi-mpi@2021.11.0 + gromacs: + spack_spec: gromacs + environments: + gromacs: + packages: + - gromacs + - intel-mpi + g2: + packages: + - gromacs + - intel-mpi +""" + workspace_name = 'test-spack-env-cache' + ws = ramble.workspace.create(workspace_name) + ws.write() + + config_path = os.path.join(ws.config_dir, ramble.workspace.config_file_name) + + with open(config_path, 'w+') as f: + f.write(test_config) + + ws._re_read() + + workspace( + 'setup', + '--dry-run', + global_args=['-w', workspace_name], + ) + + # spack env should be present only at the env_name level. + assert os.path.exists(os.path.join(ws.software_dir, 'gromacs')) + assert os.path.exists(os.path.join(ws.software_dir, 'g2')) + assert not os.path.exists(os.path.join(ws.software_dir, 'g2.water_bare')) + + # First encounter of an env_name (test1 -> gromacs, test2 -> g2) requires spack usage. + test1_log = os.path.join(ws.log_dir, 'setup.latest', 'gromacs.water_bare.test1.out') + with open(test1_log, 'r') as f: + content = f.read() + assert 'spack install' in content + assert 'spack concretize' in content + + test2_log = os.path.join(ws.log_dir, 'setup.latest', 'gromacs.water_bare.test2.out') + with open(test2_log, 'r') as f: + content = f.read() + assert 'spack install' in content + assert 'spack concretize' in content + + # Envs should already exist and can skip spack calls. + test3_log = os.path.join(ws.log_dir, 'setup.latest', 'gromacs.water_bare.test3.out') + with open(test3_log, 'r') as f: + content = f.read() + assert 'spack install' not in content + assert 'spack concretize' not in content + + test4_log = os.path.join( + ws.log_dir, 'setup.latest', 'gromacs.water_gmx50.test4.out' + ) + with open(test4_log, 'r') as f: + content = f.read() + assert 'spack install' not in content + assert 'spack concretize' not in content diff --git a/lib/ramble/ramble/test/end_to_end/tag_filtering.py b/lib/ramble/ramble/test/end_to_end/tag_filtering.py index 61ab36eef..e0b606509 100644 --- a/lib/ramble/ramble/test/end_to_end/tag_filtering.py +++ b/lib/ramble/ramble/test/end_to_end/tag_filtering.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -74,7 +74,6 @@ def test_workspace_tag_filtering(mutable_config, mutable_mock_workspace_path, variables: n_nodes: 1 spack: - concretized: true packages: {} environments: {} """ diff --git a/lib/ramble/ramble/test/end_to_end/test_configvar_dry_run.py b/lib/ramble/ramble/test/end_to_end/test_configvar_dry_run.py index 3523f2287..d05f647c3 100644 --- a/lib/ramble/ramble/test/end_to_end/test_configvar_dry_run.py +++ b/lib/ramble/ramble/test/end_to_end/test_configvar_dry_run.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -53,7 +53,6 @@ def test_configvar_dry_run(mutable_config, mutable_mock_workspace_path): variables: n_ranks: "{{{var_name}}}" spack: - concretized: true packages: gcc: spack_spec: gcc@8.5.0 @@ -92,7 +91,7 @@ def test_configvar_dry_run(mutable_config, mutable_mock_workspace_path): workspace('setup', '--dry-run', global_args=['-w', workspace_name]) - software_dir = 'openfoam.motorbike' + software_dir = 'openfoam' software_base_dir = os.path.join(ws.root, ramble.workspace.workspace_software_path) assert os.path.exists(software_base_dir) diff --git a/lib/ramble/ramble/test/end_to_end/unused_compilers_are_skipped.py b/lib/ramble/ramble/test/end_to_end/unused_compilers_are_skipped.py index 0234d8343..81d3e1464 100644 --- a/lib/ramble/ramble/test/end_to_end/unused_compilers_are_skipped.py +++ b/lib/ramble/ramble/test/end_to_end/unused_compilers_are_skipped.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -45,7 +45,6 @@ def test_unused_compilers_are_skipped(mutable_config, mutable_mock_workspace_pat variables: n_nodes: '1' spack: - concretized: true packages: gcc8: spack_spec: gcc@8.5.0 diff --git a/lib/ramble/ramble/test/end_to_end/vector_workloads.py b/lib/ramble/ramble/test/end_to_end/vector_workloads.py index c4a1977ef..47fe96fc1 100644 --- a/lib/ramble/ramble/test/end_to_end/vector_workloads.py +++ b/lib/ramble/ramble/test/end_to_end/vector_workloads.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -42,7 +42,6 @@ def test_vector_workloads(mutable_config, mutable_mock_workspace_path): application_workload: ['parallel' ,'serial', 'local'] n_nodes: 1 spack: - concretized: true packages: {} environments: {} """ diff --git a/lib/ramble/ramble/test/end_to_end/wrfv4_dry_run.py b/lib/ramble/ramble/test/end_to_end/wrfv4_dry_run.py index fc7df6ab0..734a98bee 100644 --- a/lib/ramble/ramble/test/end_to_end/wrfv4_dry_run.py +++ b/lib/ramble/ramble/test/end_to_end/wrfv4_dry_run.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -69,7 +69,6 @@ def test_wrfv4_dry_run(mutable_config, mutable_mock_workspace_path): - n_nodes - env_name spack: - concretized: true packages: gcc: spack_spec: gcc@8.5.0 @@ -137,7 +136,7 @@ def test_wrfv4_dry_run(mutable_config, mutable_mock_workspace_path): out_files, 'Would download https://www2.mmm.ucar.edu/wrf/users/benchmark/v422/v42_bench_conus12km.tar.gz') # noqa # Test software directories - software_dirs = ['wrfv4.CONUS_12km', 'wrfv4-portable.CONUS_12km'] + software_dirs = ['wrfv4', 'wrfv4-portable'] software_base_dir = os.path.join(ws1.root, ramble.workspace.workspace_software_path) assert os.path.exists(software_base_dir) for software_dir in software_dirs: diff --git a/lib/ramble/ramble/test/expander.py b/lib/ramble/ramble/test/expander.py index 1ff3614ca..a74f8dcfb 100644 --- a/lib/ramble/ramble/test/expander.py +++ b/lib/ramble/ramble/test/expander.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -30,6 +30,7 @@ def exp_dict(): 'var3': '3', 'decimal.06.var': 'foo', 'size': '"0000.96"', # Escaped as a string + 'test_mask': '"0x0"', } @@ -69,6 +70,9 @@ def exp_dict(): (r'\\{experiment_name\\}', r'\{experiment_name\}', set(), 1), (r'\\{experiment_name\\}', '{experiment_name}', set(), 2), (r'\\{experiment_name\\}', 'baz', set(), 3), + ('"2.1.1" in ["2.1.1", "3.1.1", "4.2.1"]', 'True', set(), 1), + ('"2.1.2" in ["2.1.1", "3.1.1", "4.2.1"]', 'False', set(), 1), + ('{test_mask}', '0x0', set(['test_mask']), 1), ] ) def test_expansions(input, output, no_expand_vars, passes): @@ -84,6 +88,61 @@ def test_expansions(input, output, no_expand_vars, passes): assert final_output == output +@pytest.mark.parametrize( + 'input,output,no_expand_vars,passes', + [ + ('{var1}', 3, set(), 1), + ('{var2}', 3, set(), 1), + ('{var3}', 3, set(), 1), + ('{application_name}', 'foo', set(), 1), + ('{n_nodes}', 2, set(), 1), + ('{processes_per_node}', 2, set(), 1), + ('{n_nodes}*{processes_per_node}', 4, set(), 1), + ('2**4', 16, set(), 1), + ('{((((16-10+2)/4)**2)*4)}', 16.0, set(), 1), + ('"gromacs +blas"', 'gromacs +blas', set(), 1), + ('range(0, 5)', [0, 1, 2, 3, 4], set(), 1), + ('{decimal.06.var}', 'foo', set(), 1), + ('{}', {}, set(), 1), + ('{{n_ranks}+2}', 6, set(), 1), + ('{{n_ranks}*{var{processes_per_node}}:05d}', '00012', set(), 1), + ('{{n_ranks}-1}', 3, set(), 1), + ('{{{n_ranks}/2}:0.0f}', 2, set(), 1), + ('{size}', 0.96, set(['size']), 1), + ('CPU(s)', 'CPU(s)', set(), 1), + ('str(1.5)', 1.5, set(), 1), + ('int(1.5)', 1, set(), 1), + ('float(1.5)', 1.5, set(), 1), + ('ceil(0.6)', 1, set(), 1), + ('floor(0.6)', 0, set(), 1), + ('max(1, 5)', 5, set(), 1), + ('min(1, 5)', 1, set(), 1), + ('simplify_str("a.b_c")', 'a-b-c', set(), 1), + (r'\{experiment_name\}', '{experiment_name}', set(), 1), + (r'\{experiment_name\}', 'baz', set(), 2), + (r'{\{experiment_name\}}', '{{experiment_name}}', set(), 1), + (r'\\{experiment_name\\}', r'\{experiment_name\}', set(), 1), + (r'\\{experiment_name\\}', '{experiment_name}', set(), 2), + (r'\\{experiment_name\\}', 'baz', set(), 3), + ('"2.1.1" in ["2.1.1", "3.1.1", "4.2.1"]', True, set(), 1), + ('"2.1.2" in ["2.1.1", "3.1.1", "4.2.1"]', False, set(), 1), + ('{test_mask}', 0, set(['test_mask']), 1), + ('{var3} // {processes_per_node}', 1, set(), 1), + ], +) +def test_typed_expansions(input, output, no_expand_vars, passes): + expansion_vars = exp_dict() + + expander = ramble.expander.Expander(expansion_vars, None, no_expand_vars=no_expand_vars) + + step_input = input + for _ in range(0, passes): + step_input = expander.expand_var(step_input, typed=True) + final_output = step_input + + assert final_output == output + + @pytest.mark.parametrize( 'input,output', [ diff --git a/lib/ramble/ramble/test/experiment_set.py b/lib/ramble/ramble/test/experiment_set.py index a3a47b51c..b81ce2537 100644 --- a/lib/ramble/ramble/test/experiment_set.py +++ b/lib/ramble/ramble/test/experiment_set.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -107,19 +107,18 @@ def test_vector_experiment_in_set(mutable_mock_workspace_path): assert 'basic.test_wl.series1_8' in exp_set.experiments.keys() -def test_vector_length_mismatch_errors(mutable_mock_workspace_path, capsys): - workspace('create', 'test') +def test_vector_length_mismatch_errors(request, mutable_mock_workspace_path, capsys): + ws_name = request.node.name + workspace('create', ws_name) - assert 'test' in workspace('list') + assert ws_name in workspace('list') - with ramble.workspace.read('test') as ws: + with ramble.workspace.read(ws_name) as ws: exp_set = ramble.experiment_set.ExperimentSet(ws) application_context = ramble.context.Context() application_context.context_name = 'basic' application_context.variables = { - 'app_var1': '1', - 'app_var2': '2', 'n_ranks': '{processes_per_node}*{n_nodes}', 'mpi_command': '', 'batch_submit': '' @@ -128,15 +127,11 @@ def test_vector_length_mismatch_errors(mutable_mock_workspace_path, capsys): workload_context = ramble.context.Context() workload_context.context_name = 'test_wl' workload_context.variables = { - 'wl_var1': '1', - 'wl_var2': ['2'], - 'processes_per_node': '2' + 'processes_per_node': ['2', '4', '6'] } experiment_context = ramble.context.Context() - experiment_context.context_name = 'series1_{n_ranks}' + experiment_context.context_name = 'series1_{n_ranks}_{processes_per_node}' experiment_context.variables = { - 'exp_var1': '1', - 'exp_var2': '2', 'n_nodes': ['2', '4'] } @@ -588,12 +583,13 @@ def test_experiment_names_match(mutable_mock_workspace_path): assert exp == app.expander.expand_var('{experiment_namespace}') -def test_cross_experiment_variable_references(mutable_mock_workspace_path): - workspace('create', 'test') +def test_cross_experiment_variable_references(request, mutable_mock_workspace_path): + ws_name = request.node.name + workspace('create', ws_name) - assert 'test' in workspace('list') + assert ws_name in workspace('list') - with ramble.workspace.read('test') as ws: + with ramble.workspace.read(ws_name) as ws: exp_set = ramble.experiment_set.ExperimentSet(ws) application_context = ramble.context.Context() @@ -622,6 +618,14 @@ def test_cross_experiment_variable_references(mutable_mock_workspace_path): 'n_nodes': '2', 'test_var': 'success' } + experiment1_context.internals = { + 'custom_executables': { + 'echo_test': {'template': 'echo "{test_var}"'}, + }, + 'executable_injection': [ + {'name': 'echo_test', 'order': 'after'} + ] + } experiment2_context = ramble.context.Context() experiment2_context.context_name = 'series2_{n_ranks}' @@ -632,6 +636,15 @@ def test_cross_experiment_variable_references(mutable_mock_workspace_path): 'test_var': 'test_var in basic.test_wl.series1_4' } + experiment2_context.internals = { + 'custom_executables': { + 'echo_test': {'template': 'echo "{test_var}"'}, + }, + 'executable_injection': [ + {'name': 'echo_test', 'order': 'after'} + ] + } + exp_set.set_application_context(application_context) exp_set.set_workload_context(workload_context) exp_set.set_experiment_context(experiment1_context) @@ -680,17 +693,21 @@ def test_cross_experiment_missing_experiment_errors(mutable_mock_workspace_path) 'test_var': 'processes_per_node in basic.test_wl.does_not_exist' } + experiment1_context.internals = { + 'custom_executables': { + 'echo_test': {'template': 'echo "{test_var}"'}, + }, + 'executable_injection': [ + {'name': 'echo_test', 'order': 'after'} + ] + } + exp_set.set_application_context(application_context) exp_set.set_workload_context(workload_context) - exp_set.set_experiment_context(experiment1_context) - exp_set.build_experiment_chains() - - assert 'basic.test_wl.series1_4' in exp_set.experiments.keys() - - exp1_app = exp_set.experiments['basic.test_wl.series1_4'] with pytest.raises(ramble.expander.RambleSyntaxError) as e: - exp1_app.expander.expand_var('{test_var}') + exp_set.set_experiment_context(experiment1_context) + expected = ('basic.test_wl_does_not_exist does not exist in ' f'"{experiment1_context.variables["test_var"]}"') assert e.error == expected @@ -1251,7 +1268,7 @@ def test_chained_invalid_order_errors(mutable_mock_workspace_path, capsys): assert "Invalid experiment chain defined:" in captured -def test_modifiers_set_correctly(mutable_mock_workspace_path, capsys): +def test_modifiers_set_correctly(mutable_mock_workspace_path, mock_modifiers, capsys): workspace('create', 'test') assert 'test' in workspace('list') @@ -1270,8 +1287,8 @@ def test_modifiers_set_correctly(mutable_mock_workspace_path, capsys): } application_context.modifiers = [ { - 'name': 'test_app_mod', - 'mode': 'test_app', + 'name': 'test-mod', + 'mode': 'app-scope', 'on_executable': [ 'builtin::env_vars' ] @@ -1286,8 +1303,8 @@ def test_modifiers_set_correctly(mutable_mock_workspace_path, capsys): } workload_context.modifiers = [ { - 'name': 'test_wl_mod', - 'mode': 'test_wl', + 'name': 'test-mod', + 'mode': 'wl-scope', 'on_executable': [ 'builtin::env_vars' ] @@ -1301,8 +1318,8 @@ def test_modifiers_set_correctly(mutable_mock_workspace_path, capsys): } experiment_context.modifiers = [ { - 'name': 'test_exp1_mod', - 'mode': 'test_exp1', + 'name': 'test-mod', + 'mode': 'exp-scope', 'on_executable': [ 'builtin::env_vars' ] @@ -1317,11 +1334,11 @@ def test_modifiers_set_correctly(mutable_mock_workspace_path, capsys): app_inst = exp_set.experiments['basic.test_wl.test1'] assert app_inst.modifiers is not None - expected_modifiers = set(['test_app_mod', 'test_wl_mod', 'test_exp1_mod']) + expected_modifier_modes = set(['app-scope', 'wl-scope', 'exp-scope']) for mod_def in app_inst.modifiers: - assert mod_def['name'] in expected_modifiers - expected_modifiers.remove(mod_def['name']) - assert len(expected_modifiers) == 0 + assert mod_def['mode'] in expected_modifier_modes + expected_modifier_modes.remove(mod_def['mode']) + assert len(expected_modifier_modes) == 0 def test_explicit_zips_work(mutable_mock_workspace_path): @@ -1518,7 +1535,7 @@ def test_single_var_explicit_zip(mutable_mock_workspace_path): assert 'basic.test_wl.series1_8' in exp_set.experiments.keys() -def test_zip_undefined_var_errors(mutable_mock_workspace_path, capsys): +def test_zip_multi_use_var_errors(mutable_mock_workspace_path, capsys): workspace('create', 'test') assert 'test' in workspace('list') @@ -1552,7 +1569,8 @@ def test_zip_undefined_var_errors(mutable_mock_workspace_path, capsys): 'n_nodes': ['2', '4'] } experiment_context.zips = { - 'test_zip': ['foo'], + 'test_zip1': ['n_nodes'], + 'test_zip2': ['n_nodes'], } exp_set.set_application_context(application_context) @@ -1560,10 +1578,10 @@ def test_zip_undefined_var_errors(mutable_mock_workspace_path, capsys): with pytest.raises(SystemExit): exp_set.set_experiment_context(experiment_context) captured = capsys.readouterr() - assert "An undefined variable foo is defined in zip test_zip" in captured + assert 'Variable n_nodes is used across multiple zips' in captured -def test_zip_multi_use_var_errors(mutable_mock_workspace_path, capsys): +def test_zip_non_list_var_errors(mutable_mock_workspace_path, capsys): workspace('create', 'test') assert 'test' in workspace('list') @@ -1592,13 +1610,10 @@ def test_zip_multi_use_var_errors(mutable_mock_workspace_path, capsys): experiment_context = ramble.context.Context() experiment_context.context_name = 'series1_{n_ranks}' experiment_context.variables = { - 'exp_var1': '1', - 'exp_var2': '2', 'n_nodes': ['2', '4'] } experiment_context.zips = { - 'test_zip1': ['n_nodes'], - 'test_zip2': ['n_nodes'], + 'test_zip': ['exp_var1', 'processes_per_node'], } exp_set.set_application_context(application_context) @@ -1606,10 +1621,11 @@ def test_zip_multi_use_var_errors(mutable_mock_workspace_path, capsys): with pytest.raises(SystemExit): exp_set.set_experiment_context(experiment_context) captured = capsys.readouterr() - assert 'Variable n_nodes is used across multiple zips' in captured + assert 'Variable processes_per_node in zip test_zip does not refer to a vector' in \ + captured -def test_zip_non_list_var_errors(mutable_mock_workspace_path, capsys): +def test_zip_variable_lengths_errors(mutable_mock_workspace_path, capsys): workspace('create', 'test') assert 'test' in workspace('list') @@ -1620,8 +1636,6 @@ def test_zip_non_list_var_errors(mutable_mock_workspace_path, capsys): application_context = ramble.context.Context() application_context.context_name = 'basic' application_context.variables = { - 'app_var1': '1', - 'app_var2': '2', 'n_ranks': '{processes_per_node}*{n_nodes}', 'mpi_command': '', 'batch_submit': '' @@ -1629,21 +1643,17 @@ def test_zip_non_list_var_errors(mutable_mock_workspace_path, capsys): workload_context = ramble.context.Context() workload_context.context_name = 'test_wl' - workload_context.variables = { - 'wl_var1': '1', - 'wl_var2': '2', - 'processes_per_node': '2' - } + workload_context.variables = {} experiment_context = ramble.context.Context() experiment_context.context_name = 'series1_{n_ranks}' experiment_context.variables = { 'exp_var1': '1', - 'exp_var2': '2', + 'processes_per_node': ['2'], 'n_nodes': ['2', '4'] } experiment_context.zips = { - 'test_zip': ['exp_var1'], + 'test_zip': ['n_nodes', 'processes_per_node'], } exp_set.set_application_context(application_context) @@ -1651,10 +1661,12 @@ def test_zip_non_list_var_errors(mutable_mock_workspace_path, capsys): with pytest.raises(SystemExit): exp_set.set_experiment_context(experiment_context) captured = capsys.readouterr() - assert 'Variable exp_var1 in zip test_zip does not refer to a vector' in captured + assert 'Length mismatch in zip test_zip in experiment series1_{n_ranks}' in captured + assert 'Variable processes_per_node has length 1' in captured + assert 'Variable n_nodes has length 2' in captured -def test_zip_variable_lengths_errors(mutable_mock_workspace_path, capsys): +def test_vector_experiment_with_explicit_excludes(mutable_mock_workspace_path): workspace('create', 'test') assert 'test' in workspace('list') @@ -1684,24 +1696,25 @@ def test_zip_variable_lengths_errors(mutable_mock_workspace_path, capsys): experiment_context.context_name = 'series1_{n_ranks}' experiment_context.variables = { 'exp_var1': '1', - 'exp_var2': ['2'], + 'exp_var2': '2', 'n_nodes': ['2', '4'] } - experiment_context.zips = { - 'test_zip': ['n_nodes', 'exp_var2'], + experiment_context.exclude = { + 'variables': { + 'n_nodes': ['4'] + } } exp_set.set_application_context(application_context) exp_set.set_workload_context(workload_context) - with pytest.raises(SystemExit): - exp_set.set_experiment_context(experiment_context) - captured = capsys.readouterr() - assert 'Length mismatch in zip test_zip in experiment series1_{n_ranks}' in captured - assert 'Variable exp_var has length 1' in captured - assert 'Variable n_nodes has length 2' in captured + exp_set.set_experiment_context(experiment_context) + exp_set.build_experiment_chains() + assert 'basic.test_wl.series1_4' in exp_set.experiments.keys() + assert 'basic.test_wl.series1_8' not in exp_set.experiments.keys() -def test_vector_experiment_with_explicit_excludes(mutable_mock_workspace_path): + +def test_matrix_experiments_explicit_excludes(mutable_mock_workspace_path): workspace('create', 'test') assert 'test' in workspace('list') @@ -1732,12 +1745,16 @@ def test_vector_experiment_with_explicit_excludes(mutable_mock_workspace_path): experiment_context.variables = { 'exp_var1': '1', 'exp_var2': '2', - 'n_nodes': ['2', '4'] + 'n_nodes': ['2', '3'] } + experiment_context.matrices = [ + ['n_nodes'] + ] experiment_context.exclude = { 'variables': { - 'n_nodes': ['4'] - } + 'n_nodes': ['3'], + }, + 'matrix': ['n_nodes'] } exp_set.set_application_context(application_context) @@ -1746,10 +1763,10 @@ def test_vector_experiment_with_explicit_excludes(mutable_mock_workspace_path): exp_set.build_experiment_chains() assert 'basic.test_wl.series1_4' in exp_set.experiments.keys() - assert 'basic.test_wl.series1_8' not in exp_set.experiments.keys() + assert 'basic.test_wl.series1_6' not in exp_set.experiments.keys() -def test_matrix_experiments_explicit_excludes(mutable_mock_workspace_path): +def test_vector_experiment_with_where_excludes(mutable_mock_workspace_path): workspace('create', 'test') assert 'test' in workspace('list') @@ -1780,16 +1797,12 @@ def test_matrix_experiments_explicit_excludes(mutable_mock_workspace_path): experiment_context.variables = { 'exp_var1': '1', 'exp_var2': '2', - 'n_nodes': ['2', '3'] + 'n_nodes': ['1', '2', '3', '4', '5'] } - experiment_context.matrices = [ - ['n_nodes'] - ] experiment_context.exclude = { - 'variables': { - 'n_nodes': ['3'], - }, - 'matrix': ['n_nodes'] + 'where': [ + '{n_nodes} > 2 and {n_nodes} < 5' + ] } exp_set.set_application_context(application_context) @@ -1797,11 +1810,14 @@ def test_matrix_experiments_explicit_excludes(mutable_mock_workspace_path): exp_set.set_experiment_context(experiment_context) exp_set.build_experiment_chains() + assert 'basic.test_wl.series1_2' in exp_set.experiments.keys() assert 'basic.test_wl.series1_4' in exp_set.experiments.keys() assert 'basic.test_wl.series1_6' not in exp_set.experiments.keys() + assert 'basic.test_wl.series1_8' not in exp_set.experiments.keys() + assert 'basic.test_wl.series1_10' in exp_set.experiments.keys() -def test_vector_experiment_with_where_excludes(mutable_mock_workspace_path): +def test_vector_experiment_with_multi_where_excludes(mutable_mock_workspace_path): workspace('create', 'test') assert 'test' in workspace('list') @@ -1836,7 +1852,8 @@ def test_vector_experiment_with_where_excludes(mutable_mock_workspace_path): } experiment_context.exclude = { 'where': [ - '{n_nodes} > 2 and {n_nodes} < 5' + '{n_nodes} < 2', + '{n_nodes} > 4' ] } @@ -1845,19 +1862,74 @@ def test_vector_experiment_with_where_excludes(mutable_mock_workspace_path): exp_set.set_experiment_context(experiment_context) exp_set.build_experiment_chains() - assert 'basic.test_wl.series1_2' in exp_set.experiments.keys() + assert 'basic.test_wl.series1_2' not in exp_set.experiments.keys() assert 'basic.test_wl.series1_4' in exp_set.experiments.keys() - assert 'basic.test_wl.series1_6' not in exp_set.experiments.keys() - assert 'basic.test_wl.series1_8' not in exp_set.experiments.keys() - assert 'basic.test_wl.series1_10' in exp_set.experiments.keys() + assert 'basic.test_wl.series1_6' in exp_set.experiments.keys() + assert 'basic.test_wl.series1_8' in exp_set.experiments.keys() + assert 'basic.test_wl.series1_10' not in exp_set.experiments.keys() -def test_vector_experiment_with_multi_where_excludes(mutable_mock_workspace_path): - workspace('create', 'test') +def test_unused_vector_no_error(request, mutable_mock_workspace_path): + ws_name = request.node.name + workspace('create', ws_name) - assert 'test' in workspace('list') + assert ws_name in workspace('list') - with ramble.workspace.read('test') as ws: + with ramble.workspace.read(ws_name) as ws: + exp_set = ramble.experiment_set.ExperimentSet(ws) + + application_context = ramble.context.Context() + application_context.context_name = 'basic' + application_context.variables = { + 'app_var1': '1', + 'app_var2': '2', + 'n_ranks': '{processes_per_node}*{n_nodes}', + 'mpi_command': '', + 'batch_submit': '' + } + + workload_context = ramble.context.Context() + workload_context.context_name = 'test_wl' + workload_context.variables = { + 'wl_var1': '1', + 'wl_var2': '2', + 'processes_per_node': '2' + } + + experiment_context = ramble.context.Context() + experiment_context.context_name = 'series1_{n_ranks}' + experiment_context.variables = { + 'exp_var1': '1', + 'exp_var2': '2', + 'other_var': ['1', '2,', '3'], + 'n_nodes': ['1', '2', '3', '4', '5'] + } + experiment_context.exclude = { + 'where': [ + '{n_nodes} < 2', + '{n_nodes} > 4' + ] + } + + exp_set.set_application_context(application_context) + exp_set.set_workload_context(workload_context) + exp_set.set_experiment_context(experiment_context) + exp_set.build_experiment_chains() + + assert 'basic.test_wl.series1_2' not in exp_set.experiments.keys() + assert 'basic.test_wl.series1_4' in exp_set.experiments.keys() + assert 'basic.test_wl.series1_6' in exp_set.experiments.keys() + assert 'basic.test_wl.series1_8' in exp_set.experiments.keys() + assert 'basic.test_wl.series1_10' not in exp_set.experiments.keys() + + +def test_unused_zip_no_error(request, mutable_mock_workspace_path): + ws_name = request.node.name + workspace('create', ws_name) + + assert ws_name in workspace('list') + + with ramble.workspace.read(ws_name) as ws: exp_set = ramble.experiment_set.ExperimentSet(ws) application_context = ramble.context.Context() @@ -1883,8 +1955,13 @@ def test_vector_experiment_with_multi_where_excludes(mutable_mock_workspace_path experiment_context.variables = { 'exp_var1': '1', 'exp_var2': '2', + 'other_var_1': ['1', '2,', '3'], + 'other_var_2': ['1', '2,', '3'], 'n_nodes': ['1', '2', '3', '4', '5'] } + experiment_context.zips = { + 'test_zip': ['other_var_1', 'other_var_2'], + } experiment_context.exclude = { 'where': [ '{n_nodes} < 2', @@ -1902,3 +1979,161 @@ def test_vector_experiment_with_multi_where_excludes(mutable_mock_workspace_path assert 'basic.test_wl.series1_6' in exp_set.experiments.keys() assert 'basic.test_wl.series1_8' in exp_set.experiments.keys() assert 'basic.test_wl.series1_10' not in exp_set.experiments.keys() + + +def test_unused_var_propagates_to_chain(request, mutable_mock_workspace_path): + ws_name = request.node.name + workspace('create', ws_name) + + assert ws_name in workspace('list') + + with ramble.workspace.read(ws_name) as ws: + exp_set = ramble.experiment_set.ExperimentSet(ws) + + expanded_app_context = ramble.context.Context() + expanded_app_context.context_name = 'expanded_foms' + expanded_app_context.variables = { + 'processes_per_node': '1', + 'mpi_command': '', + 'batch_submit': '' + } + + expanded_wl_context = ramble.context.Context() + expanded_wl_context.context_name = 'test_wl' + expanded_wl_context.variables = {} + + expanded_exp_context = ramble.context.Context() + expanded_exp_context.context_name = 'test' + expanded_exp_context.variables = { + 'n_ranks': '1', + 'my_var': '2.0' + } + expanded_exp_context.is_template = True + + exp_set.set_application_context(expanded_app_context) + exp_set.set_workload_context(expanded_wl_context) + exp_set.set_experiment_context(expanded_exp_context) + + application_context = ramble.context.Context() + application_context.context_name = 'basic' + application_context.variables = { + 'app_var1': '1', + 'app_var2': '2', + 'processes_per_node': '1', + 'mpi_command': '', + 'batch_submit': '' + } + + workload_context = ramble.context.Context() + workload_context.context_name = 'test_wl' + workload_context.variables = { + 'wl_var1': '1', + 'wl_var2': '2', + } + + experiment_context = ramble.context.Context() + experiment_context.context_name = 'test1' + experiment_context.variables = { + 'n_ranks': '2', + 'my_var': '5.0' + } + experiment_context.chained_experiments = [ + { + 'name': 'expanded_foms.test_wl.test', + 'order': 'after_root', + 'command': '{execute_experiment}', + 'inherit_variables': ['my_var'] + }, + ] + + exp_set.set_application_context(application_context) + exp_set.set_workload_context(workload_context) + exp_set.set_experiment_context(experiment_context) + exp_set.build_experiment_chains() + + assert 'basic.test_wl.test1' in exp_set.experiments + assert 'basic.test_wl.test1.chain.0.expanded_foms.test_wl.test' in \ + exp_set.chained_experiments + + app_inst = exp_set.get_experiment('basic.test_wl.test1.chain.0.expanded_foms.test_wl.test') + + assert app_inst.variables['my_var'] == '5.0' + + +def test_custom_executables_track_variables(request, mutable_mock_workspace_path): + ws_name = request.node.name + workspace('create', ws_name) + + assert ws_name in workspace('list') + + with ramble.workspace.read(ws_name) as ws: + exp_set = ramble.experiment_set.ExperimentSet(ws) + + expanded_app_context = ramble.context.Context() + expanded_app_context.context_name = 'expanded_foms' + expanded_app_context.variables = { + 'processes_per_node': '1', + 'mpi_command': '', + 'batch_submit': '' + } + + expanded_wl_context = ramble.context.Context() + expanded_wl_context.context_name = 'test_wl' + expanded_wl_context.variables = {} + + expanded_exp_context = ramble.context.Context() + expanded_exp_context.context_name = 'test' + expanded_exp_context.variables = { + 'n_ranks': '1', + 'my_var': '2.0' + } + expanded_exp_context.is_template = True + + exp_set.set_application_context(expanded_app_context) + exp_set.set_workload_context(expanded_wl_context) + exp_set.set_experiment_context(expanded_exp_context) + + application_context = ramble.context.Context() + application_context.context_name = 'basic' + application_context.variables = { + 'app_var1': '1', + 'app_var2': '2', + 'processes_per_node': '1', + 'mpi_command': '', + 'batch_submit': '' + } + + workload_context = ramble.context.Context() + workload_context.context_name = 'test_wl' + workload_context.variables = { + 'wl_var1': '1', + 'wl_var2': '2', + } + + experiment_context = ramble.context.Context() + experiment_context.context_name = 'test1' + experiment_context.variables = { + 'n_ranks': '2', + 'my_var': '5.0' + } + experiment_context.chained_experiments = [ + { + 'name': 'expanded_foms.test_wl.test', + 'order': 'after_root', + 'command': '{execute_experiment}', + 'inherit_variables': ['my_var'] + }, + ] + + exp_set.set_application_context(application_context) + exp_set.set_workload_context(workload_context) + exp_set.set_experiment_context(experiment_context) + exp_set.build_experiment_chains() + + assert 'basic.test_wl.test1' in exp_set.experiments + assert 'basic.test_wl.test1.chain.0.expanded_foms.test_wl.test' in \ + exp_set.chained_experiments + + app_inst = exp_set.get_experiment('basic.test_wl.test1.chain.0.expanded_foms.test_wl.test') + + assert app_inst.variables['my_var'] == '5.0' diff --git a/lib/ramble/ramble/test/gcs_fetch.py b/lib/ramble/ramble/test/gcs_fetch.py index 0e834013f..de2354198 100644 --- a/lib/ramble/ramble/test/gcs_fetch.py +++ b/lib/ramble/ramble/test/gcs_fetch.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/mirror_tests.py b/lib/ramble/ramble/test/mirror_tests.py index 703d54645..7e50adc34 100644 --- a/lib/ramble/ramble/test/mirror_tests.py +++ b/lib/ramble/ramble/test/mirror_tests.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -126,7 +126,6 @@ def test_mirror_create(tmpdir, n_nodes: '1' processes_per_node: '1' spack: - concretized: true packages: {{}} environments: {{}} """ diff --git a/lib/ramble/ramble/test/modifier_application.py b/lib/ramble/ramble/test/modifier_application.py index 17512d4fd..b45ed70e1 100644 --- a/lib/ramble/ramble/test/modifier_application.py +++ b/lib/ramble/ramble/test/modifier_application.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -45,7 +45,6 @@ def test_wrfv4_aps_test(mutable_config, mutable_mock_workspace_path): variables: n_nodes: '1' spack: - concretized: true packages: gcc: spack_spec: gcc@8.5.0 @@ -78,7 +77,7 @@ def test_wrfv4_aps_test(mutable_config, mutable_mock_workspace_path): workspace('setup', '--dry-run', global_args=['-w', workspace_name]) - software_path = os.path.join(ws1.software_dir, 'wrfv4.CONUS_12km', 'spack.yaml') + software_path = os.path.join(ws1.software_dir, 'wrfv4', 'spack.yaml') with open(software_path, 'r') as f: assert 'intel-oneapi-vtune' in f.read() diff --git a/lib/ramble/ramble/test/modifier_functionality/mock_env_var_modifiers.py b/lib/ramble/ramble/test/modifier_functionality/mock_env_var_modifiers.py index 42f612d51..492cf1ff0 100644 --- a/lib/ramble/ramble/test/modifier_functionality/mock_env_var_modifiers.py +++ b/lib/ramble/ramble/test/modifier_functionality/mock_env_var_modifiers.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/modifier_functionality/mock_modifier_dry_run.py b/lib/ramble/ramble/test/modifier_functionality/mock_modifier_dry_run.py index fa862ae76..076b277c6 100644 --- a/lib/ramble/ramble/test/modifier_functionality/mock_modifier_dry_run.py +++ b/lib/ramble/ramble/test/modifier_functionality/mock_modifier_dry_run.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/modifier_functionality/mock_modifier_phases.py b/lib/ramble/ramble/test/modifier_functionality/mock_modifier_phases.py new file mode 100644 index 000000000..2499af877 --- /dev/null +++ b/lib/ramble/ramble/test/modifier_functionality/mock_modifier_phases.py @@ -0,0 +1,94 @@ +# Copyright 2022-2024 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import re +import os +import glob + +import pytest + +from ramble.test.dry_run_helpers import dry_run_config, search_files_for_string, SCOPES +import ramble.test.modifier_functionality.modifier_helpers as modifier_helpers + +import ramble.workspace +from ramble.main import RambleCommand + +workspace = RambleCommand('workspace') + + +@pytest.mark.parametrize( + 'scope', + [ + SCOPES.workspace, + SCOPES.application, + SCOPES.workload, + SCOPES.experiment, + ] +) +def test_gromacs_dry_run_mock_mod_phase(mutable_mock_workspace_path, + mutable_applications, + mock_modifiers, + scope): + workspace_name = 'test_gromacs_dry_run_mock_mod_phase' + + test_modifiers = [ + (scope, modifier_helpers.named_modifier('mod-phase')), + ] + + with ramble.workspace.create(workspace_name) as ws1: + ws1.write() + + config_path = os.path.join(ws1.config_dir, ramble.workspace.config_file_name) + + dry_run_config('modifiers', test_modifiers, config_path, 'gromacs', 'water_bare') + + ws1._re_read() + + workspace('concretize', global_args=['-D', ws1.root]) + workspace('setup', '--dry-run', global_args=['-D', ws1.root]) + out_files = glob.glob(os.path.join(ws1.log_dir, '**', '*.out'), recursive=True) + + out_file = os.path.join(ws1.log_dir, 'setup.latest', 'gromacs.water_bare.test_exp.out') + + found_phase = False + found_make_experiments = False + found_after_phase = False + found_first_phase = False + phase_regex = re.compile('Executing phase.*') + first_phase_regex = re.compile('Executing phase first_phase') + make_experiments_regex = re.compile('Executing phase make_experiments') + after_make_experiments_regex = re.compile('Executing phase after_make_experiments') + + with open(out_file, 'r') as f: + for line in f.readlines(): + if first_phase_regex.search(line): + assert not found_phase + found_first_phase = True + + if phase_regex.search(line): + found_phase = True + + if make_experiments_regex.search(line): + assert found_phase + found_make_experiments = True + + if after_make_experiments_regex.search(line): + assert found_make_experiments + found_after_phase = True + + assert found_phase + assert found_first_phase + assert found_after_phase + + expected_str = "Inside a phase: first_phase" + + assert search_files_for_string(out_files, expected_str) + + expected_str = "Inside a phase: after_make_experiments" + + assert search_files_for_string(out_files, expected_str) diff --git a/lib/ramble/ramble/test/modifier_functionality/mock_modifier_spack_configs.py b/lib/ramble/ramble/test/modifier_functionality/mock_modifier_spack_configs.py index 66a03501b..98e7c1a29 100644 --- a/lib/ramble/ramble/test/modifier_functionality/mock_modifier_spack_configs.py +++ b/lib/ramble/ramble/test/modifier_functionality/mock_modifier_spack_configs.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -53,7 +53,7 @@ def test_gromacs_mock_spack_config_mod(mutable_mock_workspace_path, assert os.path.isfile(exp_script) - spack_yaml = os.path.join(ws1.software_dir, 'gromacs.water_bare', + spack_yaml = os.path.join(ws1.software_dir, 'gromacs', 'spack.yaml') assert os.path.isfile(spack_yaml) diff --git a/lib/ramble/ramble/test/modifier_functionality/mock_spack_modifier.py b/lib/ramble/ramble/test/modifier_functionality/mock_spack_modifier.py index 6b3bf7f14..4a0c5e835 100644 --- a/lib/ramble/ramble/test/modifier_functionality/mock_spack_modifier.py +++ b/lib/ramble/ramble/test/modifier_functionality/mock_spack_modifier.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -40,9 +40,9 @@ def test_gromacs_dry_run_mock_spack_mod(mutable_mock_workspace_path, ] software_tests = [ - ('gromacs.water_bare', 'mod_package1@1.1'), - ('gromacs.water_bare', 'mod_package2@1.1'), - ('gromacs.water_bare', 'gromacs'), + ('gromacs', 'mod_package1@1.1'), + ('gromacs', 'mod_package2@1.1'), + ('gromacs', 'gromacs'), ] with ramble.workspace.create(workspace_name) as ws1: diff --git a/lib/ramble/ramble/test/modifier_functionality/modifier_helpers.py b/lib/ramble/ramble/test/modifier_functionality/modifier_helpers.py index 86625de33..5eedd26e1 100644 --- a/lib/ramble/ramble/test/modifier_functionality/modifier_helpers.py +++ b/lib/ramble/ramble/test/modifier_functionality/modifier_helpers.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -56,8 +56,8 @@ def intel_aps_modifier(): def intel_aps_answer(): expected_software = [ - ('gromacs.water_bare', 'intel-oneapi-vtune'), - ('gromacs.water_bare', 'gromacs') + ('gromacs', 'intel-oneapi-vtune'), + ('gromacs', 'gromacs') ] expected_strs = [ 'aps -c mpi', @@ -76,7 +76,7 @@ def lscpu_modifier(): def lscpu_answer(): expected_software = [ - ('gromacs.water_bare', 'gromacs'), + ('gromacs', 'gromacs'), ] expected_strs = [ 'lscpu', @@ -94,7 +94,7 @@ def env_var_append_paths_modifier(): def env_var_append_paths_modifier_answer(): expected_software = [ - ('gromacs.water_bare', 'gromacs'), + ('gromacs', 'gromacs'), ] expected_strs = [ 'export test_var="${test_var}:test_val"' @@ -112,7 +112,7 @@ def env_var_append_vars_modifier(): def env_var_append_vars_modifier_answer(): expected_software = [ - ('gromacs.water_bare', 'gromacs'), + ('gromacs', 'gromacs'), ] expected_strs = [ 'export test_var="${test_var},test_val"', @@ -130,7 +130,7 @@ def env_var_prepend_paths_modifier(): def env_var_prepend_paths_modifier_answer(): expected_software = [ - ('gromacs.water_bare', 'gromacs'), + ('gromacs', 'gromacs'), ] expected_strs = [ 'export test_var="test_val:${test_var}"', @@ -148,10 +148,11 @@ def env_var_set_modifier(): def env_var_set_modifier_answer(): expected_software = [ - ('gromacs.water_bare', 'gromacs'), + ('gromacs', 'gromacs'), ] expected_strs = [ 'export test_var=test_val', + 'export mask_env_var="0x0"' ] return expected_software, expected_strs @@ -166,7 +167,7 @@ def env_var_unset_modifier(): def env_var_unset_modifier_answer(): expected_software = [ - ('gromacs.water_bare', 'gromacs'), + ('gromacs', 'gromacs'), ] expected_strs = [ 'unset test_var', diff --git a/lib/ramble/ramble/test/modifier_functionality/modifier_prepare_analysis.py b/lib/ramble/ramble/test/modifier_functionality/modifier_prepare_analysis.py index 0d28f4495..1aa280948 100644 --- a/lib/ramble/ramble/test/modifier_functionality/modifier_prepare_analysis.py +++ b/lib/ramble/ramble/test/modifier_functionality/modifier_prepare_analysis.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/modifier_functionality/multi_modifier_dry_run.py b/lib/ramble/ramble/test/modifier_functionality/multi_modifier_dry_run.py index ea7c6df41..70c360283 100644 --- a/lib/ramble/ramble/test/modifier_functionality/multi_modifier_dry_run.py +++ b/lib/ramble/ramble/test/modifier_functionality/multi_modifier_dry_run.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/modifier_functionality/single_modifier_dry_run.py b/lib/ramble/ramble/test/modifier_functionality/single_modifier_dry_run.py index ccbb97da1..07df00d99 100644 --- a/lib/ramble/ramble/test/modifier_functionality/single_modifier_dry_run.py +++ b/lib/ramble/ramble/test/modifier_functionality/single_modifier_dry_run.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/modifier_language.py b/lib/ramble/ramble/test/modifier_language.py index ba7bd05c3..1108ecd02 100644 --- a/lib/ramble/ramble/test/modifier_language.py +++ b/lib/ramble/ramble/test/modifier_language.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -45,7 +45,7 @@ def test_modifier_type_features(mod_class): assert hasattr(test_mod, 'modes') assert hasattr(test_mod, 'variable_modifications') assert hasattr(test_mod, 'software_specs') - assert hasattr(test_mod, 'default_compilers') + assert hasattr(test_mod, 'compilers') assert hasattr(test_mod, 'required_packages') assert hasattr(test_mod, 'success_criteria') assert hasattr(test_mod, 'builtins') @@ -233,7 +233,7 @@ def test_software_spec_directive(mod_class, func_type): assert test_def[attr] == mod_inst.software_specs[spec_name][attr] -def add_default_compiler(mod_inst, spec_num=1, func_type=func_types.directive): +def add_compiler(mod_inst, spec_num=1, func_type=func_types.directive): spec_name = f'CompilerPackage{spec_num}' spack_spec = 'compiler@1.1 target=x86_64' compiler_spec = 'compiler@1.1' @@ -247,9 +247,9 @@ def add_default_compiler(mod_inst, spec_num=1, func_type=func_types.directive): } if func_type == func_types.directive: - default_compiler(spec_name, spack_spec, compiler_spec, compiler)(mod_inst) + define_compiler(spec_name, spack_spec, compiler_spec, compiler)(mod_inst) elif func_type == func_types.method: - mod_inst.default_compiler(spec_name, spack_spec, compiler_spec, compiler) + mod_inst.define_compiler(spec_name, spack_spec, compiler_spec, compiler) else: assert False @@ -258,24 +258,24 @@ def add_default_compiler(mod_inst, spec_num=1, func_type=func_types.directive): @pytest.mark.parametrize('func_type', func_types) @pytest.mark.parametrize('mod_class', mod_types) -def test_default_compiler_directive(mod_class, func_type): +def test_define_compiler_directive(mod_class, func_type): test_class = generate_mod_class(mod_class) mod_inst = test_class('/not/a/path') test_defs = [] - test_defs.append(add_default_compiler(mod_inst, func_type=func_type).copy()) + test_defs.append(add_compiler(mod_inst, func_type=func_type).copy()) expected_attrs = ['spack_spec', 'compiler_spec', 'compiler'] if mod_inst.uses_spack: - assert hasattr(mod_inst, 'default_compilers') + assert hasattr(mod_inst, 'compilers') for test_def in test_defs: spec_name = test_def['name'] - assert spec_name in mod_inst.default_compilers + assert spec_name in mod_inst.compilers for attr in expected_attrs: - assert attr in mod_inst.default_compilers[spec_name] - assert test_def[attr] == mod_inst.default_compilers[spec_name][attr] + assert attr in mod_inst.compilers[spec_name] + assert test_def[attr] == mod_inst.compilers[spec_name][attr] def add_required_package(mod_inst, pkg_num=1, func_type=func_types.directive): @@ -547,5 +547,18 @@ def test_modifier_variable_directive(mod_class, func_type): assert mode in mod_inst.modifier_variables assert test_def['name'] in mod_inst.modifier_variables[mode] - for attr in ['description', 'default']: - assert test_def[attr] == mod_inst.modifier_variables[mode][var_name][attr] + assert test_def['description'] == mod_inst.modifier_variables[mode][var_name].description + assert test_def['default'] == mod_inst.modifier_variables[mode][var_name].default + + +@pytest.mark.parametrize('func_type', func_types) +@pytest.mark.parametrize('mod_class', mod_types) +def test_modifier_class_attributes(mod_class, func_type): + test_class = generate_mod_class(mod_class) + mod_inst = test_class('/not/a/path') + mod_copy = mod_inst.copy() + + mod_copy.mode('added_mode', description='Mode added to test attributes') + + assert 'added_mode' in mod_copy.modes + assert 'added_mode' not in mod_inst.modes diff --git a/lib/ramble/ramble/test/namespace_trie.py b/lib/ramble/ramble/test/namespace_trie.py index 9931b1b2a..7078c03d2 100644 --- a/lib/ramble/ramble/test/namespace_trie.py +++ b/lib/ramble/ramble/test/namespace_trie.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/repository.py b/lib/ramble/ramble/test/repository.py index 43af4082b..a32769c7a 100644 --- a/lib/ramble/ramble/test/repository.py +++ b/lib/ramble/ramble/test/repository.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/software_environment.py b/lib/ramble/ramble/test/software_environment.py index 42d275fda..fd3e8abcc 100644 --- a/lib/ramble/ramble/test/software_environment.py +++ b/lib/ramble/ramble/test/software_environment.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -6,12 +6,12 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. -import os import pytest import ramble.workspace import ramble.software_environments import ramble.renderer +import ramble.expander from ramble.main import RambleCommand pytestmark = pytest.mark.usefixtures('mutable_config', @@ -22,8 +22,8 @@ workspace = RambleCommand('workspace') -def test_basic_software_environment(mutable_mock_workspace_path): - ws_name = 'test_basic_software_environment' +def test_basic_software_environment(request, mutable_mock_workspace_path): + ws_name = request.node.name workspace('create', ws_name) assert ws_name in workspace('list') @@ -43,46 +43,24 @@ def test_basic_software_environment(mutable_mock_workspace_path): software_environments = ramble.software_environments.SoftwareEnvironments(ws) - assert len(software_environments._packages.keys()) == 1 - assert 'basic' in software_environments._packages.keys() - assert 'basic' in software_environments._environments.keys() - assert 'basic' in software_environments._environments['basic']['packages'] + assert 'basic' in software_environments._environment_templates + assert 'basic' in software_environments._package_templates + variables = {} + env_expander = ramble.expander.Expander(variables, None) -def test_package_vector_expansion(mutable_mock_workspace_path): - ws_name = 'test_package_vector_expansion' - workspace('create', ws_name) - - assert ws_name in workspace('list') - - with ramble.workspace.read(ws_name) as ws: - spack_dict = ws.get_spack_dict() - - spack_dict['packages'] = {} - spack_dict['packages']['basic-{arch}'] = { - 'spack_spec': 'basic@1.1 target={arch}', - 'variables': { - 'arch': ['x86_64', 'x86_64_v4'] - } - } - spack_dict['environments'] = { - 'basic': { - 'packages': ['basic-x86_64', 'basic-x86_64_v4'] - } - } - - software_environments = ramble.software_environments.SoftwareEnvironments(ws) + rendered_env = software_environments.render_environment('basic', env_expander) + assert rendered_env.name == 'basic' + pkg_found = False + for pkg in rendered_env._packages: + if pkg.name == 'basic': + pkg_found = True + assert pkg_found - assert len(software_environments._packages.keys()) == 2 - assert 'basic-x86_64' in software_environments._packages.keys() - assert 'basic-x86_64_v4' in software_environments._packages.keys() - assert 'basic' in software_environments._environments.keys() - assert 'basic-x86_64' in software_environments._environments['basic']['packages'] - assert 'basic-x86_64_v4' in software_environments._environments['basic']['packages'] +def test_software_environments_no_packages(request, mutable_mock_workspace_path): + ws_name = request.node.name -def test_package_vector_length_mismatch_errors(mutable_mock_workspace_path, capsys): - ws_name = 'test_package_vector_expansion' workspace('create', ws_name) assert ws_name in workspace('list') @@ -91,74 +69,28 @@ def test_package_vector_length_mismatch_errors(mutable_mock_workspace_path, caps spack_dict = ws.get_spack_dict() spack_dict['packages'] = {} - spack_dict['packages']['basic-{arch}'] = { - 'spack_spec': 'basic@{ver} target={arch}', - 'variables': { - 'arch': ['x86_64', 'x86_64_v4'], - 'ver': ['1.1'] - } - } spack_dict['environments'] = { - 'basic': { - 'packages': ['basic-x86_64', 'basic-x86_64_v4'] + 'basic-{env_test}': { + 'packages': [''] } } - with pytest.raises(SystemExit): - ramble.software_environments.SoftwareEnvironments(ws) - - captured = capsys.readouterr() - - assert 'Length mismatch in vector variables in package basic-{arch}' in captured - assert 'Variable arch has length 2' in captured - assert 'Variable ver has length 1' in captured - - -def test_package_matrix_expansion(mutable_mock_workspace_path): - ws_name = 'test_package_matrix_expansion' - workspace('create', ws_name) - - assert ws_name in workspace('list') + software_environments = ramble.software_environments.SoftwareEnvironments(ws) - with ramble.workspace.read(ws_name) as ws: - spack_dict = ws.get_spack_dict() + assert 'basic-{env_test}' in software_environments._environment_templates - spack_dict['packages'] = {} - spack_dict['packages']['basic-{ver}-{arch}'] = { - 'spack_spec': 'basic@{ver} target={arch}', - 'variables': { - 'arch': ['x86_64', 'x86_64_v4'], - 'ver': ['1.1', '2.0'] - }, - 'matrix': ['arch', 'ver'] - } - spack_dict['environments'] = { - 'basic': { - 'packages': [ - 'basic-1.1-x86_64', - 'basic-2.0-x86_64', - 'basic-1.1-x86_64_v4', - 'basic-2.0-x86_64_v4', - ] - } + variables = { + 'env_test': 'environment', } + env_expander = ramble.expander.Expander(variables, None) - software_environments = ramble.software_environments.SoftwareEnvironments(ws) + rendered_env = software_environments.render_environment('basic-environment', env_expander) + assert rendered_env.name == 'basic-environment' - assert len(software_environments._packages.keys()) == 4 - assert 'basic-1.1-x86_64' in software_environments._packages.keys() - assert 'basic-2.0-x86_64' in software_environments._packages.keys() - assert 'basic-1.1-x86_64_v4' in software_environments._packages.keys() - assert 'basic-2.0-x86_64_v4' in software_environments._packages.keys() - assert 'basic' in software_environments._environments.keys() - assert 'basic-1.1-x86_64' in software_environments._environments['basic']['packages'] - assert 'basic-2.0-x86_64' in software_environments._environments['basic']['packages'] - assert 'basic-1.1-x86_64_v4' in software_environments._environments['basic']['packages'] - assert 'basic-2.0-x86_64_v4' in software_environments._environments['basic']['packages'] +def test_software_environments_no_rendered_packages(request, mutable_mock_workspace_path): + ws_name = request.node.name -def test_package_matrices_expansion(mutable_mock_workspace_path): - ws_name = 'test_package_matrices_expansion' workspace('create', ws_name) assert ws_name in workspace('list') @@ -167,78 +99,29 @@ def test_package_matrices_expansion(mutable_mock_workspace_path): spack_dict = ws.get_spack_dict() spack_dict['packages'] = {} - spack_dict['packages']['basic-{ver}-{arch}'] = { - 'spack_spec': 'basic@{ver} target={arch}', - 'variables': { - 'arch': ['x86_64', 'x86_64_v4'], - 'ver': ['1.1', '2.0'] - }, - 'matrices': [['arch'], ['ver']] - } spack_dict['environments'] = { - 'basic': { - 'packages': [ - 'basic-1.1-x86_64', - 'basic-2.0-x86_64_v4', - ] + 'basic-{env_test}': { + 'packages': ['{var_pkg_name}'] } } software_environments = ramble.software_environments.SoftwareEnvironments(ws) - assert len(software_environments._packages.keys()) == 2 - assert 'basic-1.1-x86_64' in software_environments._packages.keys() - assert 'basic-2.0-x86_64_v4' in software_environments._packages.keys() - assert 'basic' in software_environments._environments.keys() - assert 'basic-1.1-x86_64' in software_environments._environments['basic']['packages'] - assert 'basic-2.0-x86_64_v4' in software_environments._environments['basic']['packages'] + assert 'basic-{env_test}' in software_environments._environment_templates - -def test_package_matrix_vector_expansion(mutable_mock_workspace_path): - ws_name = 'test_package_matrix_vector_expansion' - workspace('create', ws_name) - - assert ws_name in workspace('list') - - with ramble.workspace.read(ws_name) as ws: - spack_dict = ws.get_spack_dict() - - spack_dict['packages'] = {} - spack_dict['packages']['basic-{ver}-{arch}'] = { - 'spack_spec': 'basic@{ver} target={arch}', - 'variables': { - 'arch': ['x86_64', 'x86_64_v4'], - 'ver': ['1.1', '2.0'] - }, - 'matrices': [['arch']] - } - spack_dict['environments'] = { - 'basic': { - 'packages': [ - 'basic-1.1-x86_64', - 'basic-2.0-x86_64', - 'basic-1.1-x86_64_v4', - 'basic-2.0-x86_64_v4', - ] - } + variables = { + 'env_test': 'environment', + 'var_pkg_name': '' } + env_expander = ramble.expander.Expander(variables, None) - software_environments = ramble.software_environments.SoftwareEnvironments(ws) + rendered_env = software_environments.render_environment('basic-environment', env_expander) + assert rendered_env.name == 'basic-environment' - assert len(software_environments._packages.keys()) == 4 - assert 'basic-1.1-x86_64' in software_environments._packages.keys() - assert 'basic-2.0-x86_64' in software_environments._packages.keys() - assert 'basic-1.1-x86_64_v4' in software_environments._packages.keys() - assert 'basic-2.0-x86_64_v4' in software_environments._packages.keys() - assert 'basic' in software_environments._environments.keys() - assert 'basic-1.1-x86_64' in software_environments._environments['basic']['packages'] - assert 'basic-2.0-x86_64' in software_environments._environments['basic']['packages'] - assert 'basic-1.1-x86_64_v4' in software_environments._environments['basic']['packages'] - assert 'basic-2.0-x86_64_v4' in software_environments._environments['basic']['packages'] +def test_template_software_environments(request, mutable_mock_workspace_path): + ws_name = request.node.name -def test_environment_vector_expansion(mutable_mock_workspace_path): - ws_name = 'test_environment_vector_expansion' workspace('create', ws_name) assert ws_name in workspace('list') @@ -247,75 +130,38 @@ def test_environment_vector_expansion(mutable_mock_workspace_path): spack_dict = ws.get_spack_dict() spack_dict['packages'] = {} - spack_dict['packages']['basic-{arch}'] = { - 'spack_spec': 'basic@1.1 target={arch}', - 'variables': { - 'arch': ['x86_64', 'x86_64_v4'] - } + spack_dict['packages']['basic-{pkg_test}'] = { + 'spack_spec': 'basic@1.1' } spack_dict['environments'] = { - 'basic-{arch}': { - 'packages': ['basic-{arch}'], - 'variables': { - 'arch': ['x86_64', 'x86_64_v4'] - } + 'basic-{env_test}': { + 'packages': ['basic-{pkg_test}'] } } software_environments = ramble.software_environments.SoftwareEnvironments(ws) - assert len(software_environments._packages.keys()) == 2 - assert len(software_environments._environments.keys()) == 2 - assert 'basic-x86_64' in software_environments._packages.keys() - assert 'basic-x86_64_v4' in software_environments._packages.keys() - assert 'basic-x86_64' in software_environments._environments.keys() - assert 'basic-x86_64_v4' in software_environments._environments.keys() - assert len(software_environments._environments['basic-x86_64']['packages']) == 1 - assert 'basic-x86_64' in software_environments._environments['basic-x86_64']['packages'] - assert len(software_environments._environments['basic-x86_64_v4']['packages']) == 1 - assert 'basic-x86_64_v4' in \ - software_environments._environments['basic-x86_64_v4']['packages'] - - -def test_environment_vector_length_mismatch_errors(mutable_mock_workspace_path, capsys): - ws_name = 'test_environment_vector_expansion' - workspace('create', ws_name) + assert 'basic-{env_test}' in software_environments._environment_templates + assert 'basic-{pkg_test}' in software_environments._package_templates - assert ws_name in workspace('list') - - with ramble.workspace.read(ws_name) as ws: - spack_dict = ws.get_spack_dict() - - spack_dict['packages'] = {} - spack_dict['packages']['basic-{arch}'] = { - 'spack_spec': 'basic@1.1 target={arch}', - 'variables': { - 'arch': ['x86_64', 'x86_64_v4'] - } - } - spack_dict['environments'] = { - 'basic-{ver}-{arch}': { - 'packages': ['basic-{arch}'], - 'variables': { - 'arch': ['x86_64', 'x86_64_v4'], - 'ver': ['1.1'] - } - } + variables = { + 'env_test': 'environment', + 'pkg_test': 'package', } + env_expander = ramble.expander.Expander(variables, None) - with pytest.raises(SystemExit): - ramble.software_environments.SoftwareEnvironments(ws) + rendered_env = software_environments.render_environment('basic-environment', env_expander) + assert rendered_env.name == 'basic-environment' + pkg_found = False + for pkg in rendered_env._packages: + if pkg.name == 'basic-package': + pkg_found = True + assert pkg_found - captured = capsys.readouterr() - assert 'Length mismatch in vector variables in environment basic-{ver}-{arch}' \ - in captured - assert 'Variable arch has length 2' in captured - assert 'Variable ver has length 1' in captured +def test_multi_template_software_environments(request, mutable_mock_workspace_path): + ws_name = request.node.name - -def test_environment_matrix_expansion(mutable_mock_workspace_path): - ws_name = 'test_environment_matrix_expansion' workspace('create', ws_name) assert ws_name in workspace('list') @@ -324,270 +170,97 @@ def test_environment_matrix_expansion(mutable_mock_workspace_path): spack_dict = ws.get_spack_dict() spack_dict['packages'] = {} - spack_dict['packages']['basic-{ver}-{arch}'] = { - 'spack_spec': 'basic@{ver} target={arch}', - 'variables': { - 'arch': ['x86_64', 'x86_64_v4'], - 'ver': ['1.1', '2.0'] - }, - 'matrix': ['arch', 'ver'] - } - spack_dict['environments'] = { - 'basic-{ver}-{arch}': { - 'packages': [ - 'basic-{ver}-{arch}' - ], - 'variables': { - 'arch': ['x86_64', 'x86_64_v4'], - 'ver': ['1.1', '2.0'] - }, - 'matrix': ['arch', 'ver'] - } + spack_dict['packages']['basic1-{pkg_test}'] = { + 'spack_spec': 'basic@1.1' } - - software_environments = ramble.software_environments.SoftwareEnvironments(ws) - - assert len(software_environments._packages.keys()) == 4 - assert len(software_environments._environments.keys()) == 4 - assert 'basic-1.1-x86_64' in software_environments._packages.keys() - assert 'basic-2.0-x86_64' in software_environments._packages.keys() - assert 'basic-1.1-x86_64_v4' in software_environments._packages.keys() - assert 'basic-2.0-x86_64_v4' in software_environments._packages.keys() - assert 'basic-1.1-x86_64' in software_environments._environments.keys() - assert 'basic-2.0-x86_64' in software_environments._environments.keys() - assert 'basic-1.1-x86_64_v4' in software_environments._environments.keys() - assert 'basic-2.0-x86_64_v4' in software_environments._environments.keys() - assert 'basic-1.1-x86_64' in \ - software_environments._environments['basic-1.1-x86_64']['packages'] - assert 'basic-2.0-x86_64' in \ - software_environments._environments['basic-2.0-x86_64']['packages'] - assert 'basic-1.1-x86_64_v4' in \ - software_environments._environments['basic-1.1-x86_64_v4']['packages'] - assert 'basic-2.0-x86_64_v4' in \ - software_environments._environments['basic-2.0-x86_64_v4']['packages'] - - -def test_environment_matrices_expansion(mutable_mock_workspace_path): - ws_name = 'test_environment_matrices_expansion' - workspace('create', ws_name) - - assert ws_name in workspace('list') - - with ramble.workspace.read(ws_name) as ws: - spack_dict = ws.get_spack_dict() - - spack_dict['packages'] = {} - spack_dict['packages']['basic-{ver}-{arch}'] = { - 'spack_spec': 'basic@{ver} target={arch}', - 'variables': { - 'arch': ['x86_64', 'x86_64_v4'], - 'ver': ['1.1', '2.0'] - }, - 'matrix': ['arch', 'ver'] + spack_dict['packages']['basic2-{pkg_test}'] = { + 'spack_spec': 'basic@1.1' } spack_dict['environments'] = { - 'basic-{ver}-{arch}': { - 'packages': [ - 'basic-{ver}-{arch}' - ], - 'variables': { - 'arch': ['x86_64', 'x86_64_v4'], - 'ver': ['1.1', '2.0'] - }, - 'matrices': [['arch'], ['ver']] - } - } - - software_environments = ramble.software_environments.SoftwareEnvironments(ws) - - assert len(software_environments._packages.keys()) == 4 - assert len(software_environments._environments.keys()) == 2 - assert 'basic-1.1-x86_64' in software_environments._packages.keys() - assert 'basic-2.0-x86_64' in software_environments._packages.keys() - assert 'basic-1.1-x86_64_v4' in software_environments._packages.keys() - assert 'basic-2.0-x86_64_v4' in software_environments._packages.keys() - assert 'basic-1.1-x86_64' in software_environments._environments.keys() - assert 'basic-2.0-x86_64_v4' in software_environments._environments.keys() - assert 'basic-1.1-x86_64' in \ - software_environments._environments['basic-1.1-x86_64']['packages'] - assert 'basic-2.0-x86_64_v4' in \ - software_environments._environments['basic-2.0-x86_64_v4']['packages'] - - -def test_environment_vector_matrix_expansion(mutable_mock_workspace_path): - ws_name = 'test_environment_vector_matrix_expansion' - workspace('create', ws_name) - - assert ws_name in workspace('list') - - with ramble.workspace.read(ws_name) as ws: - spack_dict = ws.get_spack_dict() - - spack_dict['packages'] = {} - spack_dict['packages']['basic-{ver}-{arch}'] = { - 'spack_spec': 'basic@{ver} target={arch}', - 'variables': { - 'arch': ['x86_64', 'x86_64_v4'], - 'ver': ['1.1', '2.0'] + 'all-basic-{env_test}': { + 'packages': ['basic1-{pkg_test}', 'basic2-{pkg_test}'] }, - 'matrices': [['ver']] - } - spack_dict['environments'] = { - 'basic-{ver}-{arch}': { - 'packages': [ - 'basic-{ver}-{arch}' - ], - 'variables': { - 'arch': ['x86_64', 'x86_64_v4'], - 'ver': ['1.1', '2.0'] - }, - 'matrices': [['ver']] + 'basic1-{env_test}': { + 'packages': ['basic1-{pkg_test}'] + }, + 'basic2-{env_test}': { + 'packages': ['basic2-{pkg_test}'] } } software_environments = ramble.software_environments.SoftwareEnvironments(ws) - assert len(software_environments._packages.keys()) == 4 - assert len(software_environments._environments.keys()) == 4 - assert 'basic-1.1-x86_64' in software_environments._packages.keys() - assert 'basic-2.0-x86_64' in software_environments._packages.keys() - assert 'basic-1.1-x86_64_v4' in software_environments._packages.keys() - assert 'basic-2.0-x86_64_v4' in software_environments._packages.keys() - assert 'basic-1.1-x86_64' in software_environments._environments.keys() - assert 'basic-2.0-x86_64' in software_environments._environments.keys() - assert 'basic-1.1-x86_64_v4' in software_environments._environments.keys() - assert 'basic-2.0-x86_64_v4' in software_environments._environments.keys() - assert 'basic-1.1-x86_64' in \ - software_environments._environments['basic-1.1-x86_64']['packages'] - assert 'basic-2.0-x86_64' in \ - software_environments._environments['basic-2.0-x86_64']['packages'] - assert 'basic-1.1-x86_64_v4' in \ - software_environments._environments['basic-1.1-x86_64_v4']['packages'] - assert 'basic-2.0-x86_64_v4' in \ - software_environments._environments['basic-2.0-x86_64_v4']['packages'] - - -def test_package_vector_expansion_spack_level(mutable_mock_workspace_path): - ws_name = 'test_package_vector_expansion_spack_level' - workspace('create', ws_name) - - assert ws_name in workspace('list') - - with ramble.workspace.read(ws_name) as ws: - spack_dict = ws.get_spack_dict() + assert 'all-basic-{env_test}' in software_environments._environment_templates + assert 'basic1-{env_test}' in software_environments._environment_templates + assert 'basic2-{env_test}' in software_environments._environment_templates + assert 'basic1-{pkg_test}' in software_environments._package_templates + assert 'basic2-{pkg_test}' in software_environments._package_templates - spack_dict['variables'] = {} - spack_dict['variables']['arch'] = ['x86_64', 'x86_64_v4'] - spack_dict['packages'] = {} - spack_dict['packages']['basic-{arch}'] = { - 'spack_spec': 'basic@1.1 target={arch}' + variables = { + 'env_test': 'environment', + 'pkg_test': 'package', } - spack_dict['environments'] = { - 'basic': { - 'packages': ['basic-x86_64', 'basic-x86_64_v4'] - } + env_expander = ramble.expander.Expander(variables, None) + + env_tests = { + 'all-basic-environment': ['basic1-package', 'basic2-package'], + 'basic1-environment': ['basic1-package'], + 'basic2-environment': ['basic2-package'] } - software_environments = ramble.software_environments.SoftwareEnvironments(ws) + for env_name, env_packages in env_tests.items(): + rendered_env = software_environments.render_environment(env_name, env_expander) + assert rendered_env.name == env_name - assert len(software_environments._packages.keys()) == 2 - assert 'basic-x86_64' in software_environments._packages.keys() - assert 'basic-x86_64_v4' in software_environments._packages.keys() - assert 'basic' in software_environments._environments.keys() - assert 'basic-x86_64' in software_environments._environments['basic']['packages'] - assert 'basic-x86_64_v4' in software_environments._environments['basic']['packages'] + assert len(rendered_env._packages) == len(env_packages) + for pkg_name in env_packages: + pkg_found = False + for pkg in rendered_env._packages: + if pkg.name == pkg_name: + pkg_found = True + assert pkg_found -def test_package_vector_expansion_workspace_level(mutable_mock_workspace_path): - ws_name = 'test_package_vector_expansion_spack_level' - workspace('create', ws_name) +def test_undefined_package_errors(request, mutable_mock_workspace_path): + ws_name = request.node.name - test_config = """ -ramble: - variables: - arch: ['x86_64', 'x86_64_v4'] - applications: {} - spack: {} -""" + workspace('create', ws_name) assert ws_name in workspace('list') with ramble.workspace.read(ws_name) as ws: - with open(os.path.join(ws.config_dir, 'ramble.yaml'), 'w+') as f: - f.write(test_config) - - ws._re_read() spack_dict = ws.get_spack_dict() spack_dict['packages'] = {} - spack_dict['packages']['basic-{arch}'] = { - 'spack_spec': 'basic@1.1 target={arch}' + spack_dict['packages']['basic-{pkg_test}'] = { + 'spack_spec': 'basic@{pkg_ver}' } spack_dict['environments'] = { - 'basic': { - 'packages': ['basic-x86_64', 'basic-x86_64_v4'] + 'all-basic-{env_test}': { + 'packages': ['foo-basic-{pkg_test}'] } } software_environments = ramble.software_environments.SoftwareEnvironments(ws) - assert len(software_environments._packages.keys()) == 2 - assert 'basic-x86_64' in software_environments._packages.keys() - assert 'basic-x86_64_v4' in software_environments._packages.keys() - assert 'basic' in software_environments._environments.keys() - assert 'basic-x86_64' in software_environments._environments['basic']['packages'] - assert 'basic-x86_64_v4' in software_environments._environments['basic']['packages'] - - -def test_package_vector_expansion_multi_level(mutable_mock_workspace_path): - ws_name = 'test_package_vector_expansion_multi_level' - workspace('create', ws_name) - - test_config = """ -ramble: - variables: - arch: ['x86_64', 'x86_64_v4'] - applications: {} - spack: {} -""" - - assert ws_name in workspace('list') - - with ramble.workspace.read(ws_name) as ws: - with open(os.path.join(ws.config_dir, 'ramble.yaml'), 'w+') as f: - f.write(test_config) + variables = { + 'env_test': 'environment' + } - ws._re_read() - spack_dict = ws.get_spack_dict() + env_expander = ramble.expander.Expander(variables, None) - spack_dict['variables'] = {} - spack_dict['variables']['test'] = ['test1', 'test2'] - spack_dict['packages'] = {} - spack_dict['packages']['basic-{arch}-{test}-{pkg_level}'] = { - 'spack_spec': 'basic@1.1 target={arch}', - 'variables': { - 'pkg_level': ['ll1', 'll2'], - } - } - spack_dict['environments'] = { - 'basic': { - 'packages': ['basic-x86_64-test1-ll1', 'basic-x86_64_v4-test2-ll2'] - } - } + with pytest.raises(ramble.software_environments.RambleSoftwareEnvironmentError) as pkg_err: + _ = software_environments.render_environment('all-basic-environment', env_expander) - software_environments = ramble.software_environments.SoftwareEnvironments(ws) + err_str = \ + 'Environment template all-basic-{env_test} references undefined ' \ + + 'package foo-basic-{pkg_test}' + assert err_str in str(pkg_err) - assert len(software_environments._packages.keys()) == 2 - assert 'basic-x86_64-test1-ll1' in software_environments._packages.keys() - assert 'basic-x86_64_v4-test2-ll2' in software_environments._packages.keys() - assert 'basic' in software_environments._environments.keys() - assert 'basic-x86_64-test1-ll1' in software_environments._environments['basic']['packages'] - assert 'basic-x86_64_v4-test2-ll2' in \ - software_environments._environments['basic']['packages'] +def test_invalid_packages_error(request, mutable_mock_workspace_path): + ws_name = request.node.name -def test_environment_vector_expansion_spack_level(mutable_mock_workspace_path): - ws_name = 'test_environment_vector_expansion_spack_level' workspace('create', ws_name) assert ws_name in workspace('list') @@ -595,81 +268,46 @@ def test_environment_vector_expansion_spack_level(mutable_mock_workspace_path): with ramble.workspace.read(ws_name) as ws: spack_dict = ws.get_spack_dict() - spack_dict['variables'] = {} - spack_dict['variables']['arch'] = ['x86_64', 'x86_64_v4'] spack_dict['packages'] = {} - spack_dict['packages']['basic-{arch}'] = { - 'spack_spec': 'basic@1.1 target={arch}', + spack_dict['packages']['basic-{pkg_test}'] = { + 'spack_spec': 'basic@{pkg_ver}' } spack_dict['environments'] = { - 'basic-{arch}': { - 'packages': ['basic-{arch}'], + 'all-basic-{env_test}': { + 'packages': ['basic-{pkg_test}'] } } software_environments = ramble.software_environments.SoftwareEnvironments(ws) - assert len(software_environments._packages.keys()) == 2 - assert len(software_environments._environments.keys()) == 2 - assert 'basic-x86_64' in software_environments._packages.keys() - assert 'basic-x86_64_v4' in software_environments._packages.keys() - assert 'basic-x86_64' in software_environments._environments.keys() - assert 'basic-x86_64_v4' in software_environments._environments.keys() - assert len(software_environments._environments['basic-x86_64']['packages']) == 1 - assert 'basic-x86_64' in software_environments._environments['basic-x86_64']['packages'] - assert len(software_environments._environments['basic-x86_64_v4']['packages']) == 1 - assert 'basic-x86_64_v4' in \ - software_environments._environments['basic-x86_64_v4']['packages'] - - -def test_environment_vector_expansion_workspace_level(mutable_mock_workspace_path): - ws_name = 'test_environment_vector_expansion_workspace_level' - workspace('create', ws_name) + assert 'all-basic-{env_test}' in software_environments._environment_templates + assert 'basic-{pkg_test}' in software_environments._package_templates - test_config = """ -ramble: - variables: - arch: ['x86_64', 'x86_64_v4'] - applications: {} - spack: {} -""" + variables = { + 'env_test': 'environment', + 'pkg_test': 'package', + 'pkg_ver': '1.1', + } + env_expander = ramble.expander.Expander(variables, None) - assert ws_name in workspace('list') + _ = software_environments.render_environment('all-basic-environment', env_expander) - with ramble.workspace.read(ws_name) as ws: - with open(os.path.join(ws.config_dir, 'ramble.yaml'), 'w+') as f: - f.write(test_config) + with pytest.raises(ramble.software_environments.RambleSoftwareEnvironmentError) as pkg_err: + variables = { + 'env_test': 'environment', + 'pkg_test': 'package', + 'pkg_ver': '1.4', + } + env_expander = ramble.expander.Expander(variables, None) - ws._re_read() - spack_dict = ws.get_spack_dict() + _ = software_environments.render_environment('all-basic-environment', + env_expander) + assert 'Package basic-package defined multiple times' in str(pkg_err) - spack_dict['packages'] = {} - spack_dict['packages']['basic-{arch}'] = { - 'spack_spec': 'basic@1.1 target={arch}', - } - spack_dict['environments'] = { - 'basic-{arch}': { - 'packages': ['basic-{arch}'], - } - } - software_environments = ramble.software_environments.SoftwareEnvironments(ws) +def test_invalid_environment_error(request, mutable_mock_workspace_path): + ws_name = request.node.name - assert len(software_environments._packages.keys()) == 2 - assert len(software_environments._environments.keys()) == 2 - assert 'basic-x86_64' in software_environments._packages.keys() - assert 'basic-x86_64_v4' in software_environments._packages.keys() - assert 'basic-x86_64' in software_environments._environments.keys() - assert 'basic-x86_64_v4' in software_environments._environments.keys() - assert len(software_environments._environments['basic-x86_64']['packages']) == 1 - assert 'basic-x86_64' in software_environments._environments['basic-x86_64']['packages'] - assert len(software_environments._environments['basic-x86_64_v4']['packages']) == 1 - assert 'basic-x86_64_v4' in \ - software_environments._environments['basic-x86_64_v4']['packages'] - - -def test_environment_warns_with_pkg_compiler(mutable_mock_workspace_path, capsys): - ws_name = 'test_environment_warns_with_pkg_compiler' workspace('create', ws_name) assert ws_name in workspace('list') @@ -678,70 +316,48 @@ def test_environment_warns_with_pkg_compiler(mutable_mock_workspace_path, capsys spack_dict = ws.get_spack_dict() spack_dict['packages'] = {} - spack_dict['packages']['test_comp'] = { - 'spack_spec': 'test_comp@1.1' + spack_dict['packages']['basic1-{pkg_test}'] = { + 'spack_spec': 'basic@1.1' } - spack_dict['packages']['basic'] = { - 'spack_spec': 'basic@1.1', - 'compiler': 'test_comp' + spack_dict['packages']['basic2-{pkg_test}'] = { + 'spack_spec': 'basic@1.1' } spack_dict['environments'] = { - 'basic': { - 'packages': [ - 'basic', - 'test_comp' - ], + 'all-basic-{env_test}': { + 'packages': ['basic1-{pkg_test}', 'basic2-{pkg_test}'] } } - ramble.software_environments.SoftwareEnvironments(ws) - captured = capsys.readouterr() + software_environments = ramble.software_environments.SoftwareEnvironments(ws) - assert 'Environment basic contains packages and their compilers ' + \ - 'in the package list. These include:' in captured.err + assert 'all-basic-{env_test}' in software_environments._environment_templates + assert 'basic1-{pkg_test}' in software_environments._package_templates + assert 'basic2-{pkg_test}' in software_environments._package_templates - assert 'Package: basic, Compiler: test_comp' in captured.err + variables = { + 'env_test': 'environment', + 'pkg_test': 'package', + } + env_expander = ramble.expander.Expander(variables, None) + _ = software_environments.render_environment('all-basic-environment', env_expander) -def test_package_vector_expansion_exclusions(mutable_mock_workspace_path): - ws_name = 'test_package_vector_expansion_exclusions' - workspace('create', ws_name) + variables = { + 'env_test': 'environment', + 'pkg_test': 'other-package' + } - assert ws_name in workspace('list') + env_expander = ramble.expander.Expander(variables, None) - with ramble.workspace.read(ws_name) as ws: - spack_dict = ws.get_spack_dict() + with pytest.raises(ramble.software_environments.RambleSoftwareEnvironmentError) as env_err: + _ = software_environments.render_environment('all-basic-environment', env_expander) - spack_dict['packages'] = {} - spack_dict['packages']['basic-{arch}'] = { - 'spack_spec': 'basic@1.1 target={arch}', - 'variables': { - 'arch': ['x86_64', 'x86_64_v4'] - }, - 'exclude': { - 'variables': { - 'arch': 'x86_64_v4' - } - } - } - spack_dict['environments'] = { - 'basic': { - 'packages': ['basic-x86_64'] - } - } - - software_environments = ramble.software_environments.SoftwareEnvironments(ws) + assert 'Environment all-basic-environment defined multiple times' in str(env_err) - assert len(software_environments._packages.keys()) == 1 - assert 'basic-x86_64' in software_environments._packages.keys() - assert 'basic' in software_environments._environments.keys() - assert 'basic-x86_64' in software_environments._environments['basic']['packages'] - assert 'basic-x86_64_v4' not in software_environments._packages.keys() - assert 'basic-x86_64_v4' not in software_environments._environments['basic']['packages'] +def test_undefined_compiler_errors(request, mutable_mock_workspace_path): + ws_name = request.node.name -def test_package_matrix_expansion_exclusions(mutable_mock_workspace_path): - ws_name = 'test_package_matrix_expansion_exclusions' workspace('create', ws_name) assert ws_name in workspace('list') @@ -750,89 +366,33 @@ def test_package_matrix_expansion_exclusions(mutable_mock_workspace_path): spack_dict = ws.get_spack_dict() spack_dict['packages'] = {} - spack_dict['packages']['basic-{ver}-{arch}'] = { - 'spack_spec': 'basic@{ver} target={arch}', - 'variables': { - 'arch': ['x86_64', 'x86_64_v4'], - 'ver': ['1.1', '2.0'] - }, - 'matrix': ['arch', 'ver'], - 'exclude': { - 'variables': { - 'arch': ['x86_64_v4'], - 'ver': ['2.0'], - }, - 'matrix': ['arch', 'ver'], - } + spack_dict['packages']['basic'] = { + 'spack_spec': 'basic@1.1', + 'compiler': 'foo_comp' } spack_dict['environments'] = { 'basic': { - 'packages': [ - 'basic-1.1-x86_64', - 'basic-2.0-x86_64', - 'basic-1.1-x86_64_v4', - ] + 'packages': ['basic'] } } software_environments = ramble.software_environments.SoftwareEnvironments(ws) - assert len(software_environments._packages.keys()) == 3 - assert 'basic-1.1-x86_64' in software_environments._packages.keys() - assert 'basic-2.0-x86_64' in software_environments._packages.keys() - assert 'basic-1.1-x86_64_v4' in software_environments._packages.keys() - assert 'basic-2.0-x86_64_v4' not in software_environments._packages.keys() - assert 'basic' in software_environments._environments.keys() - assert 'basic-1.1-x86_64' in software_environments._environments['basic']['packages'] - assert 'basic-2.0-x86_64' in software_environments._environments['basic']['packages'] - assert 'basic-1.1-x86_64_v4' in software_environments._environments['basic']['packages'] - assert 'basic-2.0-x86_64_v4' not in \ - software_environments._environments['basic']['packages'] - - -def test_environment_vector_expansion_exclusion(mutable_mock_workspace_path): - ws_name = 'test_package_vector_expansion_exclusions' - workspace('create', ws_name) + assert 'basic' in software_environments._environment_templates + assert 'basic' in software_environments._package_templates - assert ws_name in workspace('list') + variables = {} + env_expander = ramble.expander.Expander(variables, None) - with ramble.workspace.read(ws_name) as ws: - spack_dict = ws.get_spack_dict() + with pytest.raises(ramble.software_environments.RambleSoftwareEnvironmentError) \ + as comp_err: + _ = software_environments.render_environment('basic', env_expander) + assert 'Compiler foo_comp used, but not defined' in str(comp_err) - spack_dict['packages'] = {} - spack_dict['packages']['basic-{arch}'] = { - 'spack_spec': 'basic@1.1 target={arch}', - 'variables': { - 'arch': ['x86_64', 'x86_64_v4'] - }, - } - spack_dict['environments'] = { - 'basic-{arch}': { - 'packages': ['basic-{arch}'], - 'variables': { - 'arch': ['x86_64', 'x86_64_v4'] - }, - 'exclude': { - 'variables': { - 'arch': 'x86_64_v4' - } - } - } - } - - software_environments = ramble.software_environments.SoftwareEnvironments(ws) - - assert len(software_environments._packages.keys()) == 2 - assert len(software_environments._environments.keys()) == 1 - assert 'basic-x86_64' in software_environments._packages.keys() - assert 'basic-x86_64_v4' in software_environments._packages.keys() - assert 'basic-x86_64' in software_environments._environments.keys() - assert 'basic-x86_64_v4' not in software_environments._environments.keys() - assert 'basic-x86_64' in software_environments._environments['basic-x86_64']['packages'] +def test_compiler_in_environment_warns(request, mutable_mock_workspace_path, capsys): + ws_name = request.node.name -def test_environment_with_missing_package_errors(mutable_mock_workspace_path, capsys): - ws_name = 'test_environment_with_missing_package_errors' workspace('create', ws_name) assert ws_name in workspace('list') @@ -841,20 +401,29 @@ def test_environment_with_missing_package_errors(mutable_mock_workspace_path, ca spack_dict = ws.get_spack_dict() spack_dict['packages'] = {} + spack_dict['packages']['test_comp'] = { + 'spack_spec': 'comp@2.1' + } spack_dict['packages']['basic'] = { 'spack_spec': 'basic@1.1', + 'compiler': 'test_comp' } spack_dict['environments'] = { 'basic': { - 'packages': ['basic', 'undefined_package'] + 'packages': ['basic', 'test_comp'] } } - with pytest.raises(SystemExit): - ramble.software_environments.SoftwareEnvironments(ws) - output = capsys.readouterr() + software_environments = ramble.software_environments.SoftwareEnvironments(ws) + + assert 'basic' in software_environments._environment_templates + assert 'basic' in software_environments._package_templates + + variables = {} + env_expander = ramble.expander.Expander(variables, None) - assert 'Error: Environment basic refers to the following packages' in output - assert 'undefined_package' in output - assert 'Please make sure all packages are defined before using this environment' \ - in output + _ = software_environments.render_environment('basic', env_expander) + captured = capsys.readouterr() + + assert 'Environment basic contains packages and their compilers' in captured.err + assert 'Package: basic, Compiler: test_comp' in captured.err diff --git a/lib/ramble/ramble/test/spack_runner.py b/lib/ramble/ramble/test/spack_runner.py index f22e0ccae..08bf900c6 100644 --- a/lib/ramble/ramble/test/spack_runner.py +++ b/lib/ramble/ramble/test/spack_runner.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -79,6 +79,7 @@ def test_env_concretize_skips_already_concretized_envs(tmpdir, capsys): sr.create_env(env_path) sr.activate() sr.add_spec('zlib') + sr.add_spec('intel-oneapi-mpi') # Generate an initial env file sr.generate_env_file() @@ -279,33 +280,35 @@ def test_new_compiler_installs(tmpdir, capsys): import os - compilers_config = """ + with tmpdir.as_cwd(): + compilers_config = """ compilers:: - compiler: spec: gcc@12.1.0 paths: - cc: /path/to/gcc - cxx: /path/to/g++ - f77: /path/to/gfortran - fc: /path/to/gfortran + cc: tmpdir_path/gcc + cxx: tmpdir_path/g++ + f77: tmpdir_path/gfortran + fc: tmpdir_path/gfortran flags: {} operating_system: 'ramble' target: 'x86_64' modules: [] environment: {} extra_rpaths: [] -""" +""".replace('tmpdir_path', os.path.join(os.getcwd(), 'bin')) - packages_config = """ + packages_config = f""" packages: gcc: externals: - spec: gcc@12.1.0 languages=c,fortran - prefix: /path/to + prefix: {os.getcwd()} buildable: false """ - with tmpdir.as_cwd(): + os.mkdir(os.path.join(os.getcwd(), 'bin')) + packages_path = os.path.join(os.getcwd(), 'packages.yaml') compilers_path = os.path.join(os.getcwd(), 'compilers.yaml') # Write spack_configs @@ -528,3 +531,25 @@ def test_env_create_no_view(tmpdir): ) except ramble.spack_runner.RunnerError as e: pytest.skip('%s' % e) + + +def test_multiword_args(tmpdir, capsys): + try: + env_path = tmpdir.join('spack-env') + with ramble.config.override('config:spack', + {'install': {'flags': 'install="-multiword -args"'}}): + sr = ramble.spack_runner.SpackRunner(dry_run=True) + sr.create_env(env_path) + sr.activate() + sr.add_spec('zlib') + sr.concretize() + + sr.install() + sr.get_package_path('zlib package_path="-multiword -args"') + captured = capsys.readouterr() + print(captured.out) + + assert "install=-multiword -args" in captured.out + assert "package_path=-multiword -args" in captured.out + except ramble.spack_runner.RunnerError as e: + pytest.skip('%s' % e) diff --git a/lib/ramble/ramble/test/spec_basic.py b/lib/ramble/ramble/test/spec_basic.py index 5725afd9b..1f63cab09 100644 --- a/lib/ramble/ramble/test/spec_basic.py +++ b/lib/ramble/ramble/test/spec_basic.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/stage.py b/lib/ramble/ramble/test/stage.py index 973291bb3..2d9babfbe 100644 --- a/lib/ramble/ramble/test/stage.py +++ b/lib/ramble/ramble/test/stage.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/success_criteria.py b/lib/ramble/ramble/test/success_criteria.py index ee186090a..a75b11955 100644 --- a/lib/ramble/ramble/test/success_criteria.py +++ b/lib/ramble/ramble/test/success_criteria.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/success_criteria/always_print_foms.py b/lib/ramble/ramble/test/success_criteria/always_print_foms.py index 25f0e3094..d3bb59bc7 100644 --- a/lib/ramble/ramble/test/success_criteria/always_print_foms.py +++ b/lib/ramble/ramble/test/success_criteria/always_print_foms.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -43,7 +43,6 @@ def test_always_print_foms(mutable_config, variables: n_nodes: 1 spack: - concretized: true packages: {} environments: {} """ diff --git a/lib/ramble/ramble/test/success_criteria/repeat_success_strict.py b/lib/ramble/ramble/test/success_criteria/repeat_success_strict.py index acd1db423..0546eb230 100644 --- a/lib/ramble/ramble/test/success_criteria/repeat_success_strict.py +++ b/lib/ramble/ramble/test/success_criteria/repeat_success_strict.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -46,7 +46,6 @@ def test_repeat_success_strict(mutable_config, variables: n_nodes: 1 spack: - concretized: true packages: {} environments: {} """ diff --git a/lib/ramble/ramble/test/success_criteria/success_fom_comparison.py b/lib/ramble/ramble/test/success_criteria/success_fom_comparison.py index 076de0ef7..54395f96e 100644 --- a/lib/ramble/ramble/test/success_criteria/success_fom_comparison.py +++ b/lib/ramble/ramble/test/success_criteria/success_fom_comparison.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/success_criteria/success_fom_globbing.py b/lib/ramble/ramble/test/success_criteria/success_fom_globbing.py index a213a1f0a..975055531 100644 --- a/lib/ramble/ramble/test/success_criteria/success_fom_globbing.py +++ b/lib/ramble/ramble/test/success_criteria/success_fom_globbing.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/success_criteria/success_functions.py b/lib/ramble/ramble/test/success_criteria/success_functions.py index e4eeadff6..796e66f94 100644 --- a/lib/ramble/ramble/test/success_criteria/success_functions.py +++ b/lib/ramble/ramble/test/success_criteria/success_functions.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -43,7 +43,6 @@ def test_success_function(mutable_config, variables: n_nodes: 1 spack: - concretized: true packages: {} environments: {} """ diff --git a/lib/ramble/ramble/test/success_criteria/success_modifiers.py b/lib/ramble/ramble/test/success_criteria/success_modifiers.py index be494b404..634e9a878 100644 --- a/lib/ramble/ramble/test/success_criteria/success_modifiers.py +++ b/lib/ramble/ramble/test/success_criteria/success_modifiers.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/success_criteria/success_variable_fom_comparison.py b/lib/ramble/ramble/test/success_criteria/success_variable_fom_comparison.py index 4c2b5e058..56546faf2 100644 --- a/lib/ramble/ramble/test/success_criteria/success_variable_fom_comparison.py +++ b/lib/ramble/ramble/test/success_criteria/success_variable_fom_comparison.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/util/editor.py b/lib/ramble/ramble/test/util/editor.py index 193a5b785..9952cb425 100644 --- a/lib/ramble/ramble/test/util/editor.py +++ b/lib/ramble/ramble/test/util/editor.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/util/env.py b/lib/ramble/ramble/test/util/env.py index 011793384..65e03afe3 100644 --- a/lib/ramble/ramble/test/util/env.py +++ b/lib/ramble/ramble/test/util/env.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/util/stats.py b/lib/ramble/ramble/test/util/stats.py index 4336d0658..5bd494772 100644 --- a/lib/ramble/ramble/test/util/stats.py +++ b/lib/ramble/ramble/test/util/stats.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/test/workspace_hashing/unsetup_workspace_cannot_analyze.py b/lib/ramble/ramble/test/workspace_hashing/unsetup_workspace_cannot_analyze.py index de4b94b28..b8c072ab8 100644 --- a/lib/ramble/ramble/test/workspace_hashing/unsetup_workspace_cannot_analyze.py +++ b/lib/ramble/ramble/test/workspace_hashing/unsetup_workspace_cannot_analyze.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -48,7 +48,6 @@ def test_unsetup_workspace_cannot_analyze(mutable_config, set: MY_VAR: 'TEST' spack: - concretized: true packages: zlib: spack_spec: zlib diff --git a/lib/ramble/ramble/test/workspace_hashing/workspace_name_does_not_change_hash.py b/lib/ramble/ramble/test/workspace_hashing/workspace_name_does_not_change_hash.py index 8e946384d..6500c16a7 100644 --- a/lib/ramble/ramble/test/workspace_hashing/workspace_name_does_not_change_hash.py +++ b/lib/ramble/ramble/test/workspace_hashing/workspace_name_does_not_change_hash.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -46,7 +46,6 @@ def test_workspace_name_does_not_change_hash(mutable_config, set: MY_VAR: 'TEST' spack: - concretized: true packages: {} environments: {} """ diff --git a/lib/ramble/ramble/test/workspace_hashing/workspace_setup_creates_inventory.py b/lib/ramble/ramble/test/workspace_hashing/workspace_setup_creates_inventory.py index be1fed13c..eb3bfa8a9 100644 --- a/lib/ramble/ramble/test/workspace_hashing/workspace_setup_creates_inventory.py +++ b/lib/ramble/ramble/test/workspace_hashing/workspace_setup_creates_inventory.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -47,7 +47,6 @@ def test_workspace_setup_creates_inventory(mutable_config, set: MY_VAR: 'TEST' spack: - concretized: true packages: {} environments: {} """ diff --git a/lib/ramble/ramble/test/workspace_tests.py b/lib/ramble/ramble/test/workspace_tests.py index fe3167f92..307df48d9 100644 --- a/lib/ramble/ramble/test/workspace_tests.py +++ b/lib/ramble/ramble/test/workspace_tests.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/util/__init__.py b/lib/ramble/ramble/util/__init__.py index 60cd9ddff..8fa4e1588 100644 --- a/lib/ramble/ramble/util/__init__.py +++ b/lib/ramble/ramble/util/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/util/class_attributes.py b/lib/ramble/ramble/util/class_attributes.py new file mode 100644 index 000000000..255a64f69 --- /dev/null +++ b/lib/ramble/ramble/util/class_attributes.py @@ -0,0 +1,24 @@ +# Copyright 2022-2024 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +def convert_class_attributes(obj): + """Convert class attributes defined from directives to instance attributes + Class attributes that are valid for conversion are stored in the _directive_names + attribute. + + Args: + obj (Object): Input object instance to convert attributes in + """ + + if hasattr(obj, '_directive_names'): + dir_set = dir(obj) + var_set = vars(obj) + for attr in obj._directive_names: + if attr in dir_set and attr not in var_set: + inst_val = getattr(obj, attr).copy() + setattr(obj, attr, inst_val) diff --git a/lib/ramble/ramble/util/colors.py b/lib/ramble/ramble/util/colors.py index 3f71117ee..0bff40ff1 100644 --- a/lib/ramble/ramble/util/colors.py +++ b/lib/ramble/ramble/util/colors.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -15,6 +15,19 @@ plain_format = '@.' +def level_func(level): + if level <= 0: + return section_title + elif level == 1: + return nested_1 + elif level == 2: + return nested_2 + elif level == 3: + return nested_3 + elif level >= 4: + return nested_4 + + def config_title(s): return config_color + s + plain_format diff --git a/lib/ramble/ramble/util/directives.py b/lib/ramble/ramble/util/directives.py index b8dd37acc..14c4f2185 100644 --- a/lib/ramble/ramble/util/directives.py +++ b/lib/ramble/ramble/util/directives.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/util/editor.py b/lib/ramble/ramble/util/editor.py index 3a919e0fa..6ac3ae2a5 100644 --- a/lib/ramble/ramble/util/editor.py +++ b/lib/ramble/ramble/util/editor.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/util/env.py b/lib/ramble/ramble/util/env.py index 58909f826..6a01eb7b8 100644 --- a/lib/ramble/ramble/util/env.py +++ b/lib/ramble/ramble/util/env.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/util/executable.py b/lib/ramble/ramble/util/executable.py index 423e45196..3b289e98b 100644 --- a/lib/ramble/ramble/util/executable.py +++ b/lib/ramble/ramble/util/executable.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/util/file_cache.py b/lib/ramble/ramble/util/file_cache.py index 5fef5ff89..6fd794d36 100644 --- a/lib/ramble/ramble/util/file_cache.py +++ b/lib/ramble/ramble/util/file_cache.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/util/graph.py b/lib/ramble/ramble/util/graph.py new file mode 100644 index 000000000..44af1c28c --- /dev/null +++ b/lib/ramble/ramble/util/graph.py @@ -0,0 +1,91 @@ +# Copyright 2022-2024 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + + +class GraphNode(object): + """Class representing a node of a graph, where the node can have an + attribute attached to it. + + This allows nodes to be added into a graph, have the topological order of + the graph returned, and be able to refer to the attribute of the original + node easily. + """ + def __init__(self, key, attribute=None, obj_inst=None): + """Construct a graph node + + Args: + key: The key for the graph node. This is what will be used to sort the graph. + attribute: A list of arbitrary attribute to keep associated with the key + """ + self.key = key + self.attribute = attribute + self._order_before = [] + self._order_after = [] + self.obj_inst = obj_inst + + def set_attribute(self, attr): + """Sets the attribute of a graph node + + Args: + attr: An arbitrary attribute to attach to this node. + """ + self.attribute = attr + + def order_before(self, key): + """Adds information that this node should come before another node + + Args: + key (str): Key of node that should come after this node. + """ + + self._order_before.append(key) + + def order_after(self, key): + """Adds information that this node should come after another node + + Args: + key (str): Key of node that should come before this node. + """ + + self._order_after.append(key) + + def __repr__(self): + """Return a string representation of the node + + Returns: + str: Text representation of the node + """ + return f'{self.key}' + + def __str__(self): + """Return a string representation of the node + + Returns: + str: Text representation of the node + """ + return f'{self.key}' + + def __hash__(self): + """Hash a node based on it's key + + Returns: + str: hash of the node's key + """ + return hash(self.key) + + def __eq__(self, other): + """Equivalence test of nodes + + Returns: + bool: True if nodes keys are the same, False otherwise. + """ + + if not isinstance(other, GraphNode): + return False + + return self.key == other.key diff --git a/lib/ramble/ramble/util/hashing.py b/lib/ramble/ramble/util/hashing.py index 5565619f9..00bfc73ac 100644 --- a/lib/ramble/ramble/util/hashing.py +++ b/lib/ramble/ramble/util/hashing.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/util/imp/__init__.py b/lib/ramble/ramble/util/imp/__init__.py index 9ad9a5144..c98779809 100644 --- a/lib/ramble/ramble/util/imp/__init__.py +++ b/lib/ramble/ramble/util/imp/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/util/imp/imp_importer.py b/lib/ramble/ramble/util/imp/imp_importer.py index 2cc189e77..e573f8088 100644 --- a/lib/ramble/ramble/util/imp/imp_importer.py +++ b/lib/ramble/ramble/util/imp/imp_importer.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/util/imp/importlib_importer.py b/lib/ramble/ramble/util/imp/importlib_importer.py index 7bc815324..3d0436a06 100644 --- a/lib/ramble/ramble/util/imp/importlib_importer.py +++ b/lib/ramble/ramble/util/imp/importlib_importer.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/util/install_cache.py b/lib/ramble/ramble/util/install_cache.py index b7092411b..aac31bb7d 100644 --- a/lib/ramble/ramble/util/install_cache.py +++ b/lib/ramble/ramble/util/install_cache.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/util/lock.py b/lib/ramble/ramble/util/lock.py index 5a06a76eb..77585d5ce 100644 --- a/lib/ramble/ramble/util/lock.py +++ b/lib/ramble/ramble/util/lock.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/util/logger.py b/lib/ramble/ramble/util/logger.py index 992fb4db5..b44c7184f 100644 --- a/lib/ramble/ramble/util/logger.py +++ b/lib/ramble/ramble/util/logger.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -11,6 +11,7 @@ import llnl.util.tty.color from contextlib import contextmanager +from pathlib import Path class Logger(object): @@ -35,7 +36,7 @@ def __init__(self): def add_log(self, path): """Add a log to the current log stack - Opens (with 'w+' permissions) the file provided by the 'path' argument, + Opens (with 'a+' permissions) the file provided by the 'path' argument, and stores both the path, and the opened stream object in the current stack in the active position. @@ -43,7 +44,7 @@ def add_log(self, path): path: File path for the new log file """ if isinstance(path, str) and self.enabled: - stream = None + Path(path).parent.mkdir(parents=True, exist_ok=True) stream = llnl.util.tty.log.Unbuffered(open(path, 'a+')) self.log_stack.append((path, stream)) diff --git a/lib/ramble/ramble/util/matrices.py b/lib/ramble/ramble/util/matrices.py index 7b82ba906..a94c72d53 100644 --- a/lib/ramble/ramble/util/matrices.py +++ b/lib/ramble/ramble/util/matrices.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/util/naming.py b/lib/ramble/ramble/util/naming.py index 2cae679d9..e90da211d 100644 --- a/lib/ramble/ramble/util/naming.py +++ b/lib/ramble/ramble/util/naming.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/util/path.py b/lib/ramble/ramble/util/path.py index c0a33d920..5c6451477 100644 --- a/lib/ramble/ramble/util/path.py +++ b/lib/ramble/ramble/util/path.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/util/spec_utils.py b/lib/ramble/ramble/util/spec_utils.py index 03efad22f..7248515f6 100644 --- a/lib/ramble/ramble/util/spec_utils.py +++ b/lib/ramble/ramble/util/spec_utils.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/util/stats.py b/lib/ramble/ramble/util/stats.py index eae64eace..593505501 100644 --- a/lib/ramble/ramble/util/stats.py +++ b/lib/ramble/ramble/util/stats.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/util/web.py b/lib/ramble/ramble/util/web.py index f1d233b25..d38ae5172 100644 --- a/lib/ramble/ramble/util/web.py +++ b/lib/ramble/ramble/util/web.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/util/yaml_generation.py b/lib/ramble/ramble/util/yaml_generation.py new file mode 100644 index 000000000..0d89ca7c5 --- /dev/null +++ b/lib/ramble/ramble/util/yaml_generation.py @@ -0,0 +1,158 @@ +# Copyright 2022-2024 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +"""Module representing utility functions for managing YAML configuration files +within application definition files + +These functions are intended to help read, write, and manipulate YAML based +configuration files that an experiment might use as input. + +Workload variables that represent configuration files should be defined using a +'.' delimiter between YAML object names. As an example: + +``` +foo: + bar: + baz: 1.0 +``` + +Would translate to `foo.bar.baz = 1.0` in Ramble syntax. + +""" + +from typing import Dict, Any +import ruamel.yaml as yaml + +from ramble.util.logger import logger + + +def read_config_file(conf_path: str): + """Read an existing YAML file and return its data as a dictionary + + Args: + conf_path (str): Path to input configuration file to read + + Returns: + (dict): Dictionary representation of the data contained in conf_path + """ + with open(conf_path, 'r') as base_conf: + logger.debug(f'Reading config from {conf_path}') + try: + config_dict = yaml.safe_load(base_conf) + except yaml.YAMLError: + logger.die(f'YAML Error: Failed to load data from {conf_path}') + + return config_dict + + +def all_config_options(config_data: Dict): + """Extract all config options from config_data dictionary + + Args: + config_data (dict): A config dictionary representing data read from a YAML file. + + Returns: + (set): Set containing all detected fully qualified option names + """ + + all_configs = set() + option_parts = [] + for top_level in config_data: + option_parts.append((top_level, config_data[top_level])) + + while option_parts: + cur_part = option_parts.pop(0) + + if isinstance(cur_part[1], dict): + for level in cur_part[1]: + option_parts.insert(0, (f'{cur_part[0]}.{level}', cur_part[1][level])) + else: + if len(cur_part[0].split('.')) > 1: + all_configs.add(cur_part[0]) + + return all_configs + + +def get_config_value(config_data: Dict, option_name: str): + """Get a config option based on dictionary attribute syntax + + Given an option_name of the format: attr1.attr2.attr3 return its value + from config_data. + + Args: + config_data (dict): A config dictionary representing data read from a YAML file. + option_name (str): Name of config option to get + + Returns: + (Any): Value of config option + """ + option_parts = option_name.split('.') + + option_scope = config_data + + while len(option_parts) > 1: + cur_part = option_parts.pop(0) + if cur_part in option_scope: + option_scope = option_scope[cur_part] + else: + return None + + if option_parts[0] in option_scope: + return option_scope[option_parts[0]] + return None + + +def set_config_value(config_data: Dict, option_name: str, option_value: Any): + """Set a config option based on dictionary attribute syntax + + Given an option_name of the format: attr1.attr2.attr3 set its value to + option_value in config_data. + + Args: + config_data (dict): A config dictionary representing data read from a YAML file. + option_name (str): Name of config option to set + option_value (any): Value to set config option to + """ + option_parts = option_name.split('.') + + option_scope = config_data + + while len(option_parts) > 1: + cur_part = option_parts.pop(0) + if cur_part not in option_scope: + return + option_scope = option_scope[cur_part] + + if option_parts[0] in option_scope: + option_scope[option_parts[0]] = option_value + + +def apply_default_config_values(config_data, app_inst, default_config_string): + """Apply default config values (from config_data) to an experiment + + Process all workloads variables (for the current workload in app_inst). Any + variable who's expanded value is equal to default_config_string will have + its value overwritten to the value in the config_dict dictionary. + + Args: + config_data (dict): Dictionary of config data read from a YAML file + app_inst (application): Application instance representing an experiment + default_config_string (str): String that conveys the default config_data + should be used in place of the current value. + """ + workload = app_inst.workloads[app_inst.expander.workload_name] + + # Set all '{default_config_value}' values to value from the base config + for var_name, var_def in workload.variables.items(): + if len(var_name.split('.')) > 1: + var_val = app_inst.expander.expand_var(app_inst.expander.expansion_str(var_name)) + + if var_val == default_config_string: + var_val = get_config_value(config_data, var_name) + + app_inst.define_variable(var_name, var_val) diff --git a/lib/ramble/ramble/workload.py b/lib/ramble/ramble/workload.py new file mode 100644 index 000000000..b8a715852 --- /dev/null +++ b/lib/ramble/ramble/workload.py @@ -0,0 +1,265 @@ +# Copyright 2022-2024 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +from typing import List +import ramble.util.colors as rucolor + + +class WorkloadVariable(object): + """Class representing a variable definition""" + + def __init__(self, name: str, default=None, description: str = None, + values=None, expandable: bool = True, **kwargs): + """Constructor for a new variable + + Args: + name (str): Name of variable + default: Default value of variable + description (str): Description of variable + values: List of suggested values for variable + expandable (bool): True if variable can be expanded, False otherwise + """ + self.name = name + self.default = default + self.description = description + self.values = values.copy() if isinstance(values, list) else [values] + self.expandable = expandable + + def as_str(self, n_indent: int = 0): + """String representation of this variable + + Args: + n_indent (int): Number of spaces to indent string lines with + + Returns: + (str): Representation of this variable + """ + indentation = ' ' * n_indent + + print_attrs = ['Description', 'Default', 'Values'] + + out_str = rucolor.nested_2(f'{indentation}{self.name}:\n') + for print_attr in print_attrs: + name = print_attr + if print_attr == 'Values': + name = 'Suggested Values' + attr_name = print_attr.lower() + + attr_val = getattr(self, attr_name, None) + if attr_val: + out_str += f'{indentation} {name}: {str(attr_val).replace("@", "@@")}\n' + return out_str + + +class WorkloadEnvironmentVariable(object): + """Class representing an environment variable in a workload""" + + def __init__(self, name: str, value=None, description: str = None): + """WorkloadEnvironmentVariable constructor + + Args: + name (str): Name of environment variable + value: Value to set environment variable to + description (str): Description of the environment variable + """ + self.name = name + self.value = value + self.description = description + + def as_str(self, n_indent: int = 0): + """String representation of environment variable + + Args: + n_indent (int): Number of spaces to indent string representation by + + Returns: + (str): String representing this environment variable + """ + indentation = ' ' * n_indent + + print_attrs = ['Description', 'Value'] + + out_str = rucolor.nested_2(f'{indentation}{self.name}:\n') + for name in print_attrs: + attr_name = name.lower() + attr_val = getattr(self, attr_name, None) + if attr_val: + out_str += f'{indentation} {name}: {attr_val.replace("@", "@@")}\n' + return out_str + + +class Workload(object): + """Class representing a single workload""" + + def __init__(self, name: str, executables: List[str], + inputs: List[str] = [], tags: List[str] = []): + """Constructor for a workload + + Args: + name (str): Name of this workload + executables (list(str)): List of executable names for this workload + inputs (list(str)): List of input names for this workload + tags (list(str)): List of tags for this workload + """ + self.name = name + self.variables = {} + self.environment_variables = {} + + attr_names = ['executables', 'inputs', 'tags'] + attr_vals = [executables, inputs, tags] + + for attr, vals in zip(attr_names, attr_vals): + if isinstance(vals, list): + setattr(self, attr, vals.copy()) + else: + attr_val = [] + if vals: + attr_val.append(vals) + setattr(self, attr, attr_val) + + def as_str(self, n_indent: int = 0): + """String representation of this workload + + Args: + n_indent (int): Number of spaces to indent string with + + Returns: + (str): Representation of this workload + """ + attrs = [('Executables', 'executables'), + ('Inputs', 'inputs'), + ('Tags', 'tags')] + + indentation = ' ' * n_indent + + out_str = rucolor.section_title(f'{indentation}Workload: ') + out_str += f'{self.name}\n' + for attr in attrs: + out_str += rucolor.nested_1(f'{indentation} {attr[0]}: ') + attr_val = getattr(self, attr[1], []) + out_str += f'{attr_val}\n' + + if self.variables: + out_str += rucolor.nested_1(f'{indentation} Variables:\n') + for name, var in self.variables.items(): + out_str += var.as_str(n_indent + 4) + + if self.environment_variables: + out_str += rucolor.nested_1(f'{indentation} Environment Variables:\n') + for name, env_var in self.environment_variables.items(): + out_str += env_var.as_str(n_indent + 4) + + return out_str + + def add_variable(self, variable: WorkloadVariable): + """Add a variable to this workload + + Args: + variable (WorkloadVariable): New variable to add to this workload + """ + self.variables[variable.name] = variable + + def add_environment_variable(self, env_var: WorkloadEnvironmentVariable): + """Add an environment variable to this workload + + Args: + env_var (WorkloadEnvironmentVariable): New environment variable to add to this workload + """ + self.environment_variables[env_var.name] = env_var + + def add_executable(self, executable: str): + """Add an executable to this workload + + Args: + executable (str): Name of executable to add to this workload + """ + self.executables.append(executable) + + def add_input(self, input: str): + """Add an input to this workload + + Args: + input (str): Name of input to add to this workload + """ + self.inputs.append(input) + + def add_tag(self, tag: str): + """Add a tag to this workload + + Args: + tag (str): Tag to add to this workload + """ + self.tags.append(tag) + + def is_valid(self): + """Test if this workload is considered valid + + Returns: + (bool): True if workload is valid, False otherwise + """ + if len(self.executables) == 0: + return False + + return True + + def find_executable(self, exec_name: str): + """Find an executable in this workload + + Args: + exec_name (str): Name of executable to find + + Returns: + (str / None): Name of executable if it exists, None if it is not found + """ + for executable in self.executables: + if executable == exec_name: + return executable + return None + + def find_input(self, input_name): + """Find an input in this workload + + Args: + input_name (str): Name of input to find + + Returns: + (str / None): Name of input if it exists, None if it is not found + """ + for input in self.inputs: + if input == input_name: + return input + return None + + def find_variable(self, name): + """Find a variable in this workload + + Args: + var_name (str): Name of variable to find + + Returns: + (WorkloadVariable / None): Variable instance if it exists, None if it is not found + """ + if name in self.variables: + return self.variables[name] + else: + return None + + def find_environment_variable(self, name): + """Find an environment variable in this workload + + Args: + env_var_name (str): Name of environment variable to find + + Returns: + (WorkloadEnvironmentVariable / None): Environment variable instance + if it exists, None if it is not found + """ + if name in self.environment_variables: + return self.environment_variables[name] + else: + return None diff --git a/lib/ramble/ramble/workspace/__init__.py b/lib/ramble/ramble/workspace/__init__.py index 95e9638a6..e6e54fd26 100644 --- a/lib/ramble/ramble/workspace/__init__.py +++ b/lib/ramble/ramble/workspace/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -13,6 +13,7 @@ Workspace, RambleWorkspaceError, RambleConflictingDefinitionError, + RambleActiveWorkspaceError, RambleMissingApplicationError, RambleMissingWorkloadError, RambleMissingExperimentError, @@ -50,6 +51,7 @@ 'Workspace', 'RambleWorkspaceError', 'RambleConflictingDefinitionError', + 'RambleActiveWorkspaceError', 'RambleMissingApplicationError', 'RambleMissingWorkloadError', 'RambleMissingExperimentError', diff --git a/lib/ramble/ramble/workspace/shell.py b/lib/ramble/ramble/workspace/shell.py index 522ae539f..067cf7f3a 100644 --- a/lib/ramble/ramble/workspace/shell.py +++ b/lib/ramble/ramble/workspace/shell.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/lib/ramble/ramble/workspace/workspace.py b/lib/ramble/ramble/workspace/workspace.py index e47f3e7be..42b10f974 100644 --- a/lib/ramble/ramble/workspace/workspace.py +++ b/lib/ramble/ramble/workspace/workspace.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -84,6 +84,9 @@ #: Name of the subdirectory where shared/sourale files are stored workspace_shared_license_path = 'licenses' +#: Name of the subdirectory where deployments are stored +workspace_deployments_path = 'deployments' + #: regex for validating workspace names valid_workspace_name_re = r'^\w[\w-]*$' @@ -113,8 +116,6 @@ def default_config_yaml(): # experiments. # As an example, experiments can be defined as follows. # applications: -# variables: -# processes_per_node: '30' # hostname: # Application name, as seen in `ramble list` # variables: # iterations: '5' @@ -137,7 +138,6 @@ def default_config_yaml(): processes_per_node: -1 applications: {} spack: - concretized: false packages: {} environments: {} """ @@ -455,6 +455,7 @@ def __init__(self, root, dry_run=False, read_default_template=True): self.dry_run = dry_run self.always_print_foms = False self.repeat_success_strict = True + self.force_concretize = False self.read_default_template = read_default_template self.configs = ramble.config.ConfigScope('workspace', self.config_dir) @@ -467,7 +468,7 @@ def __init__(self, root, dry_run=False, read_default_template=True): self.input_mirror_stats = None self.input_mirror_cache = None self.software_mirror_cache = None - self._software_environments = None + self.software_environments = None self.hash_inventory = { 'experiments': [], 'versions': [] @@ -503,6 +504,10 @@ def __init__(self, root, dry_run=False, read_default_template=True): self.install_cache = ramble.util.install_cache.SetCache() + # A dict mapping (concretized) package spec to its install prefix. + # This can be re-used by all experiments of the workspace. + self.pkg_path_cache = {} + self.results = self.default_results() self.success_list = ramble.success_criteria.ScopedCriteriaList() @@ -524,6 +529,8 @@ def __init__(self, root, dry_run=False, read_default_template=True): # Create a logger to redirect certain prints from screen to log file self.logger = log.log_output(echo=False, debug=tty.debug_level()) + self.deployment_name = self.name + def _re_read(self): """Reinitialize the workspace object if it has been written (this may not be true if the workspace was just created in this running @@ -861,13 +868,17 @@ def external_spack_env(self, env_name): def concretize(self): spack_dict = self.get_spack_dict() - if 'concretized' in spack_dict and spack_dict['concretized']: - raise RambleWorkspaceError('Cannot conretize an ' + - 'already concretized ' + - 'workspace') + if not self.force_concretize: + try: + if spack_dict[namespace.packages] or spack_dict[namespace.environments]: + raise RambleWorkspaceError('Cannot concretize an already concretized ' + 'workspace. To overwrite the current configuration ' + 'with the default software configuration, use ' + '\'ramble workspace concretize -f\'.') + except KeyError: + pass spack_dict = syaml.syaml_dict() - spack_dict['concretized'] = False if namespace.packages not in spack_dict or \ not spack_dict[namespace.packages]: @@ -889,9 +900,9 @@ def concretize(self): env_name_str = app_inst.expander.expansion_str(ramble.keywords.keywords.env_name) env_name = app_inst.expander.expand_var(env_name_str) - compiler_dicts = [app_inst.default_compilers] + compiler_dicts = [app_inst.compilers] for mod_inst in app_inst._modifier_instances: - compiler_dicts.append(mod_inst.default_compilers) + compiler_dicts.append(mod_inst.compilers) for compiler_dict in compiler_dicts: for comp, info in compiler_dict.items(): @@ -949,7 +960,6 @@ def concretize(self): if spec_name not in app_packages: app_packages.append(spec_name) - spack_dict['concretized'] = True ramble.config.config.update_config('spack', spack_dict, scope=self.ws_file_config_scope_name()) @@ -1059,8 +1069,9 @@ def dump_results(self, output_formats=['text']): if exp['RAMBLE_STATUS'] == 'SUCCESS' or self.always_print_foms: if exp['N_REPEATS'] > 0: # this is a base exp with summary of repeats for context in exp['CONTEXTS']: - f.write(' %s figures of merit:\n' % - context['name']) + f.write( + f' {context["display_name"]} figures of merit:\n' + ) fom_summary = {} for fom in context['foms']: @@ -1080,7 +1091,9 @@ def dump_results(self, output_formats=['text']): f.write(f' {fom_val.strip()}\n') else: for context in exp['CONTEXTS']: - f.write(f' {context["name"]} figures of merit:\n') + f.write( + f' {context["display_name"]} figures of merit:\n' + ) for fom in context['foms']: name = fom['name'] if fom['origin_type'] == 'modifier': @@ -1120,6 +1133,12 @@ def dump_results(self, output_formats=['text']): for out_file in results_written: logger.all_msg(f' {out_file}') + # Debug print the first written result file. + # Directly use tty to avoid cluttering the analyze log. + if ramble.config.get('config:debug'): + with open(results_written[0], 'r') as f: + tty.debug(f'Results from the analysis pipeline:\n{f.read()}') + return filename_base def create_mirror(self, mirror_root): @@ -1142,6 +1161,62 @@ def create_mirror(self, mirror_root): self.input_mirror_cache = ramble.caches.MirrorCache(self.input_mirror_path) self.software_mirror_cache = ramble.caches.MirrorCache(self.software_mirror_path) + def simplify(self): + # First drop unused experiment templates from app dict so environments aren't rendered + app_dict = ramble.config.config.get_config(namespace.application, + scope=self.ws_file_config_scope_name()) + + # Build experiment sets to determine which templates never get used + self.software_environments = \ + ramble.software_environments.SoftwareEnvironments(self) + experiment_set = self.build_experiment_set() + + for _, app_inst in experiment_set.template_experiments(): + if app_inst.is_template and not app_inst.generated_experiments: + app = app_inst.expander.application_name + wl = app_inst.expander.workload_name + exp = app_inst.expander.experiment_name + + try: + app_dict[app][namespace.workload][wl][namespace.experiment].pop(exp) + if not app_dict[app][namespace.workload][wl][namespace.experiment]: + app_dict[app][namespace.workload][wl].pop(namespace.experiment) + if not app_dict[app][namespace.workload][wl]: + app_dict[app][namespace.workload].pop(wl) + if not app_dict[app][namespace.workload]: + app_dict[app].pop(namespace.workload) + if not app_dict[app]: + app_dict.pop(app) + except KeyError: + continue + + ramble.config.config.update_config(namespace.application, app_dict, + scope=self.ws_file_config_scope_name()) + + # Regenerate environments without the unused templates to see which env never get rendered + self.software_environments = \ + ramble.software_environments.SoftwareEnvironments(self) + software_environments = self.software_environments + experiment_set = self.build_experiment_set() + + spack_dict = ramble.config.config.get_config(namespace.spack, + scope=self.ws_file_config_scope_name()) + package_dict = spack_dict[namespace.packages] + environments_dict = spack_dict[namespace.environments] + + tty.debug('Removing configurations that do not spark joy.') + for pkg in software_environments.unused_packages(): + if pkg.name in package_dict: + tty.debug(f'Removing {pkg.name} from Spack packages') + package_dict.pop(pkg.name) + for env in software_environments.unused_environments(): + if env.name in environments_dict: + tty.debug(f'Removing {env.name} from Spack environments') + environments_dict.pop(env.name) + + ramble.config.config.update_config(namespace.spack, spack_dict, + scope=self.ws_file_config_scope_name()) + @property def latest_archive_path(self): return os.path.join(self.archive_dir, self.latest_archive) @@ -1246,6 +1321,16 @@ def shared_dir(self): """Path to the shared directory""" return os.path.join(self.root, workspace_shared_path) + @property + def deployments_dir(self): + """Path to the deployments directory""" + return os.path.join(self.root, workspace_deployments_path) + + @property + def named_deployment(self): + """Path to the specific deployment directory""" + return os.path.join(self.deployments_dir, self.deployment_name) + @property def shared_license_dir(self): """Path to the shared license directory""" @@ -1349,13 +1434,6 @@ def _get_application_dict_config(self, key): return self.application_configs[key]['yaml'] if key \ in self.application_configs else None - def is_concretized(self): - spack_dict = self.get_spack_dict() - if 'concretized' in spack_dict: - return (True if spack_dict['concretized'] - else False) - return False - def _get_workspace_section(self, section): """Return a dict of a workspace section""" workspace_dict = self._get_workspace_dict() @@ -1390,7 +1468,7 @@ def get_workspace_zips(self): def get_spack_dict(self): """Return the spack dictionary for this workspace""" - return ramble.config.config.get_config('spack') + return ramble.config.config.get_config(namespace.spack) def get_applications(self): """Get the dictionary of applications""" @@ -1511,6 +1589,10 @@ class RambleConflictingDefinitionError(RambleWorkspaceError): """Error when conflicting software definitions are found""" +class RambleActiveWorkspaceError(RambleWorkspaceError): + """Error when an invalid workspace is activated""" + + class RambleMissingApplicationError(RambleWorkspaceError): """Error when using an undefined application in an experiment specification""" diff --git a/lib/ramble/spack/util/debug.py b/lib/ramble/spack/util/debug.py index 64f7132a6..bdeee2df6 100644 --- a/lib/ramble/spack/util/debug.py +++ b/lib/ramble/spack/util/debug.py @@ -10,6 +10,7 @@ """ import code +import io import os import pdb import signal @@ -20,13 +21,13 @@ def debug_handler(sig, frame): """Interrupt running process, and provide a python prompt for interactive debugging.""" - d = {'_frame': frame} # Allow access to frame object. - d.update(frame.f_globals) # Unless shadowed by global + d = {"_frame": frame} # Allow access to frame object. + d.update(frame.f_globals) # Unless shadowed by global d.update(frame.f_locals) i = code.InteractiveConsole(d) - message = "Signal received : entering python shell.\nTraceback:\n" - message += ''.join(traceback.format_stack(frame)) + message = "Signal received : entering python shell.\nTraceback:\n" + message += "".join(traceback.format_stack(frame)) i.interact(message) os._exit(1) # Use os._exit to avoid test harness. @@ -53,7 +54,10 @@ class and use as a drop in for Pdb, although the syntax here is slightly differe the run of Spack.install, or any where else Spack spawns a child process. """ - _original_stdin_fd = sys.stdin.fileno() + try: + _original_stdin_fd = sys.stdin.fileno() + except io.UnsupportedOperation: + _original_stdin_fd = None _original_stdin = None def __init__(self, stdout_fd=None, stderr_fd=None): diff --git a/pytest.ini b/pytest.ini index 1c3b29023..26d30a740 100644 --- a/pytest.ini +++ b/pytest.ini @@ -14,3 +14,4 @@ markers = enable_compiler_verification: enable compiler verification within unit tests enable_compiler_link_paths: verifies compiler link paths within unit tests disable_clean_stage_check: avoid failing tests if there are leftover files in the stage area + long: mark test as long running diff --git a/requirements.txt b/requirements.txt index 8b6d2e468..d32316245 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,9 +4,11 @@ google-cloud-storage # for gcs fetch test google-api-core # for gcs fetch error .execptions coverage pre-commit +graphlib-backport;python_version<"3.9" urllib3==1.26.18;python_version<="3.6" protobuf;python_version>"3.6" protobuf==3.19.4;python_version<="3.6" pyarrow==3.0.0;python_version<="3.6" google-cloud-bigquery tqdm +deprecation diff --git a/share/ramble/bash/ramble-completion.in b/share/ramble/bash/ramble-completion.in index 48a8b9616..c803b6fd2 100755 --- a/share/ramble/bash/ramble-completion.in +++ b/share/ramble/bash/ramble-completion.in @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/share/ramble/cloud-build/Dockerfile-centos7 b/share/ramble/cloud-build/Dockerfile-yum similarity index 61% rename from share/ramble/cloud-build/Dockerfile-centos7 rename to share/ramble/cloud-build/Dockerfile-yum index 7eba5545c..6d6514214 100644 --- a/share/ramble/cloud-build/Dockerfile-centos7 +++ b/share/ramble/cloud-build/Dockerfile-yum @@ -1,20 +1,26 @@ -FROM centos:7 as builder +ARG BASE_IMG=centos +ARG BASE_VER=7 +FROM ${BASE_IMG}:${BASE_VER} as builder -RUN yum install -yq git python3 python3-pip wget mercurial which svn curl gcc && rm -rf /var/lib/apt/lists/* +ARG SPACK_REF=releases/latest +ARG CONDA_VER=4.10.3 +RUN yum install -yq git python3 python3-pip wget mercurial which svn curl gcc tar bzip2 && rm -rf /var/lib/apt/lists/* RUN cd /opt && \ - git clone https://github.com/spack/spack -b v0.19.2 && \ + git clone https://github.com/spack/spack && \ + cd spack && \ + git checkout $SPACK_REF && \ . /opt/spack/share/spack/setup-env.sh && \ - spack install miniconda3 && \ + spack install miniconda3@${CONDA_VER} && \ spack clean -a RUN echo -e "export PATH=$(. /opt/spack/share/spack/setup-env.sh && spack location -i miniconda3)/bin:${PATH}\n. /opt/spack/share/spack/setup-env.sh" > /etc/profile.d/ramble.sh RUN cd /opt && \ export PATH=$(. /opt/spack/share/spack/setup-env.sh && spack location -i miniconda3)/bin:${PATH} && \ . spack/share/spack/setup-env.sh && \ wget https://raw.githubusercontent.com/GoogleCloudPlatform/ramble/develop/requirements.txt && \ - conda install -qy --file /opt/requirements.txt && \ - conda clean -a + conda install -qy pip && \ + python -m pip install -r /opt/requirements.txt -FROM centos:7 +FROM ${BASE_IMG}:${BASE_VER} COPY --from=builder / / diff --git a/share/ramble/cloud-build/ramble-image-builder.yaml b/share/ramble/cloud-build/ramble-image-builder.yaml index f6aff5a58..7746172f1 100644 --- a/share/ramble/cloud-build/ramble-image-builder.yaml +++ b/share/ramble/cloud-build/ramble-image-builder.yaml @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -13,13 +13,27 @@ steps: args: - 'build' - '-t' - - 'us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-centos7:latest' + - 'us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-conda${_CONDA_VER}:latest' - '--cache-from' - - 'us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-centos7:latest' + - 'us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-conda${_CONDA_VER}:latest' - '-f' - - 'share/ramble/cloud-build/Dockerfile-centos7' + - 'share/ramble/cloud-build/Dockerfile-${_PKG_MANAGER}' + - '--build-arg' + - 'BASE_IMG=${_BASE_IMG}' + - '--build-arg' + - 'BASE_VER=${_BASE_VER}' + - '--build-arg' + - 'SPACK_REF=${_SPACK_REF}' + - '--build-arg' + - 'CONDA_VER=${_CONDA_VER}' - '.' -images: ['us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-centos7'] +substitutions: + _SPACK_REF: v0.21.2 + _CONDA_VER: 22.11.1 + _BASE_IMG: centos + _BASE_VER: '7' + _PKG_MANAGER: 'yum' +images: ['us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-conda${_CONDA_VER}'] timeout: 1500s options: machineType: N1_HIGHCPU_8 diff --git a/share/ramble/cloud-build/ramble-pr-software-conflicts.yaml b/share/ramble/cloud-build/ramble-pr-software-conflicts.yaml index e515cdcca..5d8c66f26 100644 --- a/share/ramble/cloud-build/ramble-pr-software-conflicts.yaml +++ b/share/ramble/cloud-build/ramble-pr-software-conflicts.yaml @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -13,7 +13,7 @@ steps: - fetch - '--unshallow' id: ramble-clone - - name: us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-centos7:latest + - name: us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-conda${_CONDA_VER}:latest args: - '-c' - | @@ -26,6 +26,9 @@ steps: . /opt/spack/share/spack/setup-env.sh . /workspace/share/ramble/setup-env.sh + echo "Spack version is $(spack --version)" + echo "Python version is $(python3 --version)" + ramble software-definitions -s ramble software-definitions -c -e @@ -43,7 +46,11 @@ steps: exit $$error id: ramble-style-tests entrypoint: /bin/bash - +substitutions: + _SPACK_REF: v0.21.2 + _CONDA_VER: 22.11.1 + _BASE_IMG: centos + _BASE_VER: '7' timeout: 600s options: machineType: N1_HIGHCPU_8 diff --git a/share/ramble/cloud-build/ramble-pr-style.yaml b/share/ramble/cloud-build/ramble-pr-style.yaml index 07a30d459..e941a0a83 100644 --- a/share/ramble/cloud-build/ramble-pr-style.yaml +++ b/share/ramble/cloud-build/ramble-pr-style.yaml @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -13,7 +13,7 @@ steps: - fetch - '--unshallow' id: ramble-clone - - name: us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-centos7:latest + - name: us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-conda${_CONDA_VER}:latest args: - '-c' - | @@ -26,6 +26,9 @@ steps: . /opt/spack/share/spack/setup-env.sh . /workspace/share/ramble/setup-env.sh + echo "Spack version is $(spack --version)" + echo "Python version is $(python3 --version)" + ramble flake8 -U # $$ characters are required for cloud-build: # https://cloud.google.com/build/docs/configuring-builds/substitute-variable-values @@ -87,7 +90,11 @@ steps: exit $$error id: ramble-style-tests entrypoint: /bin/bash - +substitutions: + _SPACK_REF: v0.21.2 + _CONDA_VER: 22.11.1 + _BASE_IMG: centos + _BASE_VER: '7' timeout: 600s options: machineType: N1_HIGHCPU_8 diff --git a/share/ramble/cloud-build/ramble-pr-unit-tests.yaml b/share/ramble/cloud-build/ramble-pr-unit-tests.yaml index 363178da8..ba8fbfbb9 100644 --- a/share/ramble/cloud-build/ramble-pr-unit-tests.yaml +++ b/share/ramble/cloud-build/ramble-pr-unit-tests.yaml @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -13,7 +13,7 @@ steps: - fetch - '--unshallow' id: ramble-clone - - name: us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-centos7:latest + - name: us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-conda${_CONDA_VER}:latest args: - '-c' - | @@ -26,6 +26,9 @@ steps: . /opt/spack/share/spack/setup-env.sh . /workspace/share/ramble/setup-env.sh + echo "Spack version is $(spack --version)" + echo "Python version is $(python3 --version)" + COVERAGE=true LONG=true /workspace/share/ramble/qa/run-unit-tests # $$ characters are required for cloud-build: # https://cloud.google.com/build/docs/configuring-builds/substitute-variable-values @@ -64,7 +67,11 @@ steps: exit $$error id: ramble-unit-tests entrypoint: /bin/bash - +substitutions: + _SPACK_REF: v0.21.2 + _CONDA_VER: 22.11.1 + _BASE_IMG: centos + _BASE_VER: '7' timeout: 900s options: machineType: N1_HIGHCPU_8 diff --git a/share/ramble/csh/pathadd.csh b/share/ramble/csh/pathadd.csh index 10ba594c2..cb6769c87 100644 --- a/share/ramble/csh/pathadd.csh +++ b/share/ramble/csh/pathadd.csh @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/share/ramble/csh/ramble.csh b/share/ramble/csh/ramble.csh index dd24d7212..4d7de1357 100644 --- a/share/ramble/csh/ramble.csh +++ b/share/ramble/csh/ramble.csh @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/share/ramble/qa/completion-test.sh b/share/ramble/qa/completion-test.sh index 2aaa312a2..a05a6d7c8 100755 --- a/share/ramble/qa/completion-test.sh +++ b/share/ramble/qa/completion-test.sh @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/share/ramble/qa/git-hooks/setup.sh b/share/ramble/qa/git-hooks/setup.sh index 8dcf111c8..063588cb1 100755 --- a/share/ramble/qa/git-hooks/setup.sh +++ b/share/ramble/qa/git-hooks/setup.sh @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/share/ramble/qa/run-flake8-tests b/share/ramble/qa/run-flake8-tests index e8c6e1f4a..5955712ee 100755 --- a/share/ramble/qa/run-flake8-tests +++ b/share/ramble/qa/run-flake8-tests @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -33,7 +33,9 @@ if [ $? != 0 ]; then fi # verify that the license headers are present -ramble license verify +# Only check for modified files as this is for pre-commit. +# The other PR check ramble-pr-style still verifies all files. +ramble license verify --modified if [ $? != 0 ]; then ERROR=1 fi diff --git a/share/ramble/qa/run-unit-tests b/share/ramble/qa/run-unit-tests index f8f237626..78824301c 100755 --- a/share/ramble/qa/run-unit-tests +++ b/share/ramble/qa/run-unit-tests @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/share/ramble/qa/setup.sh b/share/ramble/qa/setup.sh index 6a11816ed..a105dc3b8 100755 --- a/share/ramble/qa/setup.sh +++ b/share/ramble/qa/setup.sh @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/share/ramble/qa/test-framework.sh b/share/ramble/qa/test-framework.sh index 2ca7540de..637af71dd 100755 --- a/share/ramble/qa/test-framework.sh +++ b/share/ramble/qa/test-framework.sh @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/share/ramble/ramble-completion.bash b/share/ramble/ramble-completion.bash index bc490878d..d301c6e6a 100755 --- a/share/ramble/ramble-completion.bash +++ b/share/ramble/ramble-completion.bash @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -267,7 +267,7 @@ _ramble() { then RAMBLE_COMPREPLY="-h --help -H --all-help --color -c --config -C --config-scope -d --debug --disable-passthrough -N --disable-logger -P --disable-progress-bar --timestamp --pdb -w --workspace -D --workspace-dir -W --no-workspace --use-workspace-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -p --profile --sorted-profile --lines -v --verbose --stacktrace -V --version --print-shell-vars" else - RAMBLE_COMPREPLY="attributes clean commands config debug edit flake8 help info license list mirror mods on python repo results software-definitions unit-test workspace" + RAMBLE_COMPREPLY="attributes clean commands config debug deployment edit flake8 help info license list mirror mods on python repo results software-definitions unit-test workspace" fi } @@ -391,6 +391,23 @@ _ramble_debug_report() { RAMBLE_COMPREPLY="-h --help" } +_ramble_deployment() { + if $list_options + then + RAMBLE_COMPREPLY="-h --help" + else + RAMBLE_COMPREPLY="push pull" + fi +} + +_ramble_deployment_push() { + RAMBLE_COMPREPLY="-h --help --tar-archive -t --deployment-name -d --upload-url -u --phases --include-phase-dependencies --where --exclude-where --filter-tags" +} + +_ramble_deployment_pull() { + RAMBLE_COMPREPLY="-h --help --deployment-path -p" +} + _ramble_edit() { if $list_options then @@ -441,7 +458,7 @@ _ramble_license_list_files() { } _ramble_license_verify() { - RAMBLE_COMPREPLY="-h --help --root" + RAMBLE_COMPREPLY="-h --help --root --modified -m" } _ramble_list() { @@ -543,7 +560,7 @@ _ramble_mods_info() { } _ramble_on() { - RAMBLE_COMPREPLY="-h --help --executor --where --exclude-where --filter-tags" + RAMBLE_COMPREPLY="-h --help --executor --enable-per-experiment-prints --suppress-run-header --where --exclude-where --filter-tags" } _ramble_python() { @@ -569,7 +586,7 @@ _ramble_repo_create() { then RAMBLE_COMPREPLY="-h --help -d --subdirectory -t --type" else - _repos + RAMBLE_COMREPLY="" fi } @@ -671,7 +688,7 @@ _ramble_workspace_create() { } _ramble_workspace_concretize() { - RAMBLE_COMPREPLY="-h --help" + RAMBLE_COMPREPLY="-h --help -f --force-concretize --simplify" } _ramble_workspace_setup() { @@ -711,7 +728,7 @@ _ramble_workspace_remove() { then RAMBLE_COMPREPLY="-h --help -y --yes-to-all" else - RAMBLE_COMREPLY="" + _workspaces fi } @@ -720,6 +737,6 @@ _ramble_workspace_rm() { then RAMBLE_COMPREPLY="-h --help -y --yes-to-all" else - RAMBLE_COMREPLY="" + _workspaces fi } diff --git a/share/ramble/setup-env.csh b/share/ramble/setup-env.csh index 267afd3f3..30eb93f70 100755 --- a/share/ramble/setup-env.csh +++ b/share/ramble/setup-env.csh @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/share/ramble/setup-env.sh b/share/ramble/setup-env.sh index f0fa9df2c..7002cfe57 100755 --- a/share/ramble/setup-env.sh +++ b/share/ramble/setup-env.sh @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin.mock/applications/basic-inherited/application.py b/var/ramble/repos/builtin.mock/applications/basic-inherited/application.py index e7b3660e7..a289a9f76 100644 --- a/var/ramble/repos/builtin.mock/applications/basic-inherited/application.py +++ b/var/ramble/repos/builtin.mock/applications/basic-inherited/application.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -18,3 +18,7 @@ class BasicInherited(BaseBasic): description='Again, not a file', extension='.log') workload('test_wl3', executable='foo', input='inherited_input') + + workload_variable('my_var', default='1.0', + description='Shadowed Example var', + workload='test_wl') diff --git a/var/ramble/repos/builtin.mock/applications/basic/application.py b/var/ramble/repos/builtin.mock/applications/basic/application.py index 2ff5bb1dd..399ea9d40 100644 --- a/var/ramble/repos/builtin.mock/applications/basic/application.py +++ b/var/ramble/repos/builtin.mock/applications/basic/application.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -23,6 +23,10 @@ class Basic(ExecutableApplication): workload('test_wl2', executable='bar', input='input') workload('working_wl', executable='echo') + workload_variable('my_base_var', default='0.0', + description='Example var', + workload='test_wl') + workload_variable('my_var', default='1.0', description='Example var', workload='test_wl') diff --git a/var/ramble/repos/builtin.mock/applications/expanded_foms/application.py b/var/ramble/repos/builtin.mock/applications/expanded_foms/application.py index 73ad8629a..e75c1afea 100644 --- a/var/ramble/repos/builtin.mock/applications/expanded_foms/application.py +++ b/var/ramble/repos/builtin.mock/applications/expanded_foms/application.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -12,7 +12,7 @@ class ExpandedFoms(ExecutableApplication): name = "expanded-Foms" - executable('foo', 'bar', use_mpi=False) + executable('foo', template=['bar', 'echo "{my_var}"'], use_mpi=False) input_file('input', url='file:///tmp/test_file.log', description='Not a file', extension='.log') diff --git a/var/ramble/repos/builtin.mock/applications/glob-patterns/application.py b/var/ramble/repos/builtin.mock/applications/glob-patterns/application.py new file mode 100644 index 000000000..9267c7532 --- /dev/null +++ b/var/ramble/repos/builtin.mock/applications/glob-patterns/application.py @@ -0,0 +1,57 @@ +# Copyright 2022-2024 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +from ramble.appkit import * + + +class GlobPatterns(ExecutableApplication): + name = "glob-patterns" + + executable('test', 'base test {test_var} {glob_var} {baz_var}', use_mpi=False) + executable('test-foo', 'test foo {test_var} {glob_var} {baz_var} {mod_var}', use_mpi=False) + executable('test-bar', 'test bar {test_var} {glob_var} {baz_var}', use_mpi=True) + executable('baz', 'baz {test_var} {glob_var} {baz_var}', use_mpi=True) + + input_file('input', url='file:///tmp/test_file.log', + description='Not a file', extension='.log') + input_file('input-foo', url='file:///tmp/test_foo_file.log', + description='Not a file', extension='.log') + input_file('input-bar', url='file:///tmp/test_bar_file.log', + description='Not a file', extension='.log') + input_file('baz', url='file:///tmp/baz_file.log', + description='Not a file', extension='.log') + + workload('test_one_exec', executables=['test'], inputs=['input']) + workload('test_three_exec', executables=['test*'], inputs=['input*']) + workload('one_baz_exec', executables=['baz'], inputs=['baz']) + + environment_variable('env_var_test', 'set', description='Test env var', + workloads=['test_one_exec']) + environment_variable('env_var_glob', 'set', description='Test env var', + workloads=['test*']) + environment_variable('env_var_baz', 'set', description='Test env var', + workloads=['one_baz_exec']) + environment_variable('env_var_mod', 'set', description='Env var to be modified', + workloads=['test_three_exec']) + + workload_variable('test_var', default='wl_var_test', + description='Example var', + workloads=['test_one_exec']) + workload_variable('glob_var', default='wl_var_glob', + description='Example var', + workloads=['test*']) + workload_variable('baz_var', default='wl_var_baz', + description='Example var', + workloads=['one_baz_exec']) + workload_variable('var_mod', default='wl_var_mod', + description='Variable to be modified', + workloads=['test_three_exec']) + + figure_of_merit('test_fom', + fom_regex=r'(?P[0-9]+\.[0-9]+).*seconds.*', + group_name='test', units='s') diff --git a/var/ramble/repos/builtin.mock/applications/input-test/application.py b/var/ramble/repos/builtin.mock/applications/input-test/application.py index 387da092c..cd13549cc 100644 --- a/var/ramble/repos/builtin.mock/applications/input-test/application.py +++ b/var/ramble/repos/builtin.mock/applications/input-test/application.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin.mock/applications/interleved-env-vars/application.py b/var/ramble/repos/builtin.mock/applications/interleved-env-vars/application.py index 4c9fd7020..5767e563d 100644 --- a/var/ramble/repos/builtin.mock/applications/interleved-env-vars/application.py +++ b/var/ramble/repos/builtin.mock/applications/interleved-env-vars/application.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -23,6 +23,9 @@ class InterlevedEnvVars(ExecutableApplication): workload('test_wl2', executables=['bar', 'builtin::env_vars'], input='input') workload('test_wl3', executables=['baz'], input='input') + environment_variable('FROM_DIRECTIVE', 'set', description='Test env var', + workloads=['test_wl', 'test_wl2', 'test_wl3']) + workload_variable('my_var', default='1.0', description='Example var', workload='test_wl') diff --git a/var/ramble/repos/builtin.mock/applications/maintained-1/application.py b/var/ramble/repos/builtin.mock/applications/maintained-1/application.py index c7aebfce2..bef19f933 100644 --- a/var/ramble/repos/builtin.mock/applications/maintained-1/application.py +++ b/var/ramble/repos/builtin.mock/applications/maintained-1/application.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin.mock/applications/maintained-2/application.py b/var/ramble/repos/builtin.mock/applications/maintained-2/application.py index 7ca34858b..9d5307462 100644 --- a/var/ramble/repos/builtin.mock/applications/maintained-2/application.py +++ b/var/ramble/repos/builtin.mock/applications/maintained-2/application.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin.mock/applications/register-builtin/application.py b/var/ramble/repos/builtin.mock/applications/register-builtin/application.py index ec1adc17b..da5bc5328 100644 --- a/var/ramble/repos/builtin.mock/applications/register-builtin/application.py +++ b/var/ramble/repos/builtin.mock/applications/register-builtin/application.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin.mock/applications/shared-context/application.py b/var/ramble/repos/builtin.mock/applications/shared-context/application.py index d051f24c6..0134582d2 100644 --- a/var/ramble/repos/builtin.mock/applications/shared-context/application.py +++ b/var/ramble/repos/builtin.mock/applications/shared-context/application.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin.mock/applications/success-function/application.py b/var/ramble/repos/builtin.mock/applications/success-function/application.py index e83b6888a..9f0fffb34 100644 --- a/var/ramble/repos/builtin.mock/applications/success-function/application.py +++ b/var/ramble/repos/builtin.mock/applications/success-function/application.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin.mock/applications/tagged-1/application.py b/var/ramble/repos/builtin.mock/applications/tagged-1/application.py index f4d418827..9a0a3d6fe 100644 --- a/var/ramble/repos/builtin.mock/applications/tagged-1/application.py +++ b/var/ramble/repos/builtin.mock/applications/tagged-1/application.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin.mock/applications/unmaintained-1/application.py b/var/ramble/repos/builtin.mock/applications/unmaintained-1/application.py index 3f587b93f..11e0cfdb2 100644 --- a/var/ramble/repos/builtin.mock/applications/unmaintained-1/application.py +++ b/var/ramble/repos/builtin.mock/applications/unmaintained-1/application.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin.mock/applications/untagged-1/application.py b/var/ramble/repos/builtin.mock/applications/untagged-1/application.py index e68aa79ba..c64725afe 100644 --- a/var/ramble/repos/builtin.mock/applications/untagged-1/application.py +++ b/var/ramble/repos/builtin.mock/applications/untagged-1/application.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin.mock/applications/workload-tags/application.py b/var/ramble/repos/builtin.mock/applications/workload-tags/application.py index 0f2400a1a..af92fd377 100644 --- a/var/ramble/repos/builtin.mock/applications/workload-tags/application.py +++ b/var/ramble/repos/builtin.mock/applications/workload-tags/application.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin.mock/applications/zlib-configs/application.py b/var/ramble/repos/builtin.mock/applications/zlib-configs/application.py index 7d61efacc..4d0f11afd 100644 --- a/var/ramble/repos/builtin.mock/applications/zlib-configs/application.py +++ b/var/ramble/repos/builtin.mock/applications/zlib-configs/application.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin.mock/applications/zlib/application.py b/var/ramble/repos/builtin.mock/applications/zlib/application.py index 056663a91..d14862b03 100644 --- a/var/ramble/repos/builtin.mock/applications/zlib/application.py +++ b/var/ramble/repos/builtin.mock/applications/zlib/application.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin.mock/modifiers/append-env-var-mod-paths/modifier.py b/var/ramble/repos/builtin.mock/modifiers/append-env-var-mod-paths/modifier.py index 30f687833..bf35ea6e4 100644 --- a/var/ramble/repos/builtin.mock/modifiers/append-env-var-mod-paths/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/append-env-var-mod-paths/modifier.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin.mock/modifiers/append-env-var-mod-vars/modifier.py b/var/ramble/repos/builtin.mock/modifiers/append-env-var-mod-vars/modifier.py index db8cd931f..c09904889 100644 --- a/var/ramble/repos/builtin.mock/modifiers/append-env-var-mod-vars/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/append-env-var-mod-vars/modifier.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin.mock/modifiers/glob-patterns-mod/modifier.py b/var/ramble/repos/builtin.mock/modifiers/glob-patterns-mod/modifier.py new file mode 100644 index 000000000..b6f8c9e2e --- /dev/null +++ b/var/ramble/repos/builtin.mock/modifiers/glob-patterns-mod/modifier.py @@ -0,0 +1,30 @@ +# Copyright 2022-2024 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +from ramble.modkit import * # noqa: F403 + + +class GlobPatternsMod(BasicModifier): + """Define a modifier to test globbing + + This modifier tests globbing in the modifier language. + """ + name = "glob-patterns-mod" + + tags('test') + + mode('base', description='This is a base mode with no modifications') + mode('test-glob', description='This test mode turns on mods using globbing') + default_mode('base') + + variable_modification('var_mod', '{mod_var}', modes=['test*']) + + env_var_modification('env_var_mod', 'modded', modes=['test*']) + + modifier_variable('mod_var', default='var_mod_modified', + description='This is a modifier variable', modes=['test*']) diff --git a/var/ramble/repos/builtin.mock/modifiers/maintained-1/modifier.py b/var/ramble/repos/builtin.mock/modifiers/maintained-1/modifier.py index f4cdeeb0f..cb9c1e49c 100644 --- a/var/ramble/repos/builtin.mock/modifiers/maintained-1/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/maintained-1/modifier.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin.mock/modifiers/maintained-2/modifier.py b/var/ramble/repos/builtin.mock/modifiers/maintained-2/modifier.py index 57418b3e3..2bbca1987 100644 --- a/var/ramble/repos/builtin.mock/modifiers/maintained-2/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/maintained-2/modifier.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin.mock/modifiers/mod-phase/modifier.py b/var/ramble/repos/builtin.mock/modifiers/mod-phase/modifier.py new file mode 100644 index 000000000..c488087e8 --- /dev/null +++ b/var/ramble/repos/builtin.mock/modifiers/mod-phase/modifier.py @@ -0,0 +1,29 @@ +# Copyright 2022-2024 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +from ramble.modkit import * # noqa: F403 +from ramble.util.logger import logger + + +class ModPhase(BasicModifier): + """Define a modifier that defines a new phase with register_phase""" + name = "mod-phase" + + tags('test') + + mode('test', description='This is a test mode') + + register_phase('first_phase', pipeline='setup', run_before=['get_inputs']) + + def _first_phase(self, workspace, app_inst=None): + logger.all_msg('Inside a phase: first_phase') + + register_phase('after_make_experiments', pipeline='setup', run_after=['make_experiments']) + + def _after_make_experiments(self, workspace, app_inst=None): + logger.all_msg('Inside a phase: after_make_experiments') diff --git a/var/ramble/repos/builtin.mock/modifiers/multiple-modes-no-default/modifier.py b/var/ramble/repos/builtin.mock/modifiers/multiple-modes-no-default/modifier.py index 79ce8588c..dc2c785ec 100644 --- a/var/ramble/repos/builtin.mock/modifiers/multiple-modes-no-default/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/multiple-modes-no-default/modifier.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin.mock/modifiers/multiple-modes-with-default/modifier.py b/var/ramble/repos/builtin.mock/modifiers/multiple-modes-with-default/modifier.py index 34a0f8e4f..da25c0115 100644 --- a/var/ramble/repos/builtin.mock/modifiers/multiple-modes-with-default/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/multiple-modes-with-default/modifier.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin.mock/modifiers/no-docstring-mod/modifier.py b/var/ramble/repos/builtin.mock/modifiers/no-docstring-mod/modifier.py index 495131dbb..773f5c765 100644 --- a/var/ramble/repos/builtin.mock/modifiers/no-docstring-mod/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/no-docstring-mod/modifier.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin.mock/modifiers/no-variable-mods/modifier.py b/var/ramble/repos/builtin.mock/modifiers/no-variable-mods/modifier.py index 09d36c86d..c1f83c8b7 100644 --- a/var/ramble/repos/builtin.mock/modifiers/no-variable-mods/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/no-variable-mods/modifier.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin.mock/modifiers/prepare-analysis/modifier.py b/var/ramble/repos/builtin.mock/modifiers/prepare-analysis/modifier.py index a3d3b21af..da5ee9ab7 100644 --- a/var/ramble/repos/builtin.mock/modifiers/prepare-analysis/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/prepare-analysis/modifier.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin.mock/modifiers/prepend-env-var-mod-paths/modifier.py b/var/ramble/repos/builtin.mock/modifiers/prepend-env-var-mod-paths/modifier.py index 7c647ef6b..81e6232c0 100644 --- a/var/ramble/repos/builtin.mock/modifiers/prepend-env-var-mod-paths/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/prepend-env-var-mod-paths/modifier.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin.mock/modifiers/set-env-var-mod/modifier.py b/var/ramble/repos/builtin.mock/modifiers/set-env-var-mod/modifier.py index 110977e40..53fa54cee 100644 --- a/var/ramble/repos/builtin.mock/modifiers/set-env-var-mod/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/set-env-var-mod/modifier.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -18,4 +18,8 @@ class SetEnvVarMod(BasicModifier): mode('test', description='This is a test mode') + modifier_variable('mask_test', default='0x0', description='Test mask var', + modes=['test'], expandable=False) + env_var_modification('test_var', modification='test_val', method='set', mode='test') + env_var_modification('mask_env_var', modification='{mask_test}', method='set', mode='test') diff --git a/var/ramble/repos/builtin.mock/modifiers/spack-failed-reqs/modifier.py b/var/ramble/repos/builtin.mock/modifiers/spack-failed-reqs/modifier.py index 10d734830..cd3fa7d0e 100644 --- a/var/ramble/repos/builtin.mock/modifiers/spack-failed-reqs/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/spack-failed-reqs/modifier.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -17,9 +17,9 @@ class SpackFailedReqs(SpackModifier): mode('default', description='This is the default mode for the spack-failed-reqs modifier') - default_compiler('mod_compiler', - spack_spec='mod_compiler@1.1 target=x86_64', - compiler_spec='mod_compiler@1.1') + define_compiler('mod_compiler', + spack_spec='mod_compiler@1.1 target=x86_64', + compiler_spec='mod_compiler@1.1') software_spec('mod_package1', spack_spec='mod_package1@1.1', diff --git a/var/ramble/repos/builtin.mock/modifiers/spack-mod/modifier.py b/var/ramble/repos/builtin.mock/modifiers/spack-mod/modifier.py index 9d6fba8d5..20c8c3bd3 100644 --- a/var/ramble/repos/builtin.mock/modifiers/spack-mod/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/spack-mod/modifier.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -19,9 +19,9 @@ class SpackMod(SpackModifier): package_manager_config('enable_debug', 'config:debug:true') - default_compiler('mod_compiler', - spack_spec='mod_compiler@1.1 target=x86_64', - compiler_spec='mod_compiler@1.1') + define_compiler('mod_compiler', + spack_spec='mod_compiler@1.1 target=x86_64', + compiler_spec='mod_compiler@1.1') software_spec('mod_package1', spack_spec='mod_package1@1.1', diff --git a/var/ramble/repos/builtin.mock/modifiers/success-criteria/modifier.py b/var/ramble/repos/builtin.mock/modifiers/success-criteria/modifier.py index 091378881..5fec219c4 100644 --- a/var/ramble/repos/builtin.mock/modifiers/success-criteria/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/success-criteria/modifier.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin.mock/modifiers/tagged-1/modifier.py b/var/ramble/repos/builtin.mock/modifiers/tagged-1/modifier.py index c30165fd5..1dd6b1bef 100644 --- a/var/ramble/repos/builtin.mock/modifiers/tagged-1/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/tagged-1/modifier.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin.mock/modifiers/test-mod/modifier.py b/var/ramble/repos/builtin.mock/modifiers/test-mod/modifier.py index b923b9864..ef4b29f4e 100644 --- a/var/ramble/repos/builtin.mock/modifiers/test-mod/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/test-mod/modifier.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -19,6 +19,13 @@ class TestMod(BasicModifier): tags('test') mode('test', description='This is a test mode') + default_mode('test') + + mode('app-scope', description='This is a test mode at the application scope') + + mode('wl-scope', description='This is a test mode at the workload scope') + + mode('exp-scope', description='This is a test mode at the experiment scope') variable_modification('mpi_command', 'echo "prefix_mpi_command" >> {log_file}; ', method='prepend', modes=['test']) @@ -40,5 +47,8 @@ class TestMod(BasicModifier): register_builtin('test_builtin', required=True, injection_method='append') + test_attr = 'test_value' + def test_builtin(self): - return ['echo "fom_contextFOM_GOES_HERE" >> {analysis_log}'] + return ['echo "fom_contextFOM_GOES_HERE" >> {analysis_log}', + f'echo "{self.test_attr}"' + ' >> {analysis_log}'] diff --git a/var/ramble/repos/builtin.mock/modifiers/unmaintained-1/modifier.py b/var/ramble/repos/builtin.mock/modifiers/unmaintained-1/modifier.py index 1a3f010b5..71ff53bbb 100644 --- a/var/ramble/repos/builtin.mock/modifiers/unmaintained-1/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/unmaintained-1/modifier.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin.mock/modifiers/unset-env-var-mod/modifier.py b/var/ramble/repos/builtin.mock/modifiers/unset-env-var-mod/modifier.py index 0a28ff51a..bdabc25fe 100644 --- a/var/ramble/repos/builtin.mock/modifiers/unset-env-var-mod/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/unset-env-var-mod/modifier.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin.mock/modifiers/untagged-1/modifier.py b/var/ramble/repos/builtin.mock/modifiers/untagged-1/modifier.py index 9e9ed8afc..8d942c46e 100644 --- a/var/ramble/repos/builtin.mock/modifiers/untagged-1/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/untagged-1/modifier.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin/applications/cloverleaf/application.py b/var/ramble/repos/builtin/applications/cloverleaf/application.py index 7345cceea..32b3491db 100644 --- a/var/ramble/repos/builtin/applications/cloverleaf/application.py +++ b/var/ramble/repos/builtin/applications/cloverleaf/application.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -16,11 +16,11 @@ class Cloverleaf(SpackApplication): '''Define CLOVERLEAF application''' name = 'cloverleaf' - maintainers('dodecatheon') + maintainers('rfbgo') tags('cfd', 'fluid', 'dynamics', 'euler', 'miniapp', 'minibenchmark', 'mini-benchmark') - default_compiler('gcc12', spack_spec='gcc@12.2.0') + define_compiler('gcc12', spack_spec='gcc@12.2.0') software_spec('ompi414', spack_spec='openmpi@4.1.4 +legacylaunchers +cxx', compiler='gcc12') @@ -82,7 +82,7 @@ class Cloverleaf(SpackApplication): figure_of_merit_context('step', regex=step_count_regex, - output_format='{step}') + output_format='Step {step}') step_summary_regex = (r'\s*step:\s+(?P[0-9]+)\s+' + r'(?P' + floating_point_regex + r')\s+' + diff --git a/var/ramble/repos/builtin/applications/elk/application.py b/var/ramble/repos/builtin/applications/elk/application.py index 7b4e7dcad..c2c2f6dae 100644 --- a/var/ramble/repos/builtin/applications/elk/application.py +++ b/var/ramble/repos/builtin/applications/elk/application.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -59,6 +59,6 @@ class Elk(ExecutableApplication): for metric in metrics: figure_of_merit(metric, log_file=output_file, - fom_regex=f'\s*(?P{metric})\s+:\s+(?P[0-9]+\.[0-9]*).*', + fom_regex=rf'\s*(?P{metric})\s+:\s+(?P[0-9]+\.[0-9]*).*', group_name='value', units='s' ) diff --git a/var/ramble/repos/builtin/applications/gromacs/application.py b/var/ramble/repos/builtin/applications/gromacs/application.py index 3f3aebb89..eeb957161 100644 --- a/var/ramble/repos/builtin/applications/gromacs/application.py +++ b/var/ramble/repos/builtin/applications/gromacs/application.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -19,7 +19,7 @@ class Gromacs(SpackApplication): tags('molecular-dynamics') - default_compiler('gcc9', spack_spec='gcc@9.3.0') + define_compiler('gcc9', spack_spec='gcc@9.3.0') software_spec('impi2018', spack_spec='intel-mpi@2018.4.274') software_spec('gromacs', spack_spec='gromacs@2020.5', compiler='gcc9') diff --git a/var/ramble/repos/builtin/applications/hmmer/application.py b/var/ramble/repos/builtin/applications/hmmer/application.py index e91da6387..e4ed969bd 100644 --- a/var/ramble/repos/builtin/applications/hmmer/application.py +++ b/var/ramble/repos/builtin/applications/hmmer/application.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -24,11 +24,11 @@ class Hmmer(SpackApplication): name = 'hmmer' - maintainers('dodecatheon') + maintainers('rfbgo') tags('molecular-dynamics', 'hidden-markov-models', 'bio-molecule') - default_compiler('gcc9', spack_spec='gcc@9.3.0') + define_compiler('gcc9', spack_spec='gcc@9.3.0') software_spec('impi_2018', spack_spec='intel-mpi@2018.4.274') diff --git a/var/ramble/repos/builtin/applications/hostname/application.py b/var/ramble/repos/builtin/applications/hostname/application.py index 106c685e5..1b708b9a7 100644 --- a/var/ramble/repos/builtin/applications/hostname/application.py +++ b/var/ramble/repos/builtin/applications/hostname/application.py @@ -1,5 +1,5 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/var/ramble/repos/builtin/applications/hpcc/application.py b/var/ramble/repos/builtin/applications/hpcc/application.py index 8a5480df0..2fba0c057 100644 --- a/var/ramble/repos/builtin/applications/hpcc/application.py +++ b/var/ramble/repos/builtin/applications/hpcc/application.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -28,7 +28,7 @@ class Hpcc(SpackApplication): tags('benchmark-app', 'mini-app', 'benchmark', 'DGEMM') - default_compiler('gcc9', spack_spec='gcc@9.3.0') + define_compiler('gcc9', spack_spec='gcc@9.3.0') software_spec('impi2018', spack_spec='intel-mpi@2018.4.274') @@ -47,12 +47,12 @@ class Hpcc(SpackApplication): executable('execute', 'hpcc', use_mpi=True) + workload('standard', executables=['copy-config', 'execute'], input='hpccinf') + workload_variable('config_file', default='https://raw.githubusercontent.com/icl-utk-edu/hpcc/1.5.0/_hpccinf.txt', description='Default config file', workloads=['standard']) - workload('standard', executables=['copy-config', 'execute'], input='hpccinf') - workload_variable('out_file', default='{experiment_run_dir}/hpccoutf.txt', description='Output file for results', workloads=['standard']) diff --git a/var/ramble/repos/builtin/applications/hpcg/application.py b/var/ramble/repos/builtin/applications/hpcg/application.py index 13f8829b1..2e91f8574 100644 --- a/var/ramble/repos/builtin/applications/hpcg/application.py +++ b/var/ramble/repos/builtin/applications/hpcg/application.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -19,7 +19,7 @@ class Hpcg(SpackApplication): tags('benchmark-app', 'mini-app', 'benchmark') - default_compiler('gcc9', spack_spec='gcc@9.3.0') + define_compiler('gcc9', spack_spec='gcc@9.3.0') software_spec('impi2018', spack_spec='intel-mpi@2018.4.274') @@ -83,7 +83,7 @@ class Hpcg(SpackApplication): fom_regex=r'Final Summary::HPCG 2\.4 rating.*=(?P[0-9]+\.*[0-9]*)', group_name='rating', units='') - def _make_experiments(self, workspace): + def _make_experiments(self, workspace, app_inst=None): super()._make_experiments(workspace) input_path = os.path.join(self.expander.expand_var_name('experiment_run_dir'), diff --git a/var/ramble/repos/builtin/applications/hpl/application.py b/var/ramble/repos/builtin/applications/hpl/application.py index 9c9153367..589132df0 100644 --- a/var/ramble/repos/builtin/applications/hpl/application.py +++ b/var/ramble/repos/builtin/applications/hpl/application.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 Google LLC +# Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 or the MIT license @@ -21,11 +21,11 @@ class Hpl(SpackApplication): '''Define HPL application''' name = 'hpl' - maintainers('douglasjacobsen', 'dodecatheon') + maintainers('douglasjacobsen') tags('benchmark-app', 'benchmark', 'linpack') - default_compiler('gcc9', spack_spec='gcc@9.3.0') + define_compiler('gcc9', spack_spec='gcc@9.3.0') software_spec('impi_2018', spack_spec='intel-mpi@2018.4.274') @@ -202,7 +202,7 @@ class Hpl(SpackApplication): group_name='gflops', units='GFLOP/s', contexts=['problem-name']) - figure_of_merit_context('problem-name', regex=r'.*\s+(?P[0-9]+)\s+(?P[0-9]+)\s+(?P

[0-9]+)\s+(?P[0-9]+)\s+(?P

[0-9]+)\s+(?P[0-9]+)\s+(?P

[0-9]+)\s+(?P[0-9]+)\s+(?P

[0-9]+)\s+(?P[0-9]+)\s+(?P