Merge pull request EESSI#166 from smoors/eessi_init_optional

make EESSI optional in common_eessi_init
reframe-hpc · Aug 12, 2024 · 3e1ecc5 · 3e1ecc5
2 parents df29715 + b9d64fe
commit 3e1ecc5
Show file tree

Hide file tree

Showing 8 changed files with 37 additions and 23 deletions.
diff --git a/config/aws_citc.py b/config/aws_citc.py
@@ -224,7 +224,7 @@
         FEATURES['CPU']
     ],
     'prepare_cmds': [
-        'source %s' % common_eessi_init(),
+        common_eessi_init(),
         # Required when using srun as launcher with --export=NONE in partition access, in order to ensure job
         # steps inherit environment. It doesn't hurt to define this even if srun is not used
         'export SLURM_EXPORT_ENV=ALL'

diff --git a/config/aws_mc.py b/config/aws_mc.py
@@ -100,7 +100,7 @@
         FEATURES['CPU']
     ] + list(SCALES.keys()),
     'prepare_cmds': [
-        'source %s' % common_eessi_init(),
+        common_eessi_init(),
         # Required when using srun as launcher with --export=NONE in partition access, in order to ensure job
         # steps inherit environment. It doesn't hurt to define this even if srun is not used
         'export SLURM_EXPORT_ENV=ALL'

diff --git a/config/it4i_karolina.py b/config/it4i_karolina.py
@@ -36,7 +36,7 @@
                     'name': 'qcpu',
                     'scheduler': 'slurm',
                     'prepare_cmds': [
-                        'source %s' % common_eessi_init(),
+                        common_eessi_init(),
                         # Pass job environment variables like $PATH, etc., into job steps
                         'export SLURM_EXPORT_ENV=ALL',
                         # Needed when using srun launcher
@@ -71,7 +71,7 @@
                 #     'name': 'qgpu',
                 #     'scheduler': 'slurm',
                 #     'prepare_cmds': [
-                #         'source %s' % common_eessi_init(),
+                #         common_eessi_init(),
                 #         # Pass job environment variables like $PATH, etc., into job steps
                 #         'export SLURM_EXPORT_ENV=ALL',
                 #         # Needed when using srun launcher

diff --git a/config/izum_vega.py b/config/izum_vega.py
@@ -36,7 +36,7 @@
                     'name': 'cpu',
                     'scheduler': 'slurm',
                     'prepare_cmds': [
-                        'source %s' % common_eessi_init(),
+                        common_eessi_init(),
                         # Pass job environment variables like $PATH, etc., into job steps
                         'export SLURM_EXPORT_ENV=ALL',
                         # Needed when using srun launcher
@@ -72,7 +72,7 @@
                 #     'name': 'gpu',
                 #     'scheduler': 'slurm',
                 #     'prepare_cmds': [
-                #         'source %s' % common_eessi_init(),
+                #         common_eessi_init(),
                 #         # Pass job environment variables like $PATH, etc., into job steps
                 #         'export SLURM_EXPORT_ENV=ALL',
                 #         # Needed when using srun launcher

diff --git a/config/macc_deucalion.py b/config/macc_deucalion.py
@@ -25,7 +25,7 @@
                         # bypass CPU autodetection for now aarch64/a64fx,
                         # see https://github.com/EESSI/software-layer/pull/608
                         'export EESSI_SOFTWARE_SUBDIR_OVERRIDE=aarch64/a64fx',
-                        'source %s' % common_eessi_init(),
+                        common_eessi_init(),
                         # Pass job environment variables like $PATH, etc., into job steps
                         'export SLURM_EXPORT_ENV=HOME,PATH,LD_LIBRARY_PATH,PYTHONPATH',
                     ],

diff --git a/config/surf_snellius.py b/config/surf_snellius.py
@@ -39,7 +39,7 @@
                 {
                     'name': 'rome',
                     'scheduler': 'slurm',
-                    'prepare_cmds': ['source %s' % common_eessi_init()],
+                    'prepare_cmds': [common_eessi_init()],
                     'launcher': 'mpirun',
                     'access': ['-p rome', '--export=None'],
                     'environs': ['default'],
@@ -63,7 +63,7 @@
                 {
                     'name': 'genoa',
                     'scheduler': 'slurm',
-                    'prepare_cmds': ['source %s' % common_eessi_init()],
+                    'prepare_cmds': [common_eessi_init()],
                     'launcher': 'mpirun',
                     'access': ['-p genoa', '--export=None'],
                     'environs': ['default'],
@@ -87,7 +87,7 @@
                 {
                     'name': 'gpu_A100',
                     'scheduler': 'slurm',
-                    'prepare_cmds': ['source %s' % common_eessi_init()],
+                    'prepare_cmds': [common_eessi_init()],
                     'launcher': 'mpirun',
                     'access': ['-p gpu_a100', '--export=None'],
                     'environs': ['default'],
@@ -123,7 +123,7 @@
                 {
                     'name': 'gpu_H100',
                     'scheduler': 'slurm',
-                    'prepare_cmds': ['source %s' % common_eessi_init()],
+                    'prepare_cmds': [common_eessi_init()],
                     'launcher': 'mpirun',
                     'access': ['-p gpu_h100', '--export=None'],
                     'environs': ['default'],

diff --git a/config/vsc_hortense.py b/config/vsc_hortense.py
@@ -33,7 +33,7 @@ def command(self, job):
                 {
                     'name': 'cpu_rome_256gb',
                     'scheduler': 'slurm',
-                    'prepare_cmds': ['source %s' % common_eessi_init()],
+                    'prepare_cmds': [common_eessi_init()],
                     'access': hortense_access + ['--partition=cpu_rome'],
                     'environs': ['default'],
                     'descr': 'CPU nodes (AMD Rome, 256GiB RAM)',
@@ -65,7 +65,7 @@ def command(self, job):
                 {
                     'name': 'cpu_rome_512gb',
                     'scheduler': 'slurm',
-                    'prepare_cmds': ['source %s' % common_eessi_init()],
+                    'prepare_cmds': [common_eessi_init()],
                     'access': hortense_access + ['--partition=cpu_rome_512'],
                     'environs': ['default'],
                     'descr': 'CPU nodes (AMD Rome, 512GiB RAM)',
@@ -97,7 +97,7 @@ def command(self, job):
                 {
                     'name': 'cpu_milan',
                     'scheduler': 'slurm',
-                    'prepare_cmds': ['source %s' % common_eessi_init()],
+                    'prepare_cmds': [common_eessi_init()],
                     'access': hortense_access + ['--partition=cpu_milan'],
                     'environs': ['default'],
                     'descr': 'CPU nodes (AMD Milan, 256GiB RAM)',
@@ -129,7 +129,7 @@ def command(self, job):
                 {
                     'name': 'gpu_rome_a100_40gb',
                     'scheduler': 'slurm',
-                    'prepare_cmds': ['source %s' % common_eessi_init()],
+                    'prepare_cmds': [common_eessi_init()],
                     'access': hortense_access + ['--partition=gpu_rome_a100_40'],
                     'environs': ['default'],
                     'descr': 'GPU nodes (A100 40GB)',
@@ -173,7 +173,7 @@ def command(self, job):
                 {
                     'name': 'gpu_rome_a100_80gb',
                     'scheduler': 'slurm',
-                    'prepare_cmds': ['source %s' % common_eessi_init()],
+                    'prepare_cmds': [common_eessi_init()],
                     'access': hortense_access + ['--partition=gpu_rome_a100_80'],
                     'environs': ['default'],
                     'descr': 'GPU nodes (A100 80GB)',

diff --git a/eessi/testsuite/common_config.py b/eessi/testsuite/common_config.py
@@ -1,4 +1,5 @@
 import os
+import warnings
 
 perflog_format = '|'.join([
     '%(check_job_completion_time)s',
@@ -93,11 +94,21 @@ def common_eessi_init(eessi_version=None):
     """
     # Check which EESSI_CVMFS_REPO we are running under
     eessi_cvmfs_repo = os.getenv('EESSI_CVMFS_REPO', None)
+
     if eessi_cvmfs_repo is None:
-        err_msg = "Environment variable 'EESSI_CVMFS_REPO' was not found."
-        err_msg += " Did you initialize the EESSI environment before running the test suite?"
-        raise ValueError(err_msg)
-    if eessi_cvmfs_repo == '/cvmfs/pilot.eessi-hpc.org':
+        warn_msg = '\n' + '\n'.join([
+            "EESSI WARNING: Environment variable 'EESSI_CVMFS_REPO' was not found.",
+            "EESSI WARNING: If you do not intend to use the EESSI software stack, this is perfectly fine.",
+            "EESSI WARNING: To use EESSI, initialize the EESSI environment before running the test suite.",
+        ])
+        warnings.warn(warn_msg)
+        return ''
+
+    eessi_init = []
+    pilot_repo = '/cvmfs/pilot.eessi-hpc.org'
+
+    if eessi_cvmfs_repo == pilot_repo:
+        eessi_init.append('export EESSI_FORCE_PILOT=1')
         if eessi_version is None:
             # Try also EESSI_VERSION for backwards compatibility with previous common_eessi_init implementation
             eessi_version = os.getenv('EESSI_PILOT_VERSION', os.getenv('EESSI_VERSION', 'latest'))
@@ -112,7 +123,10 @@ def common_eessi_init(eessi_version=None):
             err_msg += " Did you initialize the EESSI environment before running the test suite?"
             raise ValueError(err_msg)
 
-    if eessi_cvmfs_repo == '/cvmfs/pilot.eessi-hpc.org' and eessi_version == 'latest':
-        return '/cvmfs/pilot.eessi-hpc.org/latest/init/bash'
+    if eessi_cvmfs_repo == pilot_repo and eessi_version == 'latest':
+        version_string = eessi_version
     else:
-        return '%s/versions/%s/init/bash' % (eessi_cvmfs_repo, eessi_version)
+        version_string = f'versions/{eessi_version}'
+
+    eessi_init.append(f'source {eessi_cvmfs_repo}/{version_string}/init/bash')
+    return ' && '.join(eessi_init)