diff --git a/README.md b/README.md index 43b27d6..324d401 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # E3SM-Unified -A metapackage for a unified +A metapackage for a unified [conda environment](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html) -for analysis an other post-processing of results from Energy Exascale Earth +for analysis an other post-processing of results from Energy Exascale Earth System Model (E3SM) simulations. E3SM-Unified currently supports Linux and OSX, and python >=3.9,<3.11. @@ -10,17 +10,17 @@ Support for Windows is not planned. To create a new conda environment on a laptop or workstation (or on HPC that is not already supported by the E3SM team), first install -[Mambaforge](https://github.com/conda-forge/miniforge#mambaforge). Then, +[Miniforge3](https://github.com/conda-forge/miniforge#miniforge3). Then, create the E3SM-Unified environment (with MPI support from the `mpich` package in this example), use: -```bash -mamba create -n e3sm-unified -c conda-forge -c defaults -c e3sm \ +```bash +conda create -n e3sm-unified -c conda-forge -c defaults -c e3sm \ python=3.10 "e3sm-unified=*=mpi_mpich_*" -mamba activate e3sm-unified +conda activate e3sm-unified ``` Each time you want to use the environment in the future, again run: ```bash -mamba activate e3sm-unified +conda activate e3sm-unified ``` For the full list of packages in the current version of the metapackages, see: diff --git a/e3sm_supported_machines/bootstrap.py b/e3sm_supported_machines/bootstrap.py index 48a72da..d927299 100755 --- a/e3sm_supported_machines/bootstrap.py +++ b/e3sm_supported_machines/bootstrap.py @@ -139,8 +139,7 @@ def build_env(is_test, recreate, compiler, mpi, conda_mpi, version, channels = f'{channels} -c conda-forge/label/{package}_dev' # edit if not using a release candidate for a given package - dev_labels = ['e3sm_diags', 'e3sm_to_cmip', 'mache', 'mpas_analysis', - 'mpas_tools', 'zstash'] + dev_labels = ['e3sm_diags', 'e3sm_to_cmip', 'mache', 'zppy'] for package in dev_labels: channels = f'{channels} -c conda-forge/label/{package}_dev' channels = f'{channels} ' \ diff --git a/e3sm_supported_machines/default.cfg b/e3sm_supported_machines/default.cfg index 57a020b..b8d7963 100644 --- a/e3sm_supported_machines/default.cfg +++ b/e3sm_supported_machines/default.cfg @@ -22,23 +22,23 @@ ilamb = 2.7 # the version of mache to use during deployment (should match the version used # in the package itself) -mache = 1.18.0 +mache = 1.21.0 # the version of mpi4py to build if using system compilers mpi4py = 3.1.5 # the version of esmpy to build if using system compilers (must match esmf from # spack below) -esmpy = 8.4.2 +esmpy = 8.6.0 # spack package specs [spack_specs] -esmf = esmf@8.4.2+mpi+netcdf~pnetcdf~external-parallelio +esmf = esmf@8.6.0+mpi+netcdf~pnetcdf~external-parallelio hdf5 = hdf5@1.14.1+cxx+fortran+hl+mpi+shared moab = moab@5.5.1+mpi+hdf5+netcdf+pnetcdf+metis+parmetis+tempest -nco = nco@5.1.9+openmp +nco = nco@5.2.2+openmp netcdf_c = netcdf-c@4.9.2+mpi~parallel-netcdf netcdf_fortran = netcdf-fortran@4.5.4 parallel_netcdf = parallel-netcdf@1.12.3 diff --git a/e3sm_supported_machines/deploy_e3sm_unified.py b/e3sm_supported_machines/deploy_e3sm_unified.py index 49bd080..567c05e 100755 --- a/e3sm_supported_machines/deploy_e3sm_unified.py +++ b/e3sm_supported_machines/deploy_e3sm_unified.py @@ -50,7 +50,7 @@ def setup_install_env(activate_base, use_local, mache): channels = ' '.join(channels) commands = f'{activate_base} && ' \ f'conda create -y -n temp_e3sm_unified_install ' \ - f'{channels} progressbar2 jinja2 {mache}' + f'{channels} progressbar2 jinja2 {mache} pip' check_call(commands) @@ -67,14 +67,14 @@ def remove_install_env(activate_base): def install_mache_from_branch(activate_install_env, fork, branch): print('Clone and install local mache\n') commands = f'{activate_install_env} && ' \ - f'rm -rf build_mache && ' \ - f'mkdir -p build_mache && ' \ - f'cd build_mache && ' \ - f'git clone -b {branch} ' \ - f'git@github.com:{fork}.git mache && ' \ - f'cd mache && ' \ - f'conda install -y --file spec-file.txt && ' \ - f'python -m pip install --no-deps .' + f'rm -rf build_mache && ' \ + f'mkdir -p build_mache && ' \ + f'cd build_mache && ' \ + f'git clone -b {branch} ' \ + f'git@github.com:{fork}.git mache && ' \ + f'cd mache && ' \ + f'conda install -y --file spec-file.txt && ' \ + f'python -m pip install --no-deps .' check_call(commands) diff --git a/e3sm_supported_machines/pm-cpu.cfg b/e3sm_supported_machines/pm-cpu.cfg deleted file mode 100644 index a6fe199..0000000 --- a/e3sm_supported_machines/pm-cpu.cfg +++ /dev/null @@ -1,5 +0,0 @@ -# spack package specs -[spack_specs] - -# requires the version of netcdf-c without MPI support -esmf = esmf@8.4.2+mpi+netcdf~pnetcdf~external-parallelio ^netcdf-c+mpi diff --git a/e3sm_supported_machines/shared.py b/e3sm_supported_machines/shared.py index 24bd18e..532cc78 100644 --- a/e3sm_supported_machines/shared.py +++ b/e3sm_supported_machines/shared.py @@ -14,7 +14,7 @@ def parse_args(bootstrap): parser = argparse.ArgumentParser( description='Deploy E3SM-Unified') - parser.add_argument("--version", dest="version", default="1.9.2", + parser.add_argument("--version", dest="version", default="1.9.3", help="The version of E3SM-Unified to deploy") parser.add_argument("--conda", dest="conda_base", help="Path for the conda base") diff --git a/e3sm_supported_machines/templates/build.template b/e3sm_supported_machines/templates/build.template index c3f474a..d5f8794 100644 --- a/e3sm_supported_machines/templates/build.template +++ b/e3sm_supported_machines/templates/build.template @@ -14,7 +14,7 @@ fi if [ "{{ build_ilamb }}" == "True" ]; then - mamba install -y {{ ilamb_channels }} --no-deps "ilamb={{ ilamb_version }}" + conda install -y {{ ilamb_channels }} --no-deps "ilamb={{ ilamb_version }}" fi if [ "{{ build_esmpy }}" == "True" ]; then @@ -28,7 +28,7 @@ if [ "{{ build_esmpy }}" == "True" ]; then # this is an unused dumping grounds and causes disconcerting warnings rm -rf src/esmpy/fragments - python -m pip install . + python -m pip install --no-deps . cd ${build_dir} fi \ No newline at end of file diff --git a/e3sm_supported_machines/templates/load_e3sm_unified.sh.template b/e3sm_supported_machines/templates/load_e3sm_unified.sh.template index d90ab28..282fe65 100644 --- a/e3sm_supported_machines/templates/load_e3sm_unified.sh.template +++ b/e3sm_supported_machines/templates/load_e3sm_unified.sh.template @@ -1,5 +1,4 @@ source {{ conda_base }}/etc/profile.d/conda.sh -source {{ conda_base }}/etc/profile.d/mamba.sh export E3SMU_SCRIPT="{{ script_filename }}" export E3SMU_MACHINE="{{ machine }}" @@ -9,7 +8,7 @@ then # we seem to be on a login node, so load the no-MPI environment export E3SMU_MPI="NOMPI" - mamba activate {{ env_nompi }} + conda activate {{ env_nompi }} export HDF5_USE_FILE_LOCKING=FALSE @@ -17,7 +16,7 @@ else # we seem to be on a compute node, so load the MPI environment export E3SMU_MPI="{{ env_type }}" - mamba activate {{ env_name }} + conda activate {{ env_name }} {{ spack }} diff --git a/recipes/e3sm-unified/build_and_upload.bash b/recipes/e3sm-unified/build_and_upload.bash index 22e1020..b57fe69 100755 --- a/recipes/e3sm-unified/build_and_upload.bash +++ b/recipes/e3sm-unified/build_and_upload.bash @@ -2,7 +2,7 @@ set -e -conda_dir=${HOME}/miniforge +conda_dir=${HOME}/miniforge3 os_dir=linux-64 upload=False dev=False @@ -15,14 +15,12 @@ then channels="-c conda-forge/label/e3sm_diags_dev \ -c conda-forge/label/e3sm_to_cmip_dev \ -c conda-forge/label/mache_dev \ - -c conda-forge/label/mpas_analysis_dev \ - -c conda-forge/label/mpas_tools_dev \ - -c conda-forge/label/zstash_dev \ + -c conda-forge/label/zppy_dev \ -c conda-forge" for file in configs/mpi_mpich_python3.10.yaml configs/mpi_hpc_python3.10.yaml do - conda mambabuild -m $file --override-channels --use-local $channels . + conda build -m $file --override-channels --use-local $channels . done if [ $upload == "True" ] @@ -35,7 +33,7 @@ else channels="-c conda-forge" for file in configs/mpi_*_python*.yaml do - conda mambabuild -m $file --override-channels --use-local $channels . + conda build -m $file --override-channels $channels . done if [ $upload == "True" ] diff --git a/recipes/e3sm-unified/conda_first_failure.py b/recipes/e3sm-unified/conda_first_failure.py index 70801e4..b60940b 100755 --- a/recipes/e3sm-unified/conda_first_failure.py +++ b/recipes/e3sm-unified/conda_first_failure.py @@ -79,7 +79,7 @@ 'pandas' ] -base_command = ['mamba', 'create', '-y', '-n', 'test', '--dry-run', +base_command = ['conda', 'create', '-y', '-n', 'dry-run', '--dry-run', '--override-channels', '-c', 'conda-forge/label/mpas_analysis_dev', '-c', 'conda-forge/label/zstash_dev', diff --git a/recipes/e3sm-unified/meta.yaml b/recipes/e3sm-unified/meta.yaml index a28c437..52fda0c 100644 --- a/recipes/e3sm-unified/meta.yaml +++ b/recipes/e3sm-unified/meta.yaml @@ -1,5 +1,5 @@ {% set name = "E3SM-Unified" %} -{% set version = "1.9.2" %} +{% set version = "1.9.3" %} {% set build = 0 %} package: @@ -30,27 +30,35 @@ requirements: run: ### main packages ### - python - - chemdyg 0.1.4 - - e3sm_diags 2.10.1 - - e3sm_to_cmip 1.11.1 - - geometric_features 1.2.0 + - chemdyg 0.1.5 + - e3sm_diags 2.11.0 + - e3sm_to_cmip 1.11.2 + - geometric_features 1.3.0 # fixes an issues with cryptography <37 constraint - globus-cli >=3.15.0 - ilamb 2.7 # [mpi != 'nompi' and mpi != 'hpc'] - ipython - jupyter - livvkit 3.0.1 - - mache 1.18.0 + - mache 1.21.0 - moab 5.5.1 {{ mpi_prefix }}_tempest_* # [mpi != 'hpc'] - - mpas-analysis 1.9.1 - - mpas_tools 0.30.0 - - nco 5.1.9 # [mpi != 'hpc'] + - mpas-analysis 1.10.0 + - mpas_tools 0.32.0 + - nco 5.2.2 # [mpi != 'hpc'] - pcmdi_metrics 2.3.1 - tempest-remap 2.2.0 # [mpi != 'hpc'] - tempest-extremes 2.2.1 {{ mpi_prefix }}_* # [mpi != 'hpc'] - xcdat 0.6.1 - - zppy 2.3.1 + - zppy 2.4.0 - zstash 1.4.2 # [linux] + + ### mkdocs + - mkdocs-material + - pymdown-extensions + - mkdocs-monorepo-plugin + - mdutils + - mkdocs-bibtex + ### dependencies ### - {{ mpi }} # [mpi != 'nompi' and mpi != 'hpc'] - blas @@ -64,8 +72,8 @@ requirements: - dask 2023.6.0 - dogpile.cache - eofs - - esmf 8.4.2 {{ mpi_prefix }}_* - - esmpy 8.4.2 + - esmf 8.6.0 {{ mpi_prefix }}_* + - esmpy 8.6.0 - f90nml - ffmpeg - genutil 8.2.1 @@ -75,7 +83,7 @@ requirements: - ipygany - libnetcdf 4.9.2 {{ mpi_prefix }}_* - lxml - - matplotlib 3.7.1 + - matplotlib 3.8.2 - metpy - mpi4py # [mpi != 'nompi' and mpi != 'hpc'] - nb_conda