From d061e0789f1041610bdce6637cfaedbdb66b7446 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Mon, 29 Jan 2024 11:23:53 -0700 Subject: [PATCH] add USE_CONDA xml variable, add machine derecho-gpu --- config/cesm/machines/config_batch.xml | 15 +++ config/cesm/machines/config_compilers.xml | 18 +++- config/cesm/machines/config_machines.xml | 101 ++++++++++++++++++ scripts/Tools/Makefile | 11 ++ src/build_scripts/buildlib.csm_share | 5 +- src/drivers/mct/cime_config/buildexe | 3 +- .../mct/cime_config/config_component.xml | 13 +++ 7 files changed, 163 insertions(+), 3 deletions(-) diff --git a/config/cesm/machines/config_batch.xml b/config/cesm/machines/config_batch.xml index 896e37cc9cf..b0eb7039927 100644 --- a/config/cesm/machines/config_batch.xml +++ b/config/cesm/machines/config_batch.xml @@ -370,6 +370,21 @@ + + qsub + + + + + -S {{ shell }} + -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:ngpus=4 + + + main + develop + + + sbatch diff --git a/config/cesm/machines/config_compilers.xml b/config/cesm/machines/config_compilers.xml index 28cfca3cba5..75fc4a7d3f7 100644 --- a/config/cesm/machines/config_compilers.xml +++ b/config/cesm/machines/config_compilers.xml @@ -761,11 +761,27 @@ using a fortran linker. -march=core-avx2 -no-fma + + + + $ENV{NETCDF} + $ENV{PNETCDF} - -L$(CONDA_PREFIX)/lib -lpython3 -Wl,-rpath,$(CONDA_PREFIX)/lib + -Wl,--copy-dt-needed-entries -lnetcdff -lnetcdf + + + -qno-opt-dynamic-align -fp-model precise -std=gnu99 + -std=gnu89 + -march=core-avx2 -no-fma + + + -march=core-avx2 -no-fma + + + -march=core-avx2 -no-fma diff --git a/config/cesm/machines/config_machines.xml b/config/cesm/machines/config_machines.xml index 287e5152267..07ddd697a3e 100644 --- a/config/cesm/machines/config_machines.xml +++ b/config/cesm/machines/config_machines.xml @@ -1093,6 +1093,107 @@ This allows using a different mpirun command to launch unit tests + + NCAR AMD EPYC + CNL + intel + + mpich + $ENV{SCRATCH} + $ENV{CESMDATAROOT}/inputdata + $ENV{CESMDATAROOT}/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + $ENV{CESMDATAROOT}/cesm_baselines + $ENV{CESMDATAROOT}/cprnc/cprnc + 16 + pbs + cseg + 64 + 64 + TRUE + + mpiexec + + --label + --line-buffer + -n {{ total_tasks }} + + + + $LMOD_ROOT/lmod/init/perl + $LMOD_ROOT/lmod/init/env_modules_python.py + $LMOD_ROOT/lmod/init/sh + $LMOD_ROOT/lmod/init/csh + $LMOD_ROOT/lmod/libexec/lmod perl + $LMOD_ROOT/lmod/libexec/lmod python + module + module + + cesmdev/1.0 + ncarenv/23.09 + + craype + conda/latest + + + intel/2023.2.1 + mkl + spherepack/3.2 + + + intel-oneapi/2023.2.1 + mkl + + + intel-classic/2023.2.1 + mkl + + + cce/15.0.1 + cray-libsci/23.02.1.1 + + + gcc/12.2.0 + cray-libsci/23.02.1.1 + + + nvhpc/23.7 + + + ncarcompilers/1.0.0 + cmake + + + cray-mpich/8.1.27 + + + mpi-serial/2.3.0 + + + + netcdf/4.9.2 + + + + netcdf-mpi/4.9.2 + parallel-netcdf/1.12.3 + + + + esmf/8.6.0 + + + + + + 64M + hybrid + memhooks + $ENV{NCAR_ROOT_SPHEREPACK}/lib + + + + PNL IBM Xeon cluster, os is Linux (pgi), batch system is SLURM LINUX diff --git a/scripts/Tools/Makefile b/scripts/Tools/Makefile index dcf8ab3ce78..5d42f7aed35 100644 --- a/scripts/Tools/Makefile +++ b/scripts/Tools/Makefile @@ -217,6 +217,7 @@ else endif endif endif + # Set PETSc info if it is being used ifeq ($(strip $(USE_PETSC)), TRUE) ifdef PETSC_PATH @@ -546,6 +547,16 @@ ifdef LIB_MPI endif endif +# Set python info if it is being used +ifeq ($(strip $(USE_CONDA)), True) + ifdef CONDA_PREFIX + SLIBS += -L$(CONDA_PREFIX)/lib -lpython3 -Wl,-rpath,$(CONDA_PREFIX)/lib + else + $(error CONDA_PREFIX must be defined when USE_CONDA is TRUE) + endif +endif + + ifdef XIOS_PATH SLIBS += -L$(XIOS_PATH)/lib -lxios -lstdc++ endif diff --git a/src/build_scripts/buildlib.csm_share b/src/build_scripts/buildlib.csm_share index 3598a5ed18c..4859c1a8ccf 100755 --- a/src/build_scripts/buildlib.csm_share +++ b/src/build_scripts/buildlib.csm_share @@ -47,7 +47,6 @@ def buildlib(bldroot, installpath, caseroot): cimeroot = case.get_value("CIMEROOT") srcroot = case.get_value("SRCROOT") filepath = [os.path.join(caseroot,"SourceMods","src.share"), - os.path.join(srcroot, "forpy"), os.path.join(cimeroot,"src","drivers","mct","shr"), os.path.join(cimeroot,"src","share","streams"), os.path.join(cimeroot,"src","share","util"), @@ -61,6 +60,10 @@ def buildlib(bldroot, installpath, caseroot): use_esmf = "noesmf" filepath.append(os.path.join(cimeroot, "src", "share", "esmf_wrf_timemgr")) + if case.get_value("USE_CONDA"): + filepath.append(os.path.join(srcroot, "forpy")) + + comp_interface = case.get_value("COMP_INTERFACE") ninst_value = case.get_value("NINST_VALUE") diff --git a/src/drivers/mct/cime_config/buildexe b/src/drivers/mct/cime_config/buildexe index 409c7a78b09..5ee99cd3166 100755 --- a/src/drivers/mct/cime_config/buildexe +++ b/src/drivers/mct/cime_config/buildexe @@ -33,7 +33,8 @@ def _main_func(): model = case.get_value("MODEL") num_esp = case.get_value("NUM_COMP_INST_ESP") os.environ["PIO_VERSION"] = str(case.get_value("PIO_VERSION")) - + os.environ["USE_CONDA"] = str(case.get_value("USE_CONDA")) + print(f'USE_CONDA = {os.environ["USE_CONDA"]} CONDA_PREFIX = {os.environ["CONDA_PREFIX"]}') expect((num_esp is None) or (int(num_esp) == 1), "ESP component restricted to one instance") diff --git a/src/drivers/mct/cime_config/config_component.xml b/src/drivers/mct/cime_config/config_component.xml index b0fc582f855..0a574c59ac2 100644 --- a/src/drivers/mct/cime_config/config_component.xml +++ b/src/drivers/mct/cime_config/config_component.xml @@ -722,6 +722,19 @@ CLM. This is currently only supported for certain machines. + + logical + TRUE,FALSE + FALSE + build_def + env_build.xml + TRUE implies linking to a python3 library located in CONDA_PREFIX/lib + and using the file forpy_mod.F90 to allow calls between fortran and python + typical python usage might include pytorch, tensorflow, and/or numpy + + + + logical TRUE,FALSE