add USE_CONDA xml variable, add machine derecho-gpu

ESMCI · Jan 29, 2024 · d061e07 · d061e07
1 parent fef8c7e
commit d061e07
Show file tree

Hide file tree

Showing 7 changed files with 163 additions and 3 deletions.
diff --git a/config/cesm/machines/config_batch.xml b/config/cesm/machines/config_batch.xml
@@ -370,6 +370,21 @@
     </queues>
   </batch_system>
 
+  <batch_system MACH="derecho-gpu" type="pbs" >
+    <batch_submit>qsub</batch_submit>
+    <submit_args>
+      <arg flag="-l job_priority=" name="$JOB_PRIORITY"/>
+    </submit_args>
+    <directives>
+      <directive default="/bin/bash" > -S {{ shell }}  </directive>
+      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:ngpus=4</directive>
+    </directives>
+    <queues>
+      <queue walltimemax="12:00:00" nodemin="1" nodemax="82" default="true">main</queue>
+      <queue walltimemax="1:00:00" nodemin="1" nodemax="2">develop</queue>
+    </queues>
+  </batch_system>
+
 
   <batch_system MACH="eastwind" type="slurm" >
     <batch_submit>sbatch</batch_submit>

diff --git a/config/cesm/machines/config_compilers.xml b/config/cesm/machines/config_compilers.xml
@@ -761,11 +761,27 @@ using a fortran linker.
   <FFLAGS>
     <append> -march=core-avx2 -no-fma</append>
   </FFLAGS>
+</compiler>
+
+<compiler MACH="derecho-gpu">
+  <NETCDF_PATH>$ENV{NETCDF}</NETCDF_PATH>
+  <PNETCDF_PATH>$ENV{PNETCDF}</PNETCDF_PATH>
   <SLIBS>
-    <append>-L$(CONDA_PREFIX)/lib -lpython3 -Wl,-rpath,$(CONDA_PREFIX)/lib</append>
+    <append> -Wl,--copy-dt-needed-entries -lnetcdff -lnetcdf </append>
   </SLIBS>
 </compiler>
 
+<compiler MACH="derecho-gpu" COMPILER="intel">
+  <CFLAGS>
+    <base>  -qno-opt-dynamic-align -fp-model precise -std=gnu99 </base>
+    <append MODEL="mpi-serial"> -std=gnu89 </append>
+    <append> -march=core-avx2 -no-fma</append>
+  </CFLAGS>
+  <FFLAGS>
+    <append> -march=core-avx2 -no-fma</append>
+  </FFLAGS>
+</compiler>
+
 <compiler MACH="perlmutter" >
   <FFLAGS>
     <append> -march=core-avx2 -no-fma </append>

diff --git a/config/cesm/machines/config_machines.xml b/config/cesm/machines/config_machines.xml
@@ -1093,6 +1093,107 @@ This allows using a different mpirun command to launch unit tests
 
   </machine>
 
+  <machine MACH="derecho-gpu">
+    <DESC>NCAR AMD EPYC </DESC>
+    <OS>CNL</OS>
+    <COMPILERS>intel</COMPILERS>
+<!--    <COMPILERS>intel,gnu,cray,nvhpc,intel-oneapi,intel-classic</COMPILERS> -->
+    <MPILIBS>mpich</MPILIBS>
+    <CIME_OUTPUT_ROOT>$ENV{SCRATCH}</CIME_OUTPUT_ROOT>
+    <DIN_LOC_ROOT>$ENV{CESMDATAROOT}/inputdata</DIN_LOC_ROOT>
+    <DIN_LOC_ROOT_CLMFORC>$ENV{CESMDATAROOT}/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
+    <DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
+    <BASELINE_ROOT>$ENV{CESMDATAROOT}/cesm_baselines</BASELINE_ROOT>
+    <CCSM_CPRNC>$ENV{CESMDATAROOT}/cprnc/cprnc</CCSM_CPRNC>
+    <GMAKE_J>16</GMAKE_J>
+    <BATCH_SYSTEM>pbs</BATCH_SYSTEM>
+    <SUPPORTED_BY>cseg</SUPPORTED_BY>
+    <MAX_TASKS_PER_NODE>64</MAX_TASKS_PER_NODE>
+    <MAX_MPITASKS_PER_NODE>64</MAX_MPITASKS_PER_NODE>
+    <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
+    <mpirun mpilib="default">
+      <executable>mpiexec</executable>
+      <arguments>
+	<arg name="label"> --label</arg>
+	<arg name="buffer"> --line-buffer</arg>
+	<arg name="num_tasks" > -n {{ total_tasks }}</arg>
+      </arguments>
+    </mpirun>
+    <module_system type="module" allow_error="true">
+      <init_path lang="perl">$LMOD_ROOT/lmod/init/perl</init_path>
+      <init_path lang="python">$LMOD_ROOT/lmod/init/env_modules_python.py</init_path>
+      <init_path lang="sh">$LMOD_ROOT/lmod/init/sh</init_path>
+      <init_path lang="csh">$LMOD_ROOT/lmod/init/csh</init_path>
+      <cmd_path lang="perl">$LMOD_ROOT/lmod/libexec/lmod perl</cmd_path>
+      <cmd_path lang="python">$LMOD_ROOT/lmod/libexec/lmod python</cmd_path>
+      <cmd_path lang="sh">module</cmd_path>
+      <cmd_path lang="csh">module</cmd_path>
+      <modules>
+	<command name="load">cesmdev/1.0</command>
+	<command name="load">ncarenv/23.09</command>
+	<command name="purge"/>
+	<command name="load">craype</command>
+        <command name="load">conda/latest</command>
+      </modules>
+      <modules compiler="intel">
+	<command name="load">intel/2023.2.1</command>
+	<command name="load">mkl</command>
+	<command name="load">spherepack/3.2</command>
+      </modules>
+      <modules compiler="intel-oneapi">
+	<command name="load">intel-oneapi/2023.2.1</command>
+	<command name="load">mkl</command>
+      </modules>
+      <modules compiler="intel-classic">
+	<command name="load">intel-classic/2023.2.1</command>
+	<command name="load">mkl</command>
+      </modules>
+      <modules compiler="cray">
+	<command name="load">cce/15.0.1</command>
+	<command name="load">cray-libsci/23.02.1.1</command>
+      </modules>
+      <modules compiler="gnu">
+	<command name="load">gcc/12.2.0</command>
+	<command name="load">cray-libsci/23.02.1.1</command>
+      </modules>
+      <modules compiler="nvhpc">
+	<command name="load">nvhpc/23.7</command>
+      </modules>
+      <modules>
+	<command name="load">ncarcompilers/1.0.0</command>
+	<command name="load">cmake</command>
+      </modules>
+      <modules mpilib="mpich">
+	<command name="load">cray-mpich/8.1.27</command>
+      </modules>
+      <modules mpilib="mpi-serial">
+	<command name="load">mpi-serial/2.3.0</command>
+      </modules>
+
+      <modules mpilib="mpi-serial">
+	<command name="load">netcdf/4.9.2</command>
+      </modules>
+
+      <modules mpilib="!mpi-serial">
+	<command name="load">netcdf-mpi/4.9.2</command>
+	<command name="load">parallel-netcdf/1.12.3</command>
+      </modules>
+
+      <modules>
+        <command name="load">esmf/8.6.0</command>
+      </modules>
+
+    </module_system>
+
+    <environment_variables>
+      <env name="OMP_STACKSIZE">64M</env>
+      <env name="FI_CXI_RX_MATCH_MODE">hybrid</env>
+      <env name="FI_MR_CACHE_MONITOR">memhooks</env>
+      <env name="SPHEREPACK_LIBDIR">$ENV{NCAR_ROOT_SPHEREPACK}/lib</env>
+    </environment_variables>
+
+  </machine>
+
   <machine MACH="eastwind">
     <DESC>PNL IBM Xeon cluster, os is Linux (pgi), batch system is SLURM</DESC>
     <OS>LINUX</OS>

diff --git a/scripts/Tools/Makefile b/scripts/Tools/Makefile
@@ -217,6 +217,7 @@ else
     endif
   endif
 endif
+
 # Set PETSc info if it is being used
 ifeq ($(strip $(USE_PETSC)), TRUE)
   ifdef PETSC_PATH
@@ -546,6 +547,16 @@ ifdef LIB_MPI
    endif
 endif
 
+# Set python info if it is being used
+ifeq ($(strip $(USE_CONDA)), True)
+  ifdef CONDA_PREFIX
+    SLIBS += -L$(CONDA_PREFIX)/lib -lpython3 -Wl,-rpath,$(CONDA_PREFIX)/lib
+  else
+    $(error CONDA_PREFIX must be defined when USE_CONDA is TRUE)
+  endif
+endif
+
+
 ifdef XIOS_PATH
   SLIBS += -L$(XIOS_PATH)/lib -lxios -lstdc++
 endif

diff --git a/src/build_scripts/buildlib.csm_share b/src/build_scripts/buildlib.csm_share
@@ -47,7 +47,6 @@ def buildlib(bldroot, installpath, caseroot):
         cimeroot = case.get_value("CIMEROOT")
         srcroot = case.get_value("SRCROOT")
         filepath = [os.path.join(caseroot,"SourceMods","src.share"),
-                    os.path.join(srcroot, "forpy"),
                     os.path.join(cimeroot,"src","drivers","mct","shr"),
                     os.path.join(cimeroot,"src","share","streams"),
                     os.path.join(cimeroot,"src","share","util"),
@@ -61,6 +60,10 @@ def buildlib(bldroot, installpath, caseroot):
             use_esmf = "noesmf"
             filepath.append(os.path.join(cimeroot, "src", "share", "esmf_wrf_timemgr"))
 
+        if case.get_value("USE_CONDA"):
+            filepath.append(os.path.join(srcroot, "forpy"))
+
+
         comp_interface = case.get_value("COMP_INTERFACE")
         ninst_value = case.get_value("NINST_VALUE")
 

diff --git a/src/drivers/mct/cime_config/buildexe b/src/drivers/mct/cime_config/buildexe
@@ -33,7 +33,8 @@ def _main_func():
         model     = case.get_value("MODEL")
         num_esp   = case.get_value("NUM_COMP_INST_ESP")
         os.environ["PIO_VERSION"] = str(case.get_value("PIO_VERSION"))
-
+        os.environ["USE_CONDA"] = str(case.get_value("USE_CONDA"))
+    print(f'USE_CONDA = {os.environ["USE_CONDA"]} CONDA_PREFIX = {os.environ["CONDA_PREFIX"]}')
     expect((num_esp is None) or (int(num_esp) == 1), "ESP component restricted to one instance")
 
 

diff --git a/src/drivers/mct/cime_config/config_component.xml b/src/drivers/mct/cime_config/config_component.xml
@@ -722,6 +722,19 @@
     CLM. This is currently only supported for certain machines.</desc>
   </entry>
 
+  <entry id="USE_CONDA">
+    <type>logical</type>
+    <valid_values>TRUE,FALSE</valid_values>
+    <default_value>FALSE</default_value>
+    <group>build_def</group>
+    <file>env_build.xml</file>
+    <desc> TRUE implies linking to a python3 library located in CONDA_PREFIX/lib
+    and using the file forpy_mod.F90 to allow calls between fortran and python
+    typical python usage might include pytorch, tensorflow, and/or numpy
+    </desc>
+  </entry>
+
+
   <entry id="USE_ALBANY">
     <type>logical</type>
     <valid_values>TRUE,FALSE</valid_values>