Merge pull request #34 from GEOS-ESM/feature/mathomp4/sync-with-gcm10193

Sync with GCM 10.19.3;  a more recent version of MAPL is required (included).

components.yaml

@@ -11,7 +11,7 @@ env:
 cmake:
   local: ./@cmake
   remote: ../ESMA_cmake.git
-  tag: v3.5.0
+  tag: v3.5.2
   develop: develop
 
 ecbuild:
@@ -22,14 +22,14 @@ ecbuild:
 GMAO_Shared:
   local: ./src/Shared/@GMAO_Shared
   remote: ../GMAO_Shared.git
-  tag: v1.4.3
+  tag: v1.4.6
   sparse: ./config/GMAO_Shared.sparse
   develop: main
 
 MAPL:
   local: ./src/Shared/@MAPL
   remote: ../MAPL.git
-  tag: v2.7.0
+  tag: v2.8.4
   develop: develop
 
 FMS:
@@ -47,7 +47,7 @@ FVdycoreCubed_GridComp:
 fvdycore:
   local: ./src/Components/GEOSctm_GridComp/@FVdycoreCubed_GridComp/@fvdycore
   remote: ../GFDL_atmos_cubed_sphere.git
-  tag: geos/v1.1.6
+  tag: geos/v1.1.7
   develop: geos/develop
 
 GEOSchem_GridComp:

src/Applications/GEOSctm_App/CMakeLists.txt

@@ -44,6 +44,14 @@ else()
    set(CFG_HYDROSTATIC FALSE)
 endif()
 
+# Did we build for AMD Rome hardware (aka EPYC)?
+cmake_host_system_information(RESULT proc_decription QUERY PROCESSOR_DESCRIPTION)
+if (${proc_decription} MATCHES "EPYC")
+  set(CFG_BUILT_ON_ROME TRUE)
+else ()
+  set(CFG_BUILT_ON_ROME FALSE)
+endif ()
+
 set (setup_scripts
    ctm_setup
    )

src/Applications/GEOSctm_App/ctm_convert.j

@@ -25,8 +25,8 @@ limit stacksize unlimited
 
 @SETENVS
 
-# Set OMP_NUM_THREADS
-# -------------------
+# Establish safe default number of OpenMP threads
+# -----------------------------------------------
 setenv OMP_NUM_THREADS 1
 
 #######################################################################

src/Applications/GEOSctm_App/ctm_regress.j

@@ -25,8 +25,8 @@ limit stacksize unlimited
 
 @SETENVS
 
-# Set OMP_NUM_THREADS
-# -------------------
+# Establish safe default number of OpenMP threads
+# -----------------------------------------------
 setenv OMP_NUM_THREADS 1
 
 @GPUSTART

src/Applications/GEOSctm_App/ctm_run.j

@@ -203,7 +203,7 @@ set month = `echo $RSTDATE | cut -d_ -f1 | cut -b5-6`
 >>>EMIP_NEWLAND<<<# Regrid Jason-3_4 REPLAY MERRA-2 NewLand Restarts
 >>>EMIP_NEWLAND<<<# ------------------------------------------------
 set RSTID = `/bin/ls *catch* | cut -d. -f1`
-set day   = `/bin/ls *catch* | cut -d. -f3 | cut -b 7-8`
+set day   = `/bin/ls *catch* | cut -d. -f3 | awk 'match($0,/[0-9]{8}/) {print substr($0,RSTART+6,2)}'`
 $GEOSBIN/regrid.pl -np -ymd ${year}${month}${day} -hr 21 -grout C${GEOSCTM_IM} -levsout ${GEOSCTM_LM} -outdir . -d . -expid $RSTID -tagin @EMIP_BCS_IN -oceanin e -i -nobkg -lbl -nolcv -tagout @LSMBCS -rs 3 -oceanout @OCEANOUT
 >>>EMIP_OLDLAND<<</bin/rm $RSTID.*.bin
 
@@ -977,31 +977,37 @@ python bundleParser.py
 setenv YEAR $yearc
 ./linkbcs
 
+# Establish safe default number of OpenMP threads
+# -----------------------------------------------
+setenv OMP_NUM_THREADS 1
+
 # Run GEOSctm.x
 # -------------
 if( $USE_SHMEM == 1 ) $GEOSBIN/RmShmKeys_sshmpi.csh >& /dev/null
 
 if( $USE_IOSERVER == 1) then
    set IOSERVER_OPTIONS = "--npes_model $MODEL_NPES --nodes_output_server $IOS_NODES"
-   set IOSERVER_EXTRA = ""
 
-   # Rome requires some extra bits for IOSERVER as it requires
-   # multigroup server with less PEs per backend node
-   if ( -x /usr/local/bin/nas_info ) then
-      set PROCTYPE=`/usr/local/bin/nas_info --nasmodel`
-      if ( $PROCTYPE == rom ) then
-         # Per Weiyuan Jiang, the ideal number of backend PEs is based on the
-         # number of HISTORY collections and number of IO nodes
+   # Per SI Team, the multigroup server should always be used
+   # The ideal number of backend PEs is based on the number of HISTORY
+   # collections and number of IO nodes
 
          # First we figure out the number of collections in the HISTORY.rc (this is not perfect, but is close to right)
          set NUM_HIST_COLS = `cat HISTORY.rc | sed -n '/^COLLECTIONS:/,/^ *::$/{p;/^ *::$/q}' | grep -v '^ *#' | wc -l`
 
+   # Protect against divide by zero
+   if ($IOS_NODES == 0) then
+      echo "Something is wrong. IOSERVER asked for, but zero IO nodes provided"
+      exit 3
+   endif
+
          # Now we divide that number of collections by the ioserver nodes
          set NUM_BACKEND_PES = `echo "scale=1;(($NUM_HIST_COLS - 1) / $IOS_NODES)" | bc | awk '{print int($1 + 0.5)}'`
 
+   # Finally multigroup requires at least two backend pes
+   if ($NUM_BACKEND_PES < 2) set NUM_BACKEND_PES = 2
+
          set IOSERVER_EXTRA = "--oserver_type multigroup --npes_backend_pernode $NUM_BACKEND_PES"
-      endif
-   endif
 else
    set IOSERVER_OPTIONS = ""
    set IOSERVER_EXTRA = ""

src/Applications/GEOSctm_App/ctm_setup

@@ -328,6 +328,10 @@ if( $HRCODE == 'c180'  | \
     $HRCODE == 'c1536' | \
     $HRCODE == 'c3072' ) then
 
+    set DEFAULT_DO_IOS = TRUE
+    echo "Do you wish to ${C1}IOSERVER${CN}? (Default: ${C2}YES${CN} or ${C2}TRUE${CN})"
+# MVAPICH2 requires ioserver for history (issue with MPI_Put and MAPL)
+else if( $MPI == mvapich2 ) then
     set DEFAULT_DO_IOS = TRUE
     echo "Do you wish to ${C1}IOSERVER${CN}? (Default: ${C2}YES${CN} or ${C2}TRUE${CN})"
 else
@@ -391,27 +395,48 @@ if ( $SITE == 'NCCS' ) then
    endif
 
 else if ( $SITE == 'NAS' ) then
+
+   # At NAS, if you build on Rome, we currently limit you to run on
+   # Rome; this is for two reasons. First, Romes are on SLES 15 and
+   # if you build on SLES 15, you can only run on SLES 15. Second,
+   # while a built-on-Rome GEOS might work on Intel processors, it
+   # will (probably) have a different output than if built on Intel
+   # due to different optimization flags.  This would violate the GEOS
+   # capability to get the same answers at NAS and NCCS
+
+   set BUILT_ON_ROME = @CFG_BUILT_ON_ROME@
+
    echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
+   if( "$BUILT_ON_ROME" != "TRUE" ) then
    echo "   ${C2}has (Haswell)${CN}"
    echo "   ${C2}bro (Broadwell)${CN}"
    echo "   ${C2}sky (Skylake)${CN} (default)"
    echo "   ${C2}cas (Cascade Lake)${CN}"
-   #echo "   ${C2}rom (AMD Rome)${CN}"
    echo " "
    echo " NOTE: Due to how FV3 is compiled by default, Sandy Bridge"
    echo "       and Ivy Bridge are not supported by current GEOS"
+   else
+      echo "   ${C2}rom (AMD Rome)${CN} (default)"
+   endif
    echo " "
    set MODEL = `echo $<`
    set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"`
    if ( .$MODEL == .) then
+      if( "$BUILT_ON_ROME" != "TRUE" ) then
       set MODEL = 'sky'
+      else
+         set MODEL = 'rom'
+      endif
    endif
 
+   if( "$BUILT_ON_ROME" != "TRUE" ) then
    if( $MODEL != 'has' & \
        $MODEL != 'bro' & \
        $MODEL != 'sky' & \
-       $MODEL != 'cas' & \
-       $MODEL != 'rom' ) goto ASKPROC
+          $MODEL != 'cas' ) goto ASKPROC
+   else
+      if( $MODEL != 'rom' ) goto ASKPROC
+   endif
 
    # Some processors have weird names at NAS
    # ---------------------------------------
@@ -1550,24 +1575,21 @@ else if( $MPI == mvapich2 ) then
 
 if( $GPU == "TRUE" ) then
 cat > $HOMDIR/SETENV.commands << EOF
-   #setenv MV2_ON_DEMAND_THRESHOLD 8192
-   #setenv MV2_USE_SHMEM_ALLREDUCE 0
-   #setenv MV2_USE_SHMEM_COLL      0
    setenv MV2_ENABLE_AFFINITY     0
    setenv MV2_RNDV_PROTOCOL       RPUT
    setenv MV2_USE_RDMA_ONE_SIDED  1
    setenv SLURM_DISTRIBUTION block
+   setenv MV2_MPIRUN_TIMEOUT 100
+   setenv MV2_GATHERV_SSEND_THRESHOLD 256
 EOF
 
 else
 
 cat > $HOMDIR/SETENV.commands << EOF
-   #setenv MV2_ON_DEMAND_THRESHOLD 8192
-   #setenv MV2_USE_SHMEM_ALLREDUCE 0
-   #setenv MV2_USE_SHMEM_COLL      0
-   #setenv MV2_USE_UD_HYBRID       0
    setenv MV2_ENABLE_AFFINITY     0
    setenv SLURM_DISTRIBUTION block
+   setenv MV2_MPIRUN_TIMEOUT 100
+   setenv MV2_GATHERV_SSEND_THRESHOLD 256
 EOF
 
 endif # if GPU and mvapich2

src/Applications/GEOSctm_App/logging.yaml

@@ -87,8 +87,8 @@ loggers:
 
    MAPL:
        handlers: [mpi_shared]
-       level: INFO
-       root_level: DEBUG
+       level: WARNING
+       root_level: INFO
 
   # Note: When enabling another logger, make sure
   #       indentation matches that of the above