From cabcc52a3abfc3ce5cda3c710965b46d72819804 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Tue, 23 Mar 2021 09:27:21 -0400 Subject: [PATCH 01/31] Update GEOSctm to present day GEOSgcm --- components.yaml | 83 +++++++++++++++++++--------------- config/GEOSgcm_GridComp.sparse | 1 - config/GOCART.sparse | 10 ++++ 3 files changed, 57 insertions(+), 37 deletions(-) create mode 100644 config/GOCART.sparse diff --git a/components.yaml b/components.yaml index 6c27da9..7d1fa9c 100644 --- a/components.yaml +++ b/components.yaml @@ -1,53 +1,64 @@ +GEOSctm: + fixture: true + develop: main + env: - local: ./@env - remote: ../ESMA_env.git - tag: v2.0.2 - develop: master + local: ./@env + remote: ../ESMA_env.git + tag: v3.1.3 + develop: main cmake: - local: ./@cmake - remote: ../ESMA_cmake.git - tag: v2.1.2 - develop: develop + local: ./@cmake + remote: ../ESMA_cmake.git + tag: v3.3.7 + develop: develop ecbuild: - local: ./@cmake/@ecbuild - remote: ../ecbuild.git - branch: feature/FindMKL-portability-improvement + local: ./@cmake/@ecbuild + remote: ../ecbuild.git + tag: geos/v1.0.6 GMAO_Shared: - local: ./src/Shared/@GMAO_Shared - remote: ../GMAO_Shared.git - tag: v1.1.0 - sparse: ./config/GMAO_Shared.sparse - develop: master + local: ./src/Shared/@GMAO_Shared + remote: ../GMAO_Shared.git + tag: v1.3.9 + sparse: ./config/GMAO_Shared.sparse + develop: main MAPL: - local: ./src/Shared/@MAPL - remote: ../MAPL.git - tag: v2.0.0 - develop: develop + local: ./src/Shared/@MAPL + remote: ../MAPL.git + tag: v2.6.3 + develop: develop FMS: - local: ./src/Shared/@FMS - remote: ../FMS.git - tag: geos/2019.01.01 - develop: geos/master + local: ./src/Shared/@FMS + remote: ../FMS.git + tag: geos/2019.01.02+noaff.6 + develop: geos/release/2019.01 FVdycoreCubed_GridComp: - local: ./src/Components/GEOSctm_GridComp/@FVdycoreCubed_GridComp - remote: ../FVdycoreCubed_GridComp.git - tag: v1.1.0 - develop: develop + local: ./src/Components/GEOSctm_GridComp/@FVdycoreCubed_GridComp + remote: ../FVdycoreCubed_GridComp.git + tag: v1.2.10 + develop: develop fvdycore: - local: ./src/Components/GEOSctm_GridComp/@FVdycoreCubed_GridComp/@fvdycore - remote: ../GFDL_atmos_cubed_sphere.git - tag: geos/v1.1.0 - develop: geos/develop + local: ./src/Components/GEOSctm_GridComp/@FVdycoreCubed_GridComp/@fvdycore + remote: ../GFDL_atmos_cubed_sphere.git + tag: geos/v1.1.4 + develop: geos/develop GEOSchem_GridComp: - local: ./src/Components/GEOSctm_GridComp/@GEOSchem_GridComp - remote: ../GEOSchem_GridComp.git - tag: v1.2.0 - develop: develop + local: ./src/Components/GEOSctm_GridComp/@GEOSchem_GridComp + remote: ../GEOSchem_GridComp.git + tag: v1.4.4 + develop: develop + +GOCART: + local: ./src/Components/GEOSctm_GridComp/@GEOSchem_GridComp/@GOCART + remote: ../GOCART.git + sparse: ./config/GOCART.sparse + tag: v1.0.0 + develop: develop diff --git a/config/GEOSgcm_GridComp.sparse b/config/GEOSgcm_GridComp.sparse index 5c7cc60..d88ad4b 100644 --- a/config/GEOSgcm_GridComp.sparse +++ b/config/GEOSgcm_GridComp.sparse @@ -1,4 +1,3 @@ -/GEOSagcm_GridComp/GEOSsuperdyn_GridComp/FVdycoreCubed_GridComp /GEOSagcm_GridComp/GEOSsuperdyn_GridComp/CMakeLists.txt /GEOSagcm_GridComp/CMakeLists.txt /CMakeLists.txt diff --git a/config/GOCART.sparse b/config/GOCART.sparse new file mode 100644 index 0000000..c7fc83d --- /dev/null +++ b/config/GOCART.sparse @@ -0,0 +1,10 @@ + +/* +!/CCPP +!/config +!/ESMF/Aerosol_GridComp +!/ESMF/Apps +!/ESMF/GOCART2G_GridComp +!/ESMF/NUOPC +!/ESMF/Shared/Chem_Shared +!/ESMF/Shared/Chem_Base From 94a776256cda1cc8a0f01fc7e00e3489e24efa14 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Tue, 23 Mar 2021 09:37:11 -0400 Subject: [PATCH 02/31] Use MAPL not MAPL_Base in CMake --- src/Components/GEOSctm_GridComp/CMakeLists.txt | 2 +- .../GEOSctm_GridComp/CTMconvection_GridComp/CMakeLists.txt | 4 ++-- .../GEOSctm_GridComp/CTMdiffusion_GridComp/CMakeLists.txt | 2 +- .../GEOSctm_GridComp/CTMpTracers_GridComp/CMakeLists.txt | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Components/GEOSctm_GridComp/CMakeLists.txt b/src/Components/GEOSctm_GridComp/CMakeLists.txt index 5ab47da..a22173b 100644 --- a/src/Components/GEOSctm_GridComp/CMakeLists.txt +++ b/src/Components/GEOSctm_GridComp/CMakeLists.txt @@ -12,4 +12,4 @@ set (FV_PRECISION R8) esma_add_library (${this} SRCS GEOS_ctmEnvGridComp.F90 GEOS_ctmGridCompMod.F90 SUBCOMPONENTS ${alldirs} - DEPENDENCIES FVdycoreCubed_GridComp MAPL_Base) + DEPENDENCIES FVdycoreCubed_GridComp MAPL) diff --git a/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/CMakeLists.txt b/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/CMakeLists.txt index bd4fea6..8a2293e 100644 --- a/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/CMakeLists.txt +++ b/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/CMakeLists.txt @@ -1,4 +1,4 @@ -esma_set_this () + esma_set_this () if (Convection_MODE MATCHES stub) set (SRCS @@ -20,6 +20,6 @@ endif () esma_add_library (${this} SRCS ${SRCS} - DEPENDENCIES GEOSchem_GridComp Chem_Shared MAPL_Base + DEPENDENCIES GEOSchem_GridComp Chem_Shared MAPL ) diff --git a/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/CMakeLists.txt b/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/CMakeLists.txt index 8bd92fb..603f742 100644 --- a/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/CMakeLists.txt +++ b/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/CMakeLists.txt @@ -18,6 +18,6 @@ endif () esma_add_library (${this} SRCS ${SRCS} - DEPENDENCIES GEOSchem_GridComp Chem_Shared MAPL_Base + DEPENDENCIES GEOSchem_GridComp Chem_Shared MAPL ) diff --git a/src/Components/GEOSctm_GridComp/CTMpTracers_GridComp/CMakeLists.txt b/src/Components/GEOSctm_GridComp/CTMpTracers_GridComp/CMakeLists.txt index 11904bb..17551ee 100644 --- a/src/Components/GEOSctm_GridComp/CTMpTracers_GridComp/CMakeLists.txt +++ b/src/Components/GEOSctm_GridComp/CTMpTracers_GridComp/CMakeLists.txt @@ -12,6 +12,6 @@ endif () esma_add_library (${this} SRCS ${SRCS} - DEPENDENCIES FVdycoreCubed_GridComp GMIchem_GridComp Chem_Shared MAPL_Base GMAO_hermes + DEPENDENCIES FVdycoreCubed_GridComp GMIchem_GridComp Chem_Shared MAPL GMAO_hermes ) From 7fcb2cf4963f8ea47e41781c20d5ba2f8b4efedf Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Tue, 23 Mar 2021 09:37:35 -0400 Subject: [PATCH 03/31] Update to GOCART v1.0.1 --- components.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components.yaml b/components.yaml index 7d1fa9c..9e981b0 100644 --- a/components.yaml +++ b/components.yaml @@ -60,5 +60,5 @@ GOCART: local: ./src/Components/GEOSctm_GridComp/@GEOSchem_GridComp/@GOCART remote: ../GOCART.git sparse: ./config/GOCART.sparse - tag: v1.0.0 + tag: v1.0.1 develop: develop From 2f96f05fd8844eadaa364a5ed8a220a85dca8375 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Tue, 23 Mar 2021 09:46:41 -0400 Subject: [PATCH 04/31] Remove GEOSgcm_GridComp.sparse --- config/GEOSgcm_GridComp.sparse | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 config/GEOSgcm_GridComp.sparse diff --git a/config/GEOSgcm_GridComp.sparse b/config/GEOSgcm_GridComp.sparse deleted file mode 100644 index d88ad4b..0000000 --- a/config/GEOSgcm_GridComp.sparse +++ /dev/null @@ -1,5 +0,0 @@ -/GEOSagcm_GridComp/GEOSsuperdyn_GridComp/CMakeLists.txt -/GEOSagcm_GridComp/CMakeLists.txt -/CMakeLists.txt - - From e98bb3a00d73852a11abe6fcf6a5c267efc9768b Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Tue, 23 Mar 2021 11:11:24 -0400 Subject: [PATCH 05/31] Attempt to bring GEOSctm_App up to date with GEOSgcm_App --- src/Applications/GEOSctm_App/GEOSCTM.rc.tmpl | 6 +- src/Applications/GEOSctm_App/GEOSctm.F90 | 2 +- src/Applications/GEOSctm_App/ctm_archive.j | 6 +- src/Applications/GEOSctm_App/ctm_convert.j | 183 ++-- src/Applications/GEOSctm_App/ctm_moveplot.j | 10 +- src/Applications/GEOSctm_App/ctm_plot.tmpl | 10 +- src/Applications/GEOSctm_App/ctm_post.j | 20 +- .../GEOSctm_App/ctm_quickplot.csh | 9 + src/Applications/GEOSctm_App/ctm_regress.j | 227 +++-- src/Applications/GEOSctm_App/ctm_run.j | 156 +++- src/Applications/GEOSctm_App/ctm_setup | 802 +++++++++++++----- src/Applications/GEOSctm_App/fvcore_layout.rc | 114 +-- 12 files changed, 1116 insertions(+), 429 deletions(-) diff --git a/src/Applications/GEOSctm_App/GEOSCTM.rc.tmpl b/src/Applications/GEOSctm_App/GEOSCTM.rc.tmpl index 3db95c7..e38770a 100644 --- a/src/Applications/GEOSctm_App/GEOSCTM.rc.tmpl +++ b/src/Applications/GEOSctm_App/GEOSCTM.rc.tmpl @@ -3,6 +3,8 @@ # ---------------------------------------- NX: @NX NY: @NY + IOSERVER_NODES: @IOS_NDS + IM: @GEOSCTM_IM JM: @GEOSCTM_JM LM: @GEOSCTM_LM @@ -43,8 +45,8 @@ PRINTRC: 1 # Set the number of parallel I/O processes to use when # RESTART_TYPE and or CHECKPOINT_TYPE are set to pbinary or pnc4 #--------------------------------------------------------------- -PARALLEL_READFORCING: 1 -NUM_READERS: 1 +PARALLEL_READFORCING: 0 +NUM_READERS: @NUM_READERS NUM_WRITERS: 1 MAPL_ENABLE_BOOTSTRAP: @BOOT diff --git a/src/Applications/GEOSctm_App/GEOSctm.F90 b/src/Applications/GEOSctm_App/GEOSctm.F90 index f09c204..da75b3f 100644 --- a/src/Applications/GEOSctm_App/GEOSctm.F90 +++ b/src/Applications/GEOSctm_App/GEOSctm.F90 @@ -14,7 +14,7 @@ Program GEOSctm ! !USES: - use MAPL_Mod + use MAPL use GEOS_ctmGridCompMod, only: ROOT_SetServices => SetServices implicit NONE diff --git a/src/Applications/GEOSctm_App/ctm_archive.j b/src/Applications/GEOSctm_App/ctm_archive.j index b339af5..fd9975e 100644 --- a/src/Applications/GEOSctm_App/ctm_archive.j +++ b/src/Applications/GEOSctm_App/ctm_archive.j @@ -4,12 +4,12 @@ # Batch Parameters for Archive Job ####################################################################### -#PBS -l walltime=@ARCHIVE_T +#@BATCH_TIME@ARCHIVE_T #@ARCHIVE_P -#PBS -N @ARCHIVE_N +#@BATCH_JOBNAME@ARCHIVE_N #@ARCHIVE_Q #@BATCH_GROUP -#PBS -o OUTPUT +#@BATCH_OUTPUTNAMEOUTPUT ####################################################################### # System Environment Variables diff --git a/src/Applications/GEOSctm_App/ctm_convert.j b/src/Applications/GEOSctm_App/ctm_convert.j index bf43c2c..93b42b8 100755 --- a/src/Applications/GEOSctm_App/ctm_convert.j +++ b/src/Applications/GEOSctm_App/ctm_convert.j @@ -9,9 +9,9 @@ # Batch Parameters for Run Job ####################################################################### -#PBS -l walltime=@CONVERT_T +#@BATCH_TIME@CONVERT_T #@CONVERT_P -#PBS -N @CONVERT_N +#@BATCH_JOBNAME@CONVERT_N #@RUN_Q #@BATCH_GROUP @@ -25,6 +25,10 @@ limit stacksize unlimited @SETENVS +# Set OMP_NUM_THREADS +# ------------------- +setenv OMP_NUM_THREADS 1 + ####################################################################### # Architecture Specific Environment Variables ####################################################################### @@ -32,13 +36,15 @@ limit stacksize unlimited setenv ARCH `uname` setenv SITE @SITE +setenv GEOSDIR @GEOSDIR setenv GEOSBIN @GEOSBIN -setenv RUN_CMD "@RUN_CMD" setenv GCMVER @GCMVER echo "Sourcing g5_modules in $GEOSBIN" source $GEOSBIN/g5_modules >& /dev/null -setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib +setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib:${GEOSDIR}/lib + +setenv RUN_CMD "$GEOSBIN/esma_mpirun -np " ####################################################################### # Experiment Specific Environment Variables @@ -76,10 +82,12 @@ set NY = `grep "^ *NY:" $HOMDIR/AGCM.rc | cut -d':' -f2` set AGCM_IM = `grep AGCM_IM: $HOMDIR/AGCM.rc | cut -d':' -f2` set AGCM_JM = `grep AGCM_JM: $HOMDIR/AGCM.rc | cut -d':' -f2` set AGCM_LM = `grep AGCM_LM: $HOMDIR/AGCM.rc | cut -d':' -f2` -set OGCM_IM = `grep OGCM_IM: $HOMDIR/AGCM.rc | cut -d':' -f2` -set OGCM_JM = `grep OGCM_JM: $HOMDIR/AGCM.rc | cut -d':' -f2` ->>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>FVLATLON<<>>FVCUBED<<>>DATAOCEAN<<>>COUPLED<<& /dev/null - /bin/ln -sf $EXPDIR/RC/* . - /bin/cp -f $HOMDIR/*.rc . + +/bin/ln -sf $EXPDIR/RC/* . +@CPEXEC -f $HOMDIR/*.rc . +@CPEXEC -f $HOMDIR/*.nml . +@CPEXEC -f $HOMDIR/*.yaml . + +cat fvcore_layout.rc >> input.nml + if(-e ExtData.rc ) /bin/rm -f ExtData.rc set extdata_files = `/bin/ls -1 *_ExtData.rc` @@ -164,7 +177,7 @@ echo "Converting from $fromformat restarts to $toformat restarts" echo "Copying cap_restart into $SCRDIR for BCs" if (-e $FNDDIR/cap_restart) then - /bin/cp $FNDDIR/cap_restart . + @CPEXEC $FNDDIR/cap_restart . else echo "Couldn't find cap_restart in $FNDDIR. Please place" echo "cap_restart in the same directory as the restarts." @@ -178,7 +191,8 @@ endif setenv BCSDIR @BCSDIR setenv SSTDIR @SSTDIR setenv CHMDIR @CHMDIR -setenv BCRSLV ${ATMOStag}_${OCEANtag} +>>>DATAOCEAN<<>>COUPLED<<>>COUPLED<< $FILE /bin/mkdir -p ExtData /bin/ln -sf $CHMDIR/* ExtData ->>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>DATAOCEAN<<>>DATAOCEAN<<>>DATAOCEAN<<>>DATAOCEAN<<>>DATAOCEAN<<>>DATAOCEAN<<>>DATAOCEAN<<>>DATAOCEAN<<>>DATAOCEAN<<>>DATAOCEAN<<>>DATAOCEAN<<>>DATAOCEAN<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<< $FILE >>>FVCUBED<<>>FVCUBED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<<@CPEXEC $HOMDIR/*_table . +>>>COUPLED<<<@CPEXEC $GRIDDIR/INPUT/* INPUT >>>COUPLED<<>>COUPLED<<>>FVCUBED<<>>FVCUBED<<>>FVCUBED<<>>COUPLED<<>>COUPLED<<<@CPEXEC $CNVDIR/RESTART/* INPUT ####################################################################### # Set Experiment Run Parameters that were altered @@ -389,8 +448,15 @@ set NY = `grep "^ *NY:" AGCM.rc | cut -d':' -f2` # Check for Over-Specification of CPU Resources # --------------------------------------------- - if ($?PBS_NODEFILE) then + if ($?SLURM_NTASKS) then + set NCPUS = $SLURM_NTASKS + else if ($?PBS_NODEFILE) then set NCPUS = `cat $PBS_NODEFILE | wc -l` + else + set NCPUS = NULL + endif + + if ( $NCPUS != NULL ) then @ NPES = $NX * $NY if( $NPES > $NCPUS ) then echo "CPU Resources are Over-Specified" @@ -409,11 +475,11 @@ set NY = `grep "^ *NY:" AGCM.rc | cut -d':' -f2` if ( -x $GEOSBIN/rs_numtiles.x ) then - set N_SALT_TILES_EXPECTED = `/bin/grep '^\s*0' tile.data | wc -l` - set N_SALT_TILES_FOUND = `$GEOSBIN/rs_numtiles.x saltwater_internal_rst | awk '{print $3}'` + set N_SALT_TILES_EXPECTED = `grep '^ *0' tile.data | wc -l` + set N_SALT_TILES_FOUND = `$RUN_CMD 1 $GEOSBIN/rs_numtiles.x openwater_internal_rst | grep Total | awk '{print $NF}'` if ( $N_SALT_TILES_EXPECTED != $N_SALT_TILES_FOUND ) then - echo "Error! Found $N_SALT_TILES_FOUND tiles in saltwater. Expect to find $N_SALT_TILES_EXPECTED tiles." + echo "Error! Found $N_SALT_TILES_FOUND tiles in openwater. Expect to find $N_SALT_TILES_EXPECTED tiles." echo "Your restarts are probably for a different ocean." exit 7 endif @@ -455,7 +521,7 @@ endif ####################################################################### set numrst = `echo $found_rst_types | wc -w` -set numchk = `echo $chk_types | wc -w` +set numchk = `echo $chk_files | wc -w` @ n = 1 @ z = $numchk + 1 @@ -463,11 +529,8 @@ while ( $n <= $numrst ) if ( -e $found_rst_files[$n] ) then @ m = 1 while ( $m <= $numchk ) - if( $chk_types[$m] == $found_rst_types[$n] || ) then - set chk_ext = `cat AGCM.rc | grep ${chk_types[$m]}_CHECKPOINT_TYPE | cut -d: -f2` - if( $chk_ext =~ *nc4 ) set ext = nc4 - if( $chk_ext =~ *binary ) set ext = bin - /bin/mv $chk_files[$m] $RSTDIR/$found_rst_files[$n].$ext + if( $chk_files[$m] == $found_rst_types[$n] || ) then + /bin/mv $chk_file_names[$m] $RSTDIR/$found_rst_files[$n]$toext @ m = $numchk + 999 else @ m = $m + 1 @@ -475,7 +538,7 @@ while ( $n <= $numrst ) end wait if( $m == $z ) then - echo "Warning!! Could not find CHECKPOINT/RESTART match for: " $chk_types[$n] + echo "Warning!! Could not find CHECKPOINT/RESTART match for: " $chk_files[$n] exit endif endif diff --git a/src/Applications/GEOSctm_App/ctm_moveplot.j b/src/Applications/GEOSctm_App/ctm_moveplot.j index 5409e05..764adf2 100755 --- a/src/Applications/GEOSctm_App/ctm_moveplot.j +++ b/src/Applications/GEOSctm_App/ctm_moveplot.j @@ -4,12 +4,12 @@ # Batch Parameters for Plot Job ####################################################################### -#PBS -l walltime=1:00:00 -#PBS -l select=1:ncpus=1 -#PBS -N @MOVE_N +#@BATCH_TIME1:00:00 +#@MOVE_P +#@BATCH_JOBNAME@MOVE_N #@MOVE_Q #@BATCH_GROUP -#PBS -o ctm_moveplot.o +#@BATCH_OUTPUTNAMEgcm_moveplot.o ####################################################################### # System Environment Variables @@ -29,7 +29,7 @@ setenv ARCH `uname` setenv SITE @SITE setenv GEOSBIN @GEOSBIN -setenv GEOSUTIL @GEOSSRC/GMAO_Shared/GEOS_Util +setenv GEOSUTIL @GEOSSRC source $GEOSBIN/g5_modules setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib diff --git a/src/Applications/GEOSctm_App/ctm_plot.tmpl b/src/Applications/GEOSctm_App/ctm_plot.tmpl index 6496cf5..718b4b7 100755 --- a/src/Applications/GEOSctm_App/ctm_plot.tmpl +++ b/src/Applications/GEOSctm_App/ctm_plot.tmpl @@ -4,12 +4,12 @@ # Batch Parameters for Plot Job ####################################################################### -#PBS -l walltime=@PLOT_T +#@BATCH_TIME@PLOT_T #@PLOT_P -#PBS -N @PLOT_N +#@BATCH_JOBNAME@NAME #@PLOT_Q #@BATCH_GROUP -#PBS -o OUTPUT +#@BATCH_OUTPUTNAMEOUTPUT ####################################################################### # System Environment Variables @@ -29,7 +29,7 @@ setenv ARCH `uname` setenv SITE @SITE setenv GEOSBIN @GEOSBIN -setenv GEOSUTIL @GEOSSRC/GMAO_Shared/GEOS_Util +setenv GEOSUTIL @GEOSSRC source $GEOSBIN/g5_modules setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib @@ -59,5 +59,5 @@ endif ####################################################################### set MOVE = `grep PLOT_MOVE: $EXPDIR/plot/plot.rc | cut -d'#' -f1 | cut -d':' -f2` -if( $MOVE[1] == 'ON' ) qsub $EXPDIR/plot/ctm_moveplot.j +if( $MOVE[1] == 'ON' ) @BATCH_CMD $EXPDIR/plot/ctm_moveplot.j diff --git a/src/Applications/GEOSctm_App/ctm_post.j b/src/Applications/GEOSctm_App/ctm_post.j index ead897e..43fb2cf 100755 --- a/src/Applications/GEOSctm_App/ctm_post.j +++ b/src/Applications/GEOSctm_App/ctm_post.j @@ -4,12 +4,13 @@ # Batch Parameters for Post-Processing Job ####################################################################### -#PBS -l walltime=@POST_T +#@BATCH_TIME@POST_T #@POST_P -#PBS -N @POST_N +#@BATCH_JOBNAME@POST_N_@COLLECTION.@YYYYMM #@POST_Q #@BATCH_GROUP -#PBS -o @POST_O +#@BATCH_OUTPUTNAME@POST_O +#@BATCH_JOINOUTERR ####################################################################### # System Environment Variables @@ -32,16 +33,19 @@ setenv GEOSBIN @GEOSBIN setenv GEOSUTIL @GEOSSRC setenv BATCHNAME "@POST_N" -if( $?PBS_NODEFILE ) then - setenv RUN_CMD "@RUN_CMD" +source $GEOSBIN/g5_modules +setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib + +if( $?SLURM_NTASKS ) then + setenv RUN_CMD "$GEOSBIN/esma_mpirun -np " + set NCPUS = $SLURM_NTASKS +else if( $?PBS_NODEFILE ) then + setenv RUN_CMD "$GEOSBIN/esma_mpirun -np " set NCPUS = `cat $PBS_NODEFILE | wc -l` else set NCPUS = NULL endif -source $GEOSBIN/g5_modules -setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib - ####################################################################### # Perform Post Processing ####################################################################### diff --git a/src/Applications/GEOSctm_App/ctm_quickplot.csh b/src/Applications/GEOSctm_App/ctm_quickplot.csh index 93e80aa..d0dd2a7 100755 --- a/src/Applications/GEOSctm_App/ctm_quickplot.csh +++ b/src/Applications/GEOSctm_App/ctm_quickplot.csh @@ -8,8 +8,16 @@ set LISTFILE = $2 ####################################################################### while( -e $LISTFILE ) + if( -e LOCKFILE ) /bin/echo "Waiting for LOCKFILE ... " + if( -e LOCKFILE ) set LOCK = `stat -c %Z LOCKFILE` + while( -e LOCKFILE ) + set EPOCH = `date +'%s'` + @ LOCKAGE = $EPOCH - $LOCK + if( $LOCKAGE > 120 ) /bin/rm -f LOCKFILE + end lockfile -${JOBID} LOCKFILE + if( -e $LISTFILE ) then set PLOT_COMMAND = `head -1 $LISTFILE` sed 1,1d -i $LISTFILE @@ -21,5 +29,6 @@ while( -e $LISTFILE ) else /bin/rm -f LOCKFILE endif + end diff --git a/src/Applications/GEOSctm_App/ctm_regress.j b/src/Applications/GEOSctm_App/ctm_regress.j index 386d8d4..af50826 100755 --- a/src/Applications/GEOSctm_App/ctm_regress.j +++ b/src/Applications/GEOSctm_App/ctm_regress.j @@ -9,9 +9,9 @@ # Batch Parameters for Regress Job ####################################################################### -#PBS -l walltime=@RUN_T +#@BATCH_TIME@RUN_T #@RUN_P -#PBS -N @REGRESS_N +#@BATCH_JOBNAME@REGRESS_N #@RUN_Q #@BATCH_GROUP @@ -25,6 +25,10 @@ limit stacksize unlimited @SETENVS +# Set OMP_NUM_THREADS +# ------------------- +setenv OMP_NUM_THREADS 1 + @GPUSTART ####################################################################### @@ -34,11 +38,13 @@ limit stacksize unlimited setenv ARCH `uname` setenv SITE @SITE +setenv GEOSDIR @GEOSDIR setenv GEOSBIN @GEOSBIN -setenv RUN_CMD "@RUN_CMD" source $GEOSBIN/g5_modules -setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib +setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib:${GEOSDIR}/lib + +setenv RUN_CMD "$GEOSBIN/esma_mpirun -np " ####################################################################### # Experiment Specific Environment Variables @@ -63,13 +69,17 @@ cd $HOMDIR set files = `ls -1 *.rc` foreach file ($files) set fname = `echo $file | cut -d "." -f1` - /bin/cp $fname.rc $EXPDIR/regress + @CPEXEC $fname.rc $EXPDIR/regress end cd $EXPDIR/regress /bin/ln -s $EXPDIR/RC/*.rc $EXPDIR/regress -/bin/cp $EXPDIR/GEOSctm.x $EXPDIR/regress -/bin/cp $EXPDIR/linkbcs $EXPDIR/regress +@CPEXEC $EXPDIR/GEOSctm.x $EXPDIR/regress +@CPEXEC $EXPDIR/linkbcs $EXPDIR/regress +@CPEXEC $HOMDIR/*.yaml $EXPDIR/regress +>>>COUPLED<<<@CPEXEC $HOMDIR/*.nml $EXPDIR/regress +>>>MOM6<<<@CPEXEC $HOMDIR/MOM_input $EXPDIR/regress +>>>MOM6<<<@CPEXEC $HOMDIR/MOM_override $EXPDIR/regress cat fvcore_layout.rc >> input.nml @@ -81,6 +91,7 @@ cat $extdata_files > ExtData.rc # ----------------------------- set IM = `grep AGCM_IM: $HOMDIR/AGCM.rc | cut -d':' -f2` set JM = `grep AGCM_JM: $HOMDIR/AGCM.rc | cut -d':' -f2` +set LM = `grep AGCM_LM: $HOMDIR/AGCM.rc | cut -d':' -f2` @ IM6 = 6 * $IM if( $IM6 == $JM ) then @@ -91,29 +102,61 @@ endif # Create Restart List # ------------------- -set rst_types = `cat AGCM.rc | grep "RESTART_FILE" | cut -d ":" -f1 | cut -d "_" -f1-2` -set chk_types = `cat AGCM.rc | grep "CHECKPOINT_FILE" | cut -d ":" -f1 | cut -d "_" -f1-2` -set rst_files = `cat AGCM.rc | grep "RESTART_FILE" | cut -d ":" -f2` -set chk_files = `cat AGCM.rc | grep "CHECKPOINT_FILE" | cut -d ":" -f2` +set rst_files = `cat AGCM.rc | grep "RESTART_FILE" | grep -v VEGDYN | grep -v "#" | cut -d ":" -f1 | cut -d "_" -f1-2` +set rst_file_names = `cat AGCM.rc | grep "RESTART_FILE" | grep -v VEGDYN | grep -v "#" | cut -d ":" -f2` + +set chk_files = `cat AGCM.rc | grep "CHECKPOINT_FILE" | grep -v "#" | cut -d ":" -f1 | cut -d "_" -f1-2` +set chk_file_names = `cat AGCM.rc | grep "CHECKPOINT_FILE" | grep -v "#" | cut -d ":" -f2` # Remove possible bootstrap parameters (+/-) # ------------------------------------------ -set dummy = `echo $rst_files` -set rst_files = '' +set dummy = `echo $rst_file_names` +set rst_file_names = '' foreach rst ( $dummy ) set length = `echo $rst | awk '{print length($0)}'` set bit = `echo $rst | cut -c1` if( "$bit" == "+" | \ "$bit" == "-" ) set rst = `echo $rst | cut -c2-$length` - set rst_files = `echo $rst_files $rst` + set rst_file_names = `echo $rst_file_names $rst` end # Copy Restarts to Regress directory # ---------------------------------- -foreach rst ( $rst_files ) - /bin/cp $EXPDIR/$rst $EXPDIR/regress +foreach rst ( $rst_file_names ) + @CPEXEC $EXPDIR/$rst $EXPDIR/regress end -/bin/cp $EXPDIR/cap_restart $EXPDIR/regress +@CPEXEC $EXPDIR/cap_restart $EXPDIR/regress + +>>>COUPLED<<>>COUPLED<<<@CPEXEC $EXPDIR/RESTART/* INPUT + +setenv YEAR `cat cap_restart | cut -c1-4` +./linkbcs +if(! -e tile.bin) $GEOSBIN/binarytile.x tile.data tile.bin + +####################################################################### +# Split Saltwater Restart if detected +####################################################################### + +if ( -e $EXPDIR/saltwater_internal_rst ) then + + # The splitter script requires an OutData directory + # ------------------------------------------------- + if (! -d OutData ) mkdir -p OutData + + # Run the script + # -------------- + $RUN_CMD 1 $GEOSBIN/SaltIntSplitter tile.data $EXPDIR/saltwater_internal_rst + + # Move restarts + # ------------- + /bin/mv OutData/openwater_internal_rst OutData/seaicethermo_internal_rst . + + # Remove OutData + # -------------- + /bin/rmdir OutData + +endif ####################################################################### # Create Simple History for Efficiency @@ -171,6 +214,57 @@ set date = `cat cap_restart` set nymd0 = $date[1] set nhms0 = $date[2] +# Select proper MERRA-2 GOCART Emission RC Files +# ---------------------------------------------- +setenv EMISSIONS @EMISSIONS +if( @EMISSIONS =~ MERRA2* ) then + set MERRA2_Transition_Date = 20000401 + + if( $nymd0 < ${MERRA2_Transition_Date} ) then + set MERRA2_EMISSIONS_DIRECTORY = $GEOSDIR/etc/@EMISSIONS/19600101-20000331 + else + set MERRA2_EMISSIONS_DIRECTORY = $GEOSDIR/etc/@EMISSIONS/20000401-present + endif + + if( $LM == 72 ) then + @CPEXEC --remove-destination ${MERRA2_EMISSIONS_DIRECTORY}/*.rc . + else + set files = `/bin/ls -1 ${MERRA2_EMISSIONS_DIRECTORY}/*.rc` + foreach file ($files) + /bin/rm -f `basename $file` + /bin/rm -f dummy + @CPEXEC $file dummy + cat dummy | sed -e "s|/L72/|/L${LM}/|g" | sed -e "s|z72|z${LM}|g" > `basename $file` + end + endif +endif + +if(-e ExtData.rc ) /bin/rm -f ExtData.rc +set extdata_files = `/bin/ls -1 *_ExtData.rc` +cat $extdata_files > ExtData.rc + + +# If REPLAY, link necessary forcing files +# --------------------------------------- +set REPLAY_MODE = `grep REPLAY_MODE: AGCM.rc | grep -v '#' | cut -d: -f2` +if( $REPLAY_MODE == 'Exact' | $REPLAY_MODE == 'Regular' ) then + + set ANA_EXPID = `grep REPLAY_ANA_EXPID: AGCM.rc | grep -v '#' | cut -d: -f2` + set ANA_LOCATION = `grep REPLAY_ANA_LOCATION: AGCM.rc | grep -v '#' | cut -d: -f2` + + set REPLAY_FILE = `grep REPLAY_FILE: AGCM.rc | grep -v '#' | cut -d: -f2` + set REPLAY_FILE09 = `grep REPLAY_FILE09: AGCM.rc | grep -v '#' | cut -d: -f2` + set REPLAY_FILE_TYPE = `echo $REPLAY_FILE | cut -d"/" -f1 | grep -v %` + set REPLAY_FILE09_TYPE = `echo $REPLAY_FILE09 | cut -d"/" -f1 | grep -v %` + + # Link REPLAY files + # ----------------- + /bin/ln -sf ${ANA_LOCATION}/aod . + /bin/ln -sf ${ANA_LOCATION}/${REPLAY_FILE_TYPE} . + /bin/ln -sf ${ANA_LOCATION}/${REPLAY_FILE09_TYPE} . + +endif + ################################################################## ###### ###### Perform Regression Test # 1 @@ -180,9 +274,9 @@ set nhms0 = $date[2] set test_duration = 240000 -/bin/cp CAP.rc CAP.rc.orig -/bin/cp AGCM.rc AGCM.rc.orig -/bin/cp HISTORY.rc0 HISTORY.rc +@CPEXEC CAP.rc CAP.rc.orig +@CPEXEC AGCM.rc AGCM.rc.orig +@CPEXEC HISTORY.rc0 HISTORY.rc set NX0 = `grep "^ *NX:" AGCM.rc.orig | cut -d':' -f2` set NY0 = `grep "^ *NY:" AGCM.rc.orig | cut -d':' -f2` @@ -196,10 +290,6 @@ set newstring = "JOB_SGMT: 00000000 ${test_duration}" /bin/mv CAP.rc CAP.tmp cat CAP.tmp | sed -e "s?$oldstring?$newstring?g" > CAP.rc -setenv YEAR `cat cap_restart | cut -c1-4` -./linkbcs -if(! -e tile.bin) $GEOSBIN/binarytile.x tile.data tile.bin - set NX = `grep "^ *NX": AGCM.rc | cut -d':' -f2` set NY = `grep "^ *NY": AGCM.rc | cut -d':' -f2` @ NPES = $NX * $NY @@ -210,9 +300,10 @@ set date = `cat cap_restart` set nymde = $date[1] set nhmse = $date[2] -foreach chk ( $chk_files ) +foreach chk ( $chk_file_names ) /bin/mv $chk ${chk}.${nymde}_${nhmse}.1 end +>>>MOM6<<>>DATAOCEAN<<>>DATAOCEAN<<< @ test_NX = $NPES0 / 6 +>>>DATAOCEAN<<< @ test_NP = $IM / $test_NX +>>>DATAOCEAN<<< if($test_NP < 4 ) then +>>>DATAOCEAN<<< @ test_NX = $IM / 4 # To ensure enough gridpoints for HALO +>>>DATAOCEAN<<< endif +>>>DATAOCEAN<<< set test_NY = 6 +>>>DATAOCEAN<<>>DATAOCEAN<<< set test_NX = $NY0 +>>>DATAOCEAN<<< set test_NY = $NX0 +>>>DATAOCEAN<<>>COUPLED<<>>COUPLED<< cap_restart -/bin/cp CAP.rc.orig CAP.rc -/bin/cp AGCM.rc.orig AGCM.rc -/bin/cp HISTORY.rc0 HISTORY.rc +@CPEXEC CAP.rc.orig CAP.rc +@CPEXEC AGCM.rc.orig AGCM.rc +@CPEXEC HISTORY.rc0 HISTORY.rc ./strip CAP.rc set oldstring = `cat CAP.rc | grep JOB_SGMT:` @@ -264,39 +358,41 @@ set NY = `grep "^ *NY": AGCM.rc | cut -d':' -f2` @ NPES = $NX * $NY $RUN_CMD $NPES ./GEOSctm.x -foreach rst ( $rst_files ) +foreach rst ( $rst_file_names ) /bin/rm -f $rst end -set numrst = `echo $rst_types | wc -w` -set numchk = `echo $chk_types | wc -w` +set numrst = `echo $rst_files | wc -w` +set numchk = `echo $chk_files | wc -w` @ n = 1 @ z = $numrst + 1 while ( $n <= $numchk ) @ m = 1 while ( $m <= $numrst ) - if( $chk_types[$n] == $rst_types[$m] || \ - \#$chk_types[$n] == $rst_types[$m] ) then - /bin/mv $chk_files[$n] $rst_files[$m] + if( $chk_files[$n] == $rst_files[$m] || \ + \#$chk_files[$n] == $rst_files[$m] ) then + /bin/mv $chk_file_names[$n] $rst_file_names[$m] @ m = $numrst + 999 else @ m = $m + 1 endif end if( $m == $z ) then - echo "Warning!! Could not find CHECKPOINT/RESTART match for: " $chk_types[$n] + echo "Warning!! Could not find CHECKPOINT/RESTART match for: " $chk_files[$n] exit endif @ n = $n + 1 end +>>>COUPLED<<<@CPEXEC RESTART/* INPUT + ################################################################## ###### ###### Perform Regression Test # 3 ###### ################################################################## -set test_duration = 030000 +set test_duration = 060000 if( $CUBE == TRUE ) then set test_NX = 1 @@ -323,6 +419,17 @@ set oldstring = `cat AGCM.rc | grep "^ *NY:"` set newstring = "NY: ${test_NY}" /bin/mv AGCM.rc AGCM.tmp cat AGCM.tmp | sed -e "s?$oldstring?$newstring?g" > AGCM.rc +>>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<< AGCM.rc +>>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<< AGCM.rc + +>>>MOM5<<>>MOM6<<>>MOM6<<>>MOM6<<>>MOM6<<>>MOM6<<>>MOM6<<< set check = true +>>>MOM6<<< if( $check == true ) then +>>>MOM6<<< echo Comparing "MOM6 restarts" +>>>MOM6<<< cmp $file1 $file2 +>>>MOM6<<< if( $status == 0 ) then +>>>MOM6<<< echo Success! +>>>MOM6<<< echo " " +>>>MOM6<<< else +>>>MOM6<<< echo Failed! +>>>MOM6<<< echo " " +>>>MOM6<<< set pass = false +>>>MOM6<<< endif +>>>MOM6<<< endif +>>>MOM6<< $NCPUS ) then echo "CPU Resources are Over-Specified" echo "--------------------------------" echo "Allotted NCPUs: $NCPUS" - echo "Specified NX : $NX" - echo "Specified NY : $NY" + echo "Requested NCPUs: $NPES" + echo "" + echo "Specified NX: $NX" + echo "Specified NY: $NY" + echo "" + echo "Specified IOSERVER_NODES: $AGCM_IOS_NODES" + echo "Specified cores per node: $NCPUS_PER_NODE" exit endif + + else + + @ NPES = $MODEL_NPES + + if( $NPES > $NCPUS ) then + echo "CPU Resources are Over-Specified" + echo "--------------------------------" + echo "Allotted NCPUs: $NCPUS" + echo "Requested NCPUs: $NPES" + echo "" + echo "Specified NX: $NX" + echo "Specified NY: $NY" + exit endif + endif +else + # This is for the desktop path + + @ NPES = $MODEL_NPES + +endif + ####################################################################### # GCMEMIP Setup ####################################################################### @@ -133,19 +188,24 @@ chmod +x $FILE set year = `echo $RSTDATE | cut -d_ -f1 | cut -b1-4` set month = `echo $RSTDATE | cut -d_ -f1 | cut -b5-6` -# Copy MERRA-2 Restarts -# --------------------- -@CPEXEC /discover/nobackup/projects/gmao/g6dev/ltakacs/MERRA2/restarts/AMIP/M${month}/restarts.${year}${month}.tar . +>>>EMIP_OLDLAND<<<# Copy MERRA-2 Restarts +>>>EMIP_OLDLAND<<<# --------------------- +>>>EMIP_NEWLAND<<<# Copy Jason-3_4 REPLAY MERRA-2 NewLand Restarts +>>>EMIP_NEWLAND<<<# ---------------------------------------------- +@CPEXEC /discover/nobackup/projects/gmao/g6dev/ltakacs/@EMIP_MERRA2/restarts/AMIP/M${month}/restarts.${year}${month}.tar . @TAREXEC xf restarts.${year}${month}.tar /bin/rm restarts.${year}${month}.tar -/bin/rm MERRA2*bin +>>>EMIP_OLDLAND<<>>EMIP_OLDLAND<<<# Regrid MERRA-2 Restarts +>>>EMIP_OLDLAND<<<# ----------------------- +>>>EMIP_NEWLAND<<<# Regrid Jason-3_4 REPLAY MERRA-2 NewLand Restarts +>>>EMIP_NEWLAND<<<# ------------------------------------------------ +set RSTID = `/bin/ls *catch* | cut -d. -f1` +set day = `/bin/ls *catch* | cut -d. -f3 | cut -b 7-8` +$GEOSBIN/regrid.pl -np -ymd ${year}${month}${day} -hr 21 -grout C${AGCM_IM} -levsout ${AGCM_LM} -outdir . -d . -expid $RSTID -tagin @EMIP_BCS_IN -oceanin e -i -nobkg -lbl -nolcv -tagout @LSMBCS -rs 3 -oceanout @OCEANOUT +>>>EMIP_OLDLAND<<>>EMIP_OLDLAND<<>>EMIP_OLDLAND<<<$GEOSBIN/gogo.x -s $RSTID.Chem_Registry.rc.${year}${month}${day}_21z -t $EXPDIR/RC/Chem_Registry.rc -i gocart_internal_rst.merra2 -o gocart_internal_rst -r C${AGCM_IM} -l ${AGCM_LM} # Create CAP.rc and cap_restart # ----------------------------- -set nymd = ${year}${month}10 +set nymd = ${year}${month}${day} set nhms = 210000 echo $nymd $nhms > cap_restart @@ -207,6 +271,7 @@ else endif @CPEXEC -f $HOMDIR/*.rc . @CPEXEC -f $HOMDIR/*.nml . + @CPEXEC -f $HOMDIR/*.yaml . @CPEXEC $GEOSBIN/bundleParser.py . cat fvcore_layout.rc >> input.nml @@ -461,6 +526,19 @@ endif set yearc = `echo $nymdc | cut -c1-4` set yearf = `echo $nymdf | cut -c1-4` +# For Non-Reynolds SST, Modify local CAP.rc Ending date if Finish time exceeds Current year boundary +# -------------------------------------------------------------------------------------------------- +if( @OCEANtag != DE0360xPE0180 ) then + if( $yearf > $yearc ) then + @ yearf = $yearc + 1 + @ nymdf = $yearf * 10000 + 0101 + set oldstring = `cat CAP.rc | grep END_DATE:` + set newstring = "END_DATE: $nymdf $nhmsf" + /bin/mv CAP.rc CAP.tmp + cat CAP.tmp | sed -e "s?$oldstring?$newstring?g" > CAP.rc + endif +endif + # Select proper MERRA-2 GOCART Emission RC Files # (NOTE: MERRA2-DD has same transition date) # ---------------------------------------------- @@ -469,7 +547,7 @@ if( ${EMISSIONS} == MERRA2 | \ set MERRA2_Transition_Date = 20000401 if( $nymdc < ${MERRA2_Transition_Date} ) then - set MERRA2_EMISSIONS_DIRECTORY = $GEOSDIR/$ARCH/etc/$EMISSIONS/19600101-20000331 + set MERRA2_EMISSIONS_DIRECTORY = $GEOSDIR/etc/$EMISSIONS/19600101-20000331 if( $nymdf > ${MERRA2_Transition_Date} ) then set nymdf = ${MERRA2_Transition_Date} set oldstring = `cat CAP.rc | grep END_DATE:` @@ -478,7 +556,7 @@ if( ${EMISSIONS} == MERRA2 | \ cat CAP.tmp | sed -e "s?$oldstring?$newstring?g" > CAP.rc endif else - set MERRA2_EMISSIONS_DIRECTORY = $GEOSDIR/$ARCH/etc/$EMISSIONS/20000401-present + set MERRA2_EMISSIONS_DIRECTORY = $GEOSDIR/etc/$EMISSIONS/20000401-present endif if( $GEOSCTM_LM == 72 ) then @@ -883,6 +961,13 @@ endif sed -i 's/QFED\/NRT/QFED/' ${COMPNAME}_ExtData_${sYear}.rc sed -i 's/v2.5r1_0.1_deg/v2.5r1\/0.1/' ${COMPNAME}_ExtData_${sYear}.rc +# Environment variables for MPI, etc +# ---------------------------------- + +@SETENVS + +@GPUSTART + # Run bundleParser.py #--------------------- python bundleParser.py @@ -895,8 +980,15 @@ setenv YEAR $yearc # Run GEOSctm.x # ------------- if( $USE_SHMEM == 1 ) $GEOSBIN/RmShmKeys_sshmpi.csh - @ NPES = $NX * $NY -$RUN_CMD $NPES ./GEOSctm.x + +if( $USE_IOSERVER == 1) then + set IOSERVER_OPTIONS = "--npes_model $MODEL_NPES --nodes_output_server $IOS_NODES" +else + set IOSERVER_OPTIONS = "" +endif + +$RUN_CMD $NPES ./GEOSctm.x $IOSERVER_OPTIONS --logging_config 'logging.yaml' + if( $USE_SHMEM == 1 ) $GEOSBIN/RmShmKeys_sshmpi.csh set rc = $status @@ -1042,8 +1134,8 @@ endif if ( $rc == 0 ) then cd $HOMDIR if( $GCMEMIP == TRUE ) then - if( $capdate < $enddate ) qsub $HOMDIR/ctm_run.j$RSTDATE + if( $capdate < $enddate ) @BATCH_CMD $HOMDIR/ctm_run.j$RSTDATE else - if( $capdate < $enddate ) qsub $HOMDIR/ctm_run.j + if( $capdate < $enddate ) @BATCH_CMD $HOMDIR/ctm_run.j endif endif diff --git a/src/Applications/GEOSctm_App/ctm_setup b/src/Applications/GEOSctm_App/ctm_setup index ab7acf7..6a5e29c 100755 --- a/src/Applications/GEOSctm_App/ctm_setup +++ b/src/Applications/GEOSctm_App/ctm_setup @@ -59,9 +59,9 @@ set BINDIR = `dirname $CTMSETUP` set GEOSDEF = `dirname $BINDIR` set ETCDIR = ${GEOSDEF}/etc -# Test if GEOSgcm.x is here which means you are in install directory +# Test if GEOSctm.x is here which means you are in install directory if (! -x GEOSctm.x) then - echo "You are trying to run $0 in the Applications/GEOSgcm_App directory" + echo "You are trying to run $0 in the Applications/GEOSctm_App directory" echo "This is no longer supported. Please run from the bin/ directory" echo "in your installation" exit 1 @@ -74,6 +74,7 @@ endif # Set default behavior of switches set NOCVS = TRUE set GPU = FALSE +set LINKX = FALSE while ( $#argv > 0 ) set arg = $argv[1] @@ -97,6 +98,11 @@ while ( $#argv > 0 ) set NOCVS = TRUE breaksw + # Symlink GEOSctm.x + case --link: + set LINKX = TRUE + breaksw + # Here any string not above will trigger USAGE case -[Hh]: case --[Hh][Ee][Ll][Pp]: @@ -110,6 +116,7 @@ end ####################################################################### setenv NODE `uname -n` +setenv ARCH `uname -s` setenv SITE `awk '{print $2}' $ETCDIR/SITE.rc` if ( $SITE != 'NCCS' && $SITE != 'NAS' ) then @@ -136,7 +143,9 @@ if ( $SITE == 'NCCS' ) then endif else if ( $SITE == 'NAS' ) then - if ( ($NODE =~ r[0-9]*i[0-9]*n[0-9]*) || ($NODE =~ maia*) ) then + if ( ($NODE =~ r[0-9]*i[0-9]*n[0-9]*) || \ + ($NODE =~ r[0-9]*c[0-9]*t[0-9]*n[0-9]*) || \ + ($NODE =~ maia*) ) then goto ONCOMPUTE endif endif @@ -149,19 +158,12 @@ endif setenv BASEDIR `awk '{print $2}' $ETCDIR/BASEDIR.rc` -if ( `echo $BASEDIR | grep -i ifort` != '') then - set COMPILER = ifort -else if ( `echo $BASEDIR | grep -i pgfortran` != '') then - set COMPILER = pgfortran -else - # Assume default is ifort in case of older baselibs - set COMPILER = ifort -endif - if ( `echo $BASEDIR | grep -i mvapich2` != '') then set MPI = mvapich2 else if ( `echo $BASEDIR | grep -i openmpi` != '') then set MPI = openmpi +else if ( `echo $BASEDIR | grep -i hpcx` != '') then + set MPI = openmpi else if ( `echo $BASEDIR | grep -i impi` != '') then set MPI = intelmpi else if ( `echo $BASEDIR | grep -i intelmpi` != '') then @@ -173,24 +175,6 @@ else set MPI = intelmpi endif -####################################################################### -# Set up RUN_CMD for MPI Stack -####################################################################### - -if ( $MPI == mvapich2 ) then - if ( $SITE == 'NAS') then - setenv RUN_CMD 'mpirun_rsh -export -hostfile $PBS_NODEFILE -np ' # mpi run for MVAPICH2 - else if ($SITE == 'NCCS') then - setenv RUN_CMD 'srun --mpi=pmi2 -n ' # mpi run for MVAPICH2 - endif -else if ( $MPI == openmpi ) then - setenv RUN_CMD 'mpirun -map-by core -bind-to core -np ' # mpi run for OpenMPI -else if ( $MPI == intelmpi ) then - setenv RUN_CMD 'mpirun -np ' # mpi run for Intel MPI -else if ( $MPI == mpt ) then - setenv RUN_CMD 'mpiexec_mpt -np ' # mpi run for MPT -endif - ####################################################################### # Enter Experiment Specific Run Parameters ####################################################################### @@ -249,57 +233,226 @@ endif HRCODE: echo "Enter the ${C1}Atmospheric Horizontal Resolution${CN} code:" -echo "-----------------------------------------------------------" -echo " Cubed-Sphere " -echo "-----------------------------------------------------------" -echo " ${C2}c48 -- 2 deg ${CN}" -echo " ${C2}c90 -- 1 deg ${CN}" -echo " ${C2}c180 -- 1/2 deg (${C1}56-km${C2}) ${CN}" -echo " ${C2}c360 -- 1/4 deg (${C1}28-km${C2}) ${CN} " -echo " ${C2}c720 -- 1/8 deg (${C1}14-km${C2}) ${CN}" -echo " ${C2}c1440 -- 1/16 deg (${C1} 7-km${C2}) ${CN}" +echo "--------------------------------------" +echo " Cubed-Sphere" +echo "--------------------------------------" +echo " ${C2}c12 -- 8 deg ${CN}" +echo " ${C2}c24 -- 4 deg ${CN}" +echo " ${C2}c48 -- 2 deg ${CN}" +echo " ${C2}c90 -- 1 deg ${CN}" +echo " ${C2}c180 -- 1/2 deg (${C1}56-km${C2}) ${CN}" +echo " ${C2}c360 -- 1/4 deg (${C1}28-km${C2}) ${CN} " +echo " ${C2}c720 -- 1/8 deg (${C1}14-km${C2}) ${CN}" +echo " ${C2}c1440 - 1/16 deg (${C1} 7-km${C2}) ${CN}" +echo " ${C2} DYAMOND Grids " +echo " ${C2}c768 -- 1/8 deg (${C1}12-km${C2}) ${CN}" +echo " ${C2}c1536 - 1/16 deg (${C1} 6-km${C2}) ${CN}" +echo " ${C2}c3072 - 1/32 deg (${C1} 3-km${C2}) ${CN}" echo " " set HRCODE = `echo $<` set HRCODE = `echo $HRCODE | tr "[:upper:]" "[:lower:]"` -if( $HRCODE != 'c24' & \ +if( $HRCODE != 'c12' & \ + $HRCODE != 'c24' & \ $HRCODE != 'c48' & \ $HRCODE != 'c90' & \ $HRCODE != 'c180' & \ $HRCODE != 'c360' & \ $HRCODE != 'c720' & \ - $HRCODE != 'c1440' ) goto HRCODE + $HRCODE != 'c1440' & \ + $HRCODE != 'c768' & \ + $HRCODE != 'c1536' & \ + $HRCODE != 'c3072' ) goto HRCODE + +set Resolution = $HRCODE + +set CUBED = TRUE +set DYCORE = FV3 +set GEOSCTM_NF = 6 +set GRID_TYPE = "Cubed-Sphere" +if (`echo $Resolution[1] | cut -b1` == "c" ) then + set GEOSCTM_IM = $Resolution[1] +else + set GEOSCTM_IM = "c$Resolution[1]" +endif + +echo "Enter the Atmospheric Model ${C1}Vertical Resolution${CN}: ${C2}LM${CN} (Default: 72)" +set GEOSCTM_LM = $< +if( .$GEOSCTM_LM == . ) then + set GEOSCTM_LM = 72 +endif + +####################################################################### +# Test to see if using hydrostatic or non-hydrostatic atmosphere +####################################################################### -if( $HRCODE == 'c24' | \ - $HRCODE == 'c48' | \ - $HRCODE == 'c90' | \ - $HRCODE == 'c180' | \ +ASKHYDRO: + +if( $HRCODE == 'c1440' | \ + $HRCODE == 'c768' | \ + $HRCODE == 'c1536' | \ + $HRCODE == 'c3072' ) then + set DEFAULT_HYDROSTATIC = FALSE +else + set DEFAULT_HYDROSTATIC = TRUE +endif + +echo "Use ${C1}Hydrostatic Atmosphere${CN}? (Default: ${C2}YES${CN} or ${C2}TRUE${CN} for Grid Resolutions <= C720)" +set USE_HYDROSTATIC = $< + +if( .$USE_HYDROSTATIC == . ) then + set USE_HYDROSTATIC = $DEFAULT_HYDROSTATIC +else + set USE_HYDROSTATIC = `echo $USE_HYDROSTATIC | tr "[:lower:]" "[:upper:]"` + if( $USE_HYDROSTATIC == "Y" | \ + $USE_HYDROSTATIC == "YES" | \ + $USE_HYDROSTATIC == "T" | \ + $USE_HYDROSTATIC == "TRUE" ) set USE_HYDROSTATIC = TRUE + if( $USE_HYDROSTATIC == "N" | \ + $USE_HYDROSTATIC == "NO" | \ + $USE_HYDROSTATIC == "F" | \ + $USE_HYDROSTATIC == "FALSE" ) set USE_HYDROSTATIC = FALSE + + if( "$USE_HYDROSTATIC" != "TRUE" & "$USE_HYDROSTATIC" != "FALSE" ) then + echo + echo "Use ${C1}Hydrostatic Atmosphere${CN} must be set equal to ${C2}TRUE/YES${CN} or ${C2}FALSE/NO${CN}!" + goto ASKHYDRO + endif +endif + + +####################################################################### +# Test to see if you want to use ioserver +####################################################################### + +ASKIOS: + +if( $HRCODE == 'c180' | \ $HRCODE == 'c360' | \ $HRCODE == 'c720' | \ - $HRCODE == 'c1440' ) then - set Resolution = $HRCODE + $HRCODE == 'c1440' | \ + $HRCODE == 'c768' | \ + $HRCODE == 'c1536' | \ + $HRCODE == 'c3072' ) then + + set DEFAULT_DO_IOS = TRUE + echo "Do you wish to ${C1}IOSERVER${CN}? (Default: ${C2}YES${CN} or ${C2}TRUE${CN})" +else + set DEFAULT_DO_IOS = FALSE + echo "Do you wish to ${C1}IOSERVER${CN}? (Default: ${C2}NO${CN} or ${C2}FALSE${CN})" endif -if( $#Resolution == 2 ) then - set GEOSCTM_IM = $Resolution[1] - set GEOSCTM_JM = $Resolution[2] - set CUBED = FALSE - set DYCORE = FV +set DO_IOS = $< +if( .$DO_IOS == . ) then + set DO_IOS = $DEFAULT_DO_IOS else - set CUBED = TRUE - set DYCORE = FV3 - if (`echo $Resolution[1] | cut -b1` == "c" ) then - set GEOSCTM_IM = $Resolution[1] - else - set GEOSCTM_IM = "c$Resolution[1]" + set DO_IOS = `echo $DO_IOS | tr "[:lower:]" "[:upper:]"` + if( $DO_IOS == "Y" | \ + $DO_IOS == "YES" | \ + $DO_IOS == "T" | \ + $DO_IOS == "TRUE" ) set DO_IOS = TRUE + if( $DO_IOS == "N" | \ + $DO_IOS == "NO" | \ + $DO_IOS == "F" | \ + $DO_IOS == "FALSE" ) set DO_IOS = FALSE + + if( $DO_IOS != "TRUE" & $DO_IOS != "FALSE" ) then + echo + echo "${C1}IOSERVER${CN} must be set equal to ${C2}TRUE/YES${CN} or ${C2}FALSE/NO${CN}!" + goto ASKIOS endif endif +if ( $DO_IOS == "TRUE" ) then + set USE_IOSERVER = 1 +else + set USE_IOSERVER = 0 +endif -echo "Enter the Atmospheric Model ${C1}Vertical Resolution${CN}: ${C2}LM${CN} (Default: 72)" -set GEOSCTM_LM = $< -if( .$GEOSCTM_LM == . ) then - set GEOSCTM_LM = 72 +####################################################################### +# What Processor Should We Run On? +####################################################################### + +ASKPROC: + +if ( $SITE == 'NCCS' ) then + echo "Enter the ${C1}Processor Type${CN} you wish to run on:" + echo " ${C2}hasw (Haswell)${CN} (default)" + if (-e /etc/os-release) then + echo " ${C2}sky (Skylake)${CN}" + endif + echo " " + set MODEL = `echo $<` + set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"` + if ( .$MODEL == .) then + set MODEL = 'hasw' + endif + + if( $MODEL != 'hasw' & \ + $MODEL != 'sky' ) goto ASKPROC + + if ( $MODEL == 'hasw') then + set NCPUS_PER_NODE = 28 + else if ($MODEL == 'sky') then + set NCPUS_PER_NODE = 40 + endif + +else if ( $SITE == 'NAS' ) then + echo "Enter the ${C1}Processor Type${CN} you wish to run on:" + echo " ${C2}has (Haswell)${CN}" + echo " ${C2}bro (Broadwell)${CN}" + echo " ${C2}sky (Skylake)${CN} (default)" + echo " ${C2}cas (Cascade Lake)${CN}" + #echo " ${C2}rom (AMD Rome)${CN}" + echo " " + echo " NOTE: Due to how FV3 is compiled by default, Sandy Bridge" + echo " and Ivy Bridge are not supported by current GEOS" + echo " " + set MODEL = `echo $<` + set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"` + if ( .$MODEL == .) then + set MODEL = 'sky' + endif + + if( $MODEL != 'has' & \ + $MODEL != 'bro' & \ + $MODEL != 'sky' & \ + $MODEL != 'cas' & \ + $MODEL != 'rom' ) goto ASKPROC + + # Some processors have weird names at NAS + # --------------------------------------- + + if ($MODEL == bro ) then + set MODEL = 'bro_ele' + else if ($MODEL == sky) then + set MODEL = 'sky_ele' + else if ($MODEL == cas) then + set MODEL = 'cas_ait' + else if ($MODEL == rom) then + set MODEL = 'rom_ait' + endif + + if ( $MODEL == 'san') then + set NCPUS_PER_NODE = 16 + else if ($MODEL == 'ivy') then + set NCPUS_PER_NODE = 20 + else if ($MODEL == 'has') then + set NCPUS_PER_NODE = 24 + else if ($MODEL == 'bro_ele') then + set NCPUS_PER_NODE = 28 + else if ($MODEL == 'sky_ele') then + set NCPUS_PER_NODE = 40 + else if ($MODEL == 'cas_ait') then + set NCPUS_PER_NODE = 40 + else if ($MODEL == 'rom_ait') then + set NCPUS_PER_NODE = 128 + set USE_IOSERVER = 0 + echo "WARNING WARNING: At present there are issues with IOSERVER and Rome. Turning off IOSERVER!" + endif + +else + set MODEL = 'UNKNOWN' endif @@ -312,7 +465,7 @@ set BCSTAG = Icarus_Reynolds # Set IAU-Forcing and Bias Correction OFF # --------------------------------------- set FORCEDAS = "#" - set FORCEGCM = "#" + set FORCECTM = "#" # Set Default Convert Parameters # ------------------------------ @@ -324,6 +477,10 @@ set BCSTAG = Icarus_Reynolds set NUM_READERS = 1 set NUM_WRITERS = 1 +# Set DEFAULT IOSERVER Parameters +# ------------------------------- + set DEF_IOS_NDS = 1 + # Default Run Parameters # ---------------------- if( $GEOSCTM_IM == "c12" ) then @@ -336,13 +493,13 @@ if( $GEOSCTM_IM == "c12" ) then set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` set NX = 2 set NY = `expr $NX \* 6` - set HYDROSTATIC = TRUE + set HYDROSTATIC = $USE_HYDROSTATIC set HIST_IM = `expr $GEOSCTM_IM \* 4` set HIST_JM = `expr $GEOSCTM_IM \* 2 + 1` set GRID_FILE = "Gnomonic_c${GEOSCTM_IM}.dat" set JOB_SGMT = 00000015 set NUM_SGMT = 20 - set ATMOStag = CF0012x6C + set ATMOS_RES = CF0012x6C set POST_NDS = 4 set CNV_NX = 1 set CNV_NY = 6 @@ -357,13 +514,13 @@ if( $GEOSCTM_IM == "c24" ) then set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` set NX = 4 set NY = `expr $NX \* 6` - set HYDROSTATIC = TRUE + set HYDROSTATIC = $USE_HYDROSTATIC set HIST_IM = `expr $GEOSCTM_IM \* 4` set HIST_JM = `expr $GEOSCTM_IM \* 2 + 1` set GRID_FILE = "Gnomonic_c${GEOSCTM_IM}.dat" set JOB_SGMT = 00000015 set NUM_SGMT = 20 - set ATMOStag = CF0024x6C + set ATMOS_RES = CF0024x6C set POST_NDS = 4 set CNV_NX = 1 set CNV_NY = 6 @@ -378,13 +535,13 @@ if( $GEOSCTM_IM == "c48" ) then set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` set NX = 4 set NY = `expr $NX \* 6` - set HYDROSTATIC = TRUE + set HYDROSTATIC = $USE_HYDROSTATIC set HIST_IM = 180 set HIST_JM = 91 set GRID_FILE = "Gnomonic_c${GEOSCTM_IM}.dat" set JOB_SGMT = 00000015 set NUM_SGMT = 20 - set ATMOStag = CF0048x6C + set ATMOS_RES = CF0048x6C set POST_NDS = 4 endif if( $GEOSCTM_IM == "c90" ) then @@ -397,13 +554,13 @@ if( $GEOSCTM_IM == "c90" ) then set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` set NX = 4 set NY = `expr $NX \* 6` - set HYDROSTATIC = TRUE + set HYDROSTATIC = $USE_HYDROSTATIC set HIST_IM = `expr $GEOSCTM_IM \* 4` set HIST_JM = `expr $GEOSCTM_IM \* 2 + 1` set GRID_FILE = "Gnomonic_c${GEOSCTM_IM}.dat" set JOB_SGMT = 00000032 set NUM_SGMT = 4 - set ATMOStag = CF0090x6C + set ATMOS_RES = CF0090x6C set POST_NDS = 8 endif if( $GEOSCTM_IM == "c180" ) then @@ -416,15 +573,16 @@ if( $GEOSCTM_IM == "c180" ) then set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` set NX = 6 set NY = `expr $NX \* 6` - set HYDROSTATIC = TRUE + set HYDROSTATIC = $USE_HYDROSTATIC set HIST_IM = `expr $GEOSCTM_IM \* 4` set HIST_JM = `expr $GEOSCTM_IM \* 2 + 1` set GRID_FILE = "Gnomonic_c${GEOSCTM_IM}.dat" set NUM_READERS = 2 set JOB_SGMT = 00000016 set NUM_SGMT = 1 - set ATMOStag = CF0180x6C + set ATMOS_RES = CF0180x6C set POST_NDS = 8 + set DEF_IOS_NDS = 2 endif if( $GEOSCTM_IM == "c360" ) then set DT = 300 @@ -436,16 +594,17 @@ if( $GEOSCTM_IM == "c360" ) then set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` set NX = 12 set NY = `expr $NX \* 6` - set HYDROSTATIC = TRUE + set HYDROSTATIC = $USE_HYDROSTATIC set HIST_IM = `expr $GEOSCTM_IM \* 4` set HIST_JM = `expr $GEOSCTM_IM \* 2 + 1` set GRID_FILE = "Gnomonic_c${GEOSCTM_IM}.dat" set NUM_READERS = 4 set JOB_SGMT = 00000005 set NUM_SGMT = 1 - set ATMOStag = CF0360x6C + set ATMOS_RES = CF0360x6C set CNV_NX = 4 set POST_NDS = 12 + set DEF_IOS_NDS = 2 endif if( $GEOSCTM_IM == "c500" ) then set DT = 300 @@ -457,17 +616,18 @@ if( $GEOSCTM_IM == "c500" ) then set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` set NX = 12 set NY = `expr $NX \* 6` - set HYDROSTATIC = TRUE + set HYDROSTATIC = $USE_HYDROSTATIC set HIST_IM = `expr $GEOSCTM_IM \* 4` set HIST_JM = `expr $GEOSCTM_IM \* 2 + 1` set GRID_FILE = "Gnomonic_c${GEOSCTM_IM}.dat" set NUM_READERS = 4 set JOB_SGMT = 00000005 set NUM_SGMT = 1 - set ATMOStag = CF0500x6C + set ATMOS_RES = CF0500x6C set CNV_NX = 8 set POST_NDS = 12 set USE_SHMEM = 1 + set DEF_IOS_NDS = 2 endif if( $GEOSCTM_IM == "c720" ) then set DT = 225 @@ -479,17 +639,18 @@ if( $GEOSCTM_IM == "c720" ) then set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` set NX = 16 set NY = `expr $NX \* 6` - set HYDROSTATIC = TRUE + set HYDROSTATIC = $USE_HYDROSTATIC set HIST_IM = `expr $GEOSCTM_IM \* 4` set HIST_JM = `expr $GEOSCTM_IM \* 2 + 1` set GRID_FILE = "Gnomonic_c${GEOSCTM_IM}.dat" set NUM_READERS = 6 set JOB_SGMT = 00000005 set NUM_SGMT = 1 - set ATMOStag = CF0720x6C + set ATMOS_RES = CF0720x6C set CNV_NX = 8 set POST_NDS = 16 set USE_SHMEM = 1 + set DEF_IOS_NDS = 3 endif if( $GEOSCTM_IM == "c1440" ) then set DT = 150 @@ -501,17 +662,88 @@ if( $GEOSCTM_IM == "c1440" ) then set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` set NX = 30 set NY = `expr $NX \* 6` - set HYDROSTATIC = FALSE + set HYDROSTATIC = $USE_HYDROSTATIC set HIST_IM = `expr $GEOSCTM_IM \* 4` set HIST_JM = `expr $GEOSCTM_IM \* 2 + 1` set GRID_FILE = "Gnomonic_c${GEOSCTM_IM}.dat" set NUM_READERS = 6 set JOB_SGMT = 00000001 set NUM_SGMT = 1 - set ATMOStag = CF1440x6C + set ATMOS_RES = CF1440x6C set CNV_NX = 8 set POST_NDS = 32 set USE_SHMEM = 1 + set DEF_IOS_NDS = 4 +endif +# DYAMOND Resolutions +if( $GEOSCTM_IM == "c768" ) then + set DT = 180 + set SOLAR_DT = 3600 + set IRRAD_DT = 3600 + set OCEAN_DT = $IRRAD_DT + set CHEM_DT = 900 + set GEOSCTM_IM = 768 + set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` + set NX = 32 + set NY = `expr $NX \* 6` + set HYDROSTATIC = $USE_HYDROSTATIC + set HIST_IM = `expr $GEOSCTM_IM \* 4` + set HIST_JM = `expr $GEOSCTM_IM \* 2 + 1` + set GRID_FILE = "Gnomonic_c${GEOSCTM_IM}.dat" + set NUM_READERS = 6 + set JOB_SGMT = 00000005 + set NUM_SGMT = 1 + set ATMOS_RES = CF0768x6C + set CNV_NX = 8 + set POST_NDS = 16 + set USE_SHMEM = 1 + set DEF_IOS_NDS = 3 +endif +if( $GEOSCTM_IM == "c1536" ) then + set DT = 90 + set SOLAR_DT = 3600 + set IRRAD_DT = 3600 + set OCEAN_DT = $IRRAD_DT + set CHEM_DT = 900 + set GEOSCTM_IM = 1536 + set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` + set NX = 64 + set NY = `expr $NX \* 6` + set HYDROSTATIC = $USE_HYDROSTATIC + set HIST_IM = `expr $GEOSCTM_IM \* 4` + set HIST_JM = `expr $GEOSCTM_IM \* 2 + 1` + set GRID_FILE = "Gnomonic_c${GEOSCTM_IM}.dat" + set NUM_READERS = 6 + set JOB_SGMT = 00000005 + set NUM_SGMT = 1 + set ATMOS_RES = CF1536x6C + set CNV_NX = 8 + set POST_NDS = 16 + set USE_SHMEM = 1 + set DEF_IOS_NDS = 4 +endif +if( $GEOSCTM_IM == "c3072" ) then + set DT = 45 + set SOLAR_DT = 3600 + set IRRAD_DT = 3600 + set OCEAN_DT = $IRRAD_DT + set CHEM_DT = 900 + set GEOSCTM_IM = 3072 + set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` + set NX = 64 + set NY = `expr $NX \* 6` + set HYDROSTATIC = $USE_HYDROSTATIC + set HIST_IM = `expr $GEOSCTM_IM \* 4` + set HIST_JM = `expr $GEOSCTM_IM \* 2 + 1` + set GRID_FILE = "Gnomonic_c${GEOSCTM_IM}.dat" + set NUM_READERS = 6 + set JOB_SGMT = 00000005 + set NUM_SGMT = 1 + set ATMOS_RES = CF3072x6C + set CNV_NX = 8 + set POST_NDS = 16 + set USE_SHMEM = 1 + set DEF_IOS_NDS = 5 endif set IS_FCST = 0 @@ -731,102 +963,196 @@ else echo endif endif - ####################################################################### # Architecture Specific Variables ####################################################################### -@ NPES = $NX * $NY +@ MODEL_NPES = $NX * $NY + +if ( $DO_IOS == TRUE ) then + set IOS_NDS = $DEF_IOS_NDS +else + set IOS_NDS = 0 +endif + @ CNV_NPES = $CNV_NX * $CNV_NY # Here we need to convert POST_NDS to total tasks. Using 16 cores # per task as a good default @ POST_NPES = $POST_NDS * 16 - setenv RUN_N `echo $EXPID | cut -b1-11`_RUN # RUN Job Name - setenv RUN_FN `echo $EXPID | cut -b1-11`_FCST # Forecast Job Name - setenv POST_N `echo $EXPID | cut -b1-10`_POST # POST Job Name - setenv PLOT_N `echo $EXPID | cut -b1-11`_PLT # PLOT Job Name - setenv MOVE_N `echo $EXPID | cut -b1-11`_MOVE # MOVE Job Name - setenv ARCHIVE_N `echo $EXPID | cut -b1-10`_ARCH # ARCHIVE Job Name - setenv REGRESS_N `echo $EXPID | cut -b1-10`_RGRS # REGRESS Job Name - setenv CONVERT_N `echo $EXPID | cut -b1-11`_CNV # CONVERT Job Name +# Longer job names are now supported with SLURM and PBS. Limits seem to be 1024 characters with SLURM +# and 230 with PBS. To be safe, we will limit to 200 + + setenv RUN_N `echo $EXPID | cut -b1-200`_RUN # RUN Job Name + setenv RUN_FN `echo $EXPID | cut -b1-200`_FCST # Forecast Job Name + setenv POST_N `echo $EXPID | cut -b1-199`_POST # POST Job Name + setenv PLOT_N `echo $EXPID | cut -b1-200`_PLT # PLOT Job Name + setenv MOVE_N `echo $EXPID | cut -b1-200`_MOVE # MOVE Job Name + setenv ARCHIVE_N `echo $EXPID | cut -b1-199`_ARCH # ARCHIVE Job Name + setenv REGRESS_N `echo $EXPID | cut -b1-199`_RGRS # REGRESS Job Name + setenv CONVERT_N `echo $EXPID | cut -b1-200`_CNV # CONVERT Job Name # Default converter time setenv CONVERT_T "0:15:00" # Wallclock Time for ctm_convert.j if( $SITE == 'NAS' ) then + setenv BATCH_CMD "qsub" # PBS Batch command setenv BATCH_GROUP "PBS -W group_list=" # PBS Syntax for GROUP - setenv RUN_FT "6:00:00" # Wallclock Time - UNUSED - setenv RUN_T "8:00:00" # Wallclock Time for ctm_run.j - setenv POST_T "8:00:00" # Wallclock Time for ctm_post.j - setenv PLOT_T "8:00:00" # Wallclock Time for ctm_plot.j - setenv ARCHIVE_T "8:00:00" # Wallclock Time for ctm_archive.j + setenv BATCH_TIME "PBS -l walltime=" # PBS Syntax for walltime + setenv BATCH_JOBNAME "PBS -N " # PBS Syntax for job name + setenv BATCH_OUTPUTNAME "PBS -o " # PBS Syntax for job output name + setenv BATCH_JOINOUTERR "PBS -j oe -k oed" # PBS Syntax for joining output and error + setenv RUN_FT "6:00:00" # Wallclock Time for ctm_forecast.j + setenv RUN_T "8:00:00" # Wallclock Time for ctm_run.j + setenv POST_T "8:00:00" # Wallclock Time for ctm_post.j + setenv PLOT_T "8:00:00" # Wallclock Time for ctm_plot.j + setenv ARCHIVE_T "8:00:00" # Wallclock Time for ctm_archive.j if( $GPU == "TRUE" ) then - set NCPUS = 16 # CPUS per node + set NCPUS_PER_NODE = 16 # CPUS per node set MODEL = "san_gpu" # Model of CPU set QTYPE = "gpu_k40" # Queue to use else - set NCPUS = 28 # CPUS per node - set MODEL = "bro" # Model of CPU set QTYPE = "normal" # Queue to use endif - @ NCUS = `echo "($NPES + $NCPUS - 1)/$NCPUS" | bc` - setenv RUN_Q "PBS -q ${QTYPE}" # batch queue name for ctm_run.j - setenv RUN_P "PBS -l select=${NCUS}:ncpus=${NCPUS}:mpiprocs=${NCPUS}:model=${MODEL}" # PE Configuration for ctm_run.j - setenv RUN_FP "PBS -l select=24:ncpus=${NCPUS}:mpiprocs=${NCPUS}:model=${MODEL}" # PE Configuration - UNUSED - setenv POST_Q "PBS -q normal" # batch queue name for ctm_post.j - setenv PLOT_Q "PBS -q normal" # batch queue name for ctm_plot.j - setenv MOVE_Q "PBS -q normal" # batch queue name for ctm_moveplot.j - setenv ARCHIVE_Q "PBS -q normal" # batch queue name for ctm_archive.j - setenv POST_P "PBS -l select=${POST_NDS}:ncpus=${NCPUS}:mpiprocs=${NCPUS}:model=${MODEL}" # PE Configuration for ctm_post.j - setenv PLOT_P "PBS -l select=2:ncpus=${NCPUS}:mpiprocs=${NCPUS}:model=${MODEL}" # PE Configuration for ctm_plot.j - setenv ARCHIVE_P "PBS -l select=1:ncpus=${NCPUS}:mpiprocs=${NCPUS}:model=${MODEL}" # PE Configuration for ctm_archive.j - setenv CONVERT_P "PBS -l select=${CNV_NX}:ncpus=${NCPUS}:mpiprocs=${NCPUS}:model=${MODEL}" # PE Configuration for ctm_convert.j - setenv BCSDIR /nobackup/gmao_SIteam/ModelData/bcs/Icarus/$BCSTAG # location of Boundary Conditions + + #@ NODES = `echo "($MODEL_NPES + $NCPUS_PER_NODE - 1)/$NCPUS_PER_NODE" | bc` + @ NODES = `echo "( ($MODEL_NPES + $NCPUS_PER_NODE) + ($IOS_NDS * $NCPUS_PER_NODE) - 1)/$NCPUS_PER_NODE" | bc` + @ NPCUS = `echo "($POST_NPES + $NCPUS_PER_NODE - 1)/$NCPUS_PER_NODE" | bc` + + setenv RUN_Q "PBS -q ${QTYPE}" # batch queue name for ctm_run.j + setenv RUN_P "PBS -l select=${NODES}:ncpus=${NCPUS_PER_NODE}:mpiprocs=${NCPUS_PER_NODE}:model=${MODEL}" # PE Configuration for ctm_run.j + setenv RUN_FP "PBS -l select=24:ncpus=${NCPUS_PER_NODE}:mpiprocs=${NCPUS_PER_NODE}:model=${MODEL}" # PE Configuration for ctm_forecast.j + setenv POST_Q "PBS -q normal" # batch queue name for ctm_post.j + setenv PLOT_Q "PBS -q normal" # batch queue name for ctm_plot.j + setenv MOVE_Q "PBS -q normal" # batch queue name for ctm_moveplot.j + setenv ARCHIVE_Q "PBS -q normal" # batch queue name for ctm_archive.j + setenv POST_P "PBS -l select=${NPCUS}:ncpus=${NCPUS_PER_NODE}:mpiprocs=${NCPUS_PER_NODE}:model=${MODEL}" # PE Configuration for ctm_post.j + setenv PLOT_P "PBS -l select=1:ncpus=${NCPUS_PER_NODE}:mpiprocs=1:model=${MODEL}" # PE Configuration for ctm_plot.j + setenv ARCHIVE_P "PBS -l select=1:ncpus=${NCPUS_PER_NODE}:mpiprocs=${NCPUS_PER_NODE}:model=${MODEL}" # PE Configuration for ctm_archive.j + setenv CONVERT_P "PBS -l select=${CNV_NX}:ncpus=${NCPUS_PER_NODE}:mpiprocs=${NCPUS_PER_NODE}:model=${MODEL}" # PE Configuration for ctm_convert.j + setenv MOVE_P "PBS -l select=1:ncpus=1" # PE Configuration for ctm_moveplot.j + + setenv BCSDIR /nobackup/gmao_SIteam/ModelData/bcs/${LSM_BCS}/${LSM_BCS}_${OCEAN_TAG} # location of Boundary Conditions + setenv REPLAY_ANA_EXPID ONLY_MERRA2_SUPPORTED # Default Analysis Experiment for REPLAY + setenv REPLAY_ANA_LOCATION ONLY_MERRA2_SUPPORTED # Default Analysis Location for REPLAY + setenv M2_REPLAY_ANA_LOCATION /nobackup/gmao_SIteam/ModelData/verification/MERRA-2 # Default Analysis Location for M2 REPLAY + + if( ${OGCM_IM}x${OGCM_JM} == "1440x720" ) then + setenv SSTDIR /nobackup/gmao_SIteam/ModelData/fvInput/g5gcm/bcs/SST/${OGCM_IM}x${OGCM_JM} # location of SST Boundary Conditions + else + setenv SSTDIR /nobackup/gmao_SIteam/ModelData/fvInput/g5gcm/bcs/realtime/@SSTNAME/${OGCM_IM}x${OGCM_JM} # location of SST Boundary Conditions + endif setenv CHMDIR /nobackup/gmao_SIteam/ModelData/fvInput_nc3 # locations of Aerosol Chemistry BCs setenv WRKDIR /nobackup/$LOGNAME # user work directory - setenv COUPLEDIR /nobackup/gmao_SIteam/ModelData/coupled/Forcings # Coupled Ocean/Atmos Forcing - UNUSED + setenv COUPLEDIR /nobackup/gmao_SIteam/ModelData/coupled/Forcings # Coupled Ocean/Atmos Forcing setenv CPEXEC 'mcp -a' # Copy utility for large copies setenv TAREXEC mtar # Tar utility for large archives else if( $SITE == 'NCCS' ) then - setenv BATCH_GROUP "SBATCH -A " # PBS Syntax for GROUP - setenv RUN_FT "06:00:00" # Wallclock Time - UNUSED + setenv BATCH_CMD "sbatch" # SLURM Batch command + setenv BATCH_GROUP "SBATCH --account=" # SLURM Syntax for account name + setenv BATCH_TIME "SBATCH --time=" # SLURM Syntax for walltime + setenv BATCH_JOBNAME "SBATCH --job-name=" # SLURM Syntax for job name + setenv BATCH_OUTPUTNAME "SBATCH --output=" # SLURM Syntax for job output name + setenv BATCH_JOINOUTERR "DELETE" # SLURM joins out and err by default + setenv RUN_FT "06:00:00" # Wallclock Time for ctm_forecast.j setenv RUN_T "12:00:00" # Wallclock Time for ctm_run.j setenv POST_T "8:00:00" # Wallclock Time for ctm_post.j setenv PLOT_T "12:00:00" # Wallclock Time for ctm_plot.j setenv ARCHIVE_T "2:00:00" # Wallclock Time for ctm_archive.j if( $GPU == "TRUE" ) then - setenv RUN_Q "SBATCH --constraint=k40" # batch queue name for ctm_run.j + set NCPUS_PER_NODE = 16 # CPUS per node + set MODEL = "k40" # Model of CPU + endif + + #@ NODES = `echo "($MODEL_NPES + $NCPUS_PER_NODE - 1)/$NCPUS_PER_NODE" | bc` + @ NODES = `echo "( ($MODEL_NPES + $NCPUS_PER_NODE) + ($IOS_NDS * $NCPUS_PER_NODE) - 1)/$NCPUS_PER_NODE" | bc` + @ NPCUS = `echo "($POST_NPES + $NCPUS_PER_NODE - 1)/$NCPUS_PER_NODE" | bc` + + setenv RUN_Q "SBATCH --constraint=${MODEL}" # batch queue name for ctm_run.j + setenv RUN_P "SBATCH --nodes=${NODES} --ntasks-per-node=${NCPUS_PER_NODE}" # PE Configuration for ctm_run.j + setenv RUN_FP "SBATCH --nodes=${NODES} --ntasks-per-node=${NCPUS_PER_NODE}" # PE Configuration for ctm_forecast.j + setenv POST_Q "SBATCH --constraint=${MODEL}" # batch queue name for ctm_post.j + setenv PLOT_Q "SBATCH --constraint=${MODEL}" # batch queue name for ctm_plot.j + setenv MOVE_Q "SBATCH --partition=datamove" # batch queue name for ctm_moveplot.j + setenv ARCHIVE_Q "SBATCH --partition=datamove" # batch queue name for ctm_archive.j + setenv POST_P "SBATCH --nodes=${NPCUS} --ntasks-per-node=${NCPUS_PER_NODE}" # PE Configuration for ctm_post.j + setenv PLOT_P "SBATCH --nodes=4 --ntasks=4" # PE Configuration for ctm_plot.j + setenv ARCHIVE_P "SBATCH --ntasks=1" # PE Configuration for ctm_archive.j + setenv CONVERT_P "SBATCH --ntasks=${CNV_NPES}" # PE Configuration for ctm_convert.j + setenv MOVE_P "SBATCH --ntasks=1" # PE Configuration for ctm_moveplot.j + + setenv BCSDIR /discover/nobackup/ltakacs/bcs/${LSM_BCS}/${LSM_BCS}_${OCEAN_TAG} # location of Boundary Conditions + setenv REPLAY_ANA_EXPID x0039 # Default Analysis Experiment for REPLAY + setenv REPLAY_ANA_LOCATION /discover/nobackup/projects/gmao/g6dev/ltakacs/x0039 # Default Analysis Location for REPLAY + setenv M2_REPLAY_ANA_LOCATION /discover/nobackup/projects/gmao/share/gmao_ops/verification/MERRA-2 # Default Analysis Location for M2 REPLAY + + if( ${OGCM_IM}x${OGCM_JM} == "1440x720" ) then + setenv SSTDIR $SHARE/dao_ops/fvInput/g5gcm/bcs/SST/${OGCM_IM}x${OGCM_JM} # location of SST Boundary Conditions else - setenv RUN_Q "SBATCH --constraint=hasw" # batch queue name for ctm_run.j + setenv SSTDIR $SHARE/dao_ops/fvInput/g5gcm/bcs/realtime/@SSTNAME/${OGCM_IM}x${OGCM_JM} # location of SST Boundary Conditions endif - setenv RUN_P "SBATCH --ntasks=${NPES}" # PE Configuration for ctm_run.j - setenv RUN_FP "SBATCH --ntasks=\&NPES" # PE Configuration - UNUSED - setenv POST_Q "SBATCH --constraint=hasw" # batch queue name for ctm_post.j - setenv PLOT_Q "SBATCH --constraint=hasw" # batch queue name for ctm_plot.j - setenv MOVE_Q "SBATCH --partition=datamove" # batch queue name for ctm_moveplot.j - setenv ARCHIVE_Q "SBATCH --partition=datamove" # batch queue name for ctm_archive.j + setenv CHMDIR $SHARE/dao_ops/fvInput_nc3 # locations of Aerosol Chemistry BCs + setenv WRKDIR /discover/nobackup/$LOGNAME # user work directory + setenv COUPLEDIR /discover/nobackup/yvikhlia/coupled/Forcings # Coupled Ocean/Atmos Forcing + setenv CPEXEC /bin/cp # Copy utility for large copies + setenv TAREXEC tar +else if( $SITE == 'AWS' ) then + setenv BATCH_CMD "sbatch" # SLURM Batch command + setenv BATCH_GROUP DELETE # SLURM Syntax for account name + setenv BATCH_TIME "SBATCH --time=" # SLURM Syntax for walltime + setenv BATCH_JOBNAME "SBATCH --job-name=" # SLURM Syntax for job name + setenv BATCH_OUTPUTNAME "SBATCH --output=" # SLURM Syntax for job output name + setenv BATCH_JOINOUTERR "DELETE" # SLURM joins out and err by default + setenv RUN_FT "06:00:00" # Wallclock Time for ctm_forecast.j + setenv RUN_T "12:00:00" # Wallclock Time for ctm_run.j + setenv POST_T "8:00:00" # Wallclock Time for ctm_post.j + setenv PLOT_T "12:00:00" # Wallclock Time for ctm_plot.j + setenv ARCHIVE_T "1:00:00" # Wallclock Time for ctm_archive.j + setenv RUN_Q DELETE # batch queue name for ctm_run.j + setenv RUN_P "SBATCH --ntasks=${MODEL_NPES}" # PE Configuration for ctm_run.j + setenv RUN_FP "SBATCH --ntasks=${MODEL_NPES}" # PE Configuration for ctm_forecast.j + setenv POST_Q NULL # batch queue name for ctm_post.j + setenv PLOT_Q NULL # batch queue name for ctm_plot.j + setenv MOVE_Q NULL # batch queue name for ctm_moveplot.j + setenv ARCHIVE_Q NULL # batch queue name for ctm_archive.j setenv POST_P "SBATCH --ntasks=${POST_NPES}" # PE Configuration for ctm_post.j setenv PLOT_P "SBATCH --nodes=4 --ntasks=4" # PE Configuration for ctm_plot.j setenv ARCHIVE_P "SBATCH --ntasks=1" # PE Configuration for ctm_archive.j setenv CONVERT_P "SBATCH --ntasks=${CNV_NPES}" # PE Configuration for ctm_convert.j - setenv BCSDIR /discover/nobackup/ltakacs/bcs/Icarus/$BCSTAG # location of Boundary Conditions - setenv CHMDIR $SHARE/dao_ops/fvInput_nc3 # locations of Aerosol Chemistry BCs - setenv WRKDIR /discover/nobackup/$LOGNAME # user work directory - setenv COUPLEDIR /discover/nobackup/yvikhlia/coupled/Forcings # Coupled Ocean/Atmos Forcing - UNUSED + setenv MOVE_P "SBATCH --ntasks=1" # PE Configuration for ctm_moveplot.j + + setenv BCSDIR /ford1/share/gmao_SIteam/ModelData/bcs/${LSM_BCS}/${LSM_BCS}_${OCEAN_TAG} # location of Boundary Conditions + setenv REPLAY_ANA_EXPID REPLAY_UNSUPPORTED # Default Analysis Experiment for REPLAY + setenv REPLAY_ANA_LOCATION REPLAY_UNSUPPORTED # Default Analysis Location for REPLAY + setenv M2_REPLAY_ANA_LOCATION REPLAY_UNSUPPORTED # Default Analysis Location for M2 REPLAY + + setenv SSTDIR /ford1/share/gmao_SIteam/ModelData/@SSTNAME/${OGCM_IM}x${OGCM_JM} # location of SST Boundary Conditions + setenv CHMDIR /ford1/share/gmao_SIteam/ModelData/fvInput_nc3 # locations of Aerosol Chemistry BCs + setenv WRKDIR /home/$LOGNAME # user work directory + setenv COUPLEDIR /ford1/share/gmao_SIteam/ModelData/Forcings # Coupled Ocean/Atmos Forcing + setenv CPEXEC /bin/cp # Copy utility for large copies setenv TAREXEC tar # Tar utility for large archives + # By default on AWS, just ignore IOSERVER for now until testing + set USE_IOSERVER = 0 + set IOS_NDS = 0 + set NCPUS_PER_NODE = 0 else # These are defaults for the desktop - setenv BATCH_GROUP "SBATCH -A " # PBS Syntax for GROUP - setenv RUN_FT "06:00:00" # Wallclock Time - UNUSED + setenv BATCH_CMD "sbatch" # SLURM Batch command + setenv BATCH_GROUP "SBATCH --account=" # SLURM Syntax for account name + setenv BATCH_TIME "SBATCH --time=" # SLURM Syntax for walltime + setenv BATCH_JOBNAME "SBATCH --job-name=" # SLURM Syntax for job name + setenv BATCH_OUTPUTNAME "SBATCH --output=" # SLURM Syntax for job output name + setenv BATCH_JOINOUTERR "DELETE" # SLURM joins out and err by default + setenv RUN_FT "06:00:00" # Wallclock Time for ctm_forecast.j setenv RUN_T "12:00:00" # Wallclock Time for ctm_run.j setenv POST_T "8:00:00" # Wallclock Time for ctm_post.j setenv PLOT_T "12:00:00" # Wallclock Time for ctm_plot.j setenv ARCHIVE_T "1:00:00" # Wallclock Time for ctm_archive.j setenv RUN_Q NULL # batch queue name for ctm_run.j setenv RUN_P NULL # PE Configuration for ctm_run.j - setenv RUN_FP NULL # PE Configuration - UNUSED + setenv RUN_FP NULL # PE Configuration for ctm_forecast.j setenv POST_Q NULL # batch queue name for ctm_post.j setenv PLOT_Q NULL # batch queue name for ctm_plot.j setenv MOVE_Q NULL # batch queue name for ctm_moveplot.j @@ -835,22 +1161,27 @@ else setenv PLOT_P NULL # PE Configuration for ctm_post.j setenv ARCHIVE_P NULL # PE Configuration for ctm_archive.j setenv CONVERT_P NULL # PE Configuration for ctm_convert.j - setenv BCSDIR /ford1/share/gmao_SIteam/ModelData/bcs/Icarus/$BCSTAG # location of Boundary Conditions + setenv MOVE_P NULL # PE Configuration for ctm_moveplot.j + + setenv BCSDIR /ford1/share/gmao_SIteam/ModelData/bcs/${LSM_BCS}/${LSM_BCS}_${OCEAN_TAG} # location of Boundary Conditions + setenv REPLAY_ANA_EXPID REPLAY_UNSUPPORTED # Default Analysis Experiment for REPLAY + setenv REPLAY_ANA_LOCATION REPLAY_UNSUPPORTED # Default Analysis Location for REPLAY + setenv M2_REPLAY_ANA_LOCATION REPLAY_UNSUPPORTED # Default Analysis Location for M2 REPLAY + + setenv SSTDIR /ford1/share/gmao_SIteam/ModelData/@SSTNAME/${OGCM_IM}x${OGCM_JM} # location of SST Boundary Conditions setenv CHMDIR /ford1/share/gmao_SIteam/ModelData/fvInput_nc3 # locations of Aerosol Chemistry BCs setenv WRKDIR /home/$LOGNAME # user work directory - setenv COUPLEDIR /ford1/share/gmao_SIteam/ModelData/Forcings # Coupled Ocean/Atmos Forcing - UNUSED - if( $CUBED == "TRUE" ) then + setenv COUPLEDIR /ford1/share/gmao_SIteam/ModelData/Forcings # Coupled Ocean/Atmos Forcing set NX = 1 set NY = 6 - else - set NX = 2 - set NY = 4 - endif set CNV_NX = ${NX} set CNV_NY = ${NY} - setenv RUN_CMD 'mpirun -oversubscribe -np ' # mpi run for DESKTOP (assumes OpenMPI) setenv CPEXEC /bin/cp # Copy utility for large copies setenv TAREXEC tar # Tar utility for large archives + # By default on desktop, just ignore IOSERVER for now + set USE_IOSERVER = 0 + set IOS_NDS = 0 + set NCPUS_PER_NODE = 0 endif @@ -868,20 +1199,20 @@ while( $check == FALSE ) echo $HISTORYrc > $HOME/.HISTORYrc endif - echo "Enter the tag or directory (/filename) of the ${C1}HISTORY.GEOSCTM.rc.tmpl${CN} to use" + echo "Enter the directory (/filename) of the ${C1}HISTORY.GEOSCTM.rc.tmpl${CN} to use" echo "(To use ${C1}HISTORY.GEOSCTM.rc.tmpl${CN} from current build, Type: ${C2}Current${CN} )" echo "-------------------------------------------------------------------------" - echo "Hit ENTER to use Default Tag/Location: (${C2}${HISTORYrc}${CN})" + echo "Hit ENTER to use Default Location: (${C2}${HISTORYrc}${CN})" set NUHISTORY = $< if( .$NUHISTORY != . ) set HISTORYrc = $NUHISTORY - if( -e $BINDIR/HISTORY.rc.hold ) /bin/mv -f $BINDIR/HISTORY.rc.hold $BINDIR/HISTORY.rc.tmpl - if( -e $BINDIR/HISTORY.rc.tmpl ) /bin/mv -f $BINDIR/HISTORY.rc.tmpl $BINDIR/HISTORY.rc.hold + + set TMPHIST = `mktemp` if( "$HISTORYrc" == "Current" ) then set check = TRUE if( -e $HOME/.HISTORYrc ) /bin/rm $HOME/.HISTORYrc echo $HISTORYrc > $HOME/.HISTORYrc - /bin/cp -f $ETCDIR/HISTORY.GEOSCTM.rc.tmpl $BINDIR/HISTORY.rc.tmpl + /bin/cp -f $ETCDIR/HISTORY.GEOSCTM.rc.tmpl $TMPHIST endif if( "$HISTORYrc" != "Current" ) then @@ -889,29 +1220,30 @@ while( $check == FALSE ) set check = TRUE if( -e $HOME/.HISTORYrc ) /bin/rm $HOME/.HISTORYrc echo $HISTORYrc > $HOME/.HISTORYrc - /bin/cp -f $ETCDIR/$HISTORYrc $BINDIR/HISTORY.rc.tmp1 + set TMPHIST1 = `mktemp` + /bin/cp -f $ETCDIR/$HISTORYrc $TMPHIST1 - set EXPID_old = `grep "EXPID:" $BINDIR/HISTORY.rc.tmp1 | cut -d: -f2` - set EXPDSC_old = `grep "EXPDSC:" $BINDIR/HISTORY.rc.tmp1 | cut -d: -f2` + set EXPID_old = `grep "EXPID:" $TMPHIST1 | cut -d: -f2` + set EXPDSC_old = `grep "EXPDSC:" $TMPHIST1 | cut -d: -f2` - /bin/rm -f command + set TMPCMD = `mktemp` set string = "EXPID:" - echo cat $BINDIR/HISTORY.rc.tmp1 \| awk \'\{if \( \$1 \~ \"${string}\" \) \ - \{sub \( \"${EXPID_old}\" , \"${EXPID}\" \)\;print\} else print\}\' \> $BINDIR/HISTORY.rc.tmpl > command - chmod +x command - ./command - /bin/rm -f command - /bin/mv -f $BINDIR/HISTORY.rc.tmpl $BINDIR/HISTORY.rc.tmp1 - cat $BINDIR/HISTORY.rc.tmp1 | sed -e "s|${EXPDSC_old}|${EXPDSC}|g" > $BINDIR/HISTORY.rc.tmpl - /bin/rm -f $BINDIR/HISTORY.rc.tmp1 + echo cat $TMPHIST1 \| awk \'\{if \( \$1 \~ \"${string}\" \) \ + \{sub \( \"${EXPID_old}\" , \"${EXPID}\" \)\;print\} else print\}\' \> $TMPHIST > $TMPCMD + chmod +x $TMPCMD + $TMPCMD + /bin/rm -f $TMPCMD + /bin/mv -f $TMPHIST $TMPHIST1 + cat $TMPHIST1 | sed -e "s|${EXPDSC_old}|${EXPDSC}|g" > $TMPHIST + /bin/rm -f $TMPHIST1 else if( -e $HISTORYrc/HISTORY.GEOSCTM.rc.tmpl ) then set check = TRUE if( -e $HOME/.HISTORYrc ) /bin/rm $HOME/.HISTORYrc echo $HISTORYrc > $HOME/.HISTORYrc - /bin/cp -f $HISTORYrc/HISTORY.GEOSCTM.rc.tmpl $BINDIR/HISTORY.rc.tmpl + /bin/cp -f $HISTORYrc/HISTORY.GEOSCTM.rc.tmpl $TMPHIST else - echo "This condition is based on updating HISTORY.AGCM.rc.tmpl with CVS" + echo "This condition is based on updating HISTORY.GEOSCTM.rc.tmpl with CVS" echo "This has no equivalent in git at present. Please contact Matt" echo "Thompson or Larry Takacs to help resolve this." exit 2 @@ -1039,7 +1371,8 @@ setenv CNVDIR $EXPDIR/convert ####################################################################### # GEOSctm Build Directory -# ------------------------ +# ----------------------- + echo "Enter Location for ${C1}Build${CN} directory containing: bin/ etc/ include/ etc..." echo "Hit ENTER to use Default Location:" echo "----------------------------------" @@ -1050,8 +1383,9 @@ setenv CNVDIR $EXPDIR/convert if( .$NUGEOSDIR != . ) then set GEOSDIR = $NUGEOSDIR endif - + set check = FALSE + GEOSDIR: while( $check == FALSE ) @@ -1081,9 +1415,13 @@ while( $check == FALSE ) endif end +# +# GEOSBIN does point to the bin/ directory in each +# setenv GEOSSRC ${GEOSDIR} setenv GEOSBIN ${GEOSDIR}/bin setenv GEOSETC ${GEOSDIR}/etc + setenv GEOSUTIL ${GEOSSRC} setenv CTMVER `cat ${GEOSETC}/.GEOSCTM_VERSION` @@ -1125,16 +1463,27 @@ echo $GROUP > $HOME/.GROUProot set EMISSIONS = g5chem endif +if ( $LINKX == "TRUE" ) then + if ( -e $GEOSBIN/GEOSctm.x ) ln -s $GEOSBIN/GEOSctm.x $EXPDIR +else if ( -e $GEOSBIN/GEOSctm.x ) rsync -avx $GEOSBIN/GEOSctm.x $EXPDIR +endif ####################################################################### -# Create SETENV Commands +# Set Recommended MPI Stack Settings ####################################################################### +# By default do not write restarts by oserver +set RESTART_BY_OSERVER = NO + /bin/rm -f $HOMDIR/SETENV.commands if( $MPI == openmpi ) then +# Open MPI and GEOS has issues with restart writing. Having the +# oserver write them can be orders of magnitude faster +set RESTART_BY_OSERVER = YES + # This turns off an annoying warning when running # Open MPI on a system where TMPDIRs are on a networked # file system @@ -1178,7 +1527,7 @@ cat > $HOMDIR/SETENV.commands << EOF setenv MPI_COLL_REPRODUCIBLE setenv SLURM_DISTRIBUTION block - + #setenv MPI_DISPLAY_SETTINGS 1 #setenv MPI_VERBOSE 1 @@ -1191,14 +1540,40 @@ cat > $HOMDIR/SETENV.commands << EOF # with binarytile.x and other executables. unsetenv PMI_RANK + # Often when debugging on MPT, the traceback from Intel Fortran + # is "absorbed" and only MPT's errors are displayed. To allow the + # compiler's traceback to be displayed, uncomment this environment + # variable + #setenv FOR_IGNORE_EXCEPTIONS false + EOF else if( $MPI == intelmpi ) then cat > $HOMDIR/SETENV.commands << EOF setenv I_MPI_DAPL_UD enable +setenv I_MPI_ADJUST_ALLREDUCE 12 +setenv I_MPI_ADJUST_GATHERV 3 + +# This flag prints out the Intel MPI state. Uncomment if needed +#setenv I_MPI_DEBUG 9 +EOF + +# These are options determined to be useful at NCCS +# Not setting generally as they are more fabric/cluster +# specific compared to the above adjustments +if ( $SITE == 'NCCS' ) then + +cat >> $HOMDIR/SETENV.commands << EOF +setenv I_MPI_SHM_HEAP_VSIZE 512 +setenv PSM2_MEMORY large +setenv I_MPI_EXTRA_FILESYSTEM 1 +setenv I_MPI_EXTRA_FILESYSTEM_FORCE gpfs +setenv ROMIO_FSTYPE_FORCE "gpfs:" EOF +endif # if NCCS + endif # if mpi ####################################################################### @@ -1315,6 +1690,7 @@ s?@PLOT_T?$PLOT_T?g s?@PLOT_P?$PLOT_P?g s?@PLOT_Q?$PLOT_Q?g s?@MOVE_Q?$MOVE_Q?g +s?@MOVE_P?$MOVE_P?g s?@ARCHIVE_N?$ARCHIVE_N?g s?@ARCHIVE_T?$ARCHIVE_T?g s?@ARCHIVE_P?$ARCHIVE_P?g @@ -1332,6 +1708,11 @@ s?@EXPDSC?$EXPDSC?g s?@HOMDIR?$HOMDIR?g s?@CNVDIR?$CNVDIR?g s?@BATCH_GROUP?${BATCH_GROUP}${GROUP}?g +s?@BATCH_TIME?$BATCH_TIME?g +s?@BATCH_CMD?$BATCH_CMD?g +s?@BATCH_JOBNAME?$BATCH_JOBNAME?g +s?@BATCH_OUTPUTNAME?$BATCH_OUTPUTNAME?g +s?@BATCH_JOINOUTERR?$BATCH_JOINOUTERR?g s?@SITE?$SITE?g s?@GEOSDIR?$GEOSDIR?g s?@GEOSSRC?$GEOSSRC?g @@ -1346,7 +1727,7 @@ s?@OBSERVER_FRQ?0?g s?RECORD_?#RECORD_?g s?>>>FORCEDAS<<>>FORCEGCM<<>>FORCECTM<<>>PCHEM<<>>GOCART<<>>GMICHEM<< $HOMDIR/tmpfile + else if ( -e $BINDIR/$FILE ) then cat $BINDIR/$FILE > $HOMDIR/tmpfile else if ( -e $ETCDIR/$FILE ) then cat $ETCDIR/$FILE > $HOMDIR/tmpfile @@ -1490,11 +1878,7 @@ if ($GPU == "TRUE") then /bin/rm -f $HOMDIR/GPUEND.commands endif -if( -e $BINDIR/HISTORY.rc.hold ) then -/bin/mv -f $BINDIR/HISTORY.rc.hold $BINDIR/HISTORY.rc.tmpl -else -/bin/rm -f $BINDIR/HISTORY.rc.tmpl -endif +/bin/rm -f $TMPHIST echo $HOMDIR > $EXPDIR/.HOMDIR echo " " @@ -1520,7 +1904,6 @@ endif set RSNAMES = "LH2O LMAM LCARMA LGMICHEM LGEOSCHEMCHEM LSTRATCHEM LGOCART" set RSTYPES = "INTERNAL IMPORT" -/bin/rm -f command set FILE = GEOSCTM.rc.tmpl if( -e $HOMDIR/$FILE ) set LOCDIR = $HOMDIR @@ -1531,26 +1914,28 @@ foreach rsname ($RSNAMES) set test = `eval echo \$$rsname` if( $test == FALSE ) then foreach type ($RSTYPES) + set TMPCMD = `mktemp` set string = ${name}_${type} /bin/rm -f $LOCDIR/$FILE.tmp /bin/mv -f $LOCDIR/$FILE $LOCDIR/$FILE.tmp echo cat $LOCDIR/$FILE.tmp \| awk \'\{if \( \$1 \~ \"${string}\" \) \ - \{sub \( \/${string}\/ ,\"\#${string}\" \)\;print\} else print\}\' \> $LOCDIR/$FILE > command - chmod +x command - ./command - /bin/rm -f command + \{sub \( \/${string}\/ ,\"\#${string}\" \)\;print\} else print\}\' \> $LOCDIR/$FILE > $TMPCMD + chmod +x $TMPCMD + $TMPCMD + /bin/rm -f $TMPCMD end endif end if( $LGOCART == TRUE ) then + set TMPCMD = `mktemp` /bin/rm -f $LOCDIR/$FILE.tmp /bin/mv -f $LOCDIR/$FILE $LOCDIR/$FILE.tmp set string = AEROCLIM echo cat $LOCDIR/$FILE.tmp \| awk \'\{if \( \$1 \~ \"${string}\" \) \ - \{sub \( \/${string}\/ ,\"\#${string}\" \)\;print\} else print\}\' \> $LOCDIR/$FILE > command - chmod +x command - ./command - /bin/rm -f command + \{sub \( \/${string}\/ ,\"\#${string}\" \)\;print\} else print\}\' \> $LOCDIR/$FILE > $TMPCMD + chmod +x $TMPCMD + $TMPCMD + /bin/rm -f $TMPCMD endif /bin/rm -f $LOCDIR/$FILE.tmp @@ -1993,7 +2378,7 @@ echo " " echo "Build Directory: ${C2}${GEOSDIR}${CN}" echo "----------------" echo " " -if ( ! -e $GEOSDIR/$ARCH/bin/GEOSctm.x ) then +if ( ! -e $GEOSBIN/GEOSctm.x ) then echo " " echo "Note: Build directory does not contain ${C1}GEOSctm.x${CN} !" echo " You will need to put a copy of ${C1}GEOSctm.x${CN} in your Experiment directory" @@ -2003,7 +2388,7 @@ else echo " " echo "The following executable has been placed in your Experiment Directory:" echo "----------------------------------------------------------------------" - echo "${C2}$GEOSDIR/$ARCH/bin/GEOSctm.x${CN}" + echo "${C2}$GEOSBIN/GEOSctm.x${CN}" echo " " endif echo " " @@ -2268,40 +2653,35 @@ if ( ! -e $HOME/.HOMDIRroot || ! -e $HOME/.EXPDIRroot ) then echo "Please run ctm_setup in non-clone mode at least once to use this script." exit 1 endif -else - # MAT There are two thoughts here. You can either place the clone - # in the .HOMDIRroot/.EXPDIRroot, but that is only nice if you have - # one place, you put all your experiments. What if you have lots of - # directories? Instead, let us default to the directory root of the - # cloned experiment. +endif - #set HOMDIRroot=`cat $HOME/.HOMDIRroot` +# MAT There are two thoughts here. You can either place the clone in +# the .HOMDIRroot/.EXPDIRroot, but that is only nice if you have one +# single place you put all your experiments. What if you have lots of +# directories? Instead, let us default to the directory root of the +# cloned experiment if the cloned experiment is yours. If it isn't your +# experiment you are cloning, then default to the values in the dotfile + +if ( $OLDUSER == $LOGNAME) then set HOMDIRroot=`dirname $OLDHOMDIR` - #echo "Setting HOMDIR to $HOMDIRroot" - #set EXPDIRroot=`cat $HOME/.EXPDIRroot` set EXPDIRroot=`dirname $OLDEXPDIR` - #echo "Setting EXPDIR to $EXPDIRroot" +else + set HOMDIRroot=`cat $HOME/.HOMDIRroot` + set EXPDIRroot=`cat $HOME/.EXPDIRroot` endif +echo "Setting HOMDIR to $HOMDIRroot" +echo "Setting EXPDIR to $EXPDIRroot" + if ( -e $HOME/.GROUProot ) then set GROUProot=`cat $HOME/.GROUProot` - #echo "Using account $GROUProot" + echo "Using account $GROUProot" else echo "$HOME/.GROUProot not found." set GROUProot=`groups | awk '{print $1}'` echo "Based off of groups, setting account to $GROUProot" endif -# ----------------------------------- -# Find out if we are running the cube -# ----------------------------------- - -if ( -e $CLONEDIR/fvcore_layout.rc ) then - set CUBED = TRUE -else - set CUBED = FALSE -endif - # ------------------------------------------------ # Set the new EXPDIR and HOMDIR based on the roots # ------------------------------------------------ @@ -2354,6 +2734,7 @@ HOMDIR/CAP.rc HOMDIR/GEOSCTM.rc HOMDIR/HISTORY.rc HOMDIR/ctm_run.j +HOMDIR/logging.yaml HOMDIR/MAPL_ExtData.rc HOMDIR/CTM_GridComp.rc HOMDIR/pTracers_GridComp.rc @@ -2363,11 +2744,9 @@ HOMDIR/FPIT_ExtData.rc.tmpl HOMDIR/FP_ExtData.rc.tmpl EOF -if($CUBED == "TRUE") then cat >> $FILES_TO_PROCESS << EOF HOMDIR/fvcore_layout.rc EOF -endif # ------------------------------------------------ # Create two sets of files so we can copy from one @@ -2445,14 +2824,14 @@ end # Construct the new job ids # ------------------------- - set RUN_N=`echo $NEWEXPID | cut -b1-11`_RUN - set RUN_FN=`echo $NEWEXPID | cut -b1-11`_FCST - set POST_N=`echo $NEWEXPID | cut -b1-10`_POST - set PLOT_N=`echo $NEWEXPID | cut -b1-11`_PLT - set MOVE_N=`echo $NEWEXPID | cut -b1-11`_MOVE -set ARCHIVE_N=`echo $NEWEXPID | cut -b1-10`_ARCH -set REGRESS_N=`echo $NEWEXPID | cut -b1-10`_RGRS -set CONVERT_N=`echo $NEWEXPID | cut -b1-11`_CNV + set RUN_N=`echo $NEWEXPID | cut -b1-200`_RUN + set RUN_FN=`echo $NEWEXPID | cut -b1-200`_FCST + set POST_N=`echo $NEWEXPID | cut -b1-199`_POST + set PLOT_N=`echo $NEWEXPID | cut -b1-200`_PLT + set MOVE_N=`echo $NEWEXPID | cut -b1-200`_MOVE +set ARCHIVE_N=`echo $NEWEXPID | cut -b1-199`_ARCH +set REGRESS_N=`echo $NEWEXPID | cut -b1-199`_RGRS +set CONVERT_N=`echo $NEWEXPID | cut -b1-200`_CNV sed -i -e "/^#PBS -N/ s#\(PBS -N \)\(.*\)#\1$RUN_N#" \ -e "/^#SBATCH --job-name=/ s#\(SBATCH --job-name=\)\(.*\)#\1$RUN_N#" $NEWHOMDIR/ctm_run.j @@ -2735,7 +3114,6 @@ ctm_setup, a setup script for the GEOS-5 CTM -c --color Set the colors for $0:t -g --gpu Run the model using the GPUs -h --help Show usage - --nocvs Do not archive the source If invoked alone, the script runs as normal. diff --git a/src/Applications/GEOSctm_App/fvcore_layout.rc b/src/Applications/GEOSctm_App/fvcore_layout.rc index cfbab53..72db6c0 100644 --- a/src/Applications/GEOSctm_App/fvcore_layout.rc +++ b/src/Applications/GEOSctm_App/fvcore_layout.rc @@ -1,60 +1,72 @@ &fv_core_nml - fv_debug = .F. - n_sponge = 9 - n_zfilter = 25 - tau = 0. - rf_cutoff = 7.5e2 - d2_bg_k1 = 0.15 - d2_bg_k2 = 0.02 - fv_sg_adj = 450 - remap_option = 0 - kord_tm = 9 - kord_mt = 9 - kord_wz = 9 - kord_tr = 9 - hydrostatic = .T. - make_nh = .T. - beta = 0. - a_imp = 1. - p_fac = 0.1 - k_split = 2 - n_split = 0 - nwat = 1 - nord = 2 - dddmp = 0.1 - d4_bg = 0.12 - d2_bg = 0.0 - d_ext = 0.0 - vtdm4 = 0.0 - delt_max = 0.002 - ke_bg = 0.0 - do_vort_damp = .F. - d_con = 0. - hord_mt = 10 - hord_vt = 10 - hord_tm = 10 - hord_dp = 10 - hord_tr = 8 - adjust_dry_mass = .F. - consv_te = 1. - consv_am = .F. - fill = .T. - dwind_2d = .F. - z_tracer = .F. - / + k_split = 2 + @FV_MAKENH + @FV_HYDRO + @FV_SATADJ + @FV_ZTRACER + @FV_NWAT +/ - &main_nml +&main_nml / - &test_case_nml - test_case = 13 - / +&test_case_nml + test_case = 13 +/ - &fms_io_nml +&fms_io_nml / - &fms_nml - print_memory_usage=.false. - domains_stack_size = 24000000 +&fms_nml + print_memory_usage=.false. + domains_stack_size = 24000000 / +&gfdl_cloud_microphysics_nml + sedi_transport = .true. + do_sedi_heat = .false. + rad_snow = .true. + rad_graupel = .true. + rad_rain = .true. + const_vi = .F. + const_vs = .F. + const_vg = .F. + const_vr = .F. + vi_max = 1. + vs_max = 2. + vg_max = 12. + vr_max = 12. + qi_lim = 1. + prog_ccn = .false. + do_qa = .true. + fast_sat_adj = .true. + tau_l2v = 1200. + tau_v2l = 150. + tau_g2v = 900. + rthresh = 7.e-6 ! This is a key parameter for cloud water + dw_land = 0.16 + dw_ocean = 0.10 + ql_gen = 1.0e-3 + ql_mlt = 1.0e-3 + qi0_crt = 8.0E-5 + qs0_crt = 1.0e-3 + tau_i2s = 1000. + c_psaci = 0.05 + c_pgacs = 0.01 + rh_inc = 0.30 + rh_inr = 0.30 + rh_ins = 0.30 + ccn_l = 300. + ccn_o = 100. + c_paut = 1.0 + c_cracw = 0.8 + use_ppm = .false. + use_ccn = .true. + mono_prof = .true. + z_slope_liq = .true. + z_slope_ice = .true. + de_ice = .false. + fix_negative = .true. + icloud_f = 1 + mp_time = 150. +/ From 535d38972d858f3925fa771b27dd22c788902614 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Tue, 23 Mar 2021 11:13:22 -0400 Subject: [PATCH 06/31] Updates to get CMake working --- CMakeLists.txt | 15 +++++++++++++-- src/Components/GEOSctm_GridComp/CMakeLists.txt | 4 ++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2706b68..98f8a12 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,8 +25,6 @@ set (DOING_GEOS5 YES) # Should find a better place for this - used in Chem component set (ACG_FLAGS -v) -set (F2PYEXT .so) -set (F2PY_SUFFIX .so) set (FV_PRECISION R8) list (APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/@cmake") @@ -44,3 +42,16 @@ include_directories(${MPI_Fortran_INCLUDE_PATH}) add_subdirectory (@env) add_subdirectory (src) +# https://www.scivision.dev/cmake-auto-gitignore-build-dir/ +# --- auto-ignore build directory +if(NOT EXISTS ${PROJECT_BINARY_DIR}/.gitignore) + file(WRITE ${PROJECT_BINARY_DIR}/.gitignore "*") +endif() + +# Piggyback that file into install +install( + FILES ${PROJECT_BINARY_DIR}/.gitignore + DESTINATION ${CMAKE_INSTALL_PREFIX} + ) + + diff --git a/src/Components/GEOSctm_GridComp/CMakeLists.txt b/src/Components/GEOSctm_GridComp/CMakeLists.txt index a22173b..07f9a22 100644 --- a/src/Components/GEOSctm_GridComp/CMakeLists.txt +++ b/src/Components/GEOSctm_GridComp/CMakeLists.txt @@ -1,5 +1,7 @@ esma_set_this () +ecbuild_declare_project() + set (alldirs @GEOSchem_GridComp @FVdycoreCubed_GridComp @@ -13,3 +15,5 @@ esma_add_library (${this} SRCS GEOS_ctmEnvGridComp.F90 GEOS_ctmGridCompMod.F90 SUBCOMPONENTS ${alldirs} DEPENDENCIES FVdycoreCubed_GridComp MAPL) + +ecbuild_install_project(NAME GEOSctm_GridComp) From 8e80ac631df40a036f0bf5fb423e73b03c0b32db Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Tue, 23 Mar 2021 11:38:06 -0400 Subject: [PATCH 07/31] Use 'use MAPL' --- .../CTMconvection_GridComp/CTM_ConvectionGridCompMod.F90 | 2 +- .../CTMconvection_GridComp/CTM_rasCalculationsMod.F90 | 2 +- .../CTMconvection_GridComp/GenericConvectionMethod_mod.F90 | 2 +- .../CTMconvection_GridComp/GmiConvectionMethod_mod.F90 | 2 +- .../CTMdiffusion_GridComp/CTM_DiffusionGridCompMod.F90 | 2 +- .../CTMdiffusion_GridComp/GmiDiffusionMethod_mod.F90 | 2 +- .../CTMdiffusion_GridComp/updateDiffusion_mod.F90 | 2 +- .../CTMpTracers_GridComp/CTM_pTracersGridCompMod.F90 | 3 +-- src/Components/GEOSctm_GridComp/GEOS_ctmEnvGridComp.F90 | 2 +- src/Components/GEOSctm_GridComp/GEOS_ctmGridCompMod.F90 | 2 +- 10 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/CTM_ConvectionGridCompMod.F90 b/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/CTM_ConvectionGridCompMod.F90 index 5b689af..d542108 100644 --- a/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/CTM_ConvectionGridCompMod.F90 +++ b/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/CTM_ConvectionGridCompMod.F90 @@ -13,7 +13,7 @@ MODULE CTM_ConvectionGridCompMod ! !USES: ! USE ESMF - USE MAPL_Mod + USE MAPL USE GmiConvectionMethod_mod ! GMI Convection component USE GenericConvectionMethod_mod ! Generic Convection component USE m_chars, ONLY : uppercase diff --git a/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/CTM_rasCalculationsMod.F90 b/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/CTM_rasCalculationsMod.F90 index 96ccfe4..a47a057 100644 --- a/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/CTM_rasCalculationsMod.F90 +++ b/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/CTM_rasCalculationsMod.F90 @@ -11,7 +11,7 @@ MODULE CTM_rasCalculationsMod ! !USES: ! use ESMF - use MAPL_Mod + use MAPL use GEOS_Mod use GEOS_UtilsMod diff --git a/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/GenericConvectionMethod_mod.F90 b/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/GenericConvectionMethod_mod.F90 index cc3ccab..dc9c646 100644 --- a/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/GenericConvectionMethod_mod.F90 +++ b/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/GenericConvectionMethod_mod.F90 @@ -13,7 +13,7 @@ module GenericConvectionMethod_mod ! ! !USES: use ESMF - use MAPL_Mod + use MAPL !USE Chem_UtilMod USE convectiveTransport_mod use GmiArrayBundlePointer_mod diff --git a/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/GmiConvectionMethod_mod.F90 b/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/GmiConvectionMethod_mod.F90 index d9d0f31..4649349 100644 --- a/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/GmiConvectionMethod_mod.F90 +++ b/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/GmiConvectionMethod_mod.F90 @@ -13,7 +13,7 @@ module GmiConvectionMethod_mod ! ! !USES: use ESMF - use MAPL_Mod + use MAPL USE Chem_UtilMod use GmiArrayBundlePointer_mod USE GmiESMFrcFileReading_mod diff --git a/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/CTM_DiffusionGridCompMod.F90 b/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/CTM_DiffusionGridCompMod.F90 index 750eed4..9e3bf1a 100644 --- a/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/CTM_DiffusionGridCompMod.F90 +++ b/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/CTM_DiffusionGridCompMod.F90 @@ -13,7 +13,7 @@ MODULE CTM_DiffusionGridCompMod ! !USES: ! USE ESMF - USE MAPL_Mod + USE MAPL USE GmiDiffusionMethod_mod ! ESMF parent component USE m_chars, ONLY : uppercase diff --git a/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/GmiDiffusionMethod_mod.F90 b/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/GmiDiffusionMethod_mod.F90 index f10ed4e..8ba41d4 100644 --- a/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/GmiDiffusionMethod_mod.F90 +++ b/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/GmiDiffusionMethod_mod.F90 @@ -13,7 +13,7 @@ module GmiDiffusionMethod_mod ! ! !USES: use ESMF - use MAPL_Mod + use MAPL USE Chem_UtilMod USE updateDiffusion_mod use GmiArrayBundlePointer_mod diff --git a/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/updateDiffusion_mod.F90 b/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/updateDiffusion_mod.F90 index 0e01b80..1929231 100644 --- a/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/updateDiffusion_mod.F90 +++ b/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/updateDiffusion_mod.F90 @@ -13,7 +13,7 @@ module updateDiffusion_mod ! ! !USES: use ESMF - use MAPL_Mod + use MAPL use GmiArrayBundlePointer_mod, only : t_GmiArrayBundle !use GmiSpcConcentrationMethod_mod, only : isFixedConcentration ! diff --git a/src/Components/GEOSctm_GridComp/CTMpTracers_GridComp/CTM_pTracersGridCompMod.F90 b/src/Components/GEOSctm_GridComp/CTMpTracers_GridComp/CTM_pTracersGridCompMod.F90 index 09afb1b..8fc25f1 100644 --- a/src/Components/GEOSctm_GridComp/CTMpTracers_GridComp/CTM_pTracersGridCompMod.F90 +++ b/src/Components/GEOSctm_GridComp/CTMpTracers_GridComp/CTM_pTracersGridCompMod.F90 @@ -13,8 +13,7 @@ module CTM_pTracersGridCompMod ! !USES: use ESMF use m_set_eta, only: set_eta - use MAPL_Mod - use MAPL_GenericMod + use MAPL USE Chem_UtilMod USE jw, only : tracer_q, tracer_q1_q2, tracer_q3 diff --git a/src/Components/GEOSctm_GridComp/GEOS_ctmEnvGridComp.F90 b/src/Components/GEOSctm_GridComp/GEOS_ctmEnvGridComp.F90 index 4a91d08..d05c203 100644 --- a/src/Components/GEOSctm_GridComp/GEOS_ctmEnvGridComp.F90 +++ b/src/Components/GEOSctm_GridComp/GEOS_ctmEnvGridComp.F90 @@ -12,7 +12,7 @@ module GEOS_ctmEnvGridComp ! ! !USES: use ESMF - use MAPL_Mod + use MAPL use FV_StateMod, only : calcCourantNumberMassFlux => fv_computeMassFluxes use fv_arrays_mod , only: FVPRC use m_set_eta, only : set_eta diff --git a/src/Components/GEOSctm_GridComp/GEOS_ctmGridCompMod.F90 b/src/Components/GEOSctm_GridComp/GEOS_ctmGridCompMod.F90 index d3ca6d9..0cf72b7 100644 --- a/src/Components/GEOSctm_GridComp/GEOS_ctmGridCompMod.F90 +++ b/src/Components/GEOSctm_GridComp/GEOS_ctmGridCompMod.F90 @@ -14,7 +14,7 @@ module GEOS_ctmGridCompMod ! ! !USES: use ESMF - use MAPL_Mod + use MAPL use Bundle_IncrementMod use GEOS_ctmEnvGridComp, only : EctmSetServices => SetServices From 5c043c534d7f3ee299ada7dc35f99753a1cf7383 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Tue, 23 Mar 2021 11:38:50 -0400 Subject: [PATCH 08/31] Update parallel_build --- parallel_build.csh | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/parallel_build.csh b/parallel_build.csh index 7511179..b1aa7d3 100755 --- a/parallel_build.csh +++ b/parallel_build.csh @@ -20,12 +20,11 @@ setenv ESMADIR $srcdir # tcsh script maker set origargv = "$argv" -setenv external "" -setenv USEMEPO FALSE -while ($#argv) +setenv DEVELOP FALSE - if ("$1" == "-mepo") then - setenv USEMEPO TRUE +while ($#argv) + if ("$1" == "-develop") then + setenv DEVELOP TRUE endif shift @@ -33,19 +32,23 @@ end if (! -d ${ESMADIR}/@env) then if ($?PBS_JOBID || $?SLURM_JOBID) then - echo " checkout_externals must be run!" + echo " mepo clone must be run!" echo " This requires internet access but you are on a compute node" echo " Please run from a head node" exit 1 else - if ( "$USEMEPO" == "TRUE") then echo "Running mepo initialization" mepo init mepo clone - else - echo " Running checkout_externals" - checkout_externals $external + if ( "$DEVELOP" == "TRUE" ) then + echo "Checking out development branches of GEOSgcm_GridComp and GEOSgcm_App" + mepo develop GEOSgcm_GridComp GEOSgcm_App + endif endif +else + if ( "$DEVELOP" == "TRUE" ) then + echo "Checking out development branches of GEOSgcm_GridComp and GEOSgcm_App" + mepo develop GEOSgcm_GridComp GEOSgcm_App endif endif From 18008d1a5fa9d0682a1021ba64182fedae039834 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Tue, 23 Mar 2021 11:39:03 -0400 Subject: [PATCH 09/31] Remove Externals --- Externals.cfg | 54 --------------------------------------------------- 1 file changed, 54 deletions(-) delete mode 100644 Externals.cfg diff --git a/Externals.cfg b/Externals.cfg deleted file mode 100644 index dfa8bc0..0000000 --- a/Externals.cfg +++ /dev/null @@ -1,54 +0,0 @@ -[ESMA_env] -required = True -repo_url = git@github.com:GEOS-ESM/ESMA_env.git -local_path = ./@env -tag = v2.0.2 -protocol = git - -[ESMA_cmake] -required = True -repo_url = git@github.com:GEOS-ESM/ESMA_cmake.git -local_path = ./@cmake -tag = v2.1.2 -externals = Externals.cfg -protocol = git - -[GMAO_Shared] -required = True -repo_url = git@github.com:GEOS-ESM/GMAO_Shared.git -local_path = ./src/Shared/@GMAO_Shared -tag = v1.1.0 -protocol = git -sparse = ../../../config/GMAO_Shared.sparse - -[MAPL] -required = True -repo_url = git@github.com:GEOS-ESM/MAPL.git -local_path = ./src/Shared/@MAPL -tag = v2.0.0 -protocol = git - -[FVdycoreCubed_GridComp] -required = True -repo_url = git@github.com:GEOS-ESM/FVdycoreCubed_GridComp.git -local_path = ./src/Components/GEOSctm_GridComp/@FVdycoreCubed_GridComp -tag = v1.1.0 -externals = Externals.cfg -protocol = git - -[GEOSchem_GridComp] -required = True -repo_url = git@github.com:GEOS-ESM/GEOSchem_GridComp.git -local_path = ./src/Components/GEOSctm_GridComp/@GEOSchem_GridComp -tag = v1.2.0 -protocol = git - -[FMS] -required = True -repo_url = git@github.com:GEOS-ESM/FMS.git -local_path = ./src/Shared/@FMS -tag = geos/2019.01.01 -protocol = git - -[externals_description] -schema_version = 1.0.0 From c9c79fb44e9b26583cddff00af31b5e5a246101e Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Tue, 23 Mar 2021 11:41:24 -0400 Subject: [PATCH 10/31] update readme --- README.md | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index d77eb7a..9fea31f 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,9 @@ In your `.bashrc` or `.tcshrc` or other rc file add a line: -##### NCCS (SLES11) - +##### NCCS ``` -module use -a /discover/swdev/gmao_SIteam/modulefiles-SLES11 +module use -a /discover/swdev/gmao_SIteam/modulefiles-SLES12 ``` ##### NAS @@ -33,7 +32,7 @@ Now load the `GEOSenv` module: ``` module load GEOSenv ``` -which obtains the latest `git`, `CMake`, and `manage_externals` modules. +which obtains the latest `git`, `CMake`, and `mepo` modules. #### Obtain the Model @@ -45,7 +44,7 @@ git clone git@github.com:GEOS-ESM/GEOSctm.git ### Single Step Building of the Model -If all you wish is to build the model, you can run `parallel_build.csh` from a head node. Doing so will checkout all the external repositories of the model and build it. When done, the resulting model build will be found in `build/` and the installation will be found in `install/` with setup scripts like `gcm_setup` and `fvsetup` in `install/bin`. +If all you wish is to build the model, you can run `parallel_build.csh` from a head node. Doing so will checkout all the external repositories of the model and build it. When done, the resulting model build will be found in `build/` and the installation will be found in `install/` with setup scripts like `ctm_setup` in `install/bin`. #### Debug Version of GEOS @@ -57,12 +56,21 @@ To obtain a debug version, you can run `parallel_build.csh -debug` which will bu The steps detailed below are essentially those that `parallel_build.csh` performs for you. Either method should yield identical builds. -##### Checkout externals +#### Mepo + +The GEOS CTM is comprised of a set of sub-repositories. These are +managed by a tool called [mepo](https://github.com/GEOS-ESM/mepo). To +clone all the sub-repos, you can run `mepo clone` inside the fixture: + ``` cd GEOSctm -checkout_externals +mepo clone ``` +The first command initializes the multi-repository and the second one +clones and assembles all the sub-repositories according to +`components.yaml` + #### Build the Model ##### Load Compiler, MPI Stack, and Baselibs @@ -96,6 +104,14 @@ and CMake will install there. ##### Build and Install with Make ``` -make -j6 install +make -jN install ``` +where `N` is the number of parallel processes. On discover head nodes, this should only be as high as 2 due to limits on the head nodes. On a compute node, you can set `N` has high as you like, though 8-12 is about the limit of parallelism in our model's make system. +### Run the CTM + +Once the model has built successfully, you will have an `install/` directory in your checkout. To run `ctm_setup` go to the `install/bin/` directory and run it there: +``` +cd install/bin +./ctm_setup +``` From 6e99e6dbca850abc31b19438e9b22ac998f0f389 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Tue, 23 Mar 2021 11:47:30 -0400 Subject: [PATCH 11/31] Update .gitignore --- .gitignore | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index c7a54ff..bbb5eb6 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,6 @@ /@cmake/ /@modules/ /@env/ -/BUILD/ -/build*/ -/install*/ -parallel_build.o* /.mepo/ +parallel_build.o* +log.* From 74dea0b76fd05adbf42b99a7b566efabe90711a3 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Tue, 23 Mar 2021 11:47:39 -0400 Subject: [PATCH 12/31] update cmakelists --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 98f8a12..9363bbb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,7 +4,7 @@ cmake_policy (SET CMP0054 NEW) project ( GEOSctm - VERSION 1.0 + VERSION 1.1 LANGUAGES Fortran CXX C) # Note - CXX is required for ESMF if ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}") @@ -25,7 +25,7 @@ set (DOING_GEOS5 YES) # Should find a better place for this - used in Chem component set (ACG_FLAGS -v) -set (FV_PRECISION R8) +set (FV_PRECISION "R8" CACHE STRING "Precision of FV3 core (R4, R4R8, R8)") list (APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/@cmake") include (esma) From 4dae73d368179a980c80751c303c66425332aebb Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Tue, 23 Mar 2021 12:13:12 -0400 Subject: [PATCH 13/31] Fixes for ctm_setup --- src/Applications/GEOSctm_App/.GEOSCTM_VERSION | 2 +- src/Applications/GEOSctm_App/CMakeLists.txt | 14 +- src/Applications/GEOSctm_App/ctm_setup | 98 +++++++--- src/Applications/GEOSctm_App/logging.yaml | 168 ++++++++++++++++++ 4 files changed, 245 insertions(+), 37 deletions(-) create mode 100644 src/Applications/GEOSctm_App/logging.yaml diff --git a/src/Applications/GEOSctm_App/.GEOSCTM_VERSION b/src/Applications/GEOSctm_App/.GEOSCTM_VERSION index f1d9095..e403afd 100644 --- a/src/Applications/GEOSctm_App/.GEOSCTM_VERSION +++ b/src/Applications/GEOSctm_App/.GEOSCTM_VERSION @@ -1 +1 @@ -Icarus-3_2_CTM-r1 +GEOSctm-v@CMAKE_PROJECT_VERSION@ diff --git a/src/Applications/GEOSctm_App/CMakeLists.txt b/src/Applications/GEOSctm_App/CMakeLists.txt index ca481c6..f4281d9 100644 --- a/src/Applications/GEOSctm_App/CMakeLists.txt +++ b/src/Applications/GEOSctm_App/CMakeLists.txt @@ -11,12 +11,7 @@ target_link_libraries (GEOSctm.x ${OpenMP_Fortran_LIBRARIES}) set_target_properties(GEOSctm.x PROPERTIES LINK_FLAGS "${OpenMP_Fortran_FLAGS}") target_include_directories (GEOSctm.x PUBLIC ${INC_ESMF}) -file (GLOB templates *.tmpl) - -set(otheretc - fvcore_layout.rc - .GEOSCTM_VERSION - ) +file (GLOB templates CONFIGURE_DEPENDS *.tmpl) set (programs ctm_setup @@ -31,7 +26,7 @@ set (programs install ( - FILES ${templates} ${otheretc} + FILES ${templates} DESTINATION etc ) @@ -41,7 +36,10 @@ install ( ) install ( - FILES fvcore_layout.rc + FILES fvcore_layout.rc logging.yaml DESTINATION etc ) +configure_file(.GEOSCTM_VERSION .GEOSCTM_VERSION @ONLY) +install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/.GEOSCTM_VERSION DESTINATION etc) + diff --git a/src/Applications/GEOSctm_App/ctm_setup b/src/Applications/GEOSctm_App/ctm_setup index 6a5e29c..ac019fa 100755 --- a/src/Applications/GEOSctm_App/ctm_setup +++ b/src/Applications/GEOSctm_App/ctm_setup @@ -236,8 +236,8 @@ echo "Enter the ${C1}Atmospheric Horizontal Resolution${CN} code:" echo "--------------------------------------" echo " Cubed-Sphere" echo "--------------------------------------" -echo " ${C2}c12 -- 8 deg ${CN}" -echo " ${C2}c24 -- 4 deg ${CN}" +echo " ${C2}c12 -- 8 deg ${CN}" +echo " ${C2}c24 -- 4 deg ${CN}" echo " ${C2}c48 -- 2 deg ${CN}" echo " ${C2}c90 -- 1 deg ${CN}" echo " ${C2}c180 -- 1/2 deg (${C1}56-km${C2}) ${CN}" @@ -455,8 +455,12 @@ else set MODEL = 'UNKNOWN' endif +####################################################################### +# Check for Ocean +####################################################################### -set BCSTAG = Icarus_Reynolds +# The CTM for now seems to only use Reynolds +set OCEAN_TAG = Reynolds # Set DEFAULT SHMEM Parameter # --------------------------- @@ -747,18 +751,75 @@ if( $GEOSCTM_IM == "c3072" ) then endif set IS_FCST = 0 -if($CUBED == "TRUE") then - set FVCUBED = "" - set FVLATLON = "#DELETE" - set GEOSCTM_GRIDNAME = "PE${GEOSCTM_IM}x${GEOSCTM_JM}-CF" - set RES_DATELINE = '${GEOSCTM_IM}x${GEOSCTM_JM}' -endif +set FVCUBED = "" +set GEOSCTM_GRIDNAME = "PE${GEOSCTM_IM}x${GEOSCTM_JM}-CF" +set RES_DATELINE = '${GEOSCTM_IM}x${GEOSCTM_JM}' set BEG_DATE = '18910301 000000' set END_DATE = '29990302 210000' set JOB_SGMT = "$JOB_SGMT 000000" + +####################################################################### +# Setting for Microphysics & Hydrostatic Mode # +####################################################################### + +# Settings for fvcore_layout.rc +set FV_NWAT = "" +set FV_ZTRACER = "z_tracer = .T." +if ( "$USE_HYDROSTATIC" == "TRUE" ) then + set FV_MAKENH = "Make_NH = .F." + set FV_HYDRO = "hydrostatic = .T." + set FV_SATADJ = "do_sat_adj = .F." +else + set FV_MAKENH = "Make_NH = .T." + set FV_HYDRO = "hydrostatic = .F." + set FV_SATADJ = "do_sat_adj = .F." + set FV_NWAT = "nwat = 3 " +endif + + +####################################################################### +# Check for land surface and runoff routing models # +####################################################################### + +# NOTE: The CTM uses old-land for now, so hard-coding that in + +# Check for Old vs New Land Surface Model Boundary Conditions +# ----------------------------------------------------------- +#LSM_BCS: +#echo "Enter the choice of ${C1} Land Surface Boundary Conditions${CN} using: ${C2}1${CN} (Default: Icarus), ${C2}2${CN} (Latest Icarus-NL)" +#set LSM_BCS = $< +#if( .$LSM_BCS == . ) set LSM_BCS = 1 +#if( $LSM_BCS != 1 & $LSM_BCS != 2 ) then + #echo + #echo "${C1} Land Surface Boundary Conditions${CN} must be set equal to ${C2}1 (Icarus){CN} or ${C2}2 (Icarus-NL)${CN}!" + #goto LSM_BCS +#else + #echo +#endif + +set LSM_BCS = 1 + if( $LSM_BCS == 1 ) then + set LSM_BCS = "Icarus" + set LSM_PARMS = "#DELETE" + set EMIP_BCS_IN = "Ganymed-4_0" + set EMIP_OLDLAND = "" + set EMIP_NEWLAND = "#DELETE" + set EMIP_MERRA2 = "MERRA2" + endif + if( $LSM_BCS == 2 ) then + set LSM_BCS = "Icarus-NLv3" + set LSM_PARMS = "" + set EMIP_BCS_IN = "Icarus-NLv3" + set EMIP_OLDLAND = "#DELETE" + set EMIP_NEWLAND = "" + set EMIP_MERRA2 = "MERRA2_NewLand" + endif + + + #------------------------------------ # Convert DT (in seconds) into HHMMSS #------------------------------------ @@ -1037,11 +1098,6 @@ if( $SITE == 'NAS' ) then setenv REPLAY_ANA_LOCATION ONLY_MERRA2_SUPPORTED # Default Analysis Location for REPLAY setenv M2_REPLAY_ANA_LOCATION /nobackup/gmao_SIteam/ModelData/verification/MERRA-2 # Default Analysis Location for M2 REPLAY - if( ${OGCM_IM}x${OGCM_JM} == "1440x720" ) then - setenv SSTDIR /nobackup/gmao_SIteam/ModelData/fvInput/g5gcm/bcs/SST/${OGCM_IM}x${OGCM_JM} # location of SST Boundary Conditions - else - setenv SSTDIR /nobackup/gmao_SIteam/ModelData/fvInput/g5gcm/bcs/realtime/@SSTNAME/${OGCM_IM}x${OGCM_JM} # location of SST Boundary Conditions - endif setenv CHMDIR /nobackup/gmao_SIteam/ModelData/fvInput_nc3 # locations of Aerosol Chemistry BCs setenv WRKDIR /nobackup/$LOGNAME # user work directory setenv COUPLEDIR /nobackup/gmao_SIteam/ModelData/coupled/Forcings # Coupled Ocean/Atmos Forcing @@ -1086,11 +1142,6 @@ else if( $SITE == 'NCCS' ) then setenv REPLAY_ANA_LOCATION /discover/nobackup/projects/gmao/g6dev/ltakacs/x0039 # Default Analysis Location for REPLAY setenv M2_REPLAY_ANA_LOCATION /discover/nobackup/projects/gmao/share/gmao_ops/verification/MERRA-2 # Default Analysis Location for M2 REPLAY - if( ${OGCM_IM}x${OGCM_JM} == "1440x720" ) then - setenv SSTDIR $SHARE/dao_ops/fvInput/g5gcm/bcs/SST/${OGCM_IM}x${OGCM_JM} # location of SST Boundary Conditions - else - setenv SSTDIR $SHARE/dao_ops/fvInput/g5gcm/bcs/realtime/@SSTNAME/${OGCM_IM}x${OGCM_JM} # location of SST Boundary Conditions - endif setenv CHMDIR $SHARE/dao_ops/fvInput_nc3 # locations of Aerosol Chemistry BCs setenv WRKDIR /discover/nobackup/$LOGNAME # user work directory setenv COUPLEDIR /discover/nobackup/yvikhlia/coupled/Forcings # Coupled Ocean/Atmos Forcing @@ -1126,7 +1177,6 @@ else if( $SITE == 'AWS' ) then setenv REPLAY_ANA_LOCATION REPLAY_UNSUPPORTED # Default Analysis Location for REPLAY setenv M2_REPLAY_ANA_LOCATION REPLAY_UNSUPPORTED # Default Analysis Location for M2 REPLAY - setenv SSTDIR /ford1/share/gmao_SIteam/ModelData/@SSTNAME/${OGCM_IM}x${OGCM_JM} # location of SST Boundary Conditions setenv CHMDIR /ford1/share/gmao_SIteam/ModelData/fvInput_nc3 # locations of Aerosol Chemistry BCs setenv WRKDIR /home/$LOGNAME # user work directory setenv COUPLEDIR /ford1/share/gmao_SIteam/ModelData/Forcings # Coupled Ocean/Atmos Forcing @@ -1168,7 +1218,6 @@ else setenv REPLAY_ANA_LOCATION REPLAY_UNSUPPORTED # Default Analysis Location for REPLAY setenv M2_REPLAY_ANA_LOCATION REPLAY_UNSUPPORTED # Default Analysis Location for M2 REPLAY - setenv SSTDIR /ford1/share/gmao_SIteam/ModelData/@SSTNAME/${OGCM_IM}x${OGCM_JM} # location of SST Boundary Conditions setenv CHMDIR /ford1/share/gmao_SIteam/ModelData/fvInput_nc3 # locations of Aerosol Chemistry BCs setenv WRKDIR /home/$LOGNAME # user work directory setenv COUPLEDIR /ford1/share/gmao_SIteam/ModelData/Forcings # Coupled Ocean/Atmos Forcing @@ -1719,7 +1768,6 @@ s?@GEOSSRC?$GEOSSRC?g s?@GEOSBIN?$GEOSBIN?g s?@GEOSETC?$GEOSETC?g s?@GEOSUTIL?$GEOSUTIL?g -s?@RUN_CMD?$RUN_CMD?g s?@CHECKPOINT_TYPE?default?g @@ -1735,7 +1783,6 @@ s?>>>GEOSCHEMCHEM<<>>passiveTracer<<>>IdealizedPT<<>>FVCUBED<<>>FVLATLON<<>>HIST_GOCART<<>>HIST_GMICHEM<<>>HIST_GEOSCHEMCHEM<<> $HOMDIR/sedfile << EOF s^@HYDROSTATIC^$HYDROSTATIC^g s/@GRID_FILE/$GRID_FILE/g EOF -endif set FILES = "ctm_run.j \ ctm_post.j \ @@ -1837,11 +1881,9 @@ set FILES = "ctm_run.j \ MERRA2_ExtData.rc.tmpl " set FILES = `echo $FILES` -if($CUBED == "TRUE") then set FILES = "$FILES \ fvcore_layout.rc" set FILES = `echo $FILES` -endif echo " " diff --git a/src/Applications/GEOSctm_App/logging.yaml b/src/Applications/GEOSctm_App/logging.yaml new file mode 100644 index 0000000..ff5dc71 --- /dev/null +++ b/src/Applications/GEOSctm_App/logging.yaml @@ -0,0 +1,168 @@ +schema_version: 1 + +############################### +locks: + mpi: + class: MpiLock + comm: MPI_COMM_WORLD + +############################### +formatters: + legacy: + class: Formatter + format: '%(message)a' + + basic: + class: Formatter + format: '%(short_name)a15~: %(level_name)a~: %(message)a' + + mpi: + class: MpiFormatter + format: '%(mpi_rank)i4.4~: %(name)~: %(level_name)a~: %(message)a' + comm: MPI_COMM_WORLD + + column: + class: Formatter + format: '(%(i)i3.3,%(j)i3.3): %(level_name)' + + + +############################### +handlers: + + console: + class: streamhandler +# formatter: legacy + formatter: basic + unit: OUTPUT_UNIT + level: INFO + + + warnings: + class: FileHandler + filename: warnings_and_errors.log + lock: mpi + level: WARNING + formatter: basic + + errors: + class: StreamHandler + formatter: basic + unit: ERROR_UNIT + level: ERROR + + mpi_shared: + class: FileHandler + filename: allPEs.log + formatter: mpi + comm: MPI_COMM_WORLD + lock: mpi + rank_keyword: rank + level: DEBUG + + mpi_debug: + class: MpiFileHandler + formatter: basic + filename: debug_%(rank)i3.3~.log + comm: MPI_COMM_WORLD + rank_prefix: rank + level: DEBUG + +############################### +root: + handlers: [warnings,errors,console] + level: WARNING + root_level: WARNING + +############################### +loggers: + + errors: + handlers: [errors] + level: ERROR + + CAP: + level: WARNING + root_level: INFO + + MAPL: + handlers: [mpi_shared] + level: INFO + root_level: DEBUG + + # Note: When enabling another logger, make sure + # indentation matches that of the above + # loggers! + + # MAPL.base: + # MAPL.generic: + # MAPL.pfio + + # SHARED.GMAOSHARED.GEOSSHARED.QSAT: + + # CAP.GCM.DATATM: + # CAP.GCM.AGCM: + # CAP.GCM.AGCM.SCMDYNAMICS: + # CAP.GCM.AGCM.SCMDYNAMICS.DYN: + # CAP.GCM.AGCM.SCMDYNAMICS.ADV: + # CAP.GCM.AGCM.SUPERDYNAMICS: + # CAP.GCM.AGCM.SUPERDYNAMICS.DYN: + # CAP.GCM.AGCM.SUPERDYNAMICS.ADV: + # CAP.GCM.AGCM.PHYSICS: + # CAP.GCM.AGCM.PHYSICS.GWD: + # CAP.GCM.AGCM.PHYSICS.MOIST: + # CAP.GCM.AGCM.PHYSICS.TURBULENCE: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.HEMCO: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.PCHEM: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.ACHEM: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.BC: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.BRC: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.CFC: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.CH4: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.CO2: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.CO: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.DU: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.NI: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.O3: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.OC: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.Rn: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.SS: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.SU: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.data: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GAAS: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.H2O: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.STRATCHEM: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GMICHEM: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.CARMA: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GEOSCHEMCHEM: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.MATRIX: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.MAM: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.TR: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.DNA: + # CAP.GCM.AGCM.PHYSICS.SURFACE: + # CAP.GCM.AGCM.PHYSICS.SURFACE.SALTWATER: + # CAP.GCM.AGCM.PHYSICS.SURFACE.LAKE: + # CAP.GCM.AGCM.PHYSICS.SURFACE.LANDICE: + # CAP.GCM.AGCM.PHYSICS.SURFACE.LAND: + # CAP.GCM.AGCM.PHYSICS.SURFACE.LAND.VEGDYN: + # CAP.GCM.AGCM.PHYSICS.SURFACE.LAND.: + # CAP.GCM.AGCM.PHYSICS.SURFACE.LAND.: + # CAP.GCM.AGCM.PHYSICS.SURFACE.LAND.: + # CAP.GCM.AGCM.PHYSICS.RADIATION: + # CAP.GCM.AGCM.PHYSICS.RADIATION.SOLAR: + # CAP.GCM.AGCM.PHYSICS.RADIATION.IRRAD: + # CAP.GCM.AGCM.PHYSICS.RADIATION.SATSIM: + # CAP.GCM.AGCM.ORBIT: + # CAP.GCM.AIAU: + # CAP.GCM.OGCM: + # CAP.GCM.OGCM.OBIO: + # CAP.GCM.OGCM.ORAD: + # CAP.GCM.OGCM.SEAICE: + # CAP.GCM.OGCM.SEAICEdata: + # CAP.GCM.OGCM.OCEAN: + # CAP.GCM.OGCM.OCEAN.DATASEA: + # CAP.GCM.OGCM.OCEAN.MOM: + # CAP.GCM.OGCM.OCEAN.MOM6: + From a50beb81dd86d74d5900f179db06cdc5d955e993 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Tue, 23 Mar 2021 12:32:37 -0400 Subject: [PATCH 14/31] Update circleci config --- .circleci/config.yml | 51 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a75df8f..50370e1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,28 +1,63 @@ -version: 2 -jobs: - build: +version: 2.1 + +executors: + gcc-build-env: docker: - - image: gmao/geos-build-env-gcc-source:6.0.4 + - image: gmao/ubuntu20-geos-env-mkl:v6.0.27-openmpi_4.0.5-gcc_10.2.0 + auth: + username: $DOCKERHUB_USER + password: $DOCKERHUB_AUTH_TOKEN + environment: + OMPI_ALLOW_RUN_AS_ROOT: 1 + OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 + OMPI_MCA_btl_vader_single_copy_mechanism: none + #XLARGE# resource_class: xlarge + resource_class: medium + +workflows: + version: 2.1 + build-test: + jobs: + - build-GEOSctm: + context: + - docker-hub-creds + +jobs: + build-GEOSctm: + executor: gcc-build-env working_directory: /root/project steps: - - checkout + - checkout: + path: GEOSctm - run: name: "Versions etc" command: mpirun --version && gfortran --version && echo $BASEDIR && pwd && ls - run: name: "Mepo clone external repos" command: | - mepo init + cd ${CIRCLE_WORKING_DIRECTORY}/GEOSctm mepo clone mepo status + - run: + name: "Mepo checkout-if-exists" + command: | + cd ${CIRCLE_WORKING_DIRECTORY}/GEOSctm + echo "${CIRCLE_BRANCH}" + if [ "${CIRCLE_BRANCH}" != "develop" ] && [ "${CIRCLE_BRANCH}" != "master" ] && [ "${CIRCLE_BRANCH}" != "main" ] + then + mepo checkout-if-exists ${CIRCLE_BRANCH} + fi + mepo status - run: name: "CMake" command: | + cd ${CIRCLE_WORKING_DIRECTORY}/GEOSctm mkdir build cd build cmake .. -DBASEDIR=$BASEDIR/Linux -DCMAKE_Fortran_COMPILER=gfortran -DCMAKE_BUILD_TYPE=Debug - run: name: "Build" command: | - cd build - make -j2 install + cd ${CIRCLE_WORKING_DIRECTORY}/GEOSctm/build + #XLARGE# make -j"$(nproc)" install + make -j4 install From 9c961b2048fb87d270b293990248c54c3d572702 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Tue, 23 Mar 2021 12:34:32 -0400 Subject: [PATCH 15/31] Add github actions --- .github/workflows/workflow.yml | 61 ++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 .github/workflows/workflow.yml diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml new file mode 100644 index 0000000..8057345 --- /dev/null +++ b/.github/workflows/workflow.yml @@ -0,0 +1,61 @@ +name: Build Tests + +on: + pull_request: + types: [opened, synchronize, reopened, labeled, unlabeled] + # Do not run if the only files changed cannot affect the build + paths-ignore: + - "**.md" + - "parallel_build.csh" + - ".github/CODEOWNERS" + - ".codebuild/**" + - ".circleci/**" + +jobs: + build_ctm: + name: Build GEOSctm + if: "!contains(github.event.pull_request.labels.*.name, '0 diff trivial')" + runs-on: ubuntu-latest + container: + image: gmao/ubuntu20-geos-env-mkl:v6.0.27-openmpi_4.0.5-gcc_10.2.0 + credentials: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + env: + OMPI_ALLOW_RUN_AS_ROOT: 1 + OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 + OMPI_MCA_btl_vader_single_copy_mechanism: none + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.5.0 + with: + access_token: ${{ github.token }} + - name: Checkout CTM + uses: actions/checkout@v2 + with: + fetch-depth: 1 + - name: Versions etc. + run: | + gfortran --version + mpirun --version + echo $BASEDIR + - name: Mepo clone external repos + run: | + mepo clone + mepo status + mepo status + - name: Update other branches + if: + "!contains('refs/heads/main,refs/heads/develop', github.ref)" + run: | + mepo checkout-if-exists ${GITHUB_HEAD_REF} + mepo status + - name: CMake + run: | + mkdir build + cd build + cmake .. -DBASEDIR=$BASEDIR/Linux -DCMAKE_Fortran_COMPILER=gfortran -DCMAKE_BUILD_TYPE=Debug -DMPIEXEC_PREFLAGS='--oversubscribe' + - name: Build + run: | + cd build + make -j4 install From a30d3b4ba2cb07fa36cd36325f5fa481089274b5 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Wed, 7 Apr 2021 11:53:23 -0400 Subject: [PATCH 16/31] Fixes for ctm_setup and fvcore_layout --- src/Applications/GEOSctm_App/ctm_setup | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/Applications/GEOSctm_App/ctm_setup b/src/Applications/GEOSctm_App/ctm_setup index ac019fa..cbb9da4 100755 --- a/src/Applications/GEOSctm_App/ctm_setup +++ b/src/Applications/GEOSctm_App/ctm_setup @@ -1751,6 +1751,10 @@ s?@CONVERT_T?$CONVERT_T?g s?@CNV_NX?$CNV_NX?g s?@CNV_NY?$CNV_NY?g s?@BCSDIR?$BCSDIR?g +s?@LSMBCS?$LSM_BCS?g +s?@EMIP_BCS_IN?$EMIP_BCS_IN?g +s?@EMIP_MERRA2?$EMIP_MERRA2?g +s?@BCSTAG?$OCEAN_TAG?g s?@CHMDIR?$CHMDIR?g s?@EXPDIR?$EXPDIR?g s?@EXPDSC?$EXPDSC?g @@ -1774,6 +1778,8 @@ s?@CHECKPOINT_TYPE?default?g s?@OBSERVER_FRQ?0?g s?RECORD_?#RECORD_?g +s?@DASTUNING?#?g + s?>>>FORCEDAS<<>>FORCECTM<<>>PCHEM<<>>HIST_GMICHEM<<>>HIST_GEOSCHEMCHEM<<>>HIST_passiveTracer<<>>HIST_IdealizedPT<<>>EMIP_OLDLAND<<>>EMIP_NEWLAND<<>>4DIAUDAS<<>>REGULAR_REPLAY<< Date: Tue, 20 Apr 2021 12:13:48 -0400 Subject: [PATCH 17/31] Update components to match GCM 10.18 --- components.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/components.yaml b/components.yaml index 9e981b0..c1dd38a 100644 --- a/components.yaml +++ b/components.yaml @@ -5,13 +5,13 @@ GEOSctm: env: local: ./@env remote: ../ESMA_env.git - tag: v3.1.3 + tag: v3.2.1 develop: main cmake: local: ./@cmake remote: ../ESMA_cmake.git - tag: v3.3.7 + tag: v3.3.9 develop: develop ecbuild: @@ -22,14 +22,14 @@ ecbuild: GMAO_Shared: local: ./src/Shared/@GMAO_Shared remote: ../GMAO_Shared.git - tag: v1.3.9 + tag: v1.4.0 sparse: ./config/GMAO_Shared.sparse develop: main MAPL: local: ./src/Shared/@MAPL remote: ../MAPL.git - tag: v2.6.3 + tag: v2.6.4 develop: develop FMS: @@ -41,19 +41,19 @@ FMS: FVdycoreCubed_GridComp: local: ./src/Components/GEOSctm_GridComp/@FVdycoreCubed_GridComp remote: ../FVdycoreCubed_GridComp.git - tag: v1.2.10 + tag: v1.2.12 develop: develop fvdycore: local: ./src/Components/GEOSctm_GridComp/@FVdycoreCubed_GridComp/@fvdycore remote: ../GFDL_atmos_cubed_sphere.git - tag: geos/v1.1.4 + tag: geos/v1.1.6 develop: geos/develop GEOSchem_GridComp: local: ./src/Components/GEOSctm_GridComp/@GEOSchem_GridComp remote: ../GEOSchem_GridComp.git - tag: v1.4.4 + tag: v1.5.0 develop: develop GOCART: From 9dfb642b16666062157b3c6dc5eb2e8e7c238028 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Tue, 20 Apr 2021 12:14:06 -0400 Subject: [PATCH 18/31] Updates for setup to match current GCM --- src/Applications/GEOSctm_App/CMakeLists.txt | 19 ++++++++++-- src/Applications/GEOSctm_App/ctm_setup | 32 +++++++++++---------- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/src/Applications/GEOSctm_App/CMakeLists.txt b/src/Applications/GEOSctm_App/CMakeLists.txt index f4281d9..da72f8b 100644 --- a/src/Applications/GEOSctm_App/CMakeLists.txt +++ b/src/Applications/GEOSctm_App/CMakeLists.txt @@ -14,7 +14,6 @@ target_include_directories (GEOSctm.x PUBLIC ${INC_ESMF}) file (GLOB templates CONFIGURE_DEPENDS *.tmpl) set (programs - ctm_setup ctm_run.j ctm_regress.j ctm_post.j @@ -24,7 +23,6 @@ set (programs ctm_moveplot.j ) - install ( FILES ${templates} DESTINATION etc @@ -39,7 +37,22 @@ install ( FILES fvcore_layout.rc logging.yaml DESTINATION etc ) + +if(HYDROSTATIC) + set(CFG_HYDROSTATIC TRUE) +else() + set(CFG_HYDROSTATIC FALSE) +endif() + +set (setup_scripts + ctm_setup + ) + +foreach (file ${setup_scripts}) + configure_file(${file} ${file} @ONLY) + install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${file} DESTINATION bin) +endforeach () + configure_file(.GEOSCTM_VERSION .GEOSCTM_VERSION @ONLY) install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/.GEOSCTM_VERSION DESTINATION etc) - diff --git a/src/Applications/GEOSctm_App/ctm_setup b/src/Applications/GEOSctm_App/ctm_setup index cbb9da4..5948cc4 100755 --- a/src/Applications/GEOSctm_App/ctm_setup +++ b/src/Applications/GEOSctm_App/ctm_setup @@ -288,16 +288,9 @@ endif ASKHYDRO: -if( $HRCODE == 'c1440' | \ - $HRCODE == 'c768' | \ - $HRCODE == 'c1536' | \ - $HRCODE == 'c3072' ) then - set DEFAULT_HYDROSTATIC = FALSE -else - set DEFAULT_HYDROSTATIC = TRUE -endif +set DEFAULT_HYDROSTATIC = @CFG_HYDROSTATIC@ -echo "Use ${C1}Hydrostatic Atmosphere${CN}? (Default: ${C2}YES${CN} or ${C2}TRUE${CN} for Grid Resolutions <= C720)" +echo "Use ${C1}Hydrostatic Atmosphere${CN}? (Default: ${C2}${DEFAULT_HYDROSTATIC}${CN})" set USE_HYDROSTATIC = $< if( .$USE_HYDROSTATIC == . ) then @@ -447,8 +440,8 @@ else if ( $SITE == 'NAS' ) then set NCPUS_PER_NODE = 40 else if ($MODEL == 'rom_ait') then set NCPUS_PER_NODE = 128 - set USE_IOSERVER = 0 - echo "WARNING WARNING: At present there are issues with IOSERVER and Rome. Turning off IOSERVER!" + # Romes are on a different aoe + set MODEL='rom_ait:aoe=sles15' endif else @@ -773,10 +766,19 @@ if ( "$USE_HYDROSTATIC" == "TRUE" ) then set FV_HYDRO = "hydrostatic = .T." set FV_SATADJ = "do_sat_adj = .F." else - set FV_MAKENH = "Make_NH = .T." - set FV_HYDRO = "hydrostatic = .F." - set FV_SATADJ = "do_sat_adj = .F." - set FV_NWAT = "nwat = 3 " + # Logic for NH runs based on build type + if ( "$DEFAULT_HYDROSTATIC" == "TRUE") then + # If you built for hydrostatic, but want an NH run, run make_nh + set FV_MAKENH = "Make_NH = .T." + else + # If you built for non-hydrostatic and want an NH run, do not run + # NOTE: If you run regrid.pl it will make restarts that should trigger + # make_nh automatically in FV3 + set FV_MAKENH = "Make_NH = .F." + endif + set FV_HYDRO = "hydrostatic = .F." + set FV_SATADJ = "do_sat_adj = .F." + set FV_NWAT = "nwat = 3 " endif From c51037323968d397d91e233a63f9c8839225377e Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Tue, 20 Apr 2021 15:05:14 -0400 Subject: [PATCH 19/31] Move to FV3 GC 1.2.13 --- components.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components.yaml b/components.yaml index c1dd38a..56fcc7b 100644 --- a/components.yaml +++ b/components.yaml @@ -41,7 +41,7 @@ FMS: FVdycoreCubed_GridComp: local: ./src/Components/GEOSctm_GridComp/@FVdycoreCubed_GridComp remote: ../FVdycoreCubed_GridComp.git - tag: v1.2.12 + tag: v1.2.13 develop: develop fvdycore: From 5bf6f5460b440d1eae0f2ed762c3919734df8d38 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Thu, 6 May 2021 10:49:26 -0400 Subject: [PATCH 20/31] Update README with more git clone info --- README.md | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/README.md b/README.md index 9fea31f..7863a7e 100644 --- a/README.md +++ b/README.md @@ -36,10 +36,65 @@ which obtains the latest `git`, `CMake`, and `mepo` modules. #### Obtain the Model +On GitHub, there are three ways to clone the model: SSH, HTTPS, or GitHub CLI. +The first two are "git protocols" which determine how `git` communicates with +GitHub: either through https or ssh. (The latter is a CLI that uses either ssh or +https protocol underneath.) + +For developers of GEOS, the SSH git protocol is recommended as it can avoid some issues if +[two-factor authentication +(2FA)](https://docs.github.com/en/github/authenticating-to-github/securing-your-account-with-two-factor-authentication-2fa) +is enabled on GitHub. + +##### SSH + +To clone the GEOSctm using the SSH url (starts with `git@github.com`), you run: ``` git clone git@github.com:GEOS-ESM/GEOSctm.git ``` +###### Permission denied (publickey) + +If this is your first time using GitHub with any SSH URL, you might get this +error: +``` +Permission denied (publickey). +fatal: Could not read from remote repository. + +Please make sure you have the correct access rights +and the repository exists. +``` + +If you do see this, you need to [upload an ssh +key](https://docs.github.com/en/github/authenticating-to-github/adding-a-new-ssh-key-to-your-github-account) +to your GitHub account. This needs to be done on any machine that you want to +use the SSH URL through. + + +##### HTTPS + +To clone the model through HTTPS you run: + +``` +git clone https://github.com/GEOS-ESM/GEOSctm.git +``` + +Note that if you use the HTTPS URL and have 2FA set up on GitHub, you will need +to use [personal access +tokens](https://docs.github.com/en/github/authenticating-to-github/accessing-github-using-two-factor-authentication#authenticating-on-the-command-line-using-https) +as a password. + +##### GitHub CLI + +You can also use the [GitHub CLI](https://cli.github.com/) with: +``` +gh repo clone GEOS-ESM/GEOSctm +``` + +Note that when you first use `gh`, it will ask what your preferred git protocol +is (https or ssh) to use "underneath". The caveats above will apply to whichever +you choose. + --- ### Single Step Building of the Model From 6af41af60d185dd602e9db23a2743504abb7472a Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Mon, 10 May 2021 12:02:15 -0400 Subject: [PATCH 21/31] Update CircleCI to use xlarge --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 50370e1..7251e0e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,8 +11,8 @@ executors: OMPI_ALLOW_RUN_AS_ROOT: 1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 OMPI_MCA_btl_vader_single_copy_mechanism: none - #XLARGE# resource_class: xlarge - resource_class: medium + resource_class: xlarge + #MEDIUM# resource_class: medium workflows: version: 2.1 @@ -59,5 +59,5 @@ jobs: name: "Build" command: | cd ${CIRCLE_WORKING_DIRECTORY}/GEOSctm/build - #XLARGE# make -j"$(nproc)" install - make -j4 install + make -j"$(nproc)" install + #MEDIUM# make -j4 install From bb849f50304929aeb732246df0defca64e14ad82 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Fri, 14 May 2021 10:43:02 -0400 Subject: [PATCH 22/31] Fix AGCM references. Bad Matt --- src/Applications/GEOSctm_App/ctm_convert.j | 72 +++++++++--------- src/Applications/GEOSctm_App/ctm_regress.j | 86 +++++++++++----------- src/Applications/GEOSctm_App/ctm_run.j | 18 ++--- 3 files changed, 88 insertions(+), 88 deletions(-) diff --git a/src/Applications/GEOSctm_App/ctm_convert.j b/src/Applications/GEOSctm_App/ctm_convert.j index 93b42b8..0578816 100755 --- a/src/Applications/GEOSctm_App/ctm_convert.j +++ b/src/Applications/GEOSctm_App/ctm_convert.j @@ -77,26 +77,26 @@ if (! -e $ORGDIR) mkdir -p $ORGDIR # Set Experiment Run Parameters ####################################################################### -set NX = `grep "^ *NX:" $HOMDIR/AGCM.rc | cut -d':' -f2` -set NY = `grep "^ *NY:" $HOMDIR/AGCM.rc | cut -d':' -f2` -set AGCM_IM = `grep AGCM_IM: $HOMDIR/AGCM.rc | cut -d':' -f2` -set AGCM_JM = `grep AGCM_JM: $HOMDIR/AGCM.rc | cut -d':' -f2` -set AGCM_LM = `grep AGCM_LM: $HOMDIR/AGCM.rc | cut -d':' -f2` -set OGCM_IM = `grep "OGCM\.IM_WORLD:" $HOMDIR/AGCM.rc | cut -d':' -f2` -set OGCM_JM = `grep "OGCM\.JM_WORLD:" $HOMDIR/AGCM.rc | cut -d':' -f2` - ->>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>FVCUBED<<>>FVCUBED<<>>DATAOCEAN<<>>COUPLED<< ExtData.rc unsetenv FNDDIR -set fv_rst = `cat AGCM.rc | grep "^DYN_INTERNAL_RESTART_FILE" | cut -d ":" -f2` +set fv_rst = `cat GEOSCTM.rc | grep "^DYN_INTERNAL_RESTART_FILE" | cut -d ":" -f2` if (-e $CNVDIR/$fv_rst ) then setenv FNDDIR $CNVDIR @@ -195,7 +195,7 @@ setenv CHMDIR @CHMDIR >>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>DATAOCEAN<< $FILE >>>DATAOCEAN<<>>DATAOCEAN<< ExtData.rc # Define Atmospheric Resolution # ----------------------------- -set IM = `grep AGCM_IM: $HOMDIR/AGCM.rc | cut -d':' -f2` -set JM = `grep AGCM_JM: $HOMDIR/AGCM.rc | cut -d':' -f2` -set LM = `grep AGCM_LM: $HOMDIR/AGCM.rc | cut -d':' -f2` +set IM = `grep GEOSctm_IM: $HOMDIR/GEOSCTM.rc | cut -d':' -f2` +set JM = `grep GEOSctm_JM: $HOMDIR/GEOSCTM.rc | cut -d':' -f2` +set LM = `grep GEOSctm_LM: $HOMDIR/GEOSCTM.rc | cut -d':' -f2` @ IM6 = 6 * $IM if( $IM6 == $JM ) then @@ -102,11 +102,11 @@ endif # Create Restart List # ------------------- -set rst_files = `cat AGCM.rc | grep "RESTART_FILE" | grep -v VEGDYN | grep -v "#" | cut -d ":" -f1 | cut -d "_" -f1-2` -set rst_file_names = `cat AGCM.rc | grep "RESTART_FILE" | grep -v VEGDYN | grep -v "#" | cut -d ":" -f2` +set rst_files = `cat GEOSCTM.rc | grep "RESTART_FILE" | grep -v VEGDYN | grep -v "#" | cut -d ":" -f1 | cut -d "_" -f1-2` +set rst_file_names = `cat GEOSCTM.rc | grep "RESTART_FILE" | grep -v VEGDYN | grep -v "#" | cut -d ":" -f2` -set chk_files = `cat AGCM.rc | grep "CHECKPOINT_FILE" | grep -v "#" | cut -d ":" -f1 | cut -d "_" -f1-2` -set chk_file_names = `cat AGCM.rc | grep "CHECKPOINT_FILE" | grep -v "#" | cut -d ":" -f2` +set chk_files = `cat GEOSCTM.rc | grep "CHECKPOINT_FILE" | grep -v "#" | cut -d ":" -f1 | cut -d "_" -f1-2` +set chk_file_names = `cat GEOSCTM.rc | grep "CHECKPOINT_FILE" | grep -v "#" | cut -d ":" -f2` # Remove possible bootstrap parameters (+/-) # ------------------------------------------ @@ -204,7 +204,7 @@ set EXEMPT_files = `echo SOLAR_INTERNAL_CHECKPOINT_FILE \ set EXEMPT_chk = "" foreach file ( ${EXEMPT_files} ) - set file = `cat AGCM.rc | grep "$file" | cut -d ":" -f2` + set file = `cat GEOSCTM.rc | grep "$file" | cut -d ":" -f2` set EXEMPT_chk = `echo ${EXEMPT_chk} $file` end @@ -246,14 +246,14 @@ cat $extdata_files > ExtData.rc # If REPLAY, link necessary forcing files # --------------------------------------- -set REPLAY_MODE = `grep REPLAY_MODE: AGCM.rc | grep -v '#' | cut -d: -f2` +set REPLAY_MODE = `grep REPLAY_MODE: GEOSCTM.rc | grep -v '#' | cut -d: -f2` if( $REPLAY_MODE == 'Exact' | $REPLAY_MODE == 'Regular' ) then - set ANA_EXPID = `grep REPLAY_ANA_EXPID: AGCM.rc | grep -v '#' | cut -d: -f2` - set ANA_LOCATION = `grep REPLAY_ANA_LOCATION: AGCM.rc | grep -v '#' | cut -d: -f2` + set ANA_EXPID = `grep REPLAY_ANA_EXPID: GEOSCTM.rc | grep -v '#' | cut -d: -f2` + set ANA_LOCATION = `grep REPLAY_ANA_LOCATION: GEOSCTM.rc | grep -v '#' | cut -d: -f2` - set REPLAY_FILE = `grep REPLAY_FILE: AGCM.rc | grep -v '#' | cut -d: -f2` - set REPLAY_FILE09 = `grep REPLAY_FILE09: AGCM.rc | grep -v '#' | cut -d: -f2` + set REPLAY_FILE = `grep REPLAY_FILE: GEOSCTM.rc | grep -v '#' | cut -d: -f2` + set REPLAY_FILE09 = `grep REPLAY_FILE09: GEOSCTM.rc | grep -v '#' | cut -d: -f2` set REPLAY_FILE_TYPE = `echo $REPLAY_FILE | cut -d"/" -f1 | grep -v %` set REPLAY_FILE09_TYPE = `echo $REPLAY_FILE09 | cut -d"/" -f1 | grep -v %` @@ -275,11 +275,11 @@ endif set test_duration = 240000 @CPEXEC CAP.rc CAP.rc.orig -@CPEXEC AGCM.rc AGCM.rc.orig +@CPEXEC GEOSCTM.rc GEOSCTM.rc.orig @CPEXEC HISTORY.rc0 HISTORY.rc -set NX0 = `grep "^ *NX:" AGCM.rc.orig | cut -d':' -f2` -set NY0 = `grep "^ *NY:" AGCM.rc.orig | cut -d':' -f2` +set NX0 = `grep "^ *NX:" GEOSCTM.rc.orig | cut -d':' -f2` +set NY0 = `grep "^ *NY:" GEOSCTM.rc.orig | cut -d':' -f2` @ NPES0 = $NX0 * $NY0 @@ -290,8 +290,8 @@ set newstring = "JOB_SGMT: 00000000 ${test_duration}" /bin/mv CAP.rc CAP.tmp cat CAP.tmp | sed -e "s?$oldstring?$newstring?g" > CAP.rc -set NX = `grep "^ *NX": AGCM.rc | cut -d':' -f2` -set NY = `grep "^ *NY": AGCM.rc | cut -d':' -f2` +set NX = `grep "^ *NX": GEOSCTM.rc | cut -d':' -f2` +set NY = `grep "^ *NY": GEOSCTM.rc | cut -d':' -f2` @ NPES = $NX * $NY $RUN_CMD $NPES ./GEOSctm.x @@ -332,7 +332,7 @@ set test_duration = 180000 echo $nymd0 $nhms0 > cap_restart @CPEXEC CAP.rc.orig CAP.rc -@CPEXEC AGCM.rc.orig AGCM.rc +@CPEXEC GEOSCTM.rc.orig GEOSCTM.rc @CPEXEC HISTORY.rc0 HISTORY.rc ./strip CAP.rc @@ -341,20 +341,20 @@ set newstring = "JOB_SGMT: 00000000 ${test_duration}" /bin/mv CAP.rc CAP.tmp cat CAP.tmp | sed -e "s?$oldstring?$newstring?g" > CAP.rc -./strip AGCM.rc -set oldstring = `cat AGCM.rc | grep "^ *NX:"` +./strip GEOSCTM.rc +set oldstring = `cat GEOSCTM.rc | grep "^ *NX:"` set newstring = "NX: ${test_NX}" -/bin/mv AGCM.rc AGCM.tmp -cat AGCM.tmp | sed -e "s?$oldstring?$newstring?g" > AGCM.rc -set oldstring = `cat AGCM.rc | grep "^ *NY:"` +/bin/mv GEOSCTM.rc GEOSCTM.tmp +cat GEOSCTM.tmp | sed -e "s?$oldstring?$newstring?g" > GEOSCTM.rc +set oldstring = `cat GEOSCTM.rc | grep "^ *NY:"` set newstring = "NY: ${test_NY}" -/bin/mv AGCM.rc AGCM.tmp -cat AGCM.tmp | sed -e "s?$oldstring?$newstring?g" > AGCM.rc +/bin/mv GEOSCTM.rc GEOSCTM.tmp +cat GEOSCTM.tmp | sed -e "s?$oldstring?$newstring?g" > GEOSCTM.rc setenv YEAR `cat cap_restart | cut -c1-4` ./linkbcs -set NX = `grep "^ *NX": AGCM.rc | cut -d':' -f2` -set NY = `grep "^ *NY": AGCM.rc | cut -d':' -f2` +set NX = `grep "^ *NX": GEOSCTM.rc | cut -d':' -f2` +set NY = `grep "^ *NY": GEOSCTM.rc | cut -d':' -f2` @ NPES = $NX * $NY $RUN_CMD $NPES ./GEOSctm.x @@ -410,31 +410,31 @@ set newstring = "JOB_SGMT: 00000000 ${test_duration}" /bin/mv CAP.rc CAP.tmp cat CAP.tmp | sed -e "s?$oldstring?$newstring?g" > CAP.rc -./strip AGCM.rc -set oldstring = `cat AGCM.rc | grep "^ *NX:"` +./strip GEOSCTM.rc +set oldstring = `cat GEOSCTM.rc | grep "^ *NX:"` set newstring = "NX: ${test_NX}" -/bin/mv AGCM.rc AGCM.tmp -cat AGCM.tmp | sed -e "s?$oldstring?$newstring?g" > AGCM.rc -set oldstring = `cat AGCM.rc | grep "^ *NY:"` +/bin/mv GEOSCTM.rc GEOSCTM.tmp +cat GEOSCTM.tmp | sed -e "s?$oldstring?$newstring?g" > GEOSCTM.rc +set oldstring = `cat GEOSCTM.rc | grep "^ *NY:"` set newstring = "NY: ${test_NY}" -/bin/mv AGCM.rc AGCM.tmp -cat AGCM.tmp | sed -e "s?$oldstring?$newstring?g" > AGCM.rc ->>>COUPLED<< GEOSCTM.rc +>>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<< AGCM.rc ->>>COUPLED<<>>COUPLED<<>>COUPLED<< GEOSCTM.rc +>>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<< AGCM.rc +>>>COUPLED<<>>COUPLED<< GEOSCTM.rc >>>MOM5<<>>MOM6<< $NCPUS ) then @@ -131,7 +131,7 @@ if ( $NCPUS != NULL ) then echo "Specified NX: $NX" echo "Specified NY: $NY" echo "" - echo "Specified IOSERVER_NODES: $AGCM_IOS_NODES" + echo "Specified IOSERVER_NODES: $CTM_IOS_NODES" echo "Specified cores per node: $NCPUS_PER_NODE" exit endif @@ -204,10 +204,10 @@ set month = `echo $RSTDATE | cut -d_ -f1 | cut -b5-6` >>>EMIP_NEWLAND<<<# ------------------------------------------------ set RSTID = `/bin/ls *catch* | cut -d. -f1` set day = `/bin/ls *catch* | cut -d. -f3 | cut -b 7-8` -$GEOSBIN/regrid.pl -np -ymd ${year}${month}${day} -hr 21 -grout C${AGCM_IM} -levsout ${AGCM_LM} -outdir . -d . -expid $RSTID -tagin @EMIP_BCS_IN -oceanin e -i -nobkg -lbl -nolcv -tagout @LSMBCS -rs 3 -oceanout @OCEANOUT +$GEOSBIN/regrid.pl -np -ymd ${year}${month}${day} -hr 21 -grout C${GEOSCTM_IM} -levsout ${GEOSCTM_LM} -outdir . -d . -expid $RSTID -tagin @EMIP_BCS_IN -oceanin e -i -nobkg -lbl -nolcv -tagout @LSMBCS -rs 3 -oceanout @OCEANOUT >>>EMIP_OLDLAND<<>>EMIP_OLDLAND<<>>EMIP_OLDLAND<<<$GEOSBIN/gogo.x -s $RSTID.Chem_Registry.rc.${year}${month}${day}_21z -t $EXPDIR/RC/Chem_Registry.rc -i gocart_internal_rst.merra2 -o gocart_internal_rst -r C${AGCM_IM} -l ${AGCM_LM} +>>>EMIP_OLDLAND<<<$GEOSBIN/gogo.x -s $RSTID.Chem_Registry.rc.${year}${month}${day}_21z -t $EXPDIR/RC/Chem_Registry.rc -i gocart_internal_rst.merra2 -o gocart_internal_rst -r C${GEOSCTM_IM} -l ${GEOSCTM_LM} # Create CAP.rc and cap_restart From 059eb7f6af9066d1ebac42d3758d3d1b135b01ae Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Wed, 19 May 2021 11:21:09 -0400 Subject: [PATCH 23/31] Update gitignore for mepo update Soon, `mepo` will have the ability to checkout subrepos as `repo`, `repo@`, or `@repo`. This commit updates the `.gitignore` files to support this flexibility. --- .gitignore | 5 ++++- src/Components/GEOSctm_GridComp/.gitignore | 6 ++++++ src/Shared/.gitignore | 6 ++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index bbb5eb6..78f284c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,10 @@ *~ /@cmake/ -/@modules/ +/cmake/ +/cmake@/ /@env/ +/env/ +/env@/ /.mepo/ parallel_build.o* log.* diff --git a/src/Components/GEOSctm_GridComp/.gitignore b/src/Components/GEOSctm_GridComp/.gitignore index 43277ae..2a0075c 100644 --- a/src/Components/GEOSctm_GridComp/.gitignore +++ b/src/Components/GEOSctm_GridComp/.gitignore @@ -1,3 +1,9 @@ /@GEOSchem_GridComp +/GEOSchem_GridComp +/GEOSchem_GridComp@ /@GEOSgcm_GridComp +/GEOSgcm_GridComp +/GEOSgcm_GridComp@ /@FVdycoreCubed_GridComp +/FVdycoreCubed_GridComp +/FVdycoreCubed_GridComp@ diff --git a/src/Shared/.gitignore b/src/Shared/.gitignore index 9da6443..023c268 100644 --- a/src/Shared/.gitignore +++ b/src/Shared/.gitignore @@ -1,3 +1,9 @@ /@GMAO_Shared +/GMAO_Shared +/GMAO_Shared@ /@MAPL +/MAPL +/MAPL@ /@FMS +/FMS +/FMS@ From 1f969540408cc093b47543f884cf3eb3067f3585 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Wed, 19 May 2021 11:34:31 -0400 Subject: [PATCH 24/31] Update to match GEOSgcm v10.19.0 --- components.yaml | 10 ++++----- src/Applications/GEOSctm_App/ctm_convert.j | 8 +++++-- src/Applications/GEOSctm_App/ctm_moveplot.j | 2 +- src/Applications/GEOSctm_App/ctm_regress.j | 8 +++++-- src/Applications/GEOSctm_App/ctm_run.j | 22 +++++++++---------- src/Applications/GEOSctm_App/ctm_setup | 24 ++++++++++++++++++--- 6 files changed, 50 insertions(+), 24 deletions(-) diff --git a/components.yaml b/components.yaml index 56fcc7b..dc2fde5 100644 --- a/components.yaml +++ b/components.yaml @@ -11,7 +11,7 @@ env: cmake: local: ./@cmake remote: ../ESMA_cmake.git - tag: v3.3.9 + tag: v3.4.2 develop: develop ecbuild: @@ -22,20 +22,20 @@ ecbuild: GMAO_Shared: local: ./src/Shared/@GMAO_Shared remote: ../GMAO_Shared.git - tag: v1.4.0 + tag: v1.4.1 sparse: ./config/GMAO_Shared.sparse develop: main MAPL: local: ./src/Shared/@MAPL remote: ../MAPL.git - tag: v2.6.4 + tag: v2.6.7 develop: develop FMS: local: ./src/Shared/@FMS remote: ../FMS.git - tag: geos/2019.01.02+noaff.6 + tag: geos/2019.01.02+noaff.7 develop: geos/release/2019.01 FVdycoreCubed_GridComp: @@ -53,7 +53,7 @@ fvdycore: GEOSchem_GridComp: local: ./src/Components/GEOSctm_GridComp/@GEOSchem_GridComp remote: ../GEOSchem_GridComp.git - tag: v1.5.0 + tag: v1.6.0 develop: develop GOCART: diff --git a/src/Applications/GEOSctm_App/ctm_convert.j b/src/Applications/GEOSctm_App/ctm_convert.j index 0578816..3342314 100755 --- a/src/Applications/GEOSctm_App/ctm_convert.j +++ b/src/Applications/GEOSctm_App/ctm_convert.j @@ -287,7 +287,10 @@ endif # Split Saltwater Restart if detected ####################################################################### -if ( -e $EXPDIR/saltwater_internal_rst ) then +if ( (-e $SCRDIR/openwater_internal_rst) && (-e $SCRDIR/seaicethermo_internal_rst)) then + echo "Saltwater internal state is already split, good to go!" +else + if ( ( -e $SCRDIR/saltwater_internal_rst ) || ( -e $EXPDIR/saltwater_internal_rst) ) then # The splitter script requires an OutData directory # ------------------------------------------------- @@ -295,7 +298,7 @@ if ( -e $EXPDIR/saltwater_internal_rst ) then # Run the script # -------------- - $RUN_CMD 1 $GEOSBIN/SaltIntSplitter tile.data $EXPDIR/saltwater_internal_rst + $RUN_CMD 1 $GEOSBIN/SaltIntSplitter tile.data $SCRDIR/saltwater_internal_rst # Move restarts # ------------- @@ -306,6 +309,7 @@ if ( -e $EXPDIR/saltwater_internal_rst ) then /bin/rmdir OutData endif +endif ####################################################################### # Get C2L History weights/index file for Cubed-Sphere diff --git a/src/Applications/GEOSctm_App/ctm_moveplot.j b/src/Applications/GEOSctm_App/ctm_moveplot.j index 764adf2..7efdfb1 100755 --- a/src/Applications/GEOSctm_App/ctm_moveplot.j +++ b/src/Applications/GEOSctm_App/ctm_moveplot.j @@ -9,7 +9,7 @@ #@BATCH_JOBNAME@MOVE_N #@MOVE_Q #@BATCH_GROUP -#@BATCH_OUTPUTNAMEgcm_moveplot.o +#@BATCH_OUTPUTNAMEctm_moveplot.o ####################################################################### # System Environment Variables diff --git a/src/Applications/GEOSctm_App/ctm_regress.j b/src/Applications/GEOSctm_App/ctm_regress.j index 53bc27b..917205d 100755 --- a/src/Applications/GEOSctm_App/ctm_regress.j +++ b/src/Applications/GEOSctm_App/ctm_regress.j @@ -138,7 +138,10 @@ if(! -e tile.bin) $GEOSBIN/binarytile.x tile.data tile.bin # Split Saltwater Restart if detected ####################################################################### -if ( -e $EXPDIR/saltwater_internal_rst ) then +if ( (-e $EXPDIR/regress/openwater_internal_rst) && (-e $EXPDIR/regress/seaicethermo_internal_rst)) then + echo "Saltwater internal state is already split, good to go!" +else + if ( -e $EXPDIR/regress/saltwater_internal_rst ) then # The splitter script requires an OutData directory # ------------------------------------------------- @@ -146,7 +149,7 @@ if ( -e $EXPDIR/saltwater_internal_rst ) then # Run the script # -------------- - $RUN_CMD 1 $GEOSBIN/SaltIntSplitter tile.data $EXPDIR/saltwater_internal_rst + $RUN_CMD 1 $GEOSBIN/SaltIntSplitter tile.data $EXPDIR/regress/saltwater_internal_rst # Move restarts # ------------- @@ -157,6 +160,7 @@ if ( -e $EXPDIR/saltwater_internal_rst ) then /bin/rmdir OutData endif +endif ####################################################################### # Create Simple History for Efficiency diff --git a/src/Applications/GEOSctm_App/ctm_run.j b/src/Applications/GEOSctm_App/ctm_run.j index 65504f0..411dff3 100755 --- a/src/Applications/GEOSctm_App/ctm_run.j +++ b/src/Applications/GEOSctm_App/ctm_run.j @@ -56,7 +56,7 @@ setenv EXPDIR @EXPDIR setenv HOMDIR @HOMDIR setenv RSTDATE @RSTDATE -setenv GCMEMIP @GCMEMIP +setenv CTMEMIP @CTMEMIP ####################################################################### # Create Experiment Sub-Directories @@ -68,7 +68,7 @@ if (! -e $EXPDIR/archive ) mkdir -p $EXPDIR/archive if (! -e $EXPDIR/post ) mkdir -p $EXPDIR/post if (! -e $EXPDIR/plot ) mkdir -p $EXPDIR/plot -if( $GCMEMIP == TRUE ) then +if( $CTMEMIP == TRUE ) then if (! -e $EXPDIR/restarts/$RSTDATE ) mkdir -p $EXPDIR/restarts/$RSTDATE setenv SCRDIR $EXPDIR/scratch.$RSTDATE else @@ -161,10 +161,10 @@ else endif ####################################################################### -# GCMEMIP Setup +# CTMEMIP Setup ####################################################################### -if( $GCMEMIP == TRUE & ! -e $EXPDIR/restarts/$RSTDATE/cap_restart ) then +if( $CTMEMIP == TRUE & ! -e $EXPDIR/restarts/$RSTDATE/cap_restart ) then cd $EXPDIR/restarts/$RSTDATE @@ -276,7 +276,7 @@ endif cat fvcore_layout.rc >> input.nml -if( $GCMEMIP == TRUE ) then +if( $CTMEMIP == TRUE ) then @CPEXEC -f $EXPDIR/restarts/$RSTDATE/cap_restart . @CPEXEC -f $EXPDIR/restarts/$RSTDATE/CAP.rc . endif @@ -413,7 +413,7 @@ end # Copy Restarts to Scratch Directory # ---------------------------------- -if( $GCMEMIP == TRUE ) then +if( $CTMEMIP == TRUE ) then foreach rst ( $rst_file_names ) if(-e $EXPDIR/restarts/$RSTDATE/$rst ) @CPEXEC $EXPDIR/restarts/$RSTDATE/$rst . & end @@ -474,7 +474,7 @@ while ( $counter <= ${NUM_SGMT} ) /bin/rm -f EGRESS -if( $GCMEMIP == TRUE ) then +if( $CTMEMIP == TRUE ) then @CPEXEC -f $EXPDIR/restarts/$RSTDATE/CAP.rc . else @CPEXEC -f $HOMDIR/CAP.rc . @@ -979,7 +979,7 @@ setenv YEAR $yearc # Run GEOSctm.x # ------------- -if( $USE_SHMEM == 1 ) $GEOSBIN/RmShmKeys_sshmpi.csh +if( $USE_SHMEM == 1 ) $GEOSBIN/RmShmKeys_sshmpi.csh >& /dev/null if( $USE_IOSERVER == 1) then set IOSERVER_OPTIONS = "--npes_model $MODEL_NPES --nodes_output_server $IOS_NODES" @@ -989,7 +989,7 @@ endif $RUN_CMD $NPES ./GEOSctm.x $IOSERVER_OPTIONS --logging_config 'logging.yaml' -if( $USE_SHMEM == 1 ) $GEOSBIN/RmShmKeys_sshmpi.csh +if( $USE_SHMEM == 1 ) $GEOSBIN/RmShmKeys_sshmpi.csh >& /dev/null set rc = $status @@ -1108,7 +1108,7 @@ end # Re-Submit Job ####################################################################### -if( $GCMEMIP == TRUE ) then +if( $CTMEMIP == TRUE ) then foreach rst ( `/bin/ls -1 *_rst` ) /bin/rm -f $EXPDIR/restarts/$RSTDATE/$rst end @@ -1133,7 +1133,7 @@ endif if ( $rc == 0 ) then cd $HOMDIR - if( $GCMEMIP == TRUE ) then + if( $CTMEMIP == TRUE ) then if( $capdate < $enddate ) @BATCH_CMD $HOMDIR/ctm_run.j$RSTDATE else if( $capdate < $enddate ) @BATCH_CMD $HOMDIR/ctm_run.j diff --git a/src/Applications/GEOSctm_App/ctm_setup b/src/Applications/GEOSctm_App/ctm_setup index 5948cc4..c40bb38 100755 --- a/src/Applications/GEOSctm_App/ctm_setup +++ b/src/Applications/GEOSctm_App/ctm_setup @@ -791,12 +791,12 @@ endif # Check for Old vs New Land Surface Model Boundary Conditions # ----------------------------------------------------------- #LSM_BCS: -#echo "Enter the choice of ${C1} Land Surface Boundary Conditions${CN} using: ${C2}1${CN} (Default: Icarus), ${C2}2${CN} (Latest Icarus-NL)" +#echo "Enter the choice of ${C1} Land Surface Boundary Conditions${CN} using: ${C2}1${CN} (Icarus), ${C2}2${CN} (Default: Icarus-NLv3)" #set LSM_BCS = $< -#if( .$LSM_BCS == . ) set LSM_BCS = 1 +#if( .$LSM_BCS == . ) set LSM_BCS = 2 #if( $LSM_BCS != 1 & $LSM_BCS != 2 ) then #echo - #echo "${C1} Land Surface Boundary Conditions${CN} must be set equal to ${C2}1 (Icarus){CN} or ${C2}2 (Icarus-NL)${CN}!" + #echo "${C1} Land Surface Boundary Conditions${CN} must be set equal to ${C2}1 (Icarus){CN} or ${C2}2 (Icarus-NLv3)${CN}!" #goto LSM_BCS #else #echo @@ -2426,6 +2426,24 @@ if( $RATS_PROVIDER == PCHEM ) then awk '{if ( $1 ~ "doing") { if ( $1 ~ "PC") sub(/no/, "yes" ); print;} else print }' > $EXPDIR/RC/Chem_Registry.rc endif +# Update LAND_PARAMS Choices +# ------------------------------------------------------------------ +if( $LSM_BCS == "Icarus-NLv3" ) then + /bin/mv $EXPDIR/RC/GEOS_SurfaceGridComp.rc $EXPDIR/RC/GEOS_SurfaceGridComp.tmp + cat $EXPDIR/RC/GEOS_SurfaceGridComp.tmp | sed -e 's?# GEOSagcm=>? ?g' > $EXPDIR/RC/GEOS_SurfaceGridComp.rc +else + /bin/mv $EXPDIR/RC/GEOS_SurfaceGridComp.rc $EXPDIR/RC/GEOS_SurfaceGridComp.tmp + cat $EXPDIR/RC/GEOS_SurfaceGridComp.tmp | sed -e 's?# GEOSagcm=>? ?g' > $EXPDIR/RC/GEOS_SurfaceGridComp.rc + + /bin/mv $EXPDIR/RC/GEOS_SurfaceGridComp.rc $EXPDIR/RC/GEOS_SurfaceGridComp.tmp + cat $EXPDIR/RC/GEOS_SurfaceGridComp.tmp | \ + awk '{ if ($1~"LAND_PARAMS:") { sub(/NRv7.2/,"Icarus") }; print }' > $EXPDIR/RC/GEOS_SurfaceGridComp.rc + + /bin/mv $EXPDIR/RC/GEOS_SurfaceGridComp.rc $EXPDIR/RC/GEOS_SurfaceGridComp.tmp + cat $EXPDIR/RC/GEOS_SurfaceGridComp.tmp | \ + awk '{ if ($1~"Z0_FORMULATION:") { sub(/4/,"2") }; print }' > $EXPDIR/RC/GEOS_SurfaceGridComp.rc +endif + ####################################################################### # Echo Settings and Messages From 37647aef6e10e319a48127a4950259568054f51a Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Wed, 19 May 2021 11:38:18 -0400 Subject: [PATCH 25/31] Fix CI --- .circleci/config.yml | 4 ++-- .github/workflows/workflow.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7251e0e..60404f1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,7 +11,7 @@ executors: OMPI_ALLOW_RUN_AS_ROOT: 1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 OMPI_MCA_btl_vader_single_copy_mechanism: none - resource_class: xlarge + resource_class: large #MEDIUM# resource_class: medium workflows: @@ -54,7 +54,7 @@ jobs: cd ${CIRCLE_WORKING_DIRECTORY}/GEOSctm mkdir build cd build - cmake .. -DBASEDIR=$BASEDIR/Linux -DCMAKE_Fortran_COMPILER=gfortran -DCMAKE_BUILD_TYPE=Debug + cmake .. -DBASEDIR=$BASEDIR/Linux -DCMAKE_Fortran_COMPILER=gfortran -DCMAKE_BUILD_TYPE=Debug -DUSE_F2PY=OFF - run: name: "Build" command: | diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 8057345..c3af4ca 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -54,7 +54,7 @@ jobs: run: | mkdir build cd build - cmake .. -DBASEDIR=$BASEDIR/Linux -DCMAKE_Fortran_COMPILER=gfortran -DCMAKE_BUILD_TYPE=Debug -DMPIEXEC_PREFLAGS='--oversubscribe' + cmake .. -DBASEDIR=$BASEDIR/Linux -DCMAKE_Fortran_COMPILER=gfortran -DCMAKE_BUILD_TYPE=Debug -DMPIEXEC_PREFLAGS='--oversubscribe' -DUSE_F2PY=OFF - name: Build run: | cd build From ec31023ce7879eb958763d249ab4b14891c0d14d Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Wed, 19 May 2021 11:42:02 -0400 Subject: [PATCH 26/31] Forgot HEMCO and GEOS-Chem are separate now --- components.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/components.yaml b/components.yaml index dc2fde5..dfaf1af 100644 --- a/components.yaml +++ b/components.yaml @@ -56,6 +56,18 @@ GEOSchem_GridComp: tag: v1.6.0 develop: develop +HEMCO: + local: ./src/Components/GEOSctm_GridComp/@GEOSchem_GridComp/Shared/HEMCO/@HEMCO + remote: ../HEMCO.git + tag: geos/v2.2.1 + develop: geos/develop + +geos-chem: + local: ./src/Components/GEOSctm_GridComp/@GEOSchem_GridComp/GEOSCHEMchem_GridComp/@geos-chem + remote: ../geos-chem.git + tag: geos/v13.0.0-rc1 + develop: geos/develop + GOCART: local: ./src/Components/GEOSctm_GridComp/@GEOSchem_GridComp/@GOCART remote: ../GOCART.git From 9128dacd77fa31900127072e3b64ea4b099343f8 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Wed, 19 May 2021 11:56:06 -0400 Subject: [PATCH 27/31] Remove unneeded section about surfacegridcomp --- src/Applications/GEOSctm_App/ctm_setup | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/Applications/GEOSctm_App/ctm_setup b/src/Applications/GEOSctm_App/ctm_setup index c40bb38..4b263de 100755 --- a/src/Applications/GEOSctm_App/ctm_setup +++ b/src/Applications/GEOSctm_App/ctm_setup @@ -2426,24 +2426,6 @@ if( $RATS_PROVIDER == PCHEM ) then awk '{if ( $1 ~ "doing") { if ( $1 ~ "PC") sub(/no/, "yes" ); print;} else print }' > $EXPDIR/RC/Chem_Registry.rc endif -# Update LAND_PARAMS Choices -# ------------------------------------------------------------------ -if( $LSM_BCS == "Icarus-NLv3" ) then - /bin/mv $EXPDIR/RC/GEOS_SurfaceGridComp.rc $EXPDIR/RC/GEOS_SurfaceGridComp.tmp - cat $EXPDIR/RC/GEOS_SurfaceGridComp.tmp | sed -e 's?# GEOSagcm=>? ?g' > $EXPDIR/RC/GEOS_SurfaceGridComp.rc -else - /bin/mv $EXPDIR/RC/GEOS_SurfaceGridComp.rc $EXPDIR/RC/GEOS_SurfaceGridComp.tmp - cat $EXPDIR/RC/GEOS_SurfaceGridComp.tmp | sed -e 's?# GEOSagcm=>? ?g' > $EXPDIR/RC/GEOS_SurfaceGridComp.rc - - /bin/mv $EXPDIR/RC/GEOS_SurfaceGridComp.rc $EXPDIR/RC/GEOS_SurfaceGridComp.tmp - cat $EXPDIR/RC/GEOS_SurfaceGridComp.tmp | \ - awk '{ if ($1~"LAND_PARAMS:") { sub(/NRv7.2/,"Icarus") }; print }' > $EXPDIR/RC/GEOS_SurfaceGridComp.rc - - /bin/mv $EXPDIR/RC/GEOS_SurfaceGridComp.rc $EXPDIR/RC/GEOS_SurfaceGridComp.tmp - cat $EXPDIR/RC/GEOS_SurfaceGridComp.tmp | \ - awk '{ if ($1~"Z0_FORMULATION:") { sub(/4/,"2") }; print }' > $EXPDIR/RC/GEOS_SurfaceGridComp.rc -endif - ####################################################################### # Echo Settings and Messages From ccb449e98a66f123744a47445bd2a002c80d96ca Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Fri, 21 May 2021 14:32:20 -0400 Subject: [PATCH 28/31] Update CODEOWNERS --- .github/CODEOWNERS | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8e0d712..60263a3 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,6 +3,16 @@ # Order is important; the last matching pattern takes the most # precedence. +# Everything in GEOSgcm should be owned by the GCM Gatekeepers +* @GEOS-ESM/ctm-gatekeepers + +# The GCM gatekeepers and CMake should know/approve these +/.github/ @GEOS-ESM/cmake-team @GEOS-ESM/ctm-gatekeepers +/.circleci/ @GEOS-ESM/cmake-team @GEOS-ESM/ctm-gatekeepers +/.codebuild/ @GEOS-ESM/cmake-team @GEOS-ESM/ctm-gatekeepers + # The GEOS CMake Team is the CODEOWNER for the CMakeLists.txt files in this repository CMakeLists.txt @GEOS-ESM/cmake-team +# The GEOS CMake Team should be notified about and approve config changes +/config/ @GEOS-ESM/cmake-team From 2a3df881c544b064c2c26d180465d5fa7298efae Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Fri, 21 May 2021 14:37:02 -0400 Subject: [PATCH 29/31] Fix up github actions --- .github/workflows/workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index c3af4ca..f915c87 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -2,7 +2,7 @@ name: Build Tests on: pull_request: - types: [opened, synchronize, reopened, labeled, unlabeled] + types: [opened, synchronize, reopened] # Do not run if the only files changed cannot affect the build paths-ignore: - "**.md" From 0029459222422c7806f4f32d1076735c0232d9f9 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Fri, 21 May 2021 14:39:21 -0400 Subject: [PATCH 30/31] Update CI image to 6.1.0 --- .circleci/config.yml | 2 +- .github/workflows/workflow.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 60404f1..a007a8d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,7 +3,7 @@ version: 2.1 executors: gcc-build-env: docker: - - image: gmao/ubuntu20-geos-env-mkl:v6.0.27-openmpi_4.0.5-gcc_10.2.0 + - image: gmao/ubuntu20-geos-env-mkl:v6.1.0-openmpi_4.0.5-gcc_10.2.0 auth: username: $DOCKERHUB_USER password: $DOCKERHUB_AUTH_TOKEN diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index f915c87..5a7d9b5 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -17,7 +17,7 @@ jobs: if: "!contains(github.event.pull_request.labels.*.name, '0 diff trivial')" runs-on: ubuntu-latest container: - image: gmao/ubuntu20-geos-env-mkl:v6.0.27-openmpi_4.0.5-gcc_10.2.0 + image: gmao/ubuntu20-geos-env-mkl:v6.1.0-openmpi_4.0.5-gcc_10.2.0 credentials: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} From 1e673f147955ad26c926cdedefa0ac8490701029 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Fri, 21 May 2021 14:42:29 -0400 Subject: [PATCH 31/31] Bump version to 2.0 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9363bbb..9e68486 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,7 +4,7 @@ cmake_policy (SET CMP0054 NEW) project ( GEOSctm - VERSION 1.1 + VERSION 2.0 LANGUAGES Fortran CXX C) # Note - CXX is required for ESMF if ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}")