From d9e8d2c0c1c9551c3ca20c7985794788588b50c9 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Mon, 19 Aug 2024 08:51:05 -0400 Subject: [PATCH 1/3] Updates needed to run on SLES15 at NCCS --- .editorconfig | 32 + components.yaml | 6 +- src/Applications/GEOSctm_App/CMakeLists.txt | 4 +- src/Applications/GEOSctm_App/ctm_setup | 662 +++++++------------- 4 files changed, 254 insertions(+), 450 deletions(-) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..9f1cd65 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,32 @@ +# Global Editor Config for MAPL +# +# This is an ini style configuration. See http://editorconfig.org/ for more information on this file. +# +# Top level editor config. +root = true + +# Always use Unix style new lines with new line ending on every file and trim whitespace +[*] +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +# Python: PEP8 defines 4 spaces for indentation +[*.py] +indent_style = space +indent_size = 4 + +# YAML format, 2 spaces +[{*.yaml,*.yml}] +indent_style = space +indent_size = 2 + +# CMake (from KitWare: https://github.com/Kitware/CMake/blob/master/.editorconfig) +[{CMakeLists.txt,*.cmake,*.rst}] +indent_style = space +indent_size = 2 + +# Markdown +[*.md] +trim_trailing_whitespace = true +indent_style = space diff --git a/components.yaml b/components.yaml index e48d915..95d8bdd 100644 --- a/components.yaml +++ b/components.yaml @@ -5,13 +5,13 @@ GEOSctm: env: local: ./@env remote: ../ESMA_env.git - tag: v3.4.0 + branch: feature/mathomp4/3.4-sles15 develop: main cmake: local: ./@cmake remote: ../ESMA_cmake.git - tag: v3.6.2 + branch: feature/mathomp4/3.6-sles15 develop: develop ecbuild: @@ -60,7 +60,7 @@ HEMCO: local: ./src/Components/GEOSctm_GridComp/@GEOSchem_GridComp/Shared/HEMCO/@HEMCO remote: ../HEMCO.git tag: geos/v2.2.1 - develop: geos/develop + develop: geos/develop geos-chem: local: ./src/Components/GEOSctm_GridComp/@GEOSchem_GridComp/GEOSCHEMchem_GridComp/@geos-chem diff --git a/src/Applications/GEOSctm_App/CMakeLists.txt b/src/Applications/GEOSctm_App/CMakeLists.txt index e5fc6a2..b86c49f 100644 --- a/src/Applications/GEOSctm_App/CMakeLists.txt +++ b/src/Applications/GEOSctm_App/CMakeLists.txt @@ -2,7 +2,7 @@ ecbuild_add_executable ( TARGET GEOSctm.x SOURCES GEOSctm.F90 LIBS GEOSctm_GridComp - ) + ) set (CMAKE_Fortran_FLAGS_RELEASE "${GEOS_Fortran_FLAGS_VECT}") @@ -35,7 +35,7 @@ install ( FILES fvcore_layout.rc logging.yaml DESTINATION etc ) - + if(HYDROSTATIC) set(CFG_HYDROSTATIC TRUE) else() diff --git a/src/Applications/GEOSctm_App/ctm_setup b/src/Applications/GEOSctm_App/ctm_setup index ce6250b..195cb0f 100755 --- a/src/Applications/GEOSctm_App/ctm_setup +++ b/src/Applications/GEOSctm_App/ctm_setup @@ -17,7 +17,7 @@ ####################################################################### # Define Colors -# Note: For No Colors, set C1 and C2 to NONE +# Note: For No Colors, set C1 and C2 to NONE ####################################################################### set BLACK = `tput setaf 0` @@ -72,7 +72,6 @@ endif ####################################################################### # Set default behavior of switches -set NOCVS = TRUE set GPU = FALSE set LINKX = FALSE @@ -93,11 +92,6 @@ while ( $#argv > 0 ) set GPU = TRUE breaksw - # Do not archive the source - case --nocvs: - set NOCVS = TRUE - breaksw - # Symlink GEOSctm.x case --link: set LINKX = TRUE @@ -112,46 +106,13 @@ while ( $#argv > 0 ) end ####################################################################### -# CVS: Use CVS functionality only at NCCS or NAS +# Determine site ####################################################################### setenv NODE `uname -n` setenv ARCH `uname -s` setenv SITE `awk '{print $2}' $ETCDIR/SITE.rc` -if ( $SITE != 'NCCS' && $SITE != 'NAS' ) then - set NOCVS = TRUE -endif - -####################################################################### -# CVS: Test for Environment Variable -####################################################################### - -if ( $?ESMA_NOCVS ) then - set NOCVS = TRUE -endif - -####################################################################### -# CVS: Test if we are on a compute node -####################################################################### - -if ( $NOCVS != "TRUE" ) then - -if ( $SITE == 'NCCS' ) then - if ( ($NODE =~ borg*) || ($NODE =~ warp*) ) then - goto ONCOMPUTE - endif - -else if ( $SITE == 'NAS' ) then - if ( ($NODE =~ r[0-9]*i[0-9]*n[0-9]*) || \ - ($NODE =~ r[0-9]*c[0-9]*t[0-9]*n[0-9]*) || \ - ($NODE =~ maia*) ) then - goto ONCOMPUTE - endif -endif - -endif - ####################################################################### # Test for Compiler and MPI Setup ####################################################################### @@ -172,7 +133,7 @@ else if ( `echo $BASEDIR | grep -i mpt` != '') then set MPI = mpt else # Assume default is Intel MPI in case of older baselibs - set MPI = intelmpi + set MPI = intelmpi endif ####################################################################### @@ -194,7 +155,6 @@ end CTMTAG: set CTMTAG = `cat ${ETCDIR}/.GEOSCTM_VERSION` -#echo "${C1}CVS BASE Source Tag${CN} used for Experiment: ${C2}${CTMTAG}${CN}" ####################################################################### # Test to see if you want to CLONE old experiment @@ -228,7 +188,7 @@ else endif ####################################################################### -# Continue to enter in experiment parameters +# Continue to enter in experiment parameters ####################################################################### HRCODE: @@ -373,33 +333,50 @@ endif ASKPROC: if ( $SITE == 'NCCS' ) then - echo "Enter the ${C1}Processor Type${CN} you wish to run on:" - echo " ${C2}hasw (Haswell)${CN}" - echo " ${C2}sky (Skylake)${CN} (default)" - echo " ${C2}cas (Cascade Lake)${CN}" - echo " " - set MODEL = `echo $<` - set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"` - if ( .$MODEL == .) then - set MODEL = 'sky' - endif - if( $MODEL != 'hasw' & \ - $MODEL != 'sky' & \ - $MODEL != 'cas' ) goto ASKPROC + set BUILT_ON_SLES15 = @BUILT_ON_SLES15@ - if ( $MODEL == 'hasw') then - set NCPUS_PER_NODE = 28 - else if ($MODEL == 'sky') then - set NCPUS_PER_NODE = 40 - else if ($MODEL == 'cas') then - # NCCS currently recommends that users do not run with - # 48 cores per node on SCU16 due to OS issues and - # recommends that CPU-intensive works run with 46 or less - # cores. As 45 is a multiple of 3, it's the best value - # that doesn't waste too much - #set NCPUS_PER_NODE = 48 - set NCPUS_PER_NODE = 45 + if ("$BUILT_ON_SLES15" == "TRUE") then + echo "Enter the ${C1}Processor Type${CN} you wish to run on:" + echo " ${C2}mil (Milan)${CN} (default)" + echo " " + set MODEL = `echo $<` + set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"` + if ( .$MODEL == .) then + set MODEL = 'mil' + endif + + if( $MODEL != 'mil' ) goto ASKPROC + + if ($MODEL == 'mil') then + # We save a couple processes for the kernel + set NCPUS_PER_NODE = 126 + endif + else + echo "Enter the ${C1}Processor Type${CN} you wish to run on:" + echo " ${C2}sky (Skylake)${CN} (default)" + echo " ${C2}cas (Cascade Lake)${CN}" + echo " " + set MODEL = `echo $<` + set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"` + if ( .$MODEL == .) then + set MODEL = 'sky' + endif + + if( $MODEL != 'sky' & \ + $MODEL != 'cas' ) goto ASKPROC + + if ($MODEL == 'sky') then + set NCPUS_PER_NODE = 40 + else if ($MODEL == 'cas') then + # NCCS currently recommends that users do not run with + # 48 cores per node on SCU16 due to OS issues and + # recommends that CPU-intensive works run with 46 or less + # cores. As 45 is a multiple of 3, it's the best value + # that doesn't waste too much + #set NCPUS_PER_NODE = 48 + set NCPUS_PER_NODE = 45 + endif endif else if ( $SITE == 'NAS' ) then @@ -638,7 +615,7 @@ if( $GEOSCTM_IM == "c360" ) then set CHEM_DT = $DT set GEOSCTM_IM = 360 set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` - set NX = 12 + set NX = 12 set NY = `expr $NX \* 6` set HYDROSTATIC = $USE_HYDROSTATIC set HIST_IM = `expr $GEOSCTM_IM \* 4` @@ -706,7 +683,7 @@ if( $GEOSCTM_IM == "c1440" ) then set CHEM_DT = $DT set GEOSCTM_IM = 1440 set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` - set NX = 30 + set NX = 30 set NY = `expr $NX \* 6` set HYDROSTATIC = $USE_HYDROSTATIC set HIST_IM = `expr $GEOSCTM_IM \* 4` @@ -907,7 +884,7 @@ set HH = `expr $DT \/ 3600` set MM = `expr \( $DT \- $HH \* 3600 \) \/ 60` set SS = `expr $DT \- $HH \* 3600 \- $MM \* 60` set nhmsDT = `expr $HH \* 10000 \+ $MM \* 100 \+ $SS` - + ####################################################################### # Check for AEROSOL/Chemistry Models ####################################################################### @@ -1070,9 +1047,9 @@ if( $ctmTRACER == TRUE ) then endif if( $ctmIPT == TRUE ) then - set AERO_PROVIDER = - set RATS_PROVIDER = - set OANA_PROVIDER = + set AERO_PROVIDER = + set RATS_PROVIDER = + set OANA_PROVIDER = endif set PCHEM = "#" @@ -1148,13 +1125,8 @@ if( $SITE == 'NAS' ) then setenv POST_T "8:00:00" # Wallclock Time for ctm_post.j setenv PLOT_T "8:00:00" # Wallclock Time for ctm_plot.j setenv ARCHIVE_T "8:00:00" # Wallclock Time for ctm_archive.j - if( $GPU == "TRUE" ) then - set NCPUS_PER_NODE = 16 # CPUS per node - set MODEL = "san_gpu" # Model of CPU - set QTYPE = "gpu_k40" # Queue to use - else - set QTYPE = "normal" # Queue to use - endif + + set QTYPE = "normal" # Queue to use #@ NODES = `echo "($MODEL_NPES + $NCPUS_PER_NODE - 1)/$NCPUS_PER_NODE" | bc` @ NODES = `echo "( ($MODEL_NPES + $NCPUS_PER_NODE) + ($IOS_NDS * $NCPUS_PER_NODE) - 1)/$NCPUS_PER_NODE" | bc` @@ -1178,8 +1150,8 @@ if( $SITE == 'NAS' ) then setenv CHMDIR /nobackup/gmao_SIteam/ModelData/fvInput_nc3 # locations of Aerosol Chemistry BCs setenv WRKDIR /nobackup/$LOGNAME # user work directory setenv COUPLEDIR /nobackup/gmao_SIteam/ModelData/coupled/Forcings # Coupled Ocean/Atmos Forcing - setenv CPEXEC 'mcp -a' # Copy utility for large copies - setenv TAREXEC mtar # Tar utility for large archives + setenv CPEXEC 'cp -a' # Copy utility for large copies + setenv TAREXEC tar # Tar utility for large archives else if( $SITE == 'NCCS' ) then setenv BATCH_CMD "sbatch" # SLURM Batch command setenv BATCH_GROUP "SBATCH --account=" # SLURM Syntax for account name @@ -1192,10 +1164,6 @@ else if( $SITE == 'NCCS' ) then setenv POST_T "8:00:00" # Wallclock Time for ctm_post.j setenv PLOT_T "12:00:00" # Wallclock Time for ctm_plot.j setenv ARCHIVE_T "2:00:00" # Wallclock Time for ctm_archive.j - if( $GPU == "TRUE" ) then - set NCPUS_PER_NODE = 16 # CPUS per node - set MODEL = "k40" # Model of CPU - endif #@ NODES = `echo "($MODEL_NPES + $NCPUS_PER_NODE - 1)/$NCPUS_PER_NODE" | bc` @ NODES = `echo "( ($MODEL_NPES + $NCPUS_PER_NODE) + ($IOS_NDS * $NCPUS_PER_NODE) - 1)/$NCPUS_PER_NODE" | bc` @@ -1396,7 +1364,7 @@ while( $check == FALSE ) else setenv HOMDIR $HOMDIR_def endif - + if( "$EXPID" != `basename $HOMDIR` ) then echo "\!\! The ${C1}HOME${CN} Directory MUST point to the ${C1}EXPID${CN}: ${C2}${EXPID}${CN}" echo " " @@ -1429,7 +1397,7 @@ while( $check == FALSE ) else setenv EXPDIR $EXPDIR_def endif - + if( "$EXPID" != `basename $EXPDIR` ) then echo "\!\! The ${C1}EXPERIMENT${CN} Directory MUST point to the ${C1}EXPID${CN}: ${C2}${EXPID}${CN}" echo " " @@ -1484,7 +1452,7 @@ echo $EXPDIRroot > $HOME/.EXPDIRroot setenv CNVDIR $EXPDIR/convert ####################################################################### -# Locate Build Directory +# Locate Build Directory ####################################################################### # GEOSctm Build Directory @@ -1595,79 +1563,89 @@ set RESTART_BY_OSERVER = NO /bin/rm -f $HOMDIR/SETENV.commands + if( $MPI == openmpi ) then # Open MPI and GEOS has issues with restart writing. Having the # oserver write them can be orders of magnitude faster + set RESTART_BY_OSERVER = YES -# This turns off an annoying warning when running -# Open MPI on a system where TMPDIRs are on a networked -# file system +# Testing by Bill Putman determined some useful +# Open MPI parameters. Testing shows these work +# on both OSs at NCCS and on macOS cat > $HOMDIR/SETENV.commands << EOF - setenv OMPI_MCA_shmem_mmap_enable_nfs_warning 0 +# Turn off warning about TMPDIR on NFS +setenv OMPI_MCA_shmem_mmap_enable_nfs_warning 0 +# pre-connect MPI procs on mpi_init +setenv OMPI_MCA_mpi_preconnect_all 1 +setenv OMPI_MCA_coll_tuned_bcast_algorithm 7 +setenv OMPI_MCA_coll_tuned_scatter_algorithm 2 +setenv OMPI_MCA_coll_tuned_reduce_scatter_algorithm 3 +setenv OMPI_MCA_coll_tuned_allreduce_algorithm 3 +setenv OMPI_MCA_coll_tuned_allgather_algorithm 4 +setenv OMPI_MCA_coll_tuned_allgatherv_algorithm 3 +setenv OMPI_MCA_coll_tuned_gather_algorithm 1 +setenv OMPI_MCA_coll_tuned_barrier_algorithm 0 +# required for a tuned flag to be effective +setenv OMPI_MCA_coll_tuned_use_dynamic_rules 1 +# disable file locks +setenv OMPI_MCA_sharedfp "^lockedfile,individual" EOF # The below settings seem to be recommended for hybrid -# systems using MVAPICH2 but could change +# systems using MVAPICH but could change -else if( $MPI == mvapich2 ) then - -if( $GPU == "TRUE" ) then -cat > $HOMDIR/SETENV.commands << EOF - setenv MV2_ENABLE_AFFINITY 0 - setenv MV2_RNDV_PROTOCOL RPUT - setenv MV2_USE_RDMA_ONE_SIDED 1 - setenv SLURM_DISTRIBUTION block - setenv MV2_MPIRUN_TIMEOUT 100 - setenv MV2_GATHERV_SSEND_THRESHOLD 256 -EOF +else if( $MPI == mvapich ) then -else +# MVAPICH and GEOS has issues with restart writing. Having the +# oserver write them seems to...work +set RESTART_BY_OSERVER = YES cat > $HOMDIR/SETENV.commands << EOF - setenv MV2_ENABLE_AFFINITY 0 - setenv SLURM_DISTRIBUTION block - setenv MV2_MPIRUN_TIMEOUT 100 - setenv MV2_GATHERV_SSEND_THRESHOLD 256 +setenv MV2_ENABLE_AFFINITY 0 +setenv SLURM_DISTRIBUTION block +setenv MV2_MPIRUN_TIMEOUT 100 +setenv MV2_GATHERV_SSEND_THRESHOLD 256 EOF -endif # if GPU and mvapich2 - else if( $MPI == mpt ) then cat > $HOMDIR/SETENV.commands << EOF - setenv MPI_COLL_REPRODUCIBLE - setenv SLURM_DISTRIBUTION block +setenv MPI_COLL_REPRODUCIBLE +setenv SLURM_DISTRIBUTION block + +#setenv MPI_DISPLAY_SETTINGS 1 +#setenv MPI_VERBOSE 1 - #setenv MPI_DISPLAY_SETTINGS 1 - #setenv MPI_VERBOSE 1 - - unsetenv MPI_MEMMAP_OFF - unsetenv MPI_NUM_MEMORY_REGIONS - setenv MPI_XPMEM_ENABLED yes - unsetenv SUPPRESS_XPMEM_TRIM_THRESH +setenv MPI_MEMMAP_OFF +unsetenv MPI_NUM_MEMORY_REGIONS +setenv MPI_XPMEM_ENABLED yes +unsetenv SUPPRESS_XPMEM_TRIM_THRESH - setenv MPI_LAUNCH_TIMEOUT 40 +setenv MPI_LAUNCH_TIMEOUT 40 - # For some reason, PMI_RANK is randomly set and interferes - # with binarytile.x and other executables. - unsetenv PMI_RANK +setenv MPI_COMM_MAX 1024 +setenv MPI_GROUP_MAX 1024 +setenv MPI_BUFS_PER_PROC 256 - # Often when debugging on MPT, the traceback from Intel Fortran - # is "absorbed" and only MPT's errors are displayed. To allow the - # compiler's traceback to be displayed, uncomment this environment - # variable - #setenv FOR_IGNORE_EXCEPTIONS false +# For some reason, PMI_RANK is randomly set and interferes +# with binarytile.x and other executables. +unsetenv PMI_RANK + +# Often when debugging on MPT, the traceback from Intel Fortran +# is "absorbed" and only MPT's errors are displayed. To allow the +# compiler's traceback to be displayed, uncomment this environment +# variable +#setenv FOR_IGNORE_EXCEPTIONS false EOF else if( $MPI == intelmpi ) then cat > $HOMDIR/SETENV.commands << EOF -setenv I_MPI_DAPL_UD enable setenv I_MPI_ADJUST_ALLREDUCE 12 setenv I_MPI_ADJUST_GATHERV 3 @@ -1696,6 +1674,36 @@ setenv I_MPI_EXTRA_FILESYSTEM_FORCE gpfs setenv ROMIO_FSTYPE_FORCE "gpfs:" EOF +# Testing by Bill Putman found these to be +# useful flags with Intel MPI on SLES15 on the +# Milan nodes. +# Note 1: Testing by NCCS shows the PSM3 provider +# runs on the Infiniband fabric. Tests show it runs +# up to C720. +# Note 2: When the Cascade Lakes are moved to +# SLES15, these will need to be Milan-only flags +# as Intel MPI will probably work just fine with +# Intel chips. +if ("$BUILT_ON_SLES15" == "TRUE") then +cat >> $HOMDIR/SETENV.commands << EOF +setenv I_MPI_FALLBACK 0 +setenv I_MPI_FABRICS ofi +setenv I_MPI_OFI_PROVIDER psm3 +setenv I_MPI_ADJUST_SCATTER 2 +setenv I_MPI_ADJUST_SCATTERV 2 +setenv I_MPI_ADJUST_GATHER 2 +setenv I_MPI_ADJUST_GATHERV 3 +setenv I_MPI_ADJUST_ALLGATHER 3 +setenv I_MPI_ADJUST_ALLGATHERV 3 +setenv I_MPI_ADJUST_ALLREDUCE 12 +setenv I_MPI_ADJUST_REDUCE 10 +setenv I_MPI_ADJUST_BCAST 11 +setenv I_MPI_ADJUST_REDUCE_SCATTER 4 +setenv I_MPI_ADJUST_BARRIER 9 +EOF + +endif # if SLES15 + endif # if NCCS endif # if mpi @@ -1719,8 +1727,8 @@ if ( $SITE == 'NCCS' ) then cat >> $HOMDIR/GPUSTART.commands << _EOF1_ -setenv GPU_NODELIST \`scontrol show hostnames\` -setenv CUDA_TMPDIR /tmp/nvidia-hyperq-\$SLURM_JOBID +setenv GPU_NODELIST \`scontrol show hostnames\` +setenv CUDA_TMPDIR /tmp/nvidia-hyperq-\$SLURM_JOBID _EOF1_ @@ -1728,31 +1736,31 @@ else if ( $SITE == 'NAS' ) then cat >> $HOMDIR/GPUSTART.commands << _EOF1_ -setenv GPU_NODELIST \`cat \$PBS_NODEFILE | uniq\` -setenv CUDA_TMPDIR /tmp/nvidia-hyperq-\$PBS_JOBID +setenv GPU_NODELIST \`cat \$PBS_NODEFILE | uniq\` +setenv CUDA_TMPDIR /tmp/nvidia-hyperq-\$PBS_JOBID _EOF1_ endif # if site cat >> $HOMDIR/GPUSTART.commands << _EOF2_ -setenv CUDA_VISIBLE_DEVICES 0 -setenv CUDA_MPS_CLIENT 1 -setenv CUDA_MPS_PIPE_DIRECTORY \$CUDA_TMPDIR/mps_0 -setenv CUDA_MPS_LOG_DIRECTORY \$CUDA_TMPDIR/mps_log_0 - -foreach node (\$GPU_NODELIST) - ssh -f \$node "env CUDA_TMPDIR=\$CUDA_TMPDIR $HOME/bin/kill_mps_server.bash" -end - -sleep 3 - -foreach node (\$GPU_NODELIST) - echo "Running MPS Server on \$node" - ssh -f \$node "env CUDA_TMPDIR=\$CUDA_TMPDIR $HOME/bin/run_mps_server.bash" -end - -sleep 3 +setenv CUDA_VISIBLE_DEVICES 0 +setenv CUDA_MPS_CLIENT 1 +setenv CUDA_MPS_PIPE_DIRECTORY \$CUDA_TMPDIR/mps_0 +setenv CUDA_MPS_LOG_DIRECTORY \$CUDA_TMPDIR/mps_log_0 + +foreach node (\$GPU_NODELIST) + ssh -f \$node "env CUDA_TMPDIR=\$CUDA_TMPDIR $HOME/bin/kill_mps_server.bash" +end + +sleep 3 + +foreach node (\$GPU_NODELIST) + echo "Running MPS Server on \$node" + ssh -f \$node "env CUDA_TMPDIR=\$CUDA_TMPDIR $HOME/bin/run_mps_server.bash" +end + +sleep 3 _EOF2_ @@ -1760,9 +1768,9 @@ cat > $HOMDIR/GPUEND.commands << _EOF3_ # Remove the MPS Server on Each GPU Node # -------------------------------------- -foreach node (\$GPU_NODELIST) - ssh -f \$node "env CUDA_TMPDIR=\$CUDA_TMPDIR $HOME/bin/kill_mps_server.bash" -end +foreach node (\$GPU_NODELIST) + ssh -f \$node "env CUDA_TMPDIR=\$CUDA_TMPDIR $HOME/bin/kill_mps_server.bash" +end _EOF3_ @@ -1912,7 +1920,7 @@ s/@GEOSCTM_IM/$GEOSCTM_IM/g s/@GEOSCTM_JM/$GEOSCTM_JM/g s/@GEOSCTM_LM/$GEOSCTM_LM/g s/@BEG_DATE/${BEG_DATE}/g -s/@END_DATE/${END_DATE}/g +s/@END_DATE/${END_DATE}/g s/@JOB_SGMT/${JOB_SGMT}/g s/@NUM_SGMT/${NUM_SGMT}/g @@ -2128,7 +2136,7 @@ endif /bin/mv $HOMDIR/pTracers_GridComp.rc.tmpl $HOMDIR/pTracers_GridComp.rc ####################################################################### -# Modify RC Directory for LM and PCHEM/GOCART/GOCART.data Options +# Modify RC Directory for LM and PCHEM/GOCART/GOCART.data Options ####################################################################### # Modify RC Files for LM @@ -2370,7 +2378,7 @@ else if ( $ctmGMI == TRUE ) then sed -e 's|EXTDATA|/discover/nobackup/projects/gmao/share/dasilva/fvInput|g' $EXPDIR/RC/GMI_GridComp.rc > $EXPDIR/RC/GMI_GridComp.tmp /bin/mv -f $EXPDIR/RC/GMI_GridComp.tmp $EXPDIR/RC/GMI_GridComp.rc endif - + # Turn on GEOS-Chem # ----------- else if ( $ctmGEOS_Chem == TRUE ) then @@ -2555,150 +2563,31 @@ if( -e $HOMDIR/tmpfile ) /bin/rm $HOMDIR/tmpfile if( -e $HOMDIR/sedfile ) /bin/rm $HOMDIR/sedfile ####################################################################### -# Determine Experiment Specific src Files -####################################################################### - -if ( $NOCVS != "TRUE" ) then - -echo "ERROR! Repository managment is not supported yet in this model" -echo " due to the move to git. If this section is reached," -echo " something has gone wrong. Please contact Matt Thompson" -echo " or Larry Takacs" -exit 3 - -# Make a src directory under EXPDIR to hold current Experiment files -# Note: Sandbox Source Location: $GEOSDIR/src -# EXP TAG Source Location: $EXPDIR/src/GEOSctm/src -# ------------------------------------------------------------------ -/bin/rm -rf ${EXPDIR}/src -mkdir -p ${EXPDIR}/src/GEOSctm/src -cd ${EXPDIR}/src/GEOSctm/src - -echo "${C2}Copying${CN} ${C1}Sandbox Source Code${CN} ${C2}(including non-committed files) into" -echo " ${CN} ${C1}${EXPDIR}/src/GEOSctm/src${CN} ${C2}...${CN}" -# ---------------------------------------------------------------------------------------------------------------------------------- -rsync -ar --exclude '*.o' \ - --exclude '*.x' \ - --exclude '*.xx' \ - --exclude '*.a' \ - --exclude '*.d' \ - --exclude '*.nc4' \ - --exclude '*.mod' \ - --exclude 'GEOSctm.x.*' \ - ${GEOSDIR}/src/* . -echo " " - -# ----------------------------------------------------------------------------- -# Create CVS TAG for Experiment -# ----------------------------------------------------------------------------- - -# Change Dots to Underscores in EXPID for CVS Tag -# ----------------------------------------------- -set EXPIDTAG = `echo ${EXPID} | sed -e 's/\./_/g'` -set tagname = "${EXPIDTAG}__${LOGNAME}" - -if( ! $?ESMA_NOCOMMIT ) then -TAGNAME: - echo "Enter ${C1}CVS Tag${CN} to COMMIT this Experiment (Default: ${C2}${tagname}${CN})" - echo "Enter ${C1}q${CN} or ${C1}quit${CN} to QUIT or SKIP the CVS COMMIT:" - set TAGTMP = $< - if ( .$TAGTMP != . ) then - set tagname = "$TAGTMP" - endif - # Change Dots to Underscores in TAGNAME for CVS Tag - # ------------------------------------------------- - set tagname = `echo ${tagname} | sed -e 's/\./_/g'` - if( $tagname == 'q' | $tagname == 'quit' ) then - setenv ESMA_NOCOMMIT TRUE - set tagname = "${EXPIDTAG}__${LOGNAME}" - set NUM_COMMIT_STATUS = `/bin/ls -1 ${EXPDIR}/src | grep COMMIT_STATUS` - if($#NUM_COMMIT_STATUS > 0 ) /bin/rm -f ${EXPDIR}/src/COMMIT_STATUS_* - touch ${EXPDIR}/src/COMMIT_STATUS_FALSE - else - set tagtest = `cvs status -v g5_modules | grep ${tagname}` - if( $#tagtest != 0 ) then - echo " ${C1}CVS COMMIT Tag${CN} ${C2}${tagname}${CN} already exists." - set tagname = "${EXPIDTAG}__${LOGNAME}" - goto TAGNAME - endif - endif -else - set NUM_COMMIT_STATUS = `/bin/ls -1 ${EXPDIR}/src | grep COMMIT_STATUS` - if($#NUM_COMMIT_STATUS > 0 ) /bin/rm -f ${EXPDIR}/src/COMMIT_STATUS_* - touch ${EXPDIR}/src/COMMIT_STATUS_FALSE -endif - -/bin/rm -f ${EXPDIR}/src/GEOSctm/src/Applications/GEOSctm_App/.GEOSCTM_VERSION -echo $tagname > ${EXPDIR}/src/GEOSctm/src/Applications/GEOSctm_App/.GEOSCTM_VERSION - - -# Create a list of non-committed Sandbox files (ignore updated files not included in sandbox) -# ------------------------------------------------------------------------------------------- -cvs -nq upd | grep -v "U " > ${EXPDIR}/src/srcfiles - - -echo "${C2}Non-Committed Sandbox Files:${CN}" -echo "---------------------------------" - -/bin/rm -f ${EXPDIR}/src/cvs.log - -while( -e ${EXPDIR}/src/srcfiles ) - set file = `head -1 ${EXPDIR}/src/srcfiles` - set bit = `echo "$file" | cut -c1` - if( "$bit" != "?" ) then - set file = `echo "$file" | cut -d' ' -f2` - echo $file +# Copy over Source Tarfile +####################################################################### - # Local Copy of Differing Sandbox File from TAG file - # -------------------------------------------------- - set newfile = `echo $file | sed -e "s?/?^?g"` - /bin/cp -f ${EXPDIR}/src/GEOSctm/src/$file ${EXPDIR}/src/$newfile +# NOTE: This variable is set at CMake time depending on +# how the build was configured. +set INSTALL_TARFILE = @CFG_INSTALL_SOURCE_TARFILE@ +set TARFILE_NAME = "@CMAKE_PROJECT_NAME@.tar.gz" - # CVS Commit updated Sandbox files - # -------------------------------- - if( ! $?ESMA_NOCOMMIT ) then - set branchname = "BRANCH__${tagname}" - cvs tag -b ${branchname} $file >& ${EXPDIR}/src/cvs.log - cvs upd -r ${branchname} $file >& ${EXPDIR}/src/cvs.log - cvs commit -m "${tagname} ${EXPDSC}" $file >& ${EXPDIR}/src/cvs.log - endif +if ( $INSTALL_TARFILE == "TRUE" ) then - endif - sed 1,1d -i ${EXPDIR}/src/srcfiles - set nfiles = `wc -c ${EXPDIR}/src/srcfiles | cut -d" " -f 1` - if( $nfiles == 0 ) /bin/rm -f ${EXPDIR}/src/srcfiles -end - -echo " " - # Tag Final EXPDIR/src/GEOSctm/src - # --------------------------------- - if( ! $?ESMA_NOCOMMIT ) then - cvs tag ${tagname} >& ${EXPDIR}/src/cvs.log - if( $status == 0 ) then - echo "${C2}EXP:${CN} ${C1}$EXPID${CN} ${C2}successfully committed with TAG:${CN} ${C1}${tagname}${CN}" - echo "-------------------------------------------------------" - set NUM_COMMIT_STATUS = `/bin/ls -1 ${EXPDIR}/src | grep COMMIT_STATUS` - if($#NUM_COMMIT_STATUS > 0 ) /bin/rm -f ${EXPDIR}/src/COMMIT_STATUS_* - touch ${EXPDIR}/src/COMMIT_STATUS_TRUE - else - echo "${C2}EXP:${CN} ${C1}$EXPID${CN} ${C2}failed to be committed with TAG:${CN} ${C1}${tagname}${CN}" - echo "-------------------------------------------------------" - set NUM_COMMIT_STATUS = `/bin/ls -1 ${EXPDIR}/src | grep COMMIT_STATUS` - if($#NUM_COMMIT_STATUS > 0 ) /bin/rm -f ${EXPDIR}/src/COMMIT_STATUS_* - if($#NUM_COMMIT_STATUS > 0 ) /bin/rm -f ${EXPDIR}/src/COMMIT_STATUS_* - touch ${EXPDIR}/src/COMMIT_STATUS_FALSE - endif - endif - -if( -e ${EXPDIR}/src/cvs.log ) /bin/rm -f ${EXPDIR}/src/cvs.log -cd ${EXPDIR}/src + # Make a src directory under EXPDIR to hold current Experiment files + # ------------------------------------------------------------------ + /bin/rm -rf ${EXPDIR}/src + mkdir -p ${EXPDIR}/src -echo "" -echo "${C2}Tarring${CN} Experiment Source Code into Single File ${C2}...${CN}" -# --------------------------------------------------------------------------- -/bin/tar cf ${EXPID}.GEOSctm.tar GEOSctm -/bin/rm -r GEOSctm -echo "" + echo "Copying Build Source Code into ${C2}${EXPDIR}/src${CN}" + # ----------------------------------------------------------- + if (-e ${GEOSDEF}/src/${TARFILE_NAME}) then + cp ${GEOSDEF}/src/${TARFILE_NAME} ${EXPDIR}/src + else + echo "${GEOSDEF}/src/${TARFILE_NAME} not found yet CMake was asked to make and install a tarfile." + echo "Something went wrong." + exit 7 + endif + echo "" endif @@ -2730,7 +2619,7 @@ else if ( ! -d $CLONEDIR ) then endif # ------------------------------------------------------ -# To setup the clone, we need to look in a couple files, +# To setup the clone, we need to look in a couple files, # so make sure they are readable # ------------------------------------------------------ @@ -2878,7 +2767,7 @@ EOF # ------------------------------------------------ # Create two sets of files so we can copy from one -# directory to another. +# directory to another. # ------------------------------------------------ # Then alter them @@ -3007,149 +2896,32 @@ echo "" /bin/rm $COPYSCRIPT /bin/rm $SEDFILE -# -------------------------------- -# Cloned Experiment Source Control -# -------------------------------- - -if ( $NOCVS != "TRUE" ) then - -echo "ERROR! Repository managment is not supported yet in this model" -echo " due to the move to git. If this section is reached," -echo " something has gone wrong. Please contact Matt Thompson" -echo " or Larry Takacs" -exit 3 - -# Make a src directory under NEWEXPDIR to hold current Experiment files -# Note: Sandbox Source Location: $GEOSDIR/src -# EXP TAG Source Location: $NEWEXPDIR/src/GEOSctm/src -# ------------------------------------------------------------------ -/bin/rm -rf ${NEWEXPDIR}/src -mkdir -p ${NEWEXPDIR}/src/GEOSctm/src -cd ${NEWEXPDIR}/src/GEOSctm/src - -echo "${C2}Copying${CN} ${C1}Sandbox Source Code${CN} ${C2}(including non-committed files) into${CN} ${C1}${NEWEXPDIR}/src/GEOSctm/src${CN} ${C2}...${CN}" -# --------------------------------------------------------------------------------------------------------------------------------- -rsync -ar --exclude '*.o' \ - --exclude '*.x' \ - --exclude '*.xx' \ - --exclude '*.a' \ - --exclude '*.d' \ - --exclude '*.nc4' \ - --exclude '*.mod' \ - --exclude 'GEOSctm.x.*' \ - ${GEOSDIR}/src/* . -echo " " - -# ----------------------------------------------------------------------------- -# Create CVS TAG for Experiment -# ----------------------------------------------------------------------------- - -# Change Dots to Underscores in NEWEXPID for CVS Tag -# -------------------------------------------------- -set EXPIDTAG = `echo ${NEWEXPID} | sed -e 's/\./_/g'` -set tagname = "${EXPIDTAG}__${LOGNAME}" - -if( ! $?ESMA_NOCOMMIT ) then -NEWTAGNAME: - echo "Enter ${C1}CVS Tag${CN} to COMMIT this Experiment (Default: ${C2}${tagname}${CN})" - echo "Enter ${C1}q${CN} or ${C1}quit${CN} to QUIT or SKIP the CVS COMMIT:" - set TAGTMP = $< - if ( .$TAGTMP != . ) then - set tagname = "$TAGTMP" - endif - # Change Dots to Underscores in TAGNAME for CVS Tag - # ------------------------------------------------- - set tagname = `echo ${tagname} | sed -e 's/\./_/g'` - if( $tagname == 'q' | $tagname == 'quit' ) then - setenv ESMA_NOCOMMIT TRUE - set tagname = "${EXPIDTAG}__${LOGNAME}" - set NUM_COMMIT_STATUS = `/bin/ls -1 ${NEWEXPDIR}/src | grep COMMIT_STATUS` - if($#NUM_COMMIT_STATUS > 0 ) /bin/rm -f ${NEWEXPDIR}/src/COMMIT_STATUS_* - touch ${NEWEXPDIR}/src/COMMIT_STATUS_FALSE - else - set tagtest = `cvs status -v g5_modules | grep ${tagname}` - if( $#tagtest != 0 ) then - echo " ${C1}CVS COMMIT Tag${CN} ${C2}${tagname}${CN} already exists." - set tagname = "${EXPIDTAG}__${LOGNAME}" - goto NEWTAGNAME - endif - endif -else - set NUM_COMMIT_STATUS = `/bin/ls -1 ${NEWEXPDIR}/src | grep COMMIT_STATUS` - if($#NUM_COMMIT_STATUS > 0 ) /bin/rm -f ${NEWEXPDIR}/src/COMMIT_STATUS_* - touch ${NEWEXPDIR}/src/COMMIT_STATUS_FALSE -endif - -/bin/rm -f ${NEWEXPDIR}/src/GEOSctm/src/Applications/GEOSctm_App/.GEOSCTM_VERSION -echo $tagname > ${NEWEXPDIR}/src/GEOSctm/src/Applications/GEOSctm_App/.GEOSCTM_VERSION - - -# Create a list of non-committed Sandbox files -# -------------------------------------------- -cvs -nq upd > ${NEWEXPDIR}/src/srcfiles - - -echo "${C2}Non-Committed Sandbox Files:${CN}" -echo "---------------------------------" - -/bin/rm -f ${NEWEXPDIR}/src/cvs.log - -while( -e ${NEWEXPDIR}/src/srcfiles ) - set file = `head -1 ${NEWEXPDIR}/src/srcfiles` - set bit = `echo "$file" | cut -c1` - if( "$bit" != "?" ) then - set file = `echo "$file" | cut -d' ' -f2` - echo $file +# ------------------------ +# Cloned Experiment Source +# ------------------------ - # Local Copy of Differing Sandbox File from TAG file - # -------------------------------------------------- - set newfile = `echo $file | sed -e "s?/?^?g"` - /bin/cp -f ${NEWEXPDIR}/src/GEOSctm/src/$file ${NEWEXPDIR}/src/$newfile +# NOTE: This variable is set at CMake time depending on +# how the build was configured. +set INSTALL_TARFILE = @CFG_INSTALL_SOURCE_TARFILE@ +set TARFILE_NAME = "@CMAKE_PROJECT_NAME@.tar.gz" - # CVS Commit updated Sandbox files - # -------------------------------- - if( ! $?ESMA_NOCOMMIT ) then - set branchname = "BRANCH__${tagname}" - cvs tag -b ${branchname} $file >& ${NEWEXPDIR}/src/cvs.log - cvs upd -r ${branchname} $file >& ${NEWEXPDIR}/src/cvs.log - cvs commit -m "${tagname} ${EXPDSC}" $file >& ${NEWEXPDIR}/src/cvs.log - endif +if ( $INSTALL_TARFILE == "TRUE" ) then - endif - sed 1,1d -i ${NEWEXPDIR}/src/srcfiles - set nfiles = `wc -c ${NEWEXPDIR}/src/srcfiles | cut -d" " -f 1` - if( $nfiles == 0 ) /bin/rm -f ${NEWEXPDIR}/src/srcfiles -end + # Make a src directory under EXPDIR to hold current Experiment files + # ------------------------------------------------------------------ + /bin/rm -rf ${NEWEXPDIR}/src + mkdir -p ${NEWEXPDIR}/src -echo " " - # Tag Final NEWEXPDIR/src/GEOSctm/src - # ----------------------------------- - if( ! $?ESMA_NOCOMMIT ) then - cvs tag ${tagname} >& ${NEWEXPDIR}/src/cvs.log - if( $status == 0 ) then - echo "${C2}EXP:${CN} ${C1}$NEWEXPID${CN} ${C2}successfully committed with TAG:${CN} ${C1}${tagname}${CN}" - echo "-------------------------------------------------------" - set NUM_COMMIT_STATUS = `/bin/ls -1 ${NEWEXPDIR}/src | grep COMMIT_STATUS` - if($#NUM_COMMIT_STATUS > 0 ) /bin/rm -f ${NEWEXPDIR}/src/COMMIT_STATUS_* - touch ${NEWEXPDIR}/src/COMMIT_STATUS_TRUE - else - echo "${C2}EXP:${CN} ${C1}$NEWEXPID${CN} ${C2}failed to be committed with TAG:${CN} ${C1}${tagname}${CN}" - echo "-------------------------------------------------------" - set NUM_COMMIT_STATUS = `/bin/ls -1 ${NEWEXPDIR}/src | grep COMMIT_STATUS` - if($#NUM_COMMIT_STATUS > 0 ) /bin/rm -f ${NEWEXPDIR}/src/COMMIT_STATUS_* - touch ${NEWEXPDIR}/src/COMMIT_STATUS_FALSE - endif - endif - -/bin/rm -f ${NEWEXPDIR}/src/cvs.log -cd ${NEWEXPDIR}/src - -echo "" -echo "${C2}Tarring${CN} Experiment Source Code into Single File ${C2}...${CN}" -# --------------------------------------------------------------------------- - tar cf ${NEWEXPID}.GEOSctm.tar GEOSctm -/bin/rm -r GEOSctm -echo "" + echo "Copying Build Source Code into ${C2}${NEWEXPDIR}/src${CN}" + # -------------------------------------------------------------- + if (-e ${GEOSDEF}/src/${TARFILE_NAME}) then + cp ${GEOSDEF}/src/${TARFILE_NAME} ${NEWEXPDIR}/src + else + echo "${GEOSDEF}/src/${TARFILE_NAME} not found yet CMake was asked to make and install a tarfile." + echo "Something went wrong." + exit 7 + endif + echo "" endif @@ -3176,8 +2948,8 @@ cat << EOF ctm_setup requires access to internet resources, namely the CVS repository. You are attempting to run ctm_setup -on compute node ${NODE} at ${SITE}. Please re-run on a head -node, or supply the --nocvs option. +on compute node ${NODE} at ${SITE}. Please re-run on a head +node, or supply the --nocvs option. For more information, contact Matt Thompson or Larry Takacs at GMAO. @@ -3190,9 +2962,9 @@ CONTACTMATT: cat < Date: Mon, 19 Aug 2024 15:52:29 -0400 Subject: [PATCH 2/3] Update to ESMA_env and ESMA_cmake tags --- CMakeLists.txt | 4 ++-- components.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1fc8ecc..6f00e68 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,7 +4,7 @@ cmake_policy (SET CMP0054 NEW) project ( GEOSctm - VERSION 2.0.1 + VERSION 2.2.3 LANGUAGES Fortran CXX C) # Note - CXX is required for ESMF if ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}") @@ -26,7 +26,7 @@ set (DOING_GEOS5 YES) # Should find a better place for this - used in Chem component set (ACG_FLAGS -v) -# This flag at R8 means that FV3 is compiled at R8 and *linked* to +# This flag at R8 means that FV3 is compiled at R8 and *linked* to # FMS built at R8. set (FV_PRECISION "R8" CACHE STRING "Precision of FV3 core (R4, R4R8, R8)") diff --git a/components.yaml b/components.yaml index 95d8bdd..10d8e60 100644 --- a/components.yaml +++ b/components.yaml @@ -5,13 +5,13 @@ GEOSctm: env: local: ./@env remote: ../ESMA_env.git - branch: feature/mathomp4/3.4-sles15 + tag: v3.4.2 develop: main cmake: local: ./@cmake remote: ../ESMA_cmake.git - branch: feature/mathomp4/3.6-sles15 + tag: v3.6.7 develop: develop ecbuild: From 49a82ad8c59515a81f5de47fa87988ed3e5561e6 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Tue, 20 Aug 2024 07:47:27 -0400 Subject: [PATCH 3/3] Update github actions --- .circleci/config.yml | 5 +---- .github/workflows/enforce-labels.yml | 8 +++++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2f118e8..59c9cf4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,6 @@ executors: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 OMPI_MCA_btl_vader_single_copy_mechanism: none resource_class: large - #MEDIUM# resource_class: medium ifort-large: docker: @@ -21,7 +20,6 @@ executors: username: $DOCKERHUB_USER password: $DOCKERHUB_AUTH_TOKEN resource_class: large - #MEDIUM# resource_class: medium workflows: build-test: @@ -31,7 +29,7 @@ workflows: matrix: parameters: compiler: [gfortran, ifort] - context: + context: - docker-hub-creds jobs: @@ -76,7 +74,6 @@ jobs: command: | cd ${CIRCLE_WORKING_DIRECTORY}/workspace/build-GEOSctm make -j"$(nproc)" install |& tee /logfiles/make.log - #MEDIUM# make -j4 install |& tee /logfiles/make.log - run: name: "Compress artifacts" command: | diff --git a/.github/workflows/enforce-labels.yml b/.github/workflows/enforce-labels.yml index 6e1720e..8855e54 100644 --- a/.github/workflows/enforce-labels.yml +++ b/.github/workflows/enforce-labels.yml @@ -8,18 +8,18 @@ jobs: require-label: runs-on: ubuntu-latest steps: - - uses: mheap/github-action-required-labels@v2 + - uses: mheap/github-action-required-labels@v5 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: mode: minimum count: 1 - labels: "0 diff,0 diff trivial,Non 0-diff,0 diff structural,0-diff trivial,Not 0-diff,0-diff,automatic,0-diff uncoupled" + labels: "0 diff,0 diff trivial,Non 0-diff,0 diff structural,0-diff trivial,Not 0-diff,0-diff,automatic,0-diff uncoupled,github_actions" add_comment: true blocking-label: runs-on: ubuntu-latest steps: - - uses: mheap/github-action-required-labels@v2 + - uses: mheap/github-action-required-labels@v5 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: @@ -27,3 +27,5 @@ jobs: count: 0 labels: "Contingent - DNA,Needs Lead Approval,Contingent -- Do Not Approve" add_comment: true + message: "This PR is being prevented from merging because you have added one of our blocking labels: {{ provided }}. You'll need to remove it before this PR can be merged." +