From e97d26d9dd29b4fc525e646d1664564dc5677830 Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO Date: Fri, 25 Oct 2024 07:05:37 -0400 Subject: [PATCH 01/16] Updates for setup_expt and setup_xml workflow on Gaea C5 and C6 --- env/{GAEA.env => GAEAC5.env} | 0 env/GAEAC6.env | 101 ++++++++++++++++++ ...e_base.gaea.lua => module_base.gaeac5.lua} | 2 +- modulefiles/module_base.gaeac6.lua | 48 +++++++++ ...tup.gaea.lua => module_gwsetup.gaeac5.lua} | 4 +- modulefiles/module_gwsetup.gaeac6.lua | 20 ++++ parm/config/gfs/config.aero | 7 +- parm/config/gfs/config.resources | 7 +- ...resources.GAEA => config.resources.GAEAC5} | 2 +- parm/config/gfs/config.resources.GAEAC6 | 9 ++ sorc/link_workflow.sh | 3 +- ush/detect_machine.sh | 16 ++- ush/load_fv3gfs_modules.sh | 2 +- versions/{build.gaea.ver => build.gaeac5.ver} | 0 versions/{run.gaea.ver => build.gaeac6.ver} | 3 - versions/run.gaeac5.ver | 8 ++ versions/run.gaeac6.ver | 8 ++ workflow/hosts.py | 8 +- workflow/hosts/{gaea.yaml => gaeac5.yaml} | 0 workflow/hosts/gaeac6.yaml | 28 +++++ 20 files changed, 256 insertions(+), 20 deletions(-) rename env/{GAEA.env => GAEAC5.env} (100%) create mode 100755 env/GAEAC6.env rename modulefiles/{module_base.gaea.lua => module_base.gaeac5.lua} (98%) create mode 100644 modulefiles/module_base.gaeac6.lua rename modulefiles/{module_gwsetup.gaea.lua => module_gwsetup.gaeac5.lua} (79%) create mode 100644 modulefiles/module_gwsetup.gaeac6.lua rename parm/config/gfs/{config.resources.GAEA => config.resources.GAEAC5} (94%) create mode 100644 parm/config/gfs/config.resources.GAEAC6 rename versions/{build.gaea.ver => build.gaeac5.ver} (100%) rename versions/{run.gaea.ver => build.gaeac6.ver} (91%) create mode 100644 versions/run.gaeac5.ver create mode 100644 versions/run.gaeac6.ver rename workflow/hosts/{gaea.yaml => gaeac5.yaml} (100%) create mode 100644 workflow/hosts/gaeac6.yaml diff --git a/env/GAEA.env b/env/GAEAC5.env similarity index 100% rename from env/GAEA.env rename to env/GAEAC5.env diff --git a/env/GAEAC6.env b/env/GAEAC6.env new file mode 100755 index 0000000000..7736e0f1ea --- /dev/null +++ b/env/GAEAC6.env @@ -0,0 +1,101 @@ +#! /usr/bin/env bash + +if [[ $# -ne 1 ]]; then + + echo "Must specify an input argument to set runtime environment variables!" + exit 1 + +fi + +step=$1 + +export launcher="srun -l --export=ALL" +export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" + +export OMP_STACKSIZE=2048000 +export NTHSTACK=1024000000 + +ulimit -s unlimited +ulimit -a + +# Calculate common variables +# Check first if the dependent variables are set +if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:-}" ]]; then + max_threads_per_task=$((max_tasks_per_node / tasks_per_node)) + NTHREADSmax=${threads_per_task:-${max_threads_per_task}} + NTHREADS1=${threads_per_task:-1} + [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task} + [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task} + # This may be useful when Gaea is fully ported, so ignore SC warning + # shellcheck disable=SC2034 + APRUN_default="${launcher} -n ${ntasks}" +else + echo "ERROR config.resources must be sourced before sourcing GAEA.env" + exit 2 +fi + +if [[ "${step}" = "prep" ]]; then + + export POE="NO" + export BACK="NO" + export sys_tp="GAEA" + export launcher_PREP="srun" + +elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then + + export MKL_NUM_THREADS=4 + export MKL_CBWR=AUTO + + export CFP_MP=${CFP_MP:-"YES"} + export USE_CFP=${USE_CFP:-"YES"} + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" + + export NTHREADS_GSI=${NTHREADSmax} + export APRUN_GSI="${APRUN_default} --cpus-per-task=${NTHREADS_GSI}" + + export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} + [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task} + export APRUN_CALCINC="${launcher} \$ncmd --cpus-per-task=${NTHREADS_CALCINC}" + + export NTHREADS_CYCLE=${threads_per_task_cycle:-12} + [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} + ntasks_cycle=${ntiles:-6} + export APRUN_CYCLE="${launcher} -n ${ntasks_cycle} --cpus-per-task=${NTHREADS_CYCLE}" + + export NTHREADS_GAUSFCANL=1 + ntasks_gausfcanl=${ntasks_gausfcanl:-1} + export APRUN_GAUSFCANL="${launcher} -n ${ntasks_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}" + +elif [[ "${step}" = "sfcanl" ]]; then + + export NTHREADS_CYCLE=${threads_per_task:-14} + export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}" + +elif [[ "${step}" = "fcst" ]]; then + + (( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node )) + (( ufs_ntasks = nnodes*tasks_per_node )) + # With ESMF threading, the model wants to use the full node + export APRUN_UFS="${launcher} -n ${ufs_ntasks}" + unset nnodes ufs_ntasks + +elif [[ "${step}" = "upp" ]]; then + + export NTHREADS_UPP=${NTHREADS1} + export APRUN_UPP="${APRUN_default} --cpus-per-task=${NTHREADS_UPP}" + +elif [[ "${step}" = "atmos_products" ]]; then + + export USE_CFP="YES" # Use MPMD for downstream product generation on Gaea + +elif [[ "${step}" = "oceanice_products" ]]; then + + export NTHREADS_OCNICEPOST=${NTHREADS1} + export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}" + +elif [[ "${step}" = "fit2obs" ]]; then + + export NTHREADS_FIT2OBS=${NTHREADS1} + export MPIRUN="${APRUN_default} --cpus-per-task=${NTHREADS_FIT2OBS}" + +fi diff --git a/modulefiles/module_base.gaea.lua b/modulefiles/module_base.gaeac5.lua similarity index 98% rename from modulefiles/module_base.gaea.lua rename to modulefiles/module_base.gaeac5.lua index f379225380..e7a05fd095 100644 --- a/modulefiles/module_base.gaea.lua +++ b/modulefiles/module_base.gaeac5.lua @@ -1,5 +1,5 @@ help([[ -Load environment to run GFS on Gaea +Load environment to run GFS on Gaea C5 ]]) local spack_mod_path=(os.getenv("spack_mod_path") or "None") diff --git a/modulefiles/module_base.gaeac6.lua b/modulefiles/module_base.gaeac6.lua new file mode 100644 index 0000000000..9e4f069243 --- /dev/null +++ b/modulefiles/module_base.gaeac6.lua @@ -0,0 +1,48 @@ +help([[ +Load environment to run GFS on Gaea C6 +]]) + +local spack_mod_path=(os.getenv("spack_mod_path") or "None") +prepend_path("MODULEPATH", spack_mod_path) + +load(pathJoin("stack-intel", (os.getenv("stack_intel_ver") or "None"))) +load(pathJoin("stack-cray-mpich", (os.getenv("stack_cray_mpich_ver") or "None"))) +load(pathJoin("python", (os.getenv("python_ver") or "None"))) + +load(pathJoin("jasper", (os.getenv("jasper_ver") or "None"))) +load(pathJoin("libpng", (os.getenv("libpng_ver") or "None"))) +load(pathJoin("cdo", (os.getenv("cdo_ver") or "None"))) +load(pathJoin("hdf5", (os.getenv("hdf5_ver") or "None"))) +load(pathJoin("netcdf-c", (os.getenv("netcdf_c_ver") or "None"))) +load(pathJoin("netcdf-fortran", (os.getenv("netcdf_fortran_ver") or "None"))) +load(pathJoin("perlbrew", (os.getenv("perl_ver") or "None"))) + +load(pathJoin("nco", (os.getenv("nco_ver") or "None"))) +load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None"))) +load(pathJoin("grib-util", (os.getenv("grib_util_ver") or "None"))) +load(pathJoin("g2tmpl", (os.getenv("g2tmpl_ver") or "None"))) +load(pathJoin("gsi-ncdiag", (os.getenv("gsi_ncdiag_ver") or "None"))) +load(pathJoin("crtm", (os.getenv("crtm_ver") or "None"))) +load(pathJoin("bufr", (os.getenv("bufr_ver") or "None"))) +load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None"))) +load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None"))) +load(pathJoin("py-f90nml", (os.getenv("py_f90nml_ver") or "None"))) +load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None"))) +load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None"))) +load(pathJoin("py-pandas", (os.getenv("py_pandas_ver") or "None"))) +load(pathJoin("py-python-dateutil", (os.getenv("py_python_dateutil_ver") or "None"))) +load(pathJoin("met", (os.getenv("met_ver") or "None"))) +load(pathJoin("metplus", (os.getenv("metplus_ver") or "None"))) +load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) + +setenv("WGRIB2","wgrib2") +setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) + +prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) +load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) + +prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) +load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) + + +whatis("Description: GFS run setup environment") diff --git a/modulefiles/module_gwsetup.gaea.lua b/modulefiles/module_gwsetup.gaeac5.lua similarity index 79% rename from modulefiles/module_gwsetup.gaea.lua rename to modulefiles/module_gwsetup.gaeac5.lua index 0bcc689bad..0c1691d30b 100644 --- a/modulefiles/module_gwsetup.gaea.lua +++ b/modulefiles/module_gwsetup.gaeac5.lua @@ -1,5 +1,5 @@ help([[ -Load environment to run GFS workflow setup scripts on Gaea +Load environment to run GFS workflow setup scripts on Gaea C5 ]]) prepend_path("MODULEPATH", "/ncrc/proj/epic/rocoto/modulefiles") @@ -7,7 +7,7 @@ load(pathJoin("rocoto")) prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") -local stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0" +local stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0" local python_ver=os.getenv("python_ver") or "3.10.13" load(pathJoin("stack-intel", stack_intel_ver)) diff --git a/modulefiles/module_gwsetup.gaeac6.lua b/modulefiles/module_gwsetup.gaeac6.lua new file mode 100644 index 0000000000..b8945e2802 --- /dev/null +++ b/modulefiles/module_gwsetup.gaeac6.lua @@ -0,0 +1,20 @@ +help([[ +Load environment to run GFS workflow setup scripts on Gaea C6 +]]) + +prepend_path("MODULEPATH", "/ncrc/proj/epic/rocoto/modulefiles") +load(pathJoin("rocoto")) + +prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") + +local stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0" +local python_ver=os.getenv("python_ver") or "3.10.13" + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("python", python_ver)) +load("py-jinja2") +load("py-pyyaml") +load("py-numpy") +load("git-lfs") + +whatis("Description: GFS run setup environment") diff --git a/parm/config/gfs/config.aero b/parm/config/gfs/config.aero index f49593a439..72d0e556e3 100644 --- a/parm/config/gfs/config.aero +++ b/parm/config/gfs/config.aero @@ -20,8 +20,11 @@ case ${machine} in "WCOSS2") AERO_INPUTS_DIR="/lfs/h2/emc/global/noscrub/emc.global/data/gocart_emissions" ;; - "GAEA") - AERO_INPUTS_DIR="/gpfs/f5/epic/proj-shared/global/glopara/data/gocart_emissions" + "GAEAC5") + AERO_INPUTS_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/gocart_emissions" + ;; + "GAEAC6") + AERO_INPUTS_DIR="/gpfs/f6/bil-fire8/world-shared/global/glopara/data/gocart_emissions" ;; "JET") AERO_INPUTS_DIR="/lfs5/HFIP/hfv3gfs/glopara/data/gocart_emissions" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 79dbb487db..0f82fd614d 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -47,11 +47,16 @@ case ${machine} in # shellcheck disable=SC2034 mem_node_max="96GB" ;; - "GAEA") + "GAEAC5") max_tasks_per_node=128 # shellcheck disable=SC2034 mem_node_max="251GB" ;; + "GAEAC6") + max_tasks_per_node=192 + # shellcheck disable=SC2034 + mem_node_max="384GB" + ;; "ORION") max_tasks_per_node=40 # shellcheck disable=SC2034 diff --git a/parm/config/gfs/config.resources.GAEA b/parm/config/gfs/config.resources.GAEAC5 similarity index 94% rename from parm/config/gfs/config.resources.GAEA rename to parm/config/gfs/config.resources.GAEAC5 index c50601da00..cb045fedae 100644 --- a/parm/config/gfs/config.resources.GAEA +++ b/parm/config/gfs/config.resources.GAEAC5 @@ -1,6 +1,6 @@ #! /usr/bin/env bash -# Gaea-specific job resources +# GaeaC5-specific job resources case ${step} in "prep") diff --git a/parm/config/gfs/config.resources.GAEAC6 b/parm/config/gfs/config.resources.GAEAC6 new file mode 100644 index 0000000000..7fc6388159 --- /dev/null +++ b/parm/config/gfs/config.resources.GAEAC6 @@ -0,0 +1,9 @@ +#! /usr/bin/env bash + +# GaeaC6-specific job resources + +unset memory +# shellcheck disable=SC2312 +for mem_var in $(env | grep '^memory_' | cut -d= -f1); do + unset "${mem_var}" +done diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 870ddc5eba..c76ea2c231 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -75,7 +75,8 @@ case "${machine}" in "hercules") FIX_DIR="/work/noaa/global/glopara/fix" ;; "jet") FIX_DIR="/lfs5/HFIP/hfv3gfs/glopara/FIX/fix" ;; "s4") FIX_DIR="/data/prod/glopara/fix" ;; - "gaea") FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix" ;; + "gaeac5") FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix" ;; + "gaeac6") FIX_DIR="/gpfs/f6/bil-fire8/world-shared/global/glopara/data/fix" ;; "noaacloud") FIX_DIR="/contrib/global-workflow-shared-data/fix" ;; *) echo "FATAL: Unknown target machine ${machine}, couldn't set FIX_DIR" diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index 8ad217140a..9301c8492f 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -21,8 +21,11 @@ case $(hostname -f) in dlogin0[1-9].dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dogwood01-9 dlogin10.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dogwood10 - gaea5[1-8]) MACHINE_ID=gaea ;; ### gaea51-58 - gaea5[1-8].ncrc.gov) MACHINE_ID=gaea ;; ### gaea51-58 + gaea5[1-8]) MACHINE_ID=gaeac5 ;; ### gaea51-58 + gaea5[1-8].ncrc.gov) MACHINE_ID=gaeac5 ;; ### gaea51-58 + + gaea6[1-8]) MACHINE_ID=gaeac6 ;; ### gaea61-68 + gaea6[1-8].ncrc.gov) MACHINE_ID=gaeac6 ;; ### gaea61-68 hfe0[1-9]) MACHINE_ID=hera ;; ### hera01-09 hfe1[0-2]) MACHINE_ID=hera ;; ### hera10-12 @@ -81,9 +84,12 @@ elif [[ -d /work ]]; then else MACHINE_ID=orion fi -elif [[ -d /gpfs && -d /ncrc ]]; then - # We are on GAEA. - MACHINE_ID=gaea +elif [[ -d /gpfs/f5 ]]; then + # We are on GAEAC5. + MACHINE_ID=gaeac5 +elif [[ -d /gpfs/f6 ]]; then + # We are on GAEAC6. + MACHINE_ID=gaeac6 elif [[ -d /data/prod ]]; then # We are on SSEC's S4 MACHINE_ID=s4 diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh index ff6f64cece..93098322c4 100755 --- a/ush/load_fv3gfs_modules.sh +++ b/ush/load_fv3gfs_modules.sh @@ -20,7 +20,7 @@ source "${HOMEgfs}/versions/run.ver" module use "${HOMEgfs}/modulefiles" case "${MACHINE_ID}" in - "wcoss2" | "hera" | "orion" | "hercules" | "gaea" | "jet" | "s4" | "noaacloud") + "wcoss2" | "hera" | "orion" | "hercules" | "gaeac5" | "gaeac6" | "jet" | "s4" | "noaacloud") module load "module_base.${MACHINE_ID}" ;; *) diff --git a/versions/build.gaea.ver b/versions/build.gaeac5.ver similarity index 100% rename from versions/build.gaea.ver rename to versions/build.gaeac5.ver diff --git a/versions/run.gaea.ver b/versions/build.gaeac6.ver similarity index 91% rename from versions/run.gaea.ver rename to versions/build.gaeac6.ver index 81aa70df57..8b6e641eb6 100644 --- a/versions/run.gaea.ver +++ b/versions/build.gaeac6.ver @@ -1,8 +1,5 @@ export stack_intel_ver=2023.1.0 export stack_cray_mpich_ver=8.1.25 export spack_env=gsi-addon-dev - -export perl_ver=5.38.2 - source "${HOMEgfs:-}/versions/spack.ver" export spack_mod_path="/ncrc/proj/epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/versions/run.gaeac5.ver b/versions/run.gaeac5.ver new file mode 100644 index 0000000000..9d4075f334 --- /dev/null +++ b/versions/run.gaeac5.ver @@ -0,0 +1,8 @@ +export stack_intel_ver=2023.2.0 +export stack_cray_mpich_ver=8.1.28 +export spack_env=gsi-addon-dev + +export perl_ver=5.38.2 + +source "${HOMEgfs:-}/versions/spack.ver" +export spack_mod_path="/ncrc/proj/epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/versions/run.gaeac6.ver b/versions/run.gaeac6.ver new file mode 100644 index 0000000000..9d4075f334 --- /dev/null +++ b/versions/run.gaeac6.ver @@ -0,0 +1,8 @@ +export stack_intel_ver=2023.2.0 +export stack_cray_mpich_ver=8.1.28 +export spack_env=gsi-addon-dev + +export perl_ver=5.38.2 + +source "${HOMEgfs:-}/versions/spack.ver" +export spack_mod_path="/ncrc/proj/epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/workflow/hosts.py b/workflow/hosts.py index 7bde58f95f..e8694333bf 100644 --- a/workflow/hosts.py +++ b/workflow/hosts.py @@ -16,8 +16,8 @@ class Host: """ SUPPORTED_HOSTS = ['HERA', 'ORION', 'JET', 'HERCULES', - 'WCOSS2', 'S4', 'CONTAINER', 'GAEA', - 'AWSPW', 'AZUREPW', 'GOOGLEPW'] + 'WCOSS2', 'S4', 'CONTAINER', 'GAEAC5', + 'GAEAC6','AWSPW', 'AZUREPW', 'GOOGLEPW'] def __init__(self, host=None): @@ -49,7 +49,9 @@ def detect(cls): elif os.path.exists('/data/prod'): machine = 'S4' elif os.path.exists('/gpfs/f5'): - machine = 'GAEA' + machine = 'GAEAC5' + elif os.path.exists('/gpfs/f6'): + machine = 'GAEAC6' elif container is not None: machine = 'CONTAINER' elif pw_csp is not None: diff --git a/workflow/hosts/gaea.yaml b/workflow/hosts/gaeac5.yaml similarity index 100% rename from workflow/hosts/gaea.yaml rename to workflow/hosts/gaeac5.yaml diff --git a/workflow/hosts/gaeac6.yaml b/workflow/hosts/gaeac6.yaml new file mode 100644 index 0000000000..f6919f2118 --- /dev/null +++ b/workflow/hosts/gaeac6.yaml @@ -0,0 +1,28 @@ +BASE_GIT: '/gpfs/f6/bil-fire8/world-shared/global/glopara/data/git' +DMPDIR: '/gpfs/f6/bil-fire8/world-shared/global/glopara/data/dump' +BASE_DATA: '/gpfs/f6/bil-fire8/world-shared/global/glopara/data' +BASE_IC: '/gpfs/f6/bil-fire8/world-shared/global/glopara/data/ICSDIR' +PACKAGEROOT: '/gpfs/f6/bil-fire8/world-shared/global/glopara/data/nwpara' +COMROOT: '/gpfs/f6/bil-fire8/world-shared/global/glopara/data/com' +COMINsyn: '${COMROOT}/gfs/prod/syndat' +HOMEDIR: '/gpfs/f6/bil-fire8/scratch/${USER}' +STMP: '/gpfs/f6/bil-fire8/scratch/${USER}' +PTMP: '/gpfs/f6/bil-fire8/scratch/${USER}' +NOSCRUB: $HOMEDIR +ACCOUNT: bil-fire8 +SCHEDULER: slurm +QUEUE: normal +QUEUE_SERVICE: normal +PARTITION_BATCH: batch +PARTITION_SERVICE: batch +RESERVATION: '' +CLUSTERS: 'c6' +CHGRP_RSTPROD: 'NO' +CHGRP_CMD: 'chgrp rstprod' +HPSSARCH: 'NO' +HPSS_PROJECT: emc-global +LOCALARCH: 'NO' +ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' +MAKE_NSSTBUFR: 'NO' +MAKE_ACFTBUFR: 'NO' +SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] From d7e23989e5e8ad47f47c1f7a98c598fef4acff81 Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO Date: Thu, 14 Nov 2024 16:13:57 -0500 Subject: [PATCH 02/16] update submodules and some files to run on C5/C6 --- env/GAEAC5.env | 7 +++++++ parm/config/gfs/yaml/defaults.yaml | 8 ++++---- sorc/gdas.cd | 2 +- sorc/gfs_utils.fd | 2 +- sorc/gsi_enkf.fd | 2 +- sorc/gsi_monitor.fd | 2 +- sorc/gsi_utils.fd | 2 +- sorc/ufs_model.fd | 2 +- sorc/ufs_utils.fd | 2 +- 9 files changed, 18 insertions(+), 11 deletions(-) diff --git a/env/GAEAC5.env b/env/GAEAC5.env index 7736e0f1ea..91935d5812 100755 --- a/env/GAEAC5.env +++ b/env/GAEAC5.env @@ -98,4 +98,11 @@ elif [[ "${step}" = "fit2obs" ]]; then export NTHREADS_FIT2OBS=${NTHREADS1} export MPIRUN="${APRUN_default} --cpus-per-task=${NTHREADS_FIT2OBS}" +elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then + + export CFP_MP="YES" + if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi + export wavempexec=${launcher} + export wave_mpmd=${mpmd_opt} + fi diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml index dfc67d1237..b2eece3663 100644 --- a/parm/config/gfs/yaml/defaults.yaml +++ b/parm/config/gfs/yaml/defaults.yaml @@ -10,11 +10,11 @@ base: DO_GEMPAK: "NO" DO_AWIPS: "NO" DO_NPOESS: "NO" - DO_TRACKER: "YES" - DO_GENESIS: "YES" + DO_TRACKER: "NO" + DO_GENESIS: "NO" DO_GENESIS_FSU: "NO" - DO_METP: "YES" - FHMAX_GFS: 120 + DO_METP: "NO" + FHMAX_GFS: 42 FHMAX_HF_GFS: 0 FCST_BREAKPOINTS: "" DO_VRFY_OCEANDA: "NO" diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 764f58cebd..dfc871f38f 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 764f58cebdf64f3695d89538994a50183e5884d9 +Subproject commit dfc871f38fa47c525953af891eafbd237c3b33a8 diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd index 856a42076a..22ae6c315e 160000 --- a/sorc/gfs_utils.fd +++ b/sorc/gfs_utils.fd @@ -1 +1 @@ -Subproject commit 856a42076a65256aaae9b29f4891532cb4a3fbca +Subproject commit 22ae6c315e14390e1214fd058e26a6de2d2066ba diff --git a/sorc/gsi_enkf.fd b/sorc/gsi_enkf.fd index 9f44c8798c..2136b872cc 160000 --- a/sorc/gsi_enkf.fd +++ b/sorc/gsi_enkf.fd @@ -1 +1 @@ -Subproject commit 9f44c8798c2087aca06df8f629699632e57df431 +Subproject commit 2136b872cc44ccebc92b2f29b7ee45f43b12cafa diff --git a/sorc/gsi_monitor.fd b/sorc/gsi_monitor.fd index 278ee629e8..879f5900d9 160000 --- a/sorc/gsi_monitor.fd +++ b/sorc/gsi_monitor.fd @@ -1 +1 @@ -Subproject commit 278ee629e87558822e8d13b3fb3b0e16006aa856 +Subproject commit 879f5900d9236e144a18cba8e804618fa6c449e3 diff --git a/sorc/gsi_utils.fd b/sorc/gsi_utils.fd index a6ea311e5c..03d4b0ae9e 160000 --- a/sorc/gsi_utils.fd +++ b/sorc/gsi_utils.fd @@ -1 +1 @@ -Subproject commit a6ea311e5c82369d255e3afdc99c1bce0c9a3014 +Subproject commit 03d4b0ae9ec9ff52fe3dca10b0acf498fcf699ab diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index 6a4e09e947..ece273d22d 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit 6a4e09e94773ffa39ce7ab6a54a885efada91f21 +Subproject commit ece273d22de5275554a8d541c9901c77e803383b diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd index 06eec5b6f6..82f998cb5d 160000 --- a/sorc/ufs_utils.fd +++ b/sorc/ufs_utils.fd @@ -1 +1 @@ -Subproject commit 06eec5b6f636123835e2dfd9fc5229980c006735 +Subproject commit 82f998cb5d3daab6bc5af6a0e28b254dac3e2ab6 From 3dbc6258707099e689b3ac12f7f07d95a16e9077 Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO Date: Fri, 15 Nov 2024 08:30:59 -0500 Subject: [PATCH 03/16] modulefile updates for C6 --- env/GAEAC6.env | 7 +++++++ modulefiles/module_base.gaeac6.lua | 9 ++++----- sorc/link_workflow.sh | 2 +- versions/run.gaeac6.ver | 6 +++--- workflow/hosts/gaeac6.yaml | 12 ++++++------ 5 files changed, 21 insertions(+), 15 deletions(-) diff --git a/env/GAEAC6.env b/env/GAEAC6.env index 7736e0f1ea..91935d5812 100755 --- a/env/GAEAC6.env +++ b/env/GAEAC6.env @@ -98,4 +98,11 @@ elif [[ "${step}" = "fit2obs" ]]; then export NTHREADS_FIT2OBS=${NTHREADS1} export MPIRUN="${APRUN_default} --cpus-per-task=${NTHREADS_FIT2OBS}" +elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then + + export CFP_MP="YES" + if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi + export wavempexec=${launcher} + export wave_mpmd=${mpmd_opt} + fi diff --git a/modulefiles/module_base.gaeac6.lua b/modulefiles/module_base.gaeac6.lua index 9e4f069243..02823345bb 100644 --- a/modulefiles/module_base.gaeac6.lua +++ b/modulefiles/module_base.gaeac6.lua @@ -38,11 +38,10 @@ load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) setenv("WGRIB2","wgrib2") setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) -prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) -load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) - -prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) -load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) +--prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) +--load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) +--prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) +--load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) whatis("Description: GFS run setup environment") diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index f9404fb690..1a657bcc1f 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -76,7 +76,7 @@ case "${machine}" in "jet") FIX_DIR="/lfs5/HFIP/hfv3gfs/glopara/FIX/fix" ;; "s4") FIX_DIR="/data/prod/glopara/fix" ;; "gaeac5") FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix" ;; - "gaeac6") FIX_DIR="/gpfs/f6/bil-fire8/world-shared/global/glopara/data/fix" ;; + "gaeac6") FIX_DIR="/gpfs/f6/bil-fire8/world-shared/global/glopara/fix" ;; "noaacloud") FIX_DIR="/contrib/global-workflow-shared-data/fix" ;; *) echo "FATAL: Unknown target machine ${machine}, couldn't set FIX_DIR" diff --git a/versions/run.gaeac6.ver b/versions/run.gaeac6.ver index 9d4075f334..4162cc63e3 100644 --- a/versions/run.gaeac6.ver +++ b/versions/run.gaeac6.ver @@ -1,8 +1,8 @@ export stack_intel_ver=2023.2.0 -export stack_cray_mpich_ver=8.1.28 -export spack_env=gsi-addon-dev +export stack_cray_mpich_ver=8.1.29 +export spack_env=gsi-addon export perl_ver=5.38.2 source "${HOMEgfs:-}/versions/spack.ver" -export spack_mod_path="/ncrc/proj/epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" +export spack_mod_path="/ncrc/proj/epic/spack-stack/c6/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/workflow/hosts/gaeac6.yaml b/workflow/hosts/gaeac6.yaml index f6919f2118..44e5788343 100644 --- a/workflow/hosts/gaeac6.yaml +++ b/workflow/hosts/gaeac6.yaml @@ -1,9 +1,9 @@ -BASE_GIT: '/gpfs/f6/bil-fire8/world-shared/global/glopara/data/git' -DMPDIR: '/gpfs/f6/bil-fire8/world-shared/global/glopara/data/dump' -BASE_DATA: '/gpfs/f6/bil-fire8/world-shared/global/glopara/data' -BASE_IC: '/gpfs/f6/bil-fire8/world-shared/global/glopara/data/ICSDIR' -PACKAGEROOT: '/gpfs/f6/bil-fire8/world-shared/global/glopara/data/nwpara' -COMROOT: '/gpfs/f6/bil-fire8/world-shared/global/glopara/data/com' +BASE_GIT: '/gpfs/f6/bil-fire8/world-shared/global/glopara/git' +DMPDIR: '/gpfs/f6/bil-fire8/world-shared/global/glopara/dump' +BASE_DATA: '/gpfs/f6/bil-fire8/world-shared/global/glopara' +BASE_IC: '/gpfs/f6/bil-fire8/world-shared/global/glopara/ICSDIR' +PACKAGEROOT: '/gpfs/f6/bil-fire8/world-shared/global/glopara/nwpara' +COMROOT: '/gpfs/f6/bil-fire8/world-shared/global/glopara/com' COMINsyn: '${COMROOT}/gfs/prod/syndat' HOMEDIR: '/gpfs/f6/bil-fire8/scratch/${USER}' STMP: '/gpfs/f6/bil-fire8/scratch/${USER}' From 296b5c36ace754666a068f98ec84356056c9ffd1 Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO Date: Fri, 15 Nov 2024 09:35:57 -0500 Subject: [PATCH 04/16] module file updates --- env/GAEAC5.env | 8 ++++---- env/GAEAC6.env | 8 ++++---- modulefiles/module_base.gaeac6.lua | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/env/GAEAC5.env b/env/GAEAC5.env index 91935d5812..4fdb6d3546 100755 --- a/env/GAEAC5.env +++ b/env/GAEAC5.env @@ -26,11 +26,11 @@ if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:- NTHREADS1=${threads_per_task:-1} [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task} [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task} - # This may be useful when Gaea is fully ported, so ignore SC warning + # This may be useful when GaeaC5 is fully ported, so ignore SC warning # shellcheck disable=SC2034 APRUN_default="${launcher} -n ${ntasks}" else - echo "ERROR config.resources must be sourced before sourcing GAEA.env" + echo "ERROR config.resources must be sourced before sourcing GAEAC5.env" exit 2 fi @@ -38,7 +38,7 @@ if [[ "${step}" = "prep" ]]; then export POE="NO" export BACK="NO" - export sys_tp="GAEA" + export sys_tp="GAEAC5" export launcher_PREP="srun" elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then @@ -86,7 +86,7 @@ elif [[ "${step}" = "upp" ]]; then elif [[ "${step}" = "atmos_products" ]]; then - export USE_CFP="YES" # Use MPMD for downstream product generation on Gaea + export USE_CFP="YES" # Use MPMD for downstream product generation on GaeaC5 elif [[ "${step}" = "oceanice_products" ]]; then diff --git a/env/GAEAC6.env b/env/GAEAC6.env index 91935d5812..09bbd6530b 100755 --- a/env/GAEAC6.env +++ b/env/GAEAC6.env @@ -26,11 +26,11 @@ if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:- NTHREADS1=${threads_per_task:-1} [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task} [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task} - # This may be useful when Gaea is fully ported, so ignore SC warning + # This may be useful when GaeaC6 is fully ported, so ignore SC warning # shellcheck disable=SC2034 APRUN_default="${launcher} -n ${ntasks}" else - echo "ERROR config.resources must be sourced before sourcing GAEA.env" + echo "ERROR config.resources must be sourced before sourcing GAEAC6.env" exit 2 fi @@ -38,7 +38,7 @@ if [[ "${step}" = "prep" ]]; then export POE="NO" export BACK="NO" - export sys_tp="GAEA" + export sys_tp="GAEAC6" export launcher_PREP="srun" elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then @@ -86,7 +86,7 @@ elif [[ "${step}" = "upp" ]]; then elif [[ "${step}" = "atmos_products" ]]; then - export USE_CFP="YES" # Use MPMD for downstream product generation on Gaea + export USE_CFP="YES" # Use MPMD for downstream product generation on GaeaC6 elif [[ "${step}" = "oceanice_products" ]]; then diff --git a/modulefiles/module_base.gaeac6.lua b/modulefiles/module_base.gaeac6.lua index 02823345bb..741b0d93ad 100644 --- a/modulefiles/module_base.gaeac6.lua +++ b/modulefiles/module_base.gaeac6.lua @@ -38,10 +38,10 @@ load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) setenv("WGRIB2","wgrib2") setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) ---prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) ---load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) +prepend_path("MODULEPATH", pathJoin("/gpfs/f6/bil-fire8/world-shared/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) +load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) ---prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) +--prepend_path("MODULEPATH", pathJoin("/gpfs/f6/bil-fire8/world-shared/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) --load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) whatis("Description: GFS run setup environment") From 16d1ff4a2b1a2aaabfb8b73ac832290fe11b122d Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO Date: Fri, 15 Nov 2024 11:19:29 -0500 Subject: [PATCH 05/16] update C5 resources --- parm/config/gfs/config.resources.GAEAC5 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/parm/config/gfs/config.resources.GAEAC5 b/parm/config/gfs/config.resources.GAEAC5 index cb045fedae..fe3bc96153 100644 --- a/parm/config/gfs/config.resources.GAEAC5 +++ b/parm/config/gfs/config.resources.GAEAC5 @@ -26,6 +26,8 @@ case ${step} in esac +export FI_VERBS_PREFER_XRC=0 + unset memory # shellcheck disable=SC2312 for mem_var in $(env | grep '^memory_' | cut -d= -f1); do From 5973d8c83f7346a9006fff733165cbf7d2499d1d Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO Date: Fri, 15 Nov 2024 13:50:35 -0500 Subject: [PATCH 06/16] enable C5 and C6 build_all and load modules --- sorc/build_all.sh | 2 +- ush/load_ufsda_modules.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sorc/build_all.sh b/sorc/build_all.sh index e75c853c39..e8a3fe482d 100755 --- a/sorc/build_all.sh +++ b/sorc/build_all.sh @@ -149,7 +149,7 @@ build_opts["ww3prepost"]="${_wave_opt} ${_verbose_opt} ${_build_ufs_opt} ${_buil # Optional DA builds if [[ "${_build_ufsda}" == "YES" ]]; then - if [[ "${MACHINE_ID}" != "orion" && "${MACHINE_ID}" != "hera" && "${MACHINE_ID}" != "hercules" && "${MACHINE_ID}" != "wcoss2" && "${MACHINE_ID}" != "noaacloud" && "${MACHINE_ID}" != "gaea" ]]; then + if [[ "${MACHINE_ID}" != "orion" && "${MACHINE_ID}" != "hera" && "${MACHINE_ID}" != "hercules" && "${MACHINE_ID}" != "wcoss2" && "${MACHINE_ID}" != "noaacloud" && "${MACHINE_ID}" != "gaeac5" && "${MACHINE_ID}" != "gaeac6" ]]; then echo "NOTE: The GDAS App is not supported on ${MACHINE_ID}. Disabling build." else build_jobs["gdas"]=8 diff --git a/ush/load_ufsda_modules.sh b/ush/load_ufsda_modules.sh index 8117d3f359..017ff31d68 100755 --- a/ush/load_ufsda_modules.sh +++ b/ush/load_ufsda_modules.sh @@ -34,13 +34,13 @@ source "${HOMEgfs}/ush/module-setup.sh" module use "${HOMEgfs}/sorc/gdas.cd/modulefiles" case "${MACHINE_ID}" in - ("hera" | "orion" | "hercules" | "wcoss2") + ("hera" | "orion" | "hercules" | "gaeac5" | "gaeac6" | "wcoss2") module load "${MODS}/${MACHINE_ID}" ncdump=$( command -v ncdump ) NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) export NETCDF ;; - ("jet" | "gaea" | "s4" | "acorn") + ("jet" | "s4" | "acorn") echo WARNING: UFSDA NOT SUPPORTED ON THIS PLATFORM ;; *) From 3c946e348e4e25f48b159410c76d23f08a21f939 Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO Date: Fri, 15 Nov 2024 14:03:18 -0500 Subject: [PATCH 07/16] update ufs_utils hash to develop --- sorc/ufs_utils.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd index 82f998cb5d..5b2bfa39a5 160000 --- a/sorc/ufs_utils.fd +++ b/sorc/ufs_utils.fd @@ -1 +1 @@ -Subproject commit 82f998cb5d3daab6bc5af6a0e28b254dac3e2ab6 +Subproject commit 5b2bfa39a5b1fbe6bb1d247102679885685c9336 From fc56219a7e967fa0e9728f71cf5c77e10c7bc826 Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO Date: Fri, 15 Nov 2024 14:53:07 -0500 Subject: [PATCH 08/16] remove data in path for C5 fix files to be consistent with other systems --- sorc/link_workflow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 15df7ce83c..ae9c7ec7d9 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -75,7 +75,7 @@ case "${machine}" in "hercules") FIX_DIR="/work/noaa/global/glopara/fix" ;; "jet") FIX_DIR="/lfs5/HFIP/hfv3gfs/glopara/FIX/fix" ;; "s4") FIX_DIR="/data/prod/glopara/fix" ;; - "gaeac5") FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix" ;; + "gaeac5") FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/fix" ;; "gaeac6") FIX_DIR="/gpfs/f6/bil-fire8/world-shared/global/glopara/fix" ;; "noaacloud") FIX_DIR="/contrib/global-workflow-shared-data/fix" ;; *) From 1843bebfd95f72ededb2d2ba4a5d3ec42edaaf14 Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO Date: Tue, 17 Dec 2024 15:30:08 -0500 Subject: [PATCH 09/16] adjust C6 directory paths --- sorc/link_workflow.sh | 3 ++- sorc/ufs_model.fd | 2 +- workflow/hosts/gaeac6.yaml | 3 +-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index a89f070d41..9115c70757 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -76,7 +76,8 @@ case "${machine}" in "hercules") FIX_DIR="/work/noaa/global/glopara/fix" ;; "jet") FIX_DIR="/lfs5/HFIP/hfv3gfs/glopara/FIX/fix" ;; "s4") FIX_DIR="/data/prod/glopara/fix" ;; -"gaea") FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix" ;; +"gaeac5") FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix" ;; +"gaeac6") FIX_DIR="/gpfs/f6/bil-fire8/world-shared/global/glopara/fix" ;; "noaacloud") FIX_DIR="/contrib/global-workflow-shared-data/fix" ;; *) echo "FATAL: Unknown target machine ${machine}, couldn't set FIX_DIR" diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index 63ace62a36..b1937471fe 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit 63ace62a36a263f03b914a92fc5536509e862dbc +Subproject commit b1937471fe0ef4580b44cd3a01d2754110ec8c7d diff --git a/workflow/hosts/gaeac6.yaml b/workflow/hosts/gaeac6.yaml index 44e5788343..512bde15b4 100644 --- a/workflow/hosts/gaeac6.yaml +++ b/workflow/hosts/gaeac6.yaml @@ -1,9 +1,8 @@ BASE_GIT: '/gpfs/f6/bil-fire8/world-shared/global/glopara/git' DMPDIR: '/gpfs/f6/bil-fire8/world-shared/global/glopara/dump' BASE_DATA: '/gpfs/f6/bil-fire8/world-shared/global/glopara' -BASE_IC: '/gpfs/f6/bil-fire8/world-shared/global/glopara/ICSDIR' +BASE_IC: '/gpfs/f6/bil-fire8/world-shared/global/glopara/data/ICSDIR' PACKAGEROOT: '/gpfs/f6/bil-fire8/world-shared/global/glopara/nwpara' -COMROOT: '/gpfs/f6/bil-fire8/world-shared/global/glopara/com' COMINsyn: '${COMROOT}/gfs/prod/syndat' HOMEDIR: '/gpfs/f6/bil-fire8/scratch/${USER}' STMP: '/gpfs/f6/bil-fire8/scratch/${USER}' From c96c16a433be9eb2744c143dee503b08ce0b80e2 Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO Date: Wed, 18 Dec 2024 08:38:10 -0500 Subject: [PATCH 10/16] add space in hosts.py --- workflow/hosts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/hosts.py b/workflow/hosts.py index e8694333bf..37a7e53066 100644 --- a/workflow/hosts.py +++ b/workflow/hosts.py @@ -17,7 +17,7 @@ class Host: SUPPORTED_HOSTS = ['HERA', 'ORION', 'JET', 'HERCULES', 'WCOSS2', 'S4', 'CONTAINER', 'GAEAC5', - 'GAEAC6','AWSPW', 'AZUREPW', 'GOOGLEPW'] + 'GAEAC6', 'AWSPW', 'AZUREPW', 'GOOGLEPW'] def __init__(self, host=None): From 6d0c7c9ad299d4522ba5cdd7d42f39cd7287444d Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO Date: Wed, 18 Dec 2024 08:50:26 -0500 Subject: [PATCH 11/16] syndat path updates --- modulefiles/module_gwsetup.gaeac6.lua | 2 +- workflow/hosts/gaeac5.yaml | 4 ++-- workflow/hosts/gaeac6.yaml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modulefiles/module_gwsetup.gaeac6.lua b/modulefiles/module_gwsetup.gaeac6.lua index b8945e2802..1623e6f1a2 100644 --- a/modulefiles/module_gwsetup.gaeac6.lua +++ b/modulefiles/module_gwsetup.gaeac6.lua @@ -5,7 +5,7 @@ Load environment to run GFS workflow setup scripts on Gaea C6 prepend_path("MODULEPATH", "/ncrc/proj/epic/rocoto/modulefiles") load(pathJoin("rocoto")) -prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") local stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0" local python_ver=os.getenv("python_ver") or "3.10.13" diff --git a/workflow/hosts/gaeac5.yaml b/workflow/hosts/gaeac5.yaml index 23ac75ca03..cdcba537a2 100644 --- a/workflow/hosts/gaeac5.yaml +++ b/workflow/hosts/gaeac5.yaml @@ -4,7 +4,7 @@ BASE_DATA: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data' BASE_IC: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/ICSDIR' PACKAGEROOT: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/nwpara' COMROOT: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/com' -COMINsyn: '${COMROOT}/gfs/prod/syndat' +COMINsyn: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/com/gfs/prod/syndat' HOMEDIR: '/gpfs/f5/ufs-ard/scratch/${USER}' STMP: '/gpfs/f5/ufs-ard/scratch/${USER}' PTMP: '/gpfs/f5/ufs-ard/scratch/${USER}' @@ -25,5 +25,5 @@ LOCALARCH: 'NO' ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' -SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] +SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48'] AERO_INPUTS_DIR: /gpfs/f5/epic/proj-shared/global/glopara/data/gocart_emissions diff --git a/workflow/hosts/gaeac6.yaml b/workflow/hosts/gaeac6.yaml index 512bde15b4..fed31634bf 100644 --- a/workflow/hosts/gaeac6.yaml +++ b/workflow/hosts/gaeac6.yaml @@ -3,7 +3,7 @@ DMPDIR: '/gpfs/f6/bil-fire8/world-shared/global/glopara/dump' BASE_DATA: '/gpfs/f6/bil-fire8/world-shared/global/glopara' BASE_IC: '/gpfs/f6/bil-fire8/world-shared/global/glopara/data/ICSDIR' PACKAGEROOT: '/gpfs/f6/bil-fire8/world-shared/global/glopara/nwpara' -COMINsyn: '${COMROOT}/gfs/prod/syndat' +COMINsyn: '/gpfs/f6/bil-fire8/world-shared/global/glopara/com/syndat' HOMEDIR: '/gpfs/f6/bil-fire8/scratch/${USER}' STMP: '/gpfs/f6/bil-fire8/scratch/${USER}' PTMP: '/gpfs/f6/bil-fire8/scratch/${USER}' @@ -24,4 +24,4 @@ LOCALARCH: 'NO' ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' -SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] +SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48'] From 00241824cccb9e6d33815ff3e8dd004b429d4a4e Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO Date: Wed, 18 Dec 2024 09:00:10 -0500 Subject: [PATCH 12/16] update fit2obs path --- modulefiles/module_base.gaeac6.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modulefiles/module_base.gaeac6.lua b/modulefiles/module_base.gaeac6.lua index 741b0d93ad..e0388ac84b 100644 --- a/modulefiles/module_base.gaeac6.lua +++ b/modulefiles/module_base.gaeac6.lua @@ -41,7 +41,7 @@ setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) prepend_path("MODULEPATH", pathJoin("/gpfs/f6/bil-fire8/world-shared/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) ---prepend_path("MODULEPATH", pathJoin("/gpfs/f6/bil-fire8/world-shared/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) ---load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) +prepend_path("MODULEPATH", pathJoin("/gpfs/f6/bil-fire8/world-shared/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) +load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) whatis("Description: GFS run setup environment") From 1f89d7edda0caedfef47cf4122c3f29884f6835e Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO Date: Wed, 18 Dec 2024 09:36:18 -0500 Subject: [PATCH 13/16] fix aero input path --- workflow/hosts/gaeac5.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/hosts/gaeac5.yaml b/workflow/hosts/gaeac5.yaml index cdcba537a2..d43cf98d4d 100644 --- a/workflow/hosts/gaeac5.yaml +++ b/workflow/hosts/gaeac5.yaml @@ -26,4 +26,4 @@ ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48'] -AERO_INPUTS_DIR: /gpfs/f5/epic/proj-shared/global/glopara/data/gocart_emissions +AERO_INPUTS_DIR: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/gocart_emissions' From 438a06c7a6edc2c6ffb7b3c832bac38197bf9116 Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO Date: Wed, 18 Dec 2024 09:47:55 -0500 Subject: [PATCH 14/16] revert defaults.yaml --- parm/config/gfs/yaml/defaults.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml index 8c9382effa..55f4b03f50 100644 --- a/parm/config/gfs/yaml/defaults.yaml +++ b/parm/config/gfs/yaml/defaults.yaml @@ -11,11 +11,11 @@ base: DO_GEMPAK: "NO" DO_AWIPS: "NO" DO_NPOESS: "NO" - DO_TRACKER: "NO" - DO_GENESIS: "NO" + DO_TRACKER: "YES" + DO_GENESIS: "YES" DO_GENESIS_FSU: "NO" - DO_METP: "NO" - FHMAX_GFS: 42 + DO_METP: "YES" + FHMAX_GFS: 120 FHMAX_HF_GFS: 0 FCST_BREAKPOINTS: "" DO_VRFY_OCEANDA: "NO" From 8553d5428717cf266ead03e3b8e6e7cf8b81349f Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO Date: Wed, 18 Dec 2024 12:44:54 -0500 Subject: [PATCH 15/16] point ufs_model to develop --- sorc/ufs_model.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index b1937471fe..e119370476 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit b1937471fe0ef4580b44cd3a01d2754110ec8c7d +Subproject commit e1193704767800bfaece56eb2a4b058bd4d0afbc From 9ad705e3bdd62e5d8b58f27567ff44b178f2fab8 Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO Date: Mon, 30 Dec 2024 16:02:48 -0500 Subject: [PATCH 16/16] remove COMROOT from gaeac5.yaml --- workflow/hosts/gaeac5.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/workflow/hosts/gaeac5.yaml b/workflow/hosts/gaeac5.yaml index d43cf98d4d..8131a8642b 100644 --- a/workflow/hosts/gaeac5.yaml +++ b/workflow/hosts/gaeac5.yaml @@ -3,7 +3,6 @@ DMPDIR: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/dump' BASE_DATA: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data' BASE_IC: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/ICSDIR' PACKAGEROOT: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/nwpara' -COMROOT: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/com' COMINsyn: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/com/gfs/prod/syndat' HOMEDIR: '/gpfs/f5/ufs-ard/scratch/${USER}' STMP: '/gpfs/f5/ufs-ard/scratch/${USER}'