From 87db08627a060757e0d5c5f94ef58ba86a77fcae Mon Sep 17 00:00:00 2001 From: "catherine.thomas" Date: Thu, 19 Dec 2024 09:31:25 -0500 Subject: [PATCH 1/3] Updates for cycling C1152 ATM A handful of updates are needed to cycle with C1152 atmosphere, which are mostly related to configs and resources. Refs: #3173 --- parm/config/gfs/config.resources | 31 ++++++++++++++----------- parm/config/gfs/config.resources.GAEA | 2 +- parm/config/gfs/config.resources.HERA | 4 ++-- parm/config/gfs/config.resources.JET | 2 +- parm/config/gfs/config.resources.WCOSS2 | 6 ++--- parm/config/gfs/config.ufs | 2 +- 6 files changed, 26 insertions(+), 21 deletions(-) diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index e642082290..d34236b1eb 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -324,7 +324,7 @@ case ${step} in "snowanl") # below lines are for creating JEDI YAML case ${CASE} in - "C768") + "C1152" | "C768") layout_x=6 layout_y=6 ;; @@ -353,7 +353,7 @@ case ${step} in "esnowrecen") # below lines are for creating JEDI YAML case ${CASE} in - "C768") + "C1152" | "C768") layout_x=6 layout_y=6 ;; @@ -390,7 +390,7 @@ case ${step} in "aeroanlinit") # below lines are for creating JEDI YAML case ${CASE} in - "C768") + "C1152" | "C768") layout_x=8 layout_y=8 ;; @@ -423,7 +423,7 @@ case ${step} in "aeroanlvar") case ${CASE} in - "C768") + "C1152" | "C768") layout_x=8 layout_y=8 ;; @@ -457,7 +457,7 @@ case ${step} in "aeroanlgenb") case ${CASE} in - "C768") + "C1152" | "C768") layout_x=8 layout_y=8 ;; @@ -668,7 +668,7 @@ case ${step} in walltime_gdas="01:20:00" walltime_gfs="01:00:00" case ${CASE} in - "C768") + "C1152" | "C768") ntasks_gdas=780 ntasks_gfs=825 threads_per_task=5 @@ -704,7 +704,7 @@ case ${step} in export threads_per_task_echgres_gfs=12 export is_exclusive=True memory="48GB" - if [[ "${CASE}" == "C384" || "${CASE}" == "C768" ]]; then + if [[ "${CASE}" == "C384" || "${CASE}" == "C768" || "${CASE}" == "C1152" ]]; then memory="${mem_node_max}" fi ;; @@ -897,7 +897,7 @@ case ${step} in ;; "C768" | "C1152") # Not valid resolutions for ensembles - declare -x "walltime_gdas"="00:40:00" + declare -x "walltime_gdas"="00:50:00" declare -x "walltime_gfs"="06:00:00" ;; *) @@ -923,16 +923,21 @@ case ${step} in "C48" | "C96") ntasks=${CASE:1} ;; - "C192" | "C384" | "C768" ) + "C192" | "C384" | "C768") ntasks=120 memory="${mem_node_max}" ;; + "C1152") + ntasks=200 + memory="${mem_node_max}" + ;; *) echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}" exit 4 ;; esac tasks_per_node=${ntasks} + [[ ${CASE} == "C1152" ]] && tasks_per_node=40 threads_per_task=1 @@ -1005,7 +1010,7 @@ case ${step} in threads_per_task=1 tasks_per_node=1 memory="20G" - [[ ${CASE} == "C768" ]] && memory="80GB" + [[ ${CASE} == "C768" || ${CASE} == "C1152" ]] && memory="80GB" ;; "metp") @@ -1145,7 +1150,7 @@ case ${step} in fi case ${CASE} in - "C768") ntasks=200;; + "C1152" | "C768") ntasks=200;; "C384") ntasks=100;; "C192" | "C96" | "C48") ntasks=40;; *) @@ -1178,7 +1183,7 @@ case ${step} in "eupd") walltime="00:30:00" case ${CASE} in - "C768") + "C1152" | "C768") ntasks=480 threads_per_task=6 ;; @@ -1223,7 +1228,7 @@ case ${step} in "epos") walltime="00:15:00" - [[ ${CASE} == "C768" ]] && walltime="00:25:00" + [[ ${CASE} == "C768" || ${CASE} == "C1152" ]] && walltime="00:25:00" ntasks=80 threads_per_task=1 tasks_per_node=$(( max_tasks_per_node / threads_per_task )) diff --git a/parm/config/gfs/config.resources.GAEA b/parm/config/gfs/config.resources.GAEA index c50601da00..aa353b9302 100644 --- a/parm/config/gfs/config.resources.GAEA +++ b/parm/config/gfs/config.resources.GAEA @@ -12,7 +12,7 @@ case ${step} in # The number of tasks and cores used must be the same for eobs # See https://github.com/NOAA-EMC/global-workflow/issues/2092 for details case ${CASE} in - "C768" | "C384") + "C1152" | "C768" | "C384") export tasks_per_node=50 ;; *) diff --git a/parm/config/gfs/config.resources.HERA b/parm/config/gfs/config.resources.HERA index ac3067d9f9..0a190251b0 100644 --- a/parm/config/gfs/config.resources.HERA +++ b/parm/config/gfs/config.resources.HERA @@ -32,7 +32,7 @@ case ${step} in "eupd") case ${CASE} in - "C768") + "C1152" | "C768") export ntasks=80 export threads_per_task=20 ;; @@ -49,7 +49,7 @@ case ${step} in ;; "ecen") - if [[ "${CASE}" == "C768" ]]; then export threads_per_task=6; fi + if [[ "${CASE}" == "C768" || "${CASE}" == "C1152" ]]; then export threads_per_task=6; fi export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) ;; diff --git a/parm/config/gfs/config.resources.JET b/parm/config/gfs/config.resources.JET index bbd308f439..93f64c2c5b 100644 --- a/parm/config/gfs/config.resources.JET +++ b/parm/config/gfs/config.resources.JET @@ -40,7 +40,7 @@ case ${step} in ;; "ecen") - if [[ "${CASE}" == "C768" ]]; then export threads_per_task=6; fi + if [[ "${CASE}" == "C768" || "${CASE}" == "C1152" ]]; then export threads_per_task=6; fi export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) ;; diff --git a/parm/config/gfs/config.resources.WCOSS2 b/parm/config/gfs/config.resources.WCOSS2 index 3ff019068c..342286d008 100644 --- a/parm/config/gfs/config.resources.WCOSS2 +++ b/parm/config/gfs/config.resources.WCOSS2 @@ -9,7 +9,7 @@ case ${step} in ;; "anal") - if [[ "${CASE}" == "C768" ]]; then + if [[ "${CASE}" == "C768" || "${CASE}" == "C1152" ]]; then export threads_per_task=8 # Make ntasks a multiple of 16 export ntasks_gdas=784 @@ -43,7 +43,7 @@ case ${step} in "eupd") case ${CASE} in - "C768" | "C384") + "C1152" | "C768" | "C384") export ntasks=315 export threads_per_task=14 ;; @@ -55,7 +55,7 @@ case ${step} in "eobs") case ${CASE} in - "C768" | "C384") + "C1152" | "C768" | "C384") export tasks_per_node=50 ;; *) diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs index 3f8e7022fa..779efc1a50 100644 --- a/parm/config/gfs/config.ufs +++ b/parm/config/gfs/config.ufs @@ -305,7 +305,7 @@ case "${fv3_res}" in export rf_cutoff=100.0 export fv_sg_adj=450 export WRITE_GROUP_GDAS=4 - export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GDAS=10 # TODO: refine these numbers when a case is available + export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GDAS=20 # TODO: refine these numbers when a case is available export WRITE_GROUP_GFS=4 export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=20 # TODO: refine these numbers when a case is available ;; From c7b3010b758122a4d62384e1d984f38d335ce24e Mon Sep 17 00:00:00 2001 From: "catherine.thomas" Date: Mon, 6 Jan 2025 20:57:27 +0000 Subject: [PATCH 2/3] Update gsi_utils hash for C1152 cycling The calc_analysis utility was updated to handle the increased horizontal resolution of C1152 ATM. Refs: #3173 --- sorc/gsi_utils.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gsi_utils.fd b/sorc/gsi_utils.fd index a6ea311e5c..dd37ccc373 160000 --- a/sorc/gsi_utils.fd +++ b/sorc/gsi_utils.fd @@ -1 +1 @@ -Subproject commit a6ea311e5c82369d255e3afdc99c1bce0c9a3014 +Subproject commit dd37ccc3730f25b8b059fcab341f3e4e169e0e0c From a161bf268ffaa74791f8f0c79f0843dde291f9c6 Mon Sep 17 00:00:00 2001 From: "catherine.thomas" Date: Wed, 8 Jan 2025 19:16:33 +0000 Subject: [PATCH 3/3] Update gsi_utils for C1152 cycling The original calc_analysis bugfix was not suitable for JEDI-based ATM DA. This hash update allows for calc_analysis to work with both GSI and JEDI-based ATM DA. Resolves #3173 --- sorc/gsi_utils.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gsi_utils.fd b/sorc/gsi_utils.fd index dd37ccc373..f716012812 160000 --- a/sorc/gsi_utils.fd +++ b/sorc/gsi_utils.fd @@ -1 +1 @@ -Subproject commit dd37ccc3730f25b8b059fcab341f3e4e169e0e0c +Subproject commit f716012812c2564e7eab24041f7a3ec14c7aa383