From 2d77662e155a0bfd16e228c28d690e2855e26ff3 Mon Sep 17 00:00:00 2001 From: Naureen Date: Fri, 11 Oct 2024 14:36:04 +0000 Subject: [PATCH 01/18] Initial edits to coldstart and creation of run_fcst script --- parm/wflow/coldstart.yaml | 11 +-- scripts/run_fcst.py | 136 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+), 8 deletions(-) create mode 100644 scripts/run_fcst.py diff --git a/parm/wflow/coldstart.yaml b/parm/wflow/coldstart.yaml index 6fad0b8d83..4b853d1f29 100644 --- a/parm/wflow/coldstart.yaml +++ b/parm/wflow/coldstart.yaml @@ -142,17 +142,12 @@ metatask_run_ensemble: task_run_fcst_mem#mem#: <<: *default_task - command: '&LOAD_MODULES_RUN_TASK; "run_fcst" "&JOBSdir;/JREGIONAL_RUN_FCST"' - envars: - <<: *default_vars - SLASH_ENSMEM_SUBDIR: '&SLASH_ENSMEM_SUBDIR;' - nprocs: + command: !cycstr 'source &USHdir;/load_modules_wflow.sh {{ user.MACHINE }} ; &SCRIPTSdir;/run_fcst.py' join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' - nnodes: '{{ task_run_fcst.NNODES_RUN_FCST // 1 }}' + nodes: '{{ task_run_fcst.run_fcst.execution.batchargs.nodes }}:ppn={{ task_run_fcst.run_fcst.execution.batchargs.tasks_per_node }}' partition: '{% if platform.get("PARTITION_FCST") %}&PARTITION_FCST;{% else %}None{% endif %}' - ppn: '{{ task_run_fcst.PPN_RUN_FCST // 1 }}' queue: '&QUEUE_FCST;' - walltime: 04:30:00 + walltime: 04:30:00 dependency: and: taskdep_make_ics: diff --git a/scripts/run_fcst.py b/scripts/run_fcst.py new file mode 100644 index 0000000000..0579229507 --- /dev/null +++ b/scripts/run_fcst.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python +""" +The run script for run_fcst +""" + +import datetime as dt +import logging +import os +import re +import sys +from argparse import ArgumentParser +from copy import deepcopy +from pathlib import Path + +from uwtools.api.fs import link as uwlink +from uwtools.api.logging import use_uwtools_logger +from uwtools.api.fv3 import FV3 +from uwtools.api.config import get_yaml_config + + +def link_files(dest_dir, files): + """ + Link a given list of files to the destination directory using the same file names. + """ + for fpath in files: + path = Path(fpath) + linkname = dest_dir / path.name + if linkname.is_symlink(): + linkname.unlink() + logging.info(f"Linking {linkname} -> {path}") + linkname.symlink_to(path) + +def _walk_key_path(config, key_path): + """ + Navigate to the sub-config at the end of the path of given keys. + """ + keys = [] + pathstr = "" + for key in key_path: + keys.append(key) + pathstr = " -> ".join(keys) + try: + subconfig = config[key] + except KeyError: + logging.error(f"Bad config path: {pathstr}") + raise + if not isinstance(subconfig, dict): + logging.error(f"Value at {pathstr} must be a dictionary") + sys.exit(1) + config = subconfig + return config + +def parse_args(argv): + """ + Parse arguments for the script. + """ + parser = ArgumentParser( + description="Script that runs FV3 via uwtools API.", + ) + parser.add_argument( + "-c", + "--config-file", + metavar="PATH", + required=True, + help="Path to experiment config file.", + type=Path, + ) + parser.add_argument( + "--cycle", + help="The cycle in ISO8601 format (e.g. 2024-07-15T18).", + required=True, + type=dt.datetime.fromisoformat, + ) + parser.add_argument( + "--key-path", + help="Dot-separated path of keys leading through the config to the driver's YAML block.", + metavar="KEY[.KEY...]", + required=True, + type=lambda s: s.split("."), + ) + parser.add_argument( + "--member", + default="000", + help="The 3-digit ensemble member number.", + ) + return parser.parse_args(argv) + + +def run_fcst(config_file, cycle, key_path, member): + """ + Setup and run the FV3 Driver. + """ + expt_config = get_yaml_config(config_file) + + # The experiment config will have {{ MEMBER | env }} expressions in it that need to be + # dereferenced during driver initialization. + os.environ["MEMBER"] = member + + # Run the FV3 program via UW driver + fv3_driver = FV3( + config=config_file, + cycle=cycle, + key_path=key_path, + ) + rundir = Path(fv3_driver.config["rundir"]) + logging.info(f"Will run FV3 in {rundir}") + fv3_driver.run() + + if not (rundir / "runscript.fv3.done").is_file(): + logging.error("Error occurred running FV3. Please see component error logs.") + sys.exit(1) + + # Deliver output data to a common location above the rundir. + fix_lam_path = Path(expt_config["workflow"]["FIXlam"]) + + # Link output data to fix directory + _link_files( + dest_dir=fix_lam_path, + files=glob.glob(str(rundir / f"*.nc")), + ) + + # Mark the successful completion of the script on disk + Path(task_rundir / "run_fcst_task_complete.txt").touch() + + +if __name__ == "__main__": + + use_uwtools_logger() + + args = parse_args(sys.argv[1:]) + run_fv3( + config_file=args.config_file, + cycle=args.cycle, + key_path=args.key_path, + member=args.member, + ) From 1c7faec69ec4d2e20f5777611df3f37d7ffbf6de Mon Sep 17 00:00:00 2001 From: Naureen Date: Fri, 11 Oct 2024 17:09:17 +0000 Subject: [PATCH 02/18] Updates to run_fcst script --- scripts/run_fcst.py | 1 - 1 file changed, 1 deletion(-) mode change 100644 => 100755 scripts/run_fcst.py diff --git a/scripts/run_fcst.py b/scripts/run_fcst.py old mode 100644 new mode 100755 index 0579229507..660ca8b75f --- a/scripts/run_fcst.py +++ b/scripts/run_fcst.py @@ -12,7 +12,6 @@ from copy import deepcopy from pathlib import Path -from uwtools.api.fs import link as uwlink from uwtools.api.logging import use_uwtools_logger from uwtools.api.fv3 import FV3 from uwtools.api.config import get_yaml_config From 12f938c56af2b60a58d3c0633d32cb3ffa0509f9 Mon Sep 17 00:00:00 2001 From: Naureen Date: Thu, 17 Oct 2024 14:08:16 +0000 Subject: [PATCH 03/18] Update command, fail jsonschema check --- parm/wflow/coldstart.yaml | 6 +++++- scripts/make_sfc_climo.py | 1 - scripts/run_fcst.py | 22 +--------------------- ush/config_defaults.yaml | 21 +++++++++++++++++++-- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/parm/wflow/coldstart.yaml b/parm/wflow/coldstart.yaml index c829057d03..2b381daf6d 100644 --- a/parm/wflow/coldstart.yaml +++ b/parm/wflow/coldstart.yaml @@ -142,7 +142,11 @@ metatask_run_ensemble: task_run_fcst_mem#mem#: <<: *default_task - command: !cycstr 'source &USHdir;/load_modules_wflow.sh {{ user.MACHINE }} ; &SCRIPTSdir;/run_fcst.py' + command: !cycstr 'source &USHdir;/load_modules_wflow.sh {{ user.MACHINE }} ; &SCRIPTSdir;/run_fcst.py + -c &GLOBAL_VAR_DEFNS_FP; + --cycle @Y-@m-@dT@H:@M:@S + --key-path task_run_fcst + --mem #mem#' join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' nodes: '{{ task_run_fcst.run_fcst.execution.batchargs.nodes }}:ppn={{ task_run_fcst.run_fcst.execution.batchargs.tasks_per_node }}' partition: '{% if platform.get("PARTITION_FCST") %}&PARTITION_FCST;{% else %}None{% endif %}' diff --git a/scripts/make_sfc_climo.py b/scripts/make_sfc_climo.py index 95b840d236..6e0b51c198 100755 --- a/scripts/make_sfc_climo.py +++ b/scripts/make_sfc_climo.py @@ -10,7 +10,6 @@ from pathlib import Path from uwtools.api.config import get_yaml_config -from uwtools.api.fs import link as uwlink from uwtools.api.logging import use_uwtools_logger from uwtools.api.sfc_climo_gen import SfcClimoGen diff --git a/scripts/run_fcst.py b/scripts/run_fcst.py index 660ca8b75f..8be20621fd 100755 --- a/scripts/run_fcst.py +++ b/scripts/run_fcst.py @@ -29,26 +29,6 @@ def link_files(dest_dir, files): logging.info(f"Linking {linkname} -> {path}") linkname.symlink_to(path) -def _walk_key_path(config, key_path): - """ - Navigate to the sub-config at the end of the path of given keys. - """ - keys = [] - pathstr = "" - for key in key_path: - keys.append(key) - pathstr = " -> ".join(keys) - try: - subconfig = config[key] - except KeyError: - logging.error(f"Bad config path: {pathstr}") - raise - if not isinstance(subconfig, dict): - logging.error(f"Value at {pathstr} must be a dictionary") - sys.exit(1) - config = subconfig - return config - def parse_args(argv): """ Parse arguments for the script. @@ -127,7 +107,7 @@ def run_fcst(config_file, cycle, key_path, member): use_uwtools_logger() args = parse_args(sys.argv[1:]) - run_fv3( + run_fcst( config_file=args.config_file, cycle=args.cycle, key_path=args.key_path, diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 600c170a6c..46a729304e 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1471,8 +1471,8 @@ task_make_sfc_climo: stdout: "{{ user.HOMEdir }}/scripts" walltime: 00:05:00 envcmds: - - module load build_{{ user.MACHINE|lower() }}_{{ workflow.COMPILER }} - module use {{ user.HOMEdir }}/modulefiles + - module load build_{{ user.MACHINE|lower() }}_{{ workflow.COMPILER }} executable: "{{ user.EXECdir }}/sfc_climo_gen" mpicmd: '{{ platform.BATCH_RUN_CMD }}' namelist: @@ -1756,9 +1756,26 @@ task_make_lbcs: #----------------------------- task_run_fcst: # UW Placeholder additions - rundir: '{{ workflow.EXPTDIR }}/{{ timevars.yyyymmddhh }}{{ "/mem%s" % ("MEMBER"|env) if global.DO_ENSEMBLE }}' + fv3: + execution: + batchargs: + walltime: 00:10:00 + executable: '{{ user.EXECdir }}/run_fcst' + mpiargs: + - "--export=NONE" + mpicmd: '{{ platform.BATCH_RUN_CMD }}' + threads: 1 + namelist: + base_file: /path/to/base/input.nml + update_values: + fv_core_nml: + k_split: 2 + n_split: 6 + validate: true + rundir: '{{ workflow.EXPTDIR }}/run_fcst' #------------------------------------------------------------------------ # + # # NNODES_RUN_FCST: # The number of nodes to request from the job scheduler # for the forecast task. From 4249ce2dc13f2ab385b25282b680cf44a6d9fa10 Mon Sep 17 00:00:00 2001 From: Naureen Date: Fri, 18 Oct 2024 19:39:15 +0000 Subject: [PATCH 04/18] Updates to config defaults --- ush/config_defaults.yaml | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 46a729304e..1ab1e9f7b1 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1755,8 +1755,13 @@ task_make_lbcs: # IO_LAYOUT_Y FORECAST config parameters #----------------------------- task_run_fcst: - # UW Placeholder additions fv3: + diag_table: + template_file: '{{ workflow.DIAG_TABLE_TMPL_FP }}' + template_values: + starttime: '{{ timevars.yyyymmddhh }}' + cres: "{{ 'CRES' | env }}" + domain: regional execution: batchargs: walltime: 00:10:00 @@ -1765,6 +1770,24 @@ task_run_fcst: - "--export=NONE" mpicmd: '{{ platform.BATCH_RUN_CMD }}' threads: 1 + field_table: + base_file: /path/to/field_table_to_use + files_to_copy: + INPUT/gfs_ctrl.nc: '{{ DATA }}/INPUT/gfs_ctrl.nc' + INPUT/gfs_data.nc: '{{ DATA }}/INPUT/gfs_data.nc' + INPUT/sfc_data.nc: '{{ DATA }}/INPUT/sfc_data.nc' + files_to_link: + co2historicaldata_2010.txt: src/uwtools/drivers/global_co2historicaldata_2010.txt + co2historicaldata_2011.txt: src/uwtools/drivers/global_co2historicaldata_2011.txt + lateral_boundary_conditions: + interval_hours: 3 + offset: 0 + path: gfs_bndy.tile{tile}.f{forecast_hour}.nc + length: 12 + model_configure: + base_file: '{{ user.USHdir }}/create_model_configure_file.py' + update_values: + write_dopost: .false. namelist: base_file: /path/to/base/input.nml update_values: From c596c954418ebc52c4771ab1fae5407f2fcd643c Mon Sep 17 00:00:00 2001 From: Naureen Date: Tue, 22 Oct 2024 13:11:55 +0000 Subject: [PATCH 05/18] Update diag_table and file_copy config keys --- ush/config_defaults.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 1ab1e9f7b1..8c6fc567dc 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1759,7 +1759,7 @@ task_run_fcst: diag_table: template_file: '{{ workflow.DIAG_TABLE_TMPL_FP }}' template_values: - starttime: '{{ timevars.yyyymmddhh }}' + starttime: '{{ cycle }}' cres: "{{ 'CRES' | env }}" domain: regional execution: @@ -1771,11 +1771,11 @@ task_run_fcst: mpicmd: '{{ platform.BATCH_RUN_CMD }}' threads: 1 field_table: - base_file: /path/to/field_table_to_use + base_file: '{{ workflow.FIELD_TABLE_FP }}' files_to_copy: - INPUT/gfs_ctrl.nc: '{{ DATA }}/INPUT/gfs_ctrl.nc' - INPUT/gfs_data.nc: '{{ DATA }}/INPUT/gfs_data.nc' - INPUT/sfc_data.nc: '{{ DATA }}/INPUT/sfc_data.nc' + INPUT/gfs_ctrl.nc: '{{ task_run_fcst.fv3.rundir }}/INPUT/{{ nco.NET_default }}.{{ cycle }}{{ "dot_ensmem" | env }}.gfs_ctrl.nc' + INPUT/gfs_data.nc: '{{ task_run_fcst.fv.rundir }}/INPUT/{{ nco.NET_default }}.{{ cycle }}{{ "dot_ensmem" | env }}.gfs_data.tile{{ constants.TILE_RGNL }}.halo{{ constants.NH0 }}.nc' + INPUT/sfc_data.nc: '{{ task_run_fcst.fv.rundir }}/INPUT/{{ nco.NET_default }}.{{ cycle }}{{ "dot_ensmem" | env }}.sfc_data.tile{{ constants.TILE_RGNL }}.halo{{ constants.NH0 }}.nc' files_to_link: co2historicaldata_2010.txt: src/uwtools/drivers/global_co2historicaldata_2010.txt co2historicaldata_2011.txt: src/uwtools/drivers/global_co2historicaldata_2011.txt From 03e01d1f2c4b0af65c6c48fa9182205198db2bc2 Mon Sep 17 00:00:00 2001 From: Naureen Date: Tue, 22 Oct 2024 13:21:43 +0000 Subject: [PATCH 06/18] Config updates --- ush/config_defaults.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 8c6fc567dc..0624021ca5 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1780,9 +1780,9 @@ task_run_fcst: co2historicaldata_2010.txt: src/uwtools/drivers/global_co2historicaldata_2010.txt co2historicaldata_2011.txt: src/uwtools/drivers/global_co2historicaldata_2011.txt lateral_boundary_conditions: - interval_hours: 3 - offset: 0 - path: gfs_bndy.tile{tile}.f{forecast_hour}.nc + interval_hours: 1 + offset: 3 + path: '{{ task_run_fcst.fv3.rundir }}/INPUT/{{ nco.NET_default }}.{{ cycle }}{{ "dot_ensmem" | env }}.gfs_bndy.tile{{ constants.TILE_RGNL }}.f{{ task_run_fcst.fv3.lateral_boundary_conditions.offset }}.nc' length: 12 model_configure: base_file: '{{ user.USHdir }}/create_model_configure_file.py' From 16e2ac0635b922031f60953fc921c1c8ab2607c2 Mon Sep 17 00:00:00 2001 From: Naureen Date: Tue, 22 Oct 2024 18:07:42 +0000 Subject: [PATCH 07/18] Update config --- scripts/run_fcst.py | 5 +++-- ush/config_defaults.yaml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/run_fcst.py b/scripts/run_fcst.py index 8be20621fd..4e7f811f68 100755 --- a/scripts/run_fcst.py +++ b/scripts/run_fcst.py @@ -71,8 +71,9 @@ def run_fcst(config_file, cycle, key_path, member): """ expt_config = get_yaml_config(config_file) - # The experiment config will have {{ MEMBER | env }} expressions in it that need to be - # dereferenced during driver initialization. + # The experiment config will have {{ CRES | env }} and {{ MEMBER | env }} expressions in it that need to be + # dereferenced during driver initialization + os.environ["CRES"] = expt_config["workflow"]["CRES"] os.environ["MEMBER"] = member # Run the FV3 program via UW driver diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 0624021ca5..16e29e2259 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1759,7 +1759,7 @@ task_run_fcst: diag_table: template_file: '{{ workflow.DIAG_TABLE_TMPL_FP }}' template_values: - starttime: '{{ cycle }}' + starttime: '{{ timevars.yyyymmddhh }}' cres: "{{ 'CRES' | env }}" domain: regional execution: From 9947701f27c94460e86f98fc6f2da5290081054c Mon Sep 17 00:00:00 2001 From: Naureen Date: Wed, 23 Oct 2024 19:51:04 +0000 Subject: [PATCH 08/18] Update config --- parm/diag_table.FV3_RRFS_v1beta | 4 ++-- scripts/run_fcst.py | 1 + ush/config_defaults.yaml | 7 +++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/parm/diag_table.FV3_RRFS_v1beta b/parm/diag_table.FV3_RRFS_v1beta index 30bf673ef7..fed189c689 100755 --- a/parm/diag_table.FV3_RRFS_v1beta +++ b/parm/diag_table.FV3_RRFS_v1beta @@ -1,5 +1,5 @@ -{{ starttime.strftime("%Y%m%d.%H") }}Z.{{ cres }}.32bit.non-hydro.regional -{{ starttime.strftime("%Y %m %d %H %M %S") }} +{{ cycle.strftime("%Y%m%d.%H") }}Z.{{ cres }}.32bit.non-hydro.regional +{{ cycle.strftime("%Y %m %d %H %M %S") }} "grid_spec", -1, "months", 1, "days", "time" "atmos_static", -1, "hours", 1, "hours", "time" diff --git a/scripts/run_fcst.py b/scripts/run_fcst.py index 4e7f811f68..775ed6fb76 100755 --- a/scripts/run_fcst.py +++ b/scripts/run_fcst.py @@ -74,6 +74,7 @@ def run_fcst(config_file, cycle, key_path, member): # The experiment config will have {{ CRES | env }} and {{ MEMBER | env }} expressions in it that need to be # dereferenced during driver initialization os.environ["CRES"] = expt_config["workflow"]["CRES"] + os.environ["DOT_ENSMEM"] = dot_ensmem os.environ["MEMBER"] = member # Run the FV3 program via UW driver diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 16e29e2259..b1926a6796 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1759,7 +1759,6 @@ task_run_fcst: diag_table: template_file: '{{ workflow.DIAG_TABLE_TMPL_FP }}' template_values: - starttime: '{{ timevars.yyyymmddhh }}' cres: "{{ 'CRES' | env }}" domain: regional execution: @@ -1773,9 +1772,9 @@ task_run_fcst: field_table: base_file: '{{ workflow.FIELD_TABLE_FP }}' files_to_copy: - INPUT/gfs_ctrl.nc: '{{ task_run_fcst.fv3.rundir }}/INPUT/{{ nco.NET_default }}.{{ cycle }}{{ "dot_ensmem" | env }}.gfs_ctrl.nc' - INPUT/gfs_data.nc: '{{ task_run_fcst.fv.rundir }}/INPUT/{{ nco.NET_default }}.{{ cycle }}{{ "dot_ensmem" | env }}.gfs_data.tile{{ constants.TILE_RGNL }}.halo{{ constants.NH0 }}.nc' - INPUT/sfc_data.nc: '{{ task_run_fcst.fv.rundir }}/INPUT/{{ nco.NET_default }}.{{ cycle }}{{ "dot_ensmem" | env }}.sfc_data.tile{{ constants.TILE_RGNL }}.halo{{ constants.NH0 }}.nc' + INPUT/gfs_ctrl.nc: '{{ workflow.EXPTDIR }}/run_fcst/INPUT/{{ nco.NET_default }}.{{ cycle }}{{ "dot_ensmem" | env }}.gfs_ctrl.nc' + INPUT/gfs_data.nc: '{{ task_run_fcst.fv3.rundir }}/INPUT/{{ nco.NET_default }}.{{ cycle }}{{ "dot_ensmem" | env }}.gfs_data.tile{{ constants.TILE_RGNL }}.halo{{ constants.NH0 }}.nc' + INPUT/sfc_data.nc: '{{ task_run_fcst.fv3.rundir }}/INPUT/{{ nco.NET_default }}.{{ cycle }}{{ "dot_ensmem" | env }}.sfc_data.tile{{ constants.TILE_RGNL }}.halo{{ constants.NH0 }}.nc' files_to_link: co2historicaldata_2010.txt: src/uwtools/drivers/global_co2historicaldata_2010.txt co2historicaldata_2011.txt: src/uwtools/drivers/global_co2historicaldata_2011.txt From 3e7584e3130df70a2692a6e71b049da649142e12 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Fri, 25 Oct 2024 22:30:35 +0000 Subject: [PATCH 09/18] WIP --- ush/ccpp_suites_defaults.yaml | 454 +++++++++++++++++++++++++++++++++- ush/config_defaults.yaml | 92 ++++--- ush/config_defaults_aqm.yaml | 26 ++ ush/generate_FV3LAM_wflow.py | 320 ++++++++---------------- ush/setup.py | 16 +- 5 files changed, 642 insertions(+), 266 deletions(-) diff --git a/ush/ccpp_suites_defaults.yaml b/ush/ccpp_suites_defaults.yaml index c7c15f09d4..a21b879258 100644 --- a/ush/ccpp_suites_defaults.yaml +++ b/ush/ccpp_suites_defaults.yaml @@ -18,19 +18,465 @@ orog_gsl_defaults: &orog_gsl_defaults - module load build_{{ user.MACHINE|lower }}_{{ workflow.COMPILER }} rundir: "{{ task_make_orog.rundir }}/orog_gsl" -FV3_RAP: +FV3_GFS_v15p2: + task_run_fcst: + namelist: + update_values: + fv_core_nml: &gfs_v15_fv_core + agrid_vel_rst: False + d2_bg_k1: 0.15 + d2_bg_k2: 0.02 + dnats: !int '{{ 5 if cpl_aqm_parm.CPL_AQM else 1 }}' + do_sat_adj: True + fv_debug: False + fv_sg_adj: 600 + k_split: 1 + kord_mt: 9 + kord_tm: -9 + kord_tr: 9 + kord_wz: 9 + n_split: 8 + n_sponge: 30 + nord_zs_filter: !remove + nudge_qv: True + range_warn: False + rf_cutoff: 750.0 + rf_fast: False + gfdl_cloud_microphysics_nml: + <<: *gfs_gfdl_cloud_mp + sedi_transport: True + tau_l2v: 225.0 + tau_v2l: 150.0 + gfs_physics_nml: &gfs_v15_gfs_physics + bl_mynn_edmf: !remove + bl_mynn_edmf_mom: !remove + bl_mynn_tkeadvect: !remove + cnvcld: True + cnvgwd: True.... + cplflx: !remove + do_myjpbl: False + do_myjsfc: False + do_mynnedmf: !remove + do_mynnsfclay: !remove + do_tofd: False + do_ugwp: False + do_ysu: False + fhcyc: 0.0 + fhlwr: 3600.0 + fhswr: 3600.0 + hybedmf: True + iau_delthrs: !remove + iaufhrs: !remove + imfdeepcnv: 2 + imfshalcnv: 2 + imp_physics: 11 + icloud_bl: !remove + iopt_alb: 2 + iopt_btr: 1 + iopt_crs: 1 + iopt_dveg: 2 + iopt_frz: 1 + iopt_inf: 1 + iopt_rad: 1 + iopt_run: 1 + iopt_sfc: 1 + iopt_snf: 4 + iopt_stc: 1 + iopt_tbot: 2 + iopt_trs: 2 + ldiag_ugwp: False + lgfdlmprad: True + lradar: !remove + lsm: 1 + lsoil: !remove + lsoil_lsm: !remove + ltaerosol: !remove + shal_cnv: True + shinhong: False + ttendlim: !remove + xkzm_h: 1.0 + xkzm_m: 1.0 + xkzminv: 0.3 + namsfc: + landice: True + ldebug: False + surf_map_nml: !remove + +FV3_GFS_v15_thompson_mynn_lam3km: task_make_orog: orog_gsl: <<: *orog_gsl_defaults -FV3_HRRR: + task_run_fcst: + namelist: + update_values: + atmos_model_nml: + avg_max_length: 3600.0 + fv_core_nml: + agrid_vel_rst: True + full_zs_filter: !remove + n_sponge: 9 + npz_type: '' + rf_fast: False + sg_cutoff: 10000.0 + vtdm4: 0.02 + gfs_physics_nml: + avg_max_length: 3600.0 + cdmbgwd: [0.88, 0.04] + debug: True + do_deep: False + do_gsl_drag_ls_bl: False + do_gsl_drag_ss: True + do_gsl_drag_tofd: True + do_mynnsfclay: True + do_tofd: False + do_ugwp: False + do_ugwp_v0: False + do_ugwp_v0_nst_only: False + do_ugwp_v0_orog_only: False + fhswr: 900.0 + fhlwr: 900.0 + gwd_opt: 2 + iaer: 1011 + iccn: 2 + icliq_sw: 2 + imfdeepcnv: 2 + imfshalcnv: 2 + iopt_alb: 2 + iopt_btr: 1 + iopt_crs: 1 + iopt_dveg: 2 + iopt_frz: 1 + iopt_inf: 1 + iopt_rad: 1 + iopt_run: 1 + iopt_sfc: 1 + iopt_snf: 4 + iopt_stc: 1 + iopt_tbot: 2 + iopt_trs: !remove + iovr: 3 + ldiag_ugwp: False + lgfdlmprad: False + lsm: 1 + lsoil: !remove + lsoil_lsm: !remove + ltaerosol: False.... + print_diff_pgr: True + sfclay_compute_flux: !remove + xkzminv: 0.3 + xkzm_m: 1.0 + xkzm_h: 1.0 + surf_map_nml: !remove + +FV3_GFS_v16: + task_run_fcst: + namelist: + update_values: + cires_ugwp_nml: + launch_level: 27 + fv_core_nml: + <<: *gfs_v15_fv_core + agrid_vel_rst: False + d2_bg_k1: 0.2 + d2_bg_k2: 0.0 + delt_max: 0.002 + dnats: !int '{{ 5 if cpl_aqm_parm.CPL_AQM else 1 }}' + dz_min: 6 + fv_sg_adj: 450 + hord_dp: -5 + hord_mt: 5 + hord_tm: 5 + hord_tr: !int '{{ 8 if cpl_aqm_parm.CPL_AQM 10 }}' + hord_vt: 5 + k_split: 6 + make_nh: False + n_split: 6 + n_sponge: 10 + na_init: 0 + nord: !int '{{ 2 if cpl_aqm_parm.CPL_AQM 3 }}' + nudge_dz: False + res_latlon_dynamics: '' + rf_fast: !remove + tau: 10.0 + gfdl_cloud_microphysics_nml: + <<: *gfs_gfdl_cloud_mp + mp_time: 150.0 + reiflag: 2 + sedi_transport: True + tau_l2v: 225.0 + tau_v2l: 150.0 + gfs_physics_nml: + <<: *gfs_v15_gfs_physics + cdmbgwd: [4.0, 0.15, 1.0, 1.0] + do_myjpbl: !remove + do_myjsfc: !remove + do_tofd: True + do_ysu: !remove + hybedmf: False + iaer: 5111 + icliq_sw: 2 + iopt_dveg: 1 + iovr: 3 + isatmedmf: 1 + lgfdlmprad: True + lheatstrg: True + lndp_type: !remove + lsoil: 4 + n_var_lndp: !remove + prautco: [0.00015, 0.00015] + psautco: [0.0008, 0.0005] + satmedmf: True + shinhong: !remove + xkzminv: !remove + xkzm_m: !remove + xkzm_h: !remove + mpp_io_nml: + deflate_level: 1 + shuffle: 1 + namsfc: + landice: True + ldebug: False + surf_map_nml: !remove + + +FV3_GFS_v17_p8: task_make_orog: orog_gsl: <<: *orog_gsl_defaults -FV3_GFS_v15_thompson_mynn_lam3km: + task_run_fcst: + namelist: + update_values: + cires_ugwp_nml: + launch_level: 27 + fv_core_nml: + <<: *gfs_v15_fv_core + agrid_vel_rst: False + d2_bg_k1: 0.2 + d2_bg_k2: 0.0 + dnats: !int '{{ 4 if cpl_aqm_parm.CPL_AQM else 0 }}' + do_sat_adj: False + fv_sg_adj: 450 + hord_dp: -5 + hord_mt: 5 + hord_tm: 5 + hord_tr: 8 + hord_vt: 5 + k_split: 6 + make_nh: True + n_split: 6 + n_sponge: 10 + na_init: 1 + nord: !int '{{ 2 if cpl_aqm_parm.CPL_AQM else 1 }}' + nudge_dz: False + res_latlon_dynamics: '' + rf_fast: !remove + tau: 10.0 + gfs_physics_nml: + cdmbgwd: [4.0, 0.05, 1.0, 1.0] + cnvcld: True + cnvgwd: True + decfl: 10 + do_deep: True + do_gsl_drag_ls_bl: False + do_gsl_drag_ss: True + do_gsl_drag_tofd: False + do_mynnedmf: False + do_mynnsfclay: False + do_tofd: False + do_ugwp: False + do_ugwp_v0: True + do_ugwp_v0_orog_only: False + do_ugwp_v0_nst_only: False + do_ugwp_v1: False + do_ugwp_v1_orog_only: False + dt_inner: 150.0 + fhlwr: 1200.0 + fhswr: 1200.0 + frac_grid: False + gwd_opt: 2 + iaer: 1011 + ialb: 2 + icliq_sw: 2 + iems: 2 + imfdeepcnv: 2 + imfshalcnv: 2 + iopt_alb: 1 + iopt_btr: 1 + iopt_crs: 2 + iopt_dveg: 4 + iopt_frz: 1 + iopt_inf: 1 + iopt_rad: 3 + iopt_run: 1 + iopt_sfc: 3 + iopt_snf: 4 + iopt_stc: 3 + iopt_tbot: 2 + iovr: 3 + isatmedmf: 1 + ldiag_ugwp: False + lseaspray: True + lgfdlmprad: False + lheatstrg: False + lradar: False + lsm: 2 + lsoil_lsm: 4 + ltaerosol: False + min_lakeice: 0.15 + min_seaice: 0.15 + qdiag3d: False + ras: False + satmedmf: True + sedi_semi: True + shal_cnv: True + mpp_io_nml: + deflate_level: 1 + shuffle: 1 + surf_map_nml: !remove + + +FV3_HRRR: task_make_orog: orog_gsl: <<: *orog_gsl_defaults -FV3_GFS_v17_p8: + task_run_fcst: + namelist: + update_values: + fv_core_nml: &HRRR_fv_core + hord_dp: 6 + hord_mt: 6 + hord_tm: 6 + hord_vt: 6 + hord_tr: 8 + kord_mt: 9 + kord_tm: -9 + kord_tr: 9 + kord_wz: 9 + nord_tr: 0 + nrows_blend: 20 + d_con: 0.5 + n_sponge: 9 + gfs_physics_nml: &FV3_HRRR_Phys + cdmbgwd: [3.5, 1.0] + diag_log: True + do_deep: False + do_gsl_drag_ss: True + do_gsl_drag_tofd: True + do_gsl_drag_ls_bl: True + do_mynnsfclay: True + gwd_opt: 3 + iaer: 5111 + ialb: 2 + iems: 2 + icliq_sw: 2 + imfdeepcnv: -1 + imfshalcnv: -1 + iopt_alb: 2 + iopt_btr: 1 + iopt_crs: 1 + iopt_dveg: 2 + iopt_frz: 1 + iopt_inf: 1 + iopt_rad: 1 + iopt_run: 1 + iopt_sfc: 1 + iopt_snf: 4 + iopt_stc: 1 + iopt_tbot: 2 + iopt_trs: 2 + iovr: 3 + isncond_opt: 2 + isncovr_opt: 3 + lsm: 3 + lsoil: !int {{ 9 if task_get_extrn_ics.EXTRN_MDL_NAME_ICS in ("RAP", "HRRR") else 4 }} + lsoil_lsm: 9 + mosaic_lu: 0 + mosaic_soil: 0 + nst_anl: !remove + nstf_name: !remove + sfclay_compute_flux: True + thsfc_loc: False + +FV3_RAP: task_make_orog: orog_gsl: <<: *orog_gsl_defaults + task_run_fcst: + namelist: + update_values: + fv_core_nml: + <<: *HRRR_fv_core + gfs_physics_nml: + <<: *FV3_HRRR_Phys + do_deep: True + shal_cnv: True + imfdeepcnv: 3 + imfshalcnv: 3 + diag_log: True + ialb: 1 + iems: 1 + isncond_opt: !remove + isncovr_opt: !remove + mosaic_lu: !remove + mosaic_soil: !remove + thsfc_loc: !remove + +FV3_RRFS_v1beta: + task_run_fcst: + namelist: + update_values: + gfs_physics_nml: + do_deep: False + do_mynnsfclay: True + imfdeepcnv: -1 + imfshalcnv: -1 + iopt_alb: 2 + iopt_btr: 1 + iopt_crs: 1 + iopt_dveg: 2 + iopt_frz: 1 + iopt_inf: 1 + iopt_rad: 1 + iopt_run: 1 + iopt_sfc: 1 + iopt_snf: 4 + iopt_stc: 1 + iopt_tbot: 2 + iopt_trs: 2 + lsm: 2 + lsoil_lsm: 4 + +FV3_WoFS_v0: + task_run_fcst: + namelist: + update_values: + gfs_physics_nml: + do_deep: False + imfdeepcnv: 0 + imfshalcnv: 0 + iopt_alb: 2 + iopt_btr: 1 + iopt_crs: 1 + iopt_dveg: 2 + iopt_frz: 1 + iopt_inf: 1 + iopt_rad: 1 + iopt_run: 1 + iopt_sfc: 1 + iopt_snf: 4 + iopt_stc: 1 + iopt_tbot: 2 + do_mynnsfclay: True + imfdeepcnv: -1 + imfshalcnv: -1 + lsm: 1 + lsoil_lsm: 4 + imp_physics: 17 + nssl_cccn: 0.6e+9 + nssl_hail_on: True + nssl_ccn_on: True + fv_core_nml: + nwat: 7 + fv_diagnostics_nml: + do_hailcast: True + diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 16e29e2259..bbd5166d37 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1765,7 +1765,7 @@ task_run_fcst: execution: batchargs: walltime: 00:10:00 - executable: '{{ user.EXECdir }}/run_fcst' + executable: '{{ user.EXECdir }}/{{ workflow.FV3_EXEC_FN }}' mpiargs: - "--export=NONE" mpicmd: '{{ platform.BATCH_RUN_CMD }}' @@ -1785,15 +1785,61 @@ task_run_fcst: path: '{{ task_run_fcst.fv3.rundir }}/INPUT/{{ nco.NET_default }}.{{ cycle }}{{ "dot_ensmem" | env }}.gfs_bndy.tile{{ constants.TILE_RGNL }}.f{{ task_run_fcst.fv3.lateral_boundary_conditions.offset }}.nc' length: 12 model_configure: - base_file: '{{ user.USHdir }}/create_model_configure_file.py' update_values: - write_dopost: .false. + total_member: 1 + start_year: !int '{{ cycle.year }}' + start_month: !int '{{ cycle.month }}' + start_day: !int '{{ cycle.day }}' + start_hour: !int '{{ cycle.hour }}' + start_minute: !int '{{ cycle.minute }}' + start_second: !int '{{ cycle.second }}' + nhours_fcst: !int '{{ workflow.FCST_LEN_HRS }}' + fhrot: 0 + run_continue: false + ens_sps: false + dt_atmos: "" + calendar: julian + memuse_verbose: false + restart_interval: 0 + output_1st_tstep_rst: false + write_dopost: false + ideflate: 0 + nbits: 0 + ichunk2d: -1 + jchunk2d: -1 + ichunk3d: -1 + jchunk3d: -1 + kchunk3d: -1 + itasks: 1 + quilting: true + write_groups: !int {{ task_run_fcst.WRTCMP_write_groups }} + write_tasks_per_group: !int {{ task_run_fcst.WRTCMP_write_tasks_per_group }} + num_files: 2 + filename_base: 'dyn phy' + output_file: 'netcdf netcdf' + output_fh: '1 -1' + nsout: -1 + output_grid: '{{ task_run_fcst.WRTCMP_output_grid }}' namelist: - base_file: /path/to/base/input.nml + base_file: "{{ workflow.FV3_NML_BASE_SUITE_FP }}" update_values: + atmos_model_nml: + blocksize: !int {{ task_run_fcst.BLOCKSIZE }} + ccpp_suite: '{{ workflow.CCPP_PHYS_SUITE }}' fv_core_nml: - k_split: 2 - n_split: 6 + target_lon: !int {{ grid_params.LON_CTR }} + target_lat: !int {{ grid_params.LAT_CTR }} + nrows_blend: !int {{ grid_params.HALO_BLEND }} + stretch_fac: !int {{ grid_params.STRETCH_FAC }} + npx: !int {{ grid_params.NX + 1 }} + npy: !int {{ grid_params.NY + 1 }} + layout: + - !int {{ task_run_fcst.LAYOUT_X }} + - !int {{ task_run_fcst.LAYOUT_Y }} + bc_update_interval: !int {{ task_get_extrn_lbcs.LBC_SPEC_INTVL_HRS }} + gfs_physics_nml: + kice: !int {{ 9 if workflow.SDF_USES_RUC_LSM else 2 }} + print_diff_pgr: false validate: true rundir: '{{ workflow.EXPTDIR }}/run_fcst' #------------------------------------------------------------------------ @@ -1816,7 +1862,6 @@ task_run_fcst: #------------------------------------------------------------------------ NNODES_RUN_FCST: !int '{{ (task_run_fcst.PE_MEMBER01 + task_run_fcst.PPN_RUN_FCST - 1) // task_run_fcst.PPN_RUN_FCST }}' PPN_RUN_FCST: '{{ platform.NCORES_PER_NODE // task_run_fcst.OMP_NUM_THREADS_RUN_FCST }}' - FV3_EXEC_FP: '{{ user.EXECdir }}/{{ workflow.FV3_EXEC_FN }}' IO_LAYOUT_Y: 1 # #----------------------------------------------------------------------- @@ -1856,15 +1901,6 @@ task_run_fcst: # # Set model_configure parameters. Definitions: # - # DT_ATMOS: - # The main forecast model integration time step. As described in the - # forecast model documentation, "It corresponds to the frequency with - # which the top level routine in the dynamics is called as well as the - # frequency with which the physics is called." - # - # FHROT: - # Forecast hour at restart - # # RESTART_INTERVAL: # frequency of the output restart files (unit:hour). # Default=0: restart files are produced at the end of a forecast run @@ -1889,10 +1925,7 @@ task_run_fcst: # #----------------------------------------------------------------------- # - DT_ATMOS: "" - FHROT: 0 RESTART_INTERVAL: 0 - WRITE_DOPOST: false ITASKS: 1 # #----------------------------------------------------------------------- @@ -1944,14 +1977,6 @@ task_run_fcst: # # Definitions: # - # QUILTING: - # Flag that determines whether or not to use the write component for - # writing output files to disk. The regional grid requires the use of - # the write component, so users should not change the default value. - # When set to true, the forecast model will output files named dynf$HHH.nc - # and phyf$HHH.nc (where HHH is the 3-digit forecast hour) containing dynamics - # and physics fields, respectively, on the write-component grid. - # # PRINT_ESMF: # Flag that determines whether to output extra (debugging) information from # ESMF routines. Must be true or false. Note that the write @@ -2042,10 +2067,9 @@ task_run_fcst: # #----------------------------------------------------------------------- # - QUILTING: true PRINT_ESMF: false - PE_MEMBER01: '{{ task_run_fcst.LAYOUT_Y * task_run_fcst.LAYOUT_X + task_run_fcst.WRTCMP_write_groups * task_run_fcst.WRTCMP_write_tasks_per_group if task_run_fcst.QUILTING else task_run_fcst.LAYOUT_Y * task_run_fcst.LAYOUT_X}}' + PE_MEMBER01: !int '{{ task_run_fcst.LAYOUT_Y * task_run_fcst.LAYOUT_X + task_run_fcst.WRTCMP_write_groups * task_run_fcst.WRTCMP_write_tasks_per_group }}' WRTCMP_write_groups: "" WRTCMP_write_tasks_per_group: "" @@ -2454,16 +2478,6 @@ global: #----------------------------------------------------------------------- # HALO_BLEND: 10 - # - #----------------------------------------------------------------------- - # - # PRINT_DIFF_PGR: - # Option to turn on/off the pressure tendency diagnostic. - # - #----------------------------------------------------------------------- - # - PRINT_DIFF_PGR: false - #---------------------------- # verification (vx) parameters #----------------------------- diff --git a/ush/config_defaults_aqm.yaml b/ush/config_defaults_aqm.yaml index 13c1a5bfa9..8c1e36b493 100644 --- a/ush/config_defaults_aqm.yaml +++ b/ush/config_defaults_aqm.yaml @@ -9,3 +9,29 @@ task_run_post: aqf_on: true output_file_labels: - cmaq + +task_run_fcst: + namelist: + update_values: + gfs_physics_nml: + cplaqm: true + cplocn2atm: false + fscav_aero: ["aacd:0.0", "acet:0.0", "acrolein:0.0", "acro_primary:0.0", "ald2:0.0", + "ald2_primary:0.0", "aldx:0.0", "benzene:0.0", "butadiene13:0.0", "cat1:0.0", + "cl2:0.0", "clno2:0.0", "co:0.0", "cres:0.0", "cron:0.0", + "ech4:0.0", "epox:0.0", "eth:0.0", "etha:0.0", "ethy:0.0", + "etoh:0.0", "facd:0.0", "fmcl:0.0", "form:0.0", "form_primary:0.0", + "gly:0.0", "glyd:0.0", "h2o2:0.0", "hcl:0.0", "hg:0.0", + "hgiigas:0.0", "hno3:0.0", "hocl:0.0", "hono:0.0", "hpld:0.0", + "intr:0.0", "iole:0.0", "isop:0.0", "ispd:0.0", "ispx:0.0", + "ket:0.0", "meoh:0.0", "mepx:0.0", "mgly:0.0", "n2o5:0.0", + "naph:0.0", "no:0.0", "no2:0.0", "no3:0.0", "ntr1:0.0", + "ntr2:0.0", "o3:0.0", "ole:0.0", "opan:0.0", "open:0.0", + "opo3:0.0", "pacd:0.0", "pan:0.0", "panx:0.0", "par:0.0", + "pcvoc:0.0", "pna:0.0", "prpa:0.0", "rooh:0.0", "sesq:0.0", + "so2:0.0", "soaalk:0.0", "sulf:0.0", "terp:0.0", "tol:0.0", + "tolu:0.0", "vivpo1:0.0", "vlvoo1:0.0", "vlvoo2:0.0", "vlvpo1:0.0", + "vsvoo1:0.0", "vsvoo2:0.0", "vsvoo3:0.0", "vsvpo1:0.0", "vsvpo2:0.0", + "vsvpo3:0.0", "xopn:0.0", "xylmn:0.0", "*:0.2" ] + + diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index d3c2290a65..0e24427835 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -338,112 +338,6 @@ def generate_FV3LAM_wflow( lsoil = 9 if CCPP_PHYS_SUITE == "FV3_GFS_v15_thompson_mynn_lam3km": lsoil = "" - # - # Create a multiline variable that consists of a yaml-compliant string - # specifying the values that the namelist variables that are physics- - # suite-independent need to be set to. Below, this variable will be - # passed to a python script that will in turn set the values of these - # variables in the namelist file. - # - # IMPORTANT: - # If we want a namelist variable to be removed from the namelist file, - # in the "settings" variable below, we need to set its value to the - # string "null". This is equivalent to setting its value to - # !!python/none - # in the base namelist file specified by FV3_NML_BASE_SUITE_FP or the - # suite-specific yaml settings file specified by FV3_NML_YAML_CONFIG_FP. - # - # It turns out that setting the variable to an empty string also works - # to remove it from the namelist! Which is better to use?? - # - settings = {} - settings["atmos_model_nml"] = { - "blocksize": BLOCKSIZE, - "ccpp_suite": CCPP_PHYS_SUITE, - } - - fv_core_nml_dict = {} - fv_core_nml_dict.update({ - "target_lon": LON_CTR, - "target_lat": LAT_CTR, - "nrows_blend": HALO_BLEND, - # - # Question: - # For a ESGgrid type grid, what should stretch_fac be set to? This depends - # on how the FV3 code uses the stretch_fac parameter in the namelist file. - # Recall that for a ESGgrid, it gets set in the function set_gridparams_ESGgrid(.sh) - # to something like 0.9999, but is it ok to set it to that here in the - # FV3 namelist file? - # - "stretch_fac": STRETCH_FAC, - "npx": npx, - "npy": npy, - "layout": [LAYOUT_X, LAYOUT_Y], - "bc_update_interval": LBC_SPEC_INTVL_HRS, - }) - if CCPP_PHYS_SUITE == "FV3_GFS_v15p2": - if CPL_AQM: - fv_core_nml_dict.update({ - "dnats": 5 - }) - else: - fv_core_nml_dict.update({ - "dnats": 1 - }) - elif CCPP_PHYS_SUITE == "FV3_GFS_v16": - if CPL_AQM: - fv_core_nml_dict.update({ - "hord_tr": 8, - "dnats": 5, - "nord": 2 - }) - else: - fv_core_nml_dict.update({ - "dnats": 1 - }) - elif CCPP_PHYS_SUITE == "FV3_GFS_v17_p8": - if CPL_AQM: - fv_core_nml_dict.update({ - "dnats": 4 - }) - else: - fv_core_nml_dict.update({ - "dnats": 0 - }) - - settings["fv_core_nml"] = fv_core_nml_dict - - gfs_physics_nml_dict = {} - gfs_physics_nml_dict.update({ - "kice": kice or None, - "lsoil": lsoil or None, - "print_diff_pgr": PRINT_DIFF_PGR, - }) - - if CPL_AQM: - gfs_physics_nml_dict.update({ - "cplaqm": True, - "cplocn2atm": False, - "fscav_aero": [ - "aacd:0.0", "acet:0.0", "acrolein:0.0", "acro_primary:0.0", "ald2:0.0", - "ald2_primary:0.0", "aldx:0.0", "benzene:0.0", "butadiene13:0.0", "cat1:0.0", - "cl2:0.0", "clno2:0.0", "co:0.0", "cres:0.0", "cron:0.0", - "ech4:0.0", "epox:0.0", "eth:0.0", "etha:0.0", "ethy:0.0", - "etoh:0.0", "facd:0.0", "fmcl:0.0", "form:0.0", "form_primary:0.0", - "gly:0.0", "glyd:0.0", "h2o2:0.0", "hcl:0.0", "hg:0.0", - "hgiigas:0.0", "hno3:0.0", "hocl:0.0", "hono:0.0", "hpld:0.0", - "intr:0.0", "iole:0.0", "isop:0.0", "ispd:0.0", "ispx:0.0", - "ket:0.0", "meoh:0.0", "mepx:0.0", "mgly:0.0", "n2o5:0.0", - "naph:0.0", "no:0.0", "no2:0.0", "no3:0.0", "ntr1:0.0", - "ntr2:0.0", "o3:0.0", "ole:0.0", "opan:0.0", "open:0.0", - "opo3:0.0", "pacd:0.0", "pan:0.0", "panx:0.0", "par:0.0", - "pcvoc:0.0", "pna:0.0", "prpa:0.0", "rooh:0.0", "sesq:0.0", - "so2:0.0", "soaalk:0.0", "sulf:0.0", "terp:0.0", "tol:0.0", - "tolu:0.0", "vivpo1:0.0", "vlvoo1:0.0", "vlvoo2:0.0", "vlvpo1:0.0", - "vsvoo1:0.0", "vsvoo2:0.0", "vsvoo3:0.0", "vsvpo1:0.0", "vsvpo2:0.0", - "vsvpo3:0.0", "xopn:0.0", "xylmn:0.0", "*:0.2" ] - }) - settings["gfs_physics_nml"] = gfs_physics_nml_dict # # Add to "settings" the values of those namelist variables that specify @@ -462,6 +356,7 @@ def generate_FV3LAM_wflow( regex_search = "^[ ]*([^| ]+)[ ]*[|][ ]*([^| ]+)[ ]*$" num_nml_vars = len(FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING) namsfc_dict = {} + settings = {} for i in range(num_nml_vars): mapping = f"{FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING[i]}" @@ -495,8 +390,6 @@ def generate_FV3LAM_wflow( if PREDEF_GRID_NAME == "RRFS_NA_3km": settings["fms2_io_nml"] = {"netcdf_default_format": "netcdf4"} - settings_str = cfg_to_yaml_str(settings) - log_info( """ The variable 'settings' specifying values of the weather model's @@ -507,23 +400,13 @@ def generate_FV3LAM_wflow( # # ----------------------------------------------------------------------- # - # Create a new FV3 namelist file + # Update the fv3 namelist config with relevant settings # # ----------------------------------------------------------------------- # + fcst_nml_config = get_yaml_config(expt_config["task_run_fcst"]["namelist"]["update_values"]) + fcst_nml_config.update_from(settings) - physics_cfg = get_yaml_config(FV3_NML_YAML_CONFIG_FP) - base_namelist = get_nml_config(FV3_NML_BASE_SUITE_FP) - base_namelist.update_from(physics_cfg[CCPP_PHYS_SUITE]) - base_namelist.update_from(settings) - for sect, values in base_namelist.copy().items(): - if not values: - del base_namelist[sect] - continue - for k, v in values.copy().items(): - if v is None: - del base_namelist[sect][k] - base_namelist.dump(Path(FV3_NML_FP)) # # If not running the TN_MAKE_GRID task (which implies the workflow will # use pregenerated grid files), set the namelist variables specifying @@ -550,116 +433,109 @@ def generate_FV3LAM_wflow( # # ----------------------------------------------------------------------- # - settings = {} - settings["gfs_physics_nml"] = { - "do_shum": DO_SHUM, - "do_sppt": DO_SPPT, - "do_skeb": DO_SKEB, - "do_spp": DO_SPP, - "n_var_spp": N_VAR_SPP, - "n_var_lndp": N_VAR_LNDP, - "lndp_type": LNDP_TYPE, - "fhcyc": FHCYC_LSM_SPP_OR_NOT, - } - nam_stochy_dict = {} - if DO_SPPT: - nam_stochy_dict.update( - { - "iseed_sppt": ISEED_SPPT, - "new_lscale": NEW_LSCALE, - "sppt": SPPT_MAG, - "sppt_logit": SPPT_LOGIT, - "sppt_lscale": SPPT_LSCALE, - "sppt_sfclimit": SPPT_SFCLIMIT, - "sppt_tau": SPPT_TSCALE, - "spptint": SPPT_INT, - "use_zmtnblck": USE_ZMTNBLCK, - } - ) - - if DO_SHUM: - nam_stochy_dict.update( - { - "iseed_shum": ISEED_SHUM, - "new_lscale": NEW_LSCALE, - "shum": SHUM_MAG, - "shum_lscale": SHUM_LSCALE, - "shum_tau": SHUM_TSCALE, - "shumint": SHUM_INT, - } - ) - - if DO_SKEB: - nam_stochy_dict.update( - { - "iseed_skeb": ISEED_SKEB, - "new_lscale": NEW_LSCALE, - "skeb": SKEB_MAG, - "skeb_lscale": SKEB_LSCALE, - "skebnorm": SKEBNORM, - "skeb_tau": SKEB_TSCALE, - "skebint": SKEB_INT, - "skeb_vdof": SKEB_VDOF, - } - ) - - if DO_SPP or DO_LSM_SPP: - nam_stochy_dict.update({"new_lscale": NEW_LSCALE}) - - settings["nam_stochy"] = nam_stochy_dict - # - # Add the relevant SPP namelist variables to "settings" when running with - # SPP turned on. Otherwise only include an empty "nam_sppperts" stanza. - # - nam_sppperts_dict = {} - if DO_SPP: - nam_sppperts_dict = { - "iseed_spp": ISEED_SPP, - "spp_lscale": SPP_LSCALE, - "spp_prt_list": SPP_MAG_LIST, - "spp_sigtop1": SPP_SIGTOP1, - "spp_sigtop2": SPP_SIGTOP2, - "spp_stddev_cutoff": SPP_STDDEV_CUTOFF, - "spp_tau": SPP_TSCALE, - "spp_var_list": SPP_VAR_LIST, - } - - settings["nam_sppperts"] = nam_sppperts_dict - # - # Add the relevant LSM SPP namelist variables to "settings" when running with - # LSM SPP turned on. - # - nam_sfcperts_dict = {} - if DO_LSM_SPP: - nam_sfcperts_dict = { - "lndp_type": LNDP_TYPE, - "lndp_model_type": LNDP_MODEL_TYPE, - "lndp_tau": LSM_SPP_TSCALE, - "lndp_lscale": LSM_SPP_LSCALE, - "iseed_lndp": ISEED_LSM_SPP, - "lndp_var_list": LSM_SPP_VAR_LIST, - "lndp_prt_list": LSM_SPP_MAG_LIST, - } - - settings["nam_sfcperts"] = nam_sfcperts_dict - - settings_str = cfg_to_yaml_str(settings) # #----------------------------------------------------------------------- # - # Generate namelist files with stochastic physics if needed + # Update the stochastic parameters, if needed # #----------------------------------------------------------------------- # if any((DO_SPP, DO_SPPT, DO_SHUM, DO_SKEB, DO_LSM_SPP)): - realize( - input_config=FV3_NML_FP, - input_format="nml", - output_file=FV3_NML_STOCH_FP, - output_format="nml", - update_config=get_nml_config(settings), + settings = {} + settings["gfs_physics_nml"] = { + "do_shum": DO_SHUM, + "do_sppt": DO_SPPT, + "do_skeb": DO_SKEB, + "do_spp": DO_SPP, + "n_var_spp": N_VAR_SPP, + "n_var_lndp": N_VAR_LNDP, + "lndp_type": LNDP_TYPE, + "fhcyc": FHCYC_LSM_SPP_OR_NOT, + } + nam_stochy_dict = {} + if DO_SPPT: + nam_stochy_dict.update( + { + "iseed_sppt": ISEED_SPPT, + "new_lscale": NEW_LSCALE, + "sppt": SPPT_MAG, + "sppt_logit": SPPT_LOGIT, + "sppt_lscale": SPPT_LSCALE, + "sppt_sfclimit": SPPT_SFCLIMIT, + "sppt_tau": SPPT_TSCALE, + "spptint": SPPT_INT, + "use_zmtnblck": USE_ZMTNBLCK, + } + ) + + if DO_SHUM: + nam_stochy_dict.update( + { + "iseed_shum": ISEED_SHUM, + "new_lscale": NEW_LSCALE, + "shum": SHUM_MAG, + "shum_lscale": SHUM_LSCALE, + "shum_tau": SHUM_TSCALE, + "shumint": SHUM_INT, + } + ) + + if DO_SKEB: + nam_stochy_dict.update( + { + "iseed_skeb": ISEED_SKEB, + "new_lscale": NEW_LSCALE, + "skeb": SKEB_MAG, + "skeb_lscale": SKEB_LSCALE, + "skebnorm": SKEBNORM, + "skeb_tau": SKEB_TSCALE, + "skebint": SKEB_INT, + "skeb_vdof": SKEB_VDOF, + } ) + if DO_SPP or DO_LSM_SPP: + nam_stochy_dict.update({"new_lscale": NEW_LSCALE}) + + settings["nam_stochy"] = nam_stochy_dict + # + # Add the relevant SPP namelist variables to "settings" when running with + # SPP turned on. Otherwise only include an empty "nam_sppperts" stanza. + # + nam_sppperts_dict = {} + if DO_SPP: + nam_sppperts_dict = { + "iseed_spp": ISEED_SPP, + "spp_lscale": SPP_LSCALE, + "spp_prt_list": SPP_MAG_LIST, + "spp_sigtop1": SPP_SIGTOP1, + "spp_sigtop2": SPP_SIGTOP2, + "spp_stddev_cutoff": SPP_STDDEV_CUTOFF, + "spp_tau": SPP_TSCALE, + "spp_var_list": SPP_VAR_LIST, + } + + settings["nam_sppperts"] = nam_sppperts_dict + # + # Add the relevant LSM SPP namelist variables to "settings" when running with + # LSM SPP turned on. + # + nam_sfcperts_dict = {} + if DO_LSM_SPP: + nam_sfcperts_dict = { + "lndp_type": LNDP_TYPE, + "lndp_model_type": LNDP_MODEL_TYPE, + "lndp_tau": LSM_SPP_TSCALE, + "lndp_lscale": LSM_SPP_LSCALE, + "iseed_lndp": ISEED_LSM_SPP, + "lndp_var_list": LSM_SPP_VAR_LIST, + "lndp_prt_list": LSM_SPP_MAG_LIST, + } + + settings["nam_sfcperts"] = nam_sfcperts_dict + + fcst_nml_config.update_from(settings) + # # ----------------------------------------------------------------------- # diff --git a/ush/setup.py b/ush/setup.py index a1314857fc..e17e8d750e 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -722,12 +722,13 @@ def get_location(xcs, fmt, expt_cfg): """ )) + quilting = fcst_config["model_configure"]["update_values"]["quilting"] # Gather the pre-defined grid parameters, if needed if workflow_config.get("PREDEF_GRID_NAME"): grid_params = set_predef_grid_params( USHdir, workflow_config["PREDEF_GRID_NAME"], - fcst_config["QUILTING"], + quilting, ) # Users like to change these variables, so don't overwrite them @@ -762,6 +763,14 @@ def get_location(xcs, fmt, expt_cfg): else: grid_config[param] = value + # Load model write component grid settings + quilting_cfg = get_yaml_config(Path(USHdir, "quilting.yaml")) + if not quilting: + update_dict(quilting_cfg["no_quilting"], expt_config) + else: + write_grid = expt_config["task_run_fcst"]["WRTCMP_output_grid"] + update_dict(quilting_cfg[write_grid], expt_config) + run_envir = expt_config["user"].get("RUN_ENVIR", "") fcst_len_hrs = workflow_config.get("FCST_LEN_HRS") @@ -1028,6 +1037,11 @@ def get_location(xcs, fmt, expt_cfg): rem = FCST_LEN_HRS%%LBC_SPEC_INTVL_HRS = {rem}""" ) + # Configure the model namelist + + + + # # ----------------------------------------------------------------------- # From 89e9fd1e25a9c3fb1052f8ddfda426972eb7a7b2 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Wed, 30 Oct 2024 16:35:07 +0000 Subject: [PATCH 10/18] Set the fix files directly in the fv3 config. --- parm/fixed_files_mapping.yaml | 212 +++++++------------------- scripts/exregional_run_fcst.sh | 31 ++-- ush/config_defaults.yaml | 59 +++++-- ush/generate_FV3LAM_wflow.py | 95 +----------- ush/set_fv3nml_sfc_climo_filenames.py | 132 ---------------- ush/setup.py | 19 +-- 6 files changed, 119 insertions(+), 429 deletions(-) delete mode 100644 ush/set_fv3nml_sfc_climo_filenames.py diff --git a/parm/fixed_files_mapping.yaml b/parm/fixed_files_mapping.yaml index 49d3191de5..c39ed4cdde 100644 --- a/parm/fixed_files_mapping.yaml +++ b/parm/fixed_files_mapping.yaml @@ -1,31 +1,4 @@ fixed_files: - # - #----------------------------------------------------------------------- - # - # FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING: - # This array is used to set some of the namelist variables in the forecast - # model's namelist file that represent the relative or absolute paths of - # various fixed files (the first column of the array, where columns are - # delineated by the pipe symbol "|") to the full paths to surface climatology - # files (on the native FV3-LAM grid) in the FIXlam directory derived from - # the corresponding surface climatology fields (the second column of the - # array). - # - #----------------------------------------------------------------------- - # - FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING: [ - "FNALBC | snowfree_albedo", - "FNALBC2 | facsf", - "FNTG3C | substrate_temperature", - "FNVEGC | vegetation_greenness", - "FNVETC | vegetation_type", - "FNSOTC | soil_type", - "FNVMNC | vegetation_greenness", - "FNVMXC | vegetation_greenness", - "FNSLPC | slope_type", - "FNABSC | maximum_snow_albedo" - ] - # #----------------------------------------------------------------------- # @@ -34,150 +7,67 @@ fixed_files: # #----------------------------------------------------------------------- # - SFC_CLIMO_FIELDS: [ - "facsf", - "maximum_snow_albedo", - "slope_type", - "snowfree_albedo", - "soil_type", - "substrate_temperature", - "vegetation_greenness", - "vegetation_type", - ] + SFC_CLIMO_FIELDS: + - facsf + - maximum_snow_albedo + - slope_type + - snowfree_albedo + - soil_type + - substrate_temperature + - vegetation_greenness + - vegetation_type - # - #----------------------------------------------------------------------- - # - # FNGLAC, ..., FNMSKH: - # Names of (some of the) global data files that are assumed to exist in - # a system directory specified (this directory is machine-dependent; - # the experiment generation scripts will set it and store it in the - # variable FIXgsm). These file names also appear directly in the forecast - # model's input namelist file. - # - #----------------------------------------------------------------------- - # - FNGLAC: &FNGLAC "global_glacier.2x2.grb" - FNMXIC: &FNMXIC "global_maxice.2x2.grb" - FNTSFC: &FNTSFC "RTGSST.1982.2012.monthly.clim.grb" - FNSNOC: &FNSNOC "global_snoclim.1.875.grb" - FNZORC: &FNZORC "igbp" - FNAISC: &FNAISC "CFSR.SEAICE.1982.2012.monthly.clim.grb" - FNSMCC: &FNSMCC "global_soilmgldas.t126.384.190.grb" - FNMSKH: &FNMSKH "seaice_newland.grb" # #----------------------------------------------------------------------- # # FIXgsm_FILES_TO_COPY_TO_FIXam: # If not running in NCO mode, this array contains the names of the files # to copy from the FIXgsm system directory to the FIXam directory under - # the experiment directory. Note that the last element has a dummy value. - # This last element will get reset by the workflow generation scripts to - # the name of the ozone production/loss file to copy from FIXgsm. The - # name of this file depends on the ozone parameterization being used, - # and that in turn depends on the CCPP physics suite specified for the - # experiment. Thus, the CCPP physics suite XML must first be read in to - # determine the ozone parameterizaton and then the name of the ozone - # production/loss file. These steps are carried out elsewhere (in one - # of the workflow generation scripts/functions). + # the experiment directory. + # The workflow generation step may add the name of the ozone + # production/loss file to copy from FIXgsm. The name of this file + # depends on the ozone parameterization being used, and that in turn + # depends on the CCPP physics suite specified for the experiment. Thus, + # the CCPP physics suite XML must first be read in to determine the + # ozone parameterizaton and then the name of the ozone production/loss + # file. These steps are carried out elsewhere (in one of the workflow + # generation scripts/functions). # #----------------------------------------------------------------------- # - FIXgsm_FILES_TO_COPY_TO_FIXam: [ - *FNGLAC, - *FNMXIC, - *FNTSFC, - *FNSNOC, - *FNAISC, - *FNSMCC, - *FNMSKH, - "global_climaeropac_global.txt", - "fix_co2_proj/global_co2historicaldata_2010.txt", - "fix_co2_proj/global_co2historicaldata_2011.txt", - "fix_co2_proj/global_co2historicaldata_2012.txt", - "fix_co2_proj/global_co2historicaldata_2013.txt", - "fix_co2_proj/global_co2historicaldata_2014.txt", - "fix_co2_proj/global_co2historicaldata_2015.txt", - "fix_co2_proj/global_co2historicaldata_2016.txt", - "fix_co2_proj/global_co2historicaldata_2017.txt", - "fix_co2_proj/global_co2historicaldata_2018.txt", - "fix_co2_proj/global_co2historicaldata_2019.txt", - "fix_co2_proj/global_co2historicaldata_2020.txt", - "fix_co2_proj/global_co2historicaldata_2021.txt", - "global_co2historicaldata_glob.txt", - "co2monthlycyc.txt", - "global_h2o_pltc.f77", - "global_hyblev.l65.txt", - "global_zorclim.1x1.grb", - "global_sfc_emissivity_idx.txt", - "global_tg3clim.2.6x1.5.grb", - "global_solarconstant_noaa_an.txt", - "global_albedo4.1x1.grb", - "geo_em.d01.lat-lon.2.5m.HGT_M.nc", - "HGT.Beljaars_filtered.lat-lon.30s_res.nc", - "replace_with_FIXgsm_ozone_prodloss_filename" - ] - # - #----------------------------------------------------------------------- - # - # FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING: - # This array is used to set some of the namelist variables in the forecast - # model's namelist file that represent the relative or absolute paths of - # various fixed files (the first column of the array, where columns are - # delineated by the pipe symbol "|") to the full paths to these files in - # the FIXam directory derived from the corresponding workflow variables - # containing file names (the second column of the array). - # - #----------------------------------------------------------------------- - # - FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING: [ - !join_str ["FNGLAC | ",*FNGLAC], - !join_str ["FNMXIC | ",*FNMXIC], - !join_str ["FNTSFC | ",*FNTSFC], - !join_str ["FNSNOC | ",*FNSNOC], - !join_str ["FNAISC | ",*FNAISC], - !join_str ["FNSMCC | ",*FNSMCC], - !join_str ["FNMSKH | ",*FNMSKH] - ] - - # - #----------------------------------------------------------------------- - # - # CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING: - # This array specifies the mapping to use between the symlinks that need - # to be created in each cycle directory (these are the "files" that FV3 - # looks for) and their targets in the FIXam directory. The first column - # of the array specifies the symlink to be created, and the second column - # specifies its target file in FIXam (where columns are delineated by the - # pipe symbol "|"). - # - #----------------------------------------------------------------------- - # - CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING: [ - "aerosol.dat | global_climaeropac_global.txt", - "co2historicaldata_2010.txt | fix_co2_proj/global_co2historicaldata_2010.txt", - "co2historicaldata_2011.txt | fix_co2_proj/global_co2historicaldata_2011.txt", - "co2historicaldata_2012.txt | fix_co2_proj/global_co2historicaldata_2012.txt", - "co2historicaldata_2013.txt | fix_co2_proj/global_co2historicaldata_2013.txt", - "co2historicaldata_2014.txt | fix_co2_proj/global_co2historicaldata_2014.txt", - "co2historicaldata_2015.txt | fix_co2_proj/global_co2historicaldata_2015.txt", - "co2historicaldata_2016.txt | fix_co2_proj/global_co2historicaldata_2016.txt", - "co2historicaldata_2017.txt | fix_co2_proj/global_co2historicaldata_2017.txt", - "co2historicaldata_2018.txt | fix_co2_proj/global_co2historicaldata_2018.txt", - "co2historicaldata_2019.txt | fix_co2_proj/global_co2historicaldata_2019.txt", - "co2historicaldata_2020.txt | fix_co2_proj/global_co2historicaldata_2020.txt", - "co2historicaldata_2021.txt | fix_co2_proj/global_co2historicaldata_2021.txt", - "co2historicaldata_glob.txt | global_co2historicaldata_glob.txt", - "co2monthlycyc.txt | co2monthlycyc.txt", - "global_h2oprdlos.f77 | global_h2o_pltc.f77", - "global_albedo4.1x1.grb | global_albedo4.1x1.grb", - "global_zorclim.1x1.grb | global_zorclim.1x1.grb", - "global_tg3clim.2.6x1.5.grb | global_tg3clim.2.6x1.5.grb", - "sfc_emissivity_idx.txt | global_sfc_emissivity_idx.txt", - "solarconstant_noaa_an.txt | global_solarconstant_noaa_an.txt", - "global_o3prdlos.f77 | ozprdlos_2015_new_sbuvO3_tclm15_nuchem.f77" - ] - + FIXgsm_FILES_TO_COPY_TO_FIXam: + - '{{ fixed_files.FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING.FNGLAC }}' + - '{{ fixed_files.FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING.FNMXIC }}' + - '{{ fixed_files.FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING.FNTSFC }}' + - '{{ fixed_files.FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING.FNSNOC }}' + - '{{ fixed_files.FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING.FNZORC }}' + - '{{ fixed_files.FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING.FNAISC }}' + - '{{ fixed_files.FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING.FNSMCC }}' + - '{{ fixed_files.FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING.FNMSKH }}' + - global_climaeropac_global.txt + - fix_co2_proj/global_co2historicaldata_2010.txt + - fix_co2_proj/global_co2historicaldata_2011.txt + - fix_co2_proj/global_co2historicaldata_2012.txt + - fix_co2_proj/global_co2historicaldata_2013.txt + - fix_co2_proj/global_co2historicaldata_2014.txt + - fix_co2_proj/global_co2historicaldata_2015.txt + - fix_co2_proj/global_co2historicaldata_2016.txt + - fix_co2_proj/global_co2historicaldata_2017.txt + - fix_co2_proj/global_co2historicaldata_2018.txt + - fix_co2_proj/global_co2historicaldata_2019.txt + - fix_co2_proj/global_co2historicaldata_2020.txt + - fix_co2_proj/global_co2historicaldata_2021.txt + - global_co2historicaldata_glob.txt + - co2monthlycyc.txt + - global_h2o_pltc.f77 + - global_hyblev.l65.txt + - global_zorclim.1x1.grb + - global_sfc_emissivity_idx.txt + - global_tg3clim.2.6x1.5.grb + - global_solarconstant_noaa_an.txt + - global_albedo4.1x1.grb + - geo_em.d01.lat-lon.2.5m.HGT_M.nc + - HGT.Beljaars_filtered.lat-lon.30s_res.nc # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 261bbf07cc..f71727f741 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -429,21 +429,22 @@ else relative_link_flag="FALSE" fi -regex_search="^[ ]*([^| ]+)[ ]*[|][ ]*([^| ]+)[ ]*$" -num_symlinks=${#CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING[@]} -for (( i=0; i<${num_symlinks}; i++ )); do - - mapping="${CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING[$i]}" - symlink=$( printf "%s\n" "$mapping" | \ - $SED -n -r -e "s/${regex_search}/\1/p" ) - target=$( printf "%s\n" "$mapping" | \ - $SED -n -r -e "s/${regex_search}/\2/p" ) - - symlink="${DATA}/$symlink" - target="$FIXam/$target" - create_symlink_to_file $target $symlink ${relative_link_flag} - -done +# CRH added these to the config_defaults files_to_link section. +#regex_search="^[ ]*([^| ]+)[ ]*[|][ ]*([^| ]+)[ ]*$" +#num_symlinks=${#CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING[@]} +#for (( i=0; i<${num_symlinks}; i++ )); do +# +# mapping="${CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING[$i]}" +# symlink=$( printf "%s\n" "$mapping" | \ +# $SED -n -r -e "s/${regex_search}/\1/p" ) +# target=$( printf "%s\n" "$mapping" | \ +# $SED -n -r -e "s/${regex_search}/\2/p" ) +# +# symlink="${DATA}/$symlink" +# target="$FIXam/$target" +# create_symlink_to_file $target $symlink ${relative_link_flag} +# +#done # #----------------------------------------------------------------------- # diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index bbd5166d37..0bedb3caec 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -565,13 +565,7 @@ workflow: # a "ESGgrid" type of regional grid. # # FV3_NML_FN: - # Name of the forecast model's namelist file. It includes the information in - # FV3_NML_BASE_SUITE_FN (i.e., input.nml.FV3), FV3_NML_YAML_CONFIG_FN (i.e., FV3.input.yml), and the user configuration file (i.e., config.yaml). - # - # FV3_NML_BASE_SUITE_FN: - # Name of Fortran namelist file containing the forecast model's base suite - # namelist, i.e. the portion of the namelist that is common to all physics - # suites. + # Name of the forecast model's namelist file. # # FV3_NML_YAML_CONFIG_FN: # Name of YAML configuration file containing the forecast model's namelist @@ -654,9 +648,6 @@ workflow: # #----------------------------------------------------------------------- - # FV3_NML_BASE_SUITE_FP: - # Path to the FV3_NML_BASE_SUITE_FN file. - # # FV3_NML_YAML_CONFIG_FP: # Path to the FV3_NML_YAML_CONFIG_FN file. # @@ -684,7 +675,6 @@ workflow: #----------------------------------------------------------------------- # - FV3_NML_BASE_SUITE_FP: '{{ user.PARMdir }}/{{ workflow.FV3_NML_BASE_SUITE_FN }}' FV3_NML_YAML_CONFIG_FP: '{{ user.PARMdir }}/{{ workflow.FV3_NML_YAML_CONFIG_FN }}' FV3_NML_BASE_ENS_FP: '{{ workflow.EXPTDIR }}/{{ workflow.FV3_NML_BASE_ENS_FN }}' DATA_TABLE_TMPL_FP: '{{ user.PARMdir }}/{{ workflow.DATA_TABLE_FN }}' @@ -1777,8 +1767,28 @@ task_run_fcst: INPUT/gfs_data.nc: '{{ task_run_fcst.fv.rundir }}/INPUT/{{ nco.NET_default }}.{{ cycle }}{{ "dot_ensmem" | env }}.gfs_data.tile{{ constants.TILE_RGNL }}.halo{{ constants.NH0 }}.nc' INPUT/sfc_data.nc: '{{ task_run_fcst.fv.rundir }}/INPUT/{{ nco.NET_default }}.{{ cycle }}{{ "dot_ensmem" | env }}.sfc_data.tile{{ constants.TILE_RGNL }}.halo{{ constants.NH0 }}.nc' files_to_link: - co2historicaldata_2010.txt: src/uwtools/drivers/global_co2historicaldata_2010.txt - co2historicaldata_2011.txt: src/uwtools/drivers/global_co2historicaldata_2011.txt + aerosol.dat: '{{ workflow.FIXam }}/global_climaeropac_global.txt' + co2historicaldata_2010.txt: '{{ workflow.FIXam }}/fix_co2_proj/global_co2historicaldata_2010.txt' + co2historicaldata_2011.txt: '{{ workflow.FIXam }}/fix_co2_proj/global_co2historicaldata_2011.txt' + co2historicaldata_2012.txt: '{{ workflow.FIXam }}/fix_co2_proj/global_co2historicaldata_2012.txt' + co2historicaldata_2013.txt: '{{ workflow.FIXam }}/fix_co2_proj/global_co2historicaldata_2013.txt' + co2historicaldata_2014.txt: '{{ workflow.FIXam }}/fix_co2_proj/global_co2historicaldata_2014.txt' + co2historicaldata_2015.txt: '{{ workflow.FIXam }}/fix_co2_proj/global_co2historicaldata_2015.txt' + co2historicaldata_2016.txt: '{{ workflow.FIXam }}/fix_co2_proj/global_co2historicaldata_2016.txt' + co2historicaldata_2017.txt: '{{ workflow.FIXam }}/fix_co2_proj/global_co2historicaldata_2017.txt' + co2historicaldata_2018.txt: '{{ workflow.FIXam }}/fix_co2_proj/global_co2historicaldata_2018.txt' + co2historicaldata_2019.txt: '{{ workflow.FIXam }}/fix_co2_proj/global_co2historicaldata_2019.txt' + co2historicaldata_2020.txt: '{{ workflow.FIXam }}/fix_co2_proj/global_co2historicaldata_2020.txt' + co2historicaldata_2021.txt: '{{ workflow.FIXam }}/fix_co2_proj/global_co2historicaldata_2021.txt' + co2historicaldata_glob.txt: '{{ workflow.FIXam }}/global_co2historicaldata_glob.txt' + co2monthlycyc.txt: '{{ workflow.FIXam }}/co2monthlycyc.txt' + global_h2oprdlos.f77: '{{ workflow.FIXam }}/global_h2o_pltc.f77' + global_albedo4.1x1.grb: '{{ workflow.FIXam }}/global_albedo4.1x1.grb' + global_zorclim.1x1.grb: '{{ workflow.FIXam }}/global_zorclim.1x1.grb' + global_tg3clim.2.6x1.5.grb: '{{ workflow.FIXam }}/global_tg3clim.2.6x1.5.grb' + sfc_emissivity_idx.txt: '{{ workflow.FIXam }}/global_sfc_emissivity_idx.txt' + solarconstant_noaa_an.txt: '{{ workflow.FIXam }}/global_solarconstant_noaa_an.txt' + global_o3prdlos.f77: '{{ workflow.FIXam }}/ozprdlos_2015_new_sbuvO3_tclm15_nuchem.f77' lateral_boundary_conditions: interval_hours: 1 offset: 3 @@ -1821,7 +1831,7 @@ task_run_fcst: nsout: -1 output_grid: '{{ task_run_fcst.WRTCMP_output_grid }}' namelist: - base_file: "{{ workflow.FV3_NML_BASE_SUITE_FP }}" + base_file: "{{ user.PARMdir }}/input.nml.FV3" update_values: atmos_model_nml: blocksize: !int {{ task_run_fcst.BLOCKSIZE }} @@ -1840,6 +1850,27 @@ task_run_fcst: gfs_physics_nml: kice: !int {{ 9 if workflow.SDF_USES_RUC_LSM else 2 }} print_diff_pgr: false + namsfc: + # From sfc_climo_gen + fnalbc: '{{ workflow.FIXlam }}/{{ "CRES" | env }}.snowfree_albedo.tileX.nc' + fnalbc2: '{{ workflow.FIXlam }}/{{ "CRES" | env }}.facsf.tileX.nc' + fntg3c: '{{ workflow.FIXlam }}/{{ "CRES" | env }}.substrate_temperature.tileX.nc' + fnvegc: '{{ workflow.FIXlam }}/{{ "CRES" | env }}.vegetation_greenness.tileX.nc' + fnvetc: '{{ workflow.FIXlam }}/{{ "CRES" | env }}.vegetation_type.tileX.nc' + fnsotc: '{{ workflow.FIXlam }}/{{ "CRES" | env }}.soil_type.tileX.nc' + fnvmnc: '{{ workflow.FIXlam }}/{{ "CRES" | env }}.vegetation_greenness.tileX.nc' + fnvmxc: '{{ workflow.FIXlam }}/{{ "CRES" | env }}.vegetation_greenness.tileX.nc' + fnslpc: '{{ workflow.FIXlam }}/{{ "CRES" | env }}.slope_type.tileX.nc' + fnabsc: '{{ workflow.FIXlam }}/{{ "CRES" | env }}.maximum_snow_albedo.tileX.nc' + # General fix files + fnglac: '{{ workflow.FIXlam }}/global_glacier.2x2.grb' + fnmxic: '{{ workflow.FIXlam }}/global_maxice.2x2.grb' + fntsfc: '{{ workflow.FIXlam }}/RTGSST.1982.2012.monthly.clim.grb' + fnsnoc: '{{ workflow.FIXlam }}/global_snoclim.1.875.grb' + fnzorc: '{{ workflow.FIXlam }}/igbp' + fnaisc: '{{ workflow.FIXlam }}/CFSR.SEAICE.1982.2012.monthly.clim.grb' + fnsmcc: '{{ workflow.FIXlam }}/global_soilmgldas.t126.384.190.grb' + fnmskh: '{{ workflow.FIXlam }}/seaice_newland.grb' validate: true rundir: '{{ workflow.EXPTDIR }}/run_fcst' #------------------------------------------------------------------------ diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 0e24427835..efa78f087b 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -290,100 +290,7 @@ def generate_FV3LAM_wflow( verbose=debug, ) cp_vrfy(FIELD_DICT_IN_UWM_FP, FIELD_DICT_FP) - # - # ----------------------------------------------------------------------- - # - # Set parameters in the FV3-LAM namelist file. - # - # ----------------------------------------------------------------------- - # - log_info( - f""" - Setting parameters in weather model's namelist file (FV3_NML_FP): - FV3_NML_FP = '{FV3_NML_FP}'""", - verbose=debug, - ) - # - # Set npx and npy, which are just NX plus 1 and NY plus 1, respectively. - # These need to be set in the FV3-LAM Fortran namelist file. They represent - # the number of cell vertices in the x and y directions on the regional - # grid. - # - npx = NX + 1 - npy = NY + 1 - # - # For the physics suites that use RUC LSM, set the parameter kice to 9, - # Otherwise, leave it unspecified (which means it gets set to the default - # value in the forecast model). - # - kice = None - if SDF_USES_RUC_LSM: - kice = 9 - # - # Set lsoil, which is the number of input soil levels provided in the - # chgres_cube output NetCDF file. This is the same as the parameter - # nsoill_out in the namelist file for chgres_cube. [On the other hand, - # the parameter lsoil_lsm (not set here but set in input.nml.FV3 and/or - # FV3.input.yml) is the number of soil levels that the LSM scheme in the - # forecast model will run with.] Here, we use the same approach to set - # lsoil as the one used to set nsoill_out in exregional_make_ics.sh. - # See that script for details. - # - # NOTE: - # May want to remove lsoil from FV3.input.yml (and maybe input.nml.FV3). - # Also, may want to set lsm here as well depending on SDF_USES_RUC_LSM. - # - lsoil = 4 - if EXTRN_MDL_NAME_ICS in ("HRRR", "RAP") and SDF_USES_RUC_LSM: - lsoil = 9 - if CCPP_PHYS_SUITE == "FV3_GFS_v15_thompson_mynn_lam3km": - lsoil = "" - - # - # Add to "settings" the values of those namelist variables that specify - # the paths to fixed files in the FIXam directory. As above, these namelist - # variables are physcs-suite-independent. - # - # Note that the array FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING contains - # the mapping between the namelist variables and the names of the files - # in the FIXam directory. Here, we loop through this array and process - # each element to construct each line of "settings". - # - dummy_run_dir = os.path.join(EXPTDIR, "any_cyc") - if DO_ENSEMBLE: - dummy_run_dir = os.path.join(dummy_run_dir, "any_ensmem") - - regex_search = "^[ ]*([^| ]+)[ ]*[|][ ]*([^| ]+)[ ]*$" - num_nml_vars = len(FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING) - namsfc_dict = {} - settings = {} - for i in range(num_nml_vars): - - mapping = f"{FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING[i]}" - tup = find_pattern_in_str(regex_search, mapping) - nml_var_name = tup[0] - FIXam_fn = tup[1] - - fp = '""' - if FIXam_fn: - fp = os.path.join(FIXam, FIXam_fn) - # - # If not in NCO mode, for portability and brevity, change fp so that it - # is a relative path (relative to any cycle directory immediately under - # the experiment directory). - # - if RUN_ENVIR != "nco": - fp = os.path.relpath(os.path.realpath(fp), start=dummy_run_dir) - # - # Add a line to the variable "settings" that specifies (in a yaml-compliant - # format) the name of the current namelist variable and the value it should - # be set to. - # - namsfc_dict[nml_var_name] = fp - # - # Add namsfc_dict to settings - # - settings["namsfc"] = namsfc_dict + # # Use netCDF4 when running the North American 3-km domain due to file size. # diff --git a/ush/set_fv3nml_sfc_climo_filenames.py b/ush/set_fv3nml_sfc_climo_filenames.py deleted file mode 100644 index 90d686235d..0000000000 --- a/ush/set_fv3nml_sfc_climo_filenames.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env python3 - -""" -Update filenames for surface climotology files in the namelist. -""" - -import argparse -import os -import re -import sys -from textwrap import dedent - -from python_utils import ( - cfg_to_yaml_str, - check_var_valid_value, - flatten_dict, - import_vars, - print_info_msg, -) - -from uwtools.api.config import get_nml_config, get_yaml_config, realize - - -VERBOSE = os.environ.get("VERBOSE", "true") - -NEEDED_VARS = [ - "CRES", - "DO_ENSEMBLE", - "EXPTDIR", - "FIXlam", - "FV3_NML_FP", - "PARMdir", - "RUN_ENVIR", - ] - - -# pylint: disable=undefined-variable - -def set_fv3nml_sfc_climo_filenames(config, debug=False): - """ - This function sets the values of the variables in - the forecast model's namelist file that specify the paths to the surface - climatology files on the FV3LAM native grid (which are either pregenerated - or created by the TN_MAKE_SFC_CLIMO task). Note that the workflow - generation scripts create symlinks to these surface climatology files - in the FIXlam directory, and the values in the namelist file that get - set by this function are relative or full paths to these links. - - Args: - debug (bool): Enable extra output for debugging - Returns: - None - """ - - import_vars(dictionary=config, env_vars=NEEDED_VARS) - - fixed_cfg = get_yaml_config(os.path.join(PARMdir, "fixed_files_mapping.yaml"))["fixed_files"] - - # The regular expression regex_search set below will be used to extract - # from the elements of the array FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING - # the name of the namelist variable to set and the corresponding surface - # climatology field from which to form the name of the surface climatology file - regex_search = "^[ ]*([^| ]+)[ ]*[|][ ]*([^| ]+)[ ]*$" - - # Set the suffix of the surface climatology files. - suffix = "tileX.nc" - - # create yaml-compliant string - settings = {} - - dummy_run_dir = os.path.join(EXPTDIR, "any_cyc") - if DO_ENSEMBLE == "TRUE": - dummy_run_dir += os.sep + "any_ensmem" - - namsfc_dict = {} - for mapping in fixed_cfg["FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING"]: - nml_var_name, sfc_climo_field_name = re.search(regex_search, mapping).groups() - - check_var_valid_value(sfc_climo_field_name, fixed_cfg["SFC_CLIMO_FIELDS"]) - - file_path = os.path.join(FIXlam, f"{CRES}.{sfc_climo_field_name}.{suffix}") - if RUN_ENVIR != "nco": - file_path = os.path.relpath(os.path.realpath(file_path), start=dummy_run_dir) - - namsfc_dict[nml_var_name] = file_path - - settings["namsfc_dict"] = namsfc_dict - settings_str = cfg_to_yaml_str(settings) - - print_info_msg( - dedent( - f""" - The variable 'settings' specifying values of the namelist variables - has been set as follows:\n - settings = - - {settings_str} - """ - ), - verbose=debug, - ) - - realize( - input_config=FV3_NML_FP, - input_format="nml", - output_file=FV3_NML_FP, - output_format="nml", - update_config=get_nml_config(settings), - ) - -def parse_args(argv): - """Parse command line arguments""" - parser = argparse.ArgumentParser(description="Set surface climatology fields.") - - parser.add_argument( - "-p", - "--path-to-defns", - dest="path_to_defns", - required=True, - help="Path to var_defns file.", - ) - parser.add_argument('-d', '--debug', action='store_true', - help='Script will be run in debug mode with more verbose output') - - return parser.parse_args(argv) - - -if __name__ == "__main__": - args = parse_args(sys.argv[1:]) - cfg = get_yaml_config(args.path_to_defns) - cfg = flatten_dict(cfg) - set_fv3nml_sfc_climo_filenames(cfg, args.debug) diff --git a/ush/setup.py b/ush/setup.py index e17e8d750e..fcabdc7b10 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -1377,27 +1377,20 @@ def dict_find(user_dict, substring): workflow_config["SDF_USES_THOMPSON_MP"] = has_tag_with_value(ccpp_suite_xml, "scheme", "mp_thompson") if workflow_config["SDF_USES_THOMPSON_MP"]: - logging.debug(f'Selected CCPP suite ({workflow_config["CCPP_PHYS_SUITE"]}) uses Thompson MP') logging.debug(f'Setting up links for additional fix files') # If the model ICs or BCs are not from RAP or HRRR, they will not contain aerosol # climatology data needed by the Thompson scheme, so we need to provide a separate file + thompson_files = fixed_files["THOMPSON_FIX_FILES"] if (get_extrn_ics["EXTRN_MDL_NAME_ICS"] not in ["HRRR", "RAP"] or get_extrn_lbcs["EXTRN_MDL_NAME_LBCS"] not in ["HRRR", "RAP"]): - fixed_files["THOMPSON_FIX_FILES"].append(workflow_config["THOMPSON_MP_CLIMO_FN"]) - - # Add thompson-specific fix files to CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING and - # FIXgsm_FILES_TO_COPY_TO_FIXam; see parm/fixed_files_mapping.yaml for more info on these variables - - fixed_files["FIXgsm_FILES_TO_COPY_TO_FIXam"].extend(fixed_files["THOMPSON_FIX_FILES"]) - - for fix_file in fixed_files["THOMPSON_FIX_FILES"]: - fixed_files["CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING"].append(f"{fix_file} | {fix_file}") - - logging.debug(f'New fix file list:\n{fixed_files["FIXgsm_FILES_TO_COPY_TO_FIXam"]=}') - logging.debug(f'New fix file mapping:\n{fixed_files["CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING"]=}') + thompson_files.append(workflow_config["THOMPSON_MP_CLIMO_FN"]) + # Add thompson-specific fix files to the FV3 configuration + fixam = workflow_config["FIXam"] + thompson_fix_links = {fn: f"{fixam}/{fn}" in thompson_files} + expt_config["task_run_fcst"]["fv3"]["files_to_link"].update(thompson_fix_links) # # ----------------------------------------------------------------------- From caec46967958b5f1997e3fb0e942941bf38ef917 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Wed, 30 Oct 2024 16:37:53 +0000 Subject: [PATCH 11/18] Adding a quilting YAML. --- ush/quilting.yaml | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 ush/quilting.yaml diff --git a/ush/quilting.yaml b/ush/quilting.yaml new file mode 100644 index 0000000000..bf54aff728 --- /dev/null +++ b/ush/quilting.yaml @@ -0,0 +1,46 @@ + + +no_quilting: + task_run_fcst: + PE_MEMBER01: !int {{ task_run_fcst.LAYOUT_Y * task_run_fcst.LAYOUT_X }} + model_configure: + update_values: + write_groups: !remove + write_tasks_per_group: !remove + num_files: !remove + filename_base: !remove + output_file: !remove + output_fh: !remove + nsout: !remove + output_grid: !remove + +lambert_conformal: + task_run_fcst: + model_configure: + update_values: + cen_lon: '{{ task_run_fcst.WRTCMP_cen_lon }}' + cen_lat: '{{ task_run_fcst.WRTCMP_cen_lat }}' + stdlat1: '{{ task_run_fcst.WRTCMP_stdlat1 }}' + stdlat2: '{{ task_run_fcst.WRTCMP_stdlat2 }}' + nx: '{{ task_run_fcst.WRTCMP_nx }}' + ny: '{{ task_run_fcst.WRTCMP_ny }}' + lon1: '{{ task_run_fcst.WRTCMP_lon_lwr_left }}' + lat1: '{{ task_run_fcst.WRTCMP_lat_lwr_left }}' + dx: '{{ task_run_fcst.WRTCMP_dx }}' + dy: '{{ task_run_fcst.WRTCMP_dy }}' + +regional_latlon: + task_run_fcst: &latlon_reference + model_configure: + update_values: + cen_lon: '{{ task_run_fcst.WRTCMP_cen_lon }}' + cen_lat: '{{ task_run_fcst.WRTCMP_cen_lat }}' + lon1: '{{ task_run_fcst.WRTCMP_lon_lwr_left }}' + lat1: '{{ task_run_fcst.WRTCMP_lat_lwr_left }}' + lon2: '{{ task_run_fcst.WRTCMP_lon_upr_rght }}' + lat2: '{{ task_run_fcst.WRTCMP_lat_upr_rght }}' + dlon: '{{ task_run_fcst.WRTCMP_dlon }}' + dlat: '{{ task_run_fcst.WRTCMP_dlat }}' + +rotated_latlon: + <<: * latlon_reference From 2e4eec1a91adc24ef867bd02834f3b262135b296 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Wed, 30 Oct 2024 22:12:35 +0000 Subject: [PATCH 12/18] Light testing fixes. Fixing stochastic namelist settings. --- ush/ccpp_suites_defaults.yaml | 52 ++++++++++- ush/config_defaults.yaml | 101 ++++---------------- ush/generate_FV3LAM_wflow.py | 141 +--------------------------- ush/quilting.yaml | 4 +- ush/setup.py | 169 ++++++++++++---------------------- ush/stochastic_params.yaml | 112 ++++++++++++++++++++++ 6 files changed, 243 insertions(+), 336 deletions(-) create mode 100644 ush/stochastic_params.yaml diff --git a/ush/ccpp_suites_defaults.yaml b/ush/ccpp_suites_defaults.yaml index a21b879258..a0f2ce0dae 100644 --- a/ush/ccpp_suites_defaults.yaml +++ b/ush/ccpp_suites_defaults.yaml @@ -18,6 +18,56 @@ orog_gsl_defaults: &orog_gsl_defaults - module load build_{{ user.MACHINE|lower }}_{{ workflow.COMPILER }} rundir: "{{ task_make_orog.rundir }}/orog_gsl" +fv3_nml_gfs_gfdl_cloud_mp_defaults: + gfdl_cloud_microphysics_nml: &gfs_gfdl_cloud_mp + c_cracw: 0.8 + c_paut: 0.5 + c_pgacs: 0.01 + c_psaci: 0.05 + ccn_l: 300.0 + ccn_o: 100.0 + const_vg: False + const_vi: False + const_vr: False + const_vs: False + de_ice: False + do_qa: True + do_sedi_heat: False + dw_land: 0.16 + dw_ocean: 0.1 + fast_sat_adj: True + fix_negative: True + icloud_f: 1 + mono_prof: True + mp_time: 90.0 + prog_ccn: False + qi0_crt: 8.0e-05 + qi_lim: 1.0 + ql_gen: 0.001 + ql_mlt: 0.001 + qs0_crt: 0.001 + rad_graupel: True + rad_rain: True + rad_snow: True + rh_inc: 0.3 + rh_inr: 0.3 + rh_ins: 0.3 + rthresh: 1.0e-05 + sedi_transport: False + tau_g2v: 900.0 + tau_i2s: 1000.0 + tau_l2v: 180.0 + tau_v2l: 90.0 + use_ccn: True + use_ppm: False + vg_max: 12.0 + vi_max: 1.0 + vr_max: 12.0 + vs_max: 2.0 + z_slope_ice: True + z_slope_liq: True + + FV3_GFS_v15p2: task_run_fcst: namelist: @@ -388,7 +438,7 @@ FV3_HRRR: isncond_opt: 2 isncovr_opt: 3 lsm: 3 - lsoil: !int {{ 9 if task_get_extrn_ics.EXTRN_MDL_NAME_ICS in ("RAP", "HRRR") else 4 }} + lsoil: !int '{{ 9 if task_get_extrn_ics.EXTRN_MDL_NAME_ICS in ("RAP", "HRRR") else 4 }}' lsoil_lsm: 9 mosaic_lu: 0 mosaic_soil: 0 diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 0bedb3caec..976c079c01 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1822,8 +1822,8 @@ task_run_fcst: kchunk3d: -1 itasks: 1 quilting: true - write_groups: !int {{ task_run_fcst.WRTCMP_write_groups }} - write_tasks_per_group: !int {{ task_run_fcst.WRTCMP_write_tasks_per_group }} + write_groups: !int '{{ task_run_fcst.WRTCMP_write_groups }}' + write_tasks_per_group: !int '{{ task_run_fcst.WRTCMP_write_tasks_per_group }}' num_files: 2 filename_base: 'dyn phy' output_file: 'netcdf netcdf' @@ -1834,21 +1834,21 @@ task_run_fcst: base_file: "{{ user.PARMdir }}/input.nml.FV3" update_values: atmos_model_nml: - blocksize: !int {{ task_run_fcst.BLOCKSIZE }} + blocksize: !int '{{ task_run_fcst.BLOCKSIZE }}' ccpp_suite: '{{ workflow.CCPP_PHYS_SUITE }}' fv_core_nml: - target_lon: !int {{ grid_params.LON_CTR }} - target_lat: !int {{ grid_params.LAT_CTR }} - nrows_blend: !int {{ grid_params.HALO_BLEND }} - stretch_fac: !int {{ grid_params.STRETCH_FAC }} - npx: !int {{ grid_params.NX + 1 }} - npy: !int {{ grid_params.NY + 1 }} + target_lon: !int '{{ grid_params.LON_CTR }}' + target_lat: !int '{{ grid_params.LAT_CTR }}' + nrows_blend: !int '{{ grid_params.HALO_BLEND }}' + stretch_fac: !int '{{ grid_params.STRETCH_FAC }}' + npx: !int '{{ grid_params.NX + 1 }}' + npy: !int '{{ grid_params.NY + 1 }}' layout: - - !int {{ task_run_fcst.LAYOUT_X }} - - !int {{ task_run_fcst.LAYOUT_Y }} - bc_update_interval: !int {{ task_get_extrn_lbcs.LBC_SPEC_INTVL_HRS }} + - !int '{{ task_run_fcst.LAYOUT_X }}' + - !int '{{ task_run_fcst.LAYOUT_Y }}' + bc_update_interval: !int '{{ task_get_extrn_lbcs.LBC_SPEC_INTVL_HRS }}' gfs_physics_nml: - kice: !int {{ 9 if workflow.SDF_USES_RUC_LSM else 2 }} + kice: !int '{{ 9 if workflow.SDF_USES_RUC_LSM else 2 }}' print_diff_pgr: false namsfc: # From sfc_climo_gen @@ -2411,65 +2411,16 @@ global: # #----------------------------------------------------------------------- # - # Set default ad-hoc stochastic physics options. - # For detailed documentation of these parameters, see: - # https://stochastic-physics.readthedocs.io/en/ufs_public_release/namelist_options.html + # Switches for using ad-hoc stochastic physics options. + # + # For specific namelist settings associated with these switches, see + # stochastic_params.yaml # #----------------------------------------------------------------------- # DO_SHUM: false DO_SPPT: false DO_SKEB: false - ISEED_SPPT: 1 - ISEED_SHUM: 2 - ISEED_SKEB: 3 - NEW_LSCALE: true - SHUM_MAG: 0.006 #Variable "shum" in input.nml - SHUM_LSCALE: 150000 - SHUM_TSCALE: 21600 #Variable "shum_tau" in input.nml - SHUM_INT: 3600 #Variable "shumint" in input.nml - SPPT_MAG: 0.7 #Variable "sppt" in input.nml - SPPT_LOGIT: true - SPPT_LSCALE: 150000 - SPPT_TSCALE: 21600 #Variable "sppt_tau" in input.nml - SPPT_INT: 3600 #Variable "spptint" in input.nml - SPPT_SFCLIMIT: true - SKEB_MAG: 0.5 #Variable "skeb" in input.nml - SKEB_LSCALE: 150000 - SKEB_TSCALE: 21600 #Variable "skeb_tau" in input.nml - SKEB_INT: 3600 #Variable "skebint" in input.nml - SKEBNORM: 1 - SKEB_VDOF: 10 - USE_ZMTNBLCK: false - # - #----------------------------------------------------------------------- - # - # Set default SPP stochastic physics options. - # SPP perturbs specific tuning parameters within a physics parameterization - # (unlike SPPT, which multiplies overall physics tendencies by a random - # perturbation field *after* the call to the physics suite).Patterns evolve - # and are applied at each time step. Each SPP option is an array, - # applicable (in order) to the HRRR-based parameterization listed in SPP_VAR_LIST. - # Enter each value of the array in config.yaml as shown below without commas - # or single quotes (e.g., SPP_VAR_LIST: [ "pbl" "sfc" "mp" "rad" "gwd" ] ). - # Both commas and single quotes will be added by Jinja when creating the - # namelist. - # - # Note that SPP is currently only available for specific physics schemes - # used in the RAP/HRRR physics suite. Users need to be aware of which SDF - # is chosen when turning this option on. - # - #----------------------------------------------------------------------- - # - DO_SPP: false - SPP_VAR_LIST: [ "pbl", "sfc", "mp", "rad", "gwd" ] - SPP_MAG_LIST: [ 0.2, 0.2, 0.75, 0.2, 0.2 ] #Variable "spp_prt_list" in input.nml - SPP_LSCALE: [ 150000.0, 150000.0, 150000.0, 150000.0, 150000.0 ] - SPP_TSCALE: [ 21600.0, 21600.0, 21600.0, 21600.0, 21600.0 ] #Variable "spp_tau" in input.nml - SPP_SIGTOP1: [ 0.1, 0.1, 0.1, 0.1, 0.1 ] - SPP_SIGTOP2: [ 0.025, 0.025, 0.025, 0.025, 0.025 ] - SPP_STDDEV_CUTOFF: [ 1.5, 1.5, 2.5, 1.5, 1.5 ] - ISEED_SPP: [ 4, 5, 6, 7, 8 ] # #----------------------------------------------------------------------- # @@ -2477,26 +2428,10 @@ global: # Please be aware of the SDF that you choose if you wish to turn on LSM # SPP. # - # SPP in LSM schemes is handled in the &nam_sfcperts namelist block - # instead of in &nam_sppperts, where all other SPP is implemented. - # # Perturbations to soil moisture content (SMC) are only applied at the # first time step. # - # LSM perturbations include SMC - soil moisture content (volume fraction), - # VGF - vegetation fraction, ALB - albedo, SAL - salinity, - # EMI - emissivity, ZOL - surface roughness (cm), and STC - soil temperature. - # - # Only five perturbations at a time can be applied currently, but all seven - # are shown below. In addition, only one unique iseed value is allowed - # at the moment, and is used for each pattern. - # - DO_LSM_SPP: false #If true, sets lndp_type=2 - LSM_SPP_TSCALE: [ 21600, 21600, 21600, 21600, 21600, 21600, 21600 ] - LSM_SPP_LSCALE: [ 150000, 150000, 150000, 150000, 150000, 150000, 150000 ] - ISEED_LSM_SPP: [ 9 ] - LSM_SPP_VAR_LIST: [ "smc", "vgf", "alb", "sal", "emi", "zol", "stc" ] - LSM_SPP_MAG_LIST: [ 0.017, 0.001, 0.001, 0.001, 0.001, 0.001, 0.2 ] + DO_LSM_SPP: false # #----------------------------------------------------------------------- # diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index efa78f087b..0ba32d320c 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -26,17 +26,14 @@ mkdir_vrfy, mv_vrfy, check_for_preexist_dir_file, - cfg_to_yaml_str, - find_pattern_in_str, flatten_dict, ) from check_python_version import check_python_version from get_crontab_contents import add_crontab_line from setup import setup -from set_fv3nml_sfc_climo_filenames import set_fv3nml_sfc_climo_filenames -from uwtools.api.config import get_nml_config, get_yaml_config, realize +from uwtools.api.config import get_yaml_config from uwtools.api.template import render @@ -291,29 +288,6 @@ def generate_FV3LAM_wflow( ) cp_vrfy(FIELD_DICT_IN_UWM_FP, FIELD_DICT_FP) - # - # Use netCDF4 when running the North American 3-km domain due to file size. - # - if PREDEF_GRID_NAME == "RRFS_NA_3km": - settings["fms2_io_nml"] = {"netcdf_default_format": "netcdf4"} - - log_info( - """ - The variable 'settings' specifying values of the weather model's - namelist variables has been set as follows:\n""", - verbose=debug, - ) - log_info("\nsettings =\n\n" + settings_str, verbose=debug) - # - # ----------------------------------------------------------------------- - # - # Update the fv3 namelist config with relevant settings - # - # ----------------------------------------------------------------------- - # - fcst_nml_config = get_yaml_config(expt_config["task_run_fcst"]["namelist"]["update_values"]) - fcst_nml_config.update_from(settings) - # # If not running the TN_MAKE_GRID task (which implies the workflow will # use pregenerated grid files), set the namelist variables specifying @@ -330,119 +304,6 @@ def generate_FV3LAM_wflow( set_fv3nml_sfc_climo_filenames(flatten_dict(expt_config), debug) - # - # ----------------------------------------------------------------------- - # - # Add the relevant tendency-based stochastic physics namelist variables to - # "settings" when running with SPPT, SHUM, or SKEB turned on. If running - # with SPP or LSM SPP, set the "new_lscale" variable. Otherwise only - # include an empty "nam_stochy" stanza. - # - # ----------------------------------------------------------------------- - # - # - #----------------------------------------------------------------------- - # - # Update the stochastic parameters, if needed - # - #----------------------------------------------------------------------- - # - if any((DO_SPP, DO_SPPT, DO_SHUM, DO_SKEB, DO_LSM_SPP)): - settings = {} - settings["gfs_physics_nml"] = { - "do_shum": DO_SHUM, - "do_sppt": DO_SPPT, - "do_skeb": DO_SKEB, - "do_spp": DO_SPP, - "n_var_spp": N_VAR_SPP, - "n_var_lndp": N_VAR_LNDP, - "lndp_type": LNDP_TYPE, - "fhcyc": FHCYC_LSM_SPP_OR_NOT, - } - nam_stochy_dict = {} - if DO_SPPT: - nam_stochy_dict.update( - { - "iseed_sppt": ISEED_SPPT, - "new_lscale": NEW_LSCALE, - "sppt": SPPT_MAG, - "sppt_logit": SPPT_LOGIT, - "sppt_lscale": SPPT_LSCALE, - "sppt_sfclimit": SPPT_SFCLIMIT, - "sppt_tau": SPPT_TSCALE, - "spptint": SPPT_INT, - "use_zmtnblck": USE_ZMTNBLCK, - } - ) - - if DO_SHUM: - nam_stochy_dict.update( - { - "iseed_shum": ISEED_SHUM, - "new_lscale": NEW_LSCALE, - "shum": SHUM_MAG, - "shum_lscale": SHUM_LSCALE, - "shum_tau": SHUM_TSCALE, - "shumint": SHUM_INT, - } - ) - - if DO_SKEB: - nam_stochy_dict.update( - { - "iseed_skeb": ISEED_SKEB, - "new_lscale": NEW_LSCALE, - "skeb": SKEB_MAG, - "skeb_lscale": SKEB_LSCALE, - "skebnorm": SKEBNORM, - "skeb_tau": SKEB_TSCALE, - "skebint": SKEB_INT, - "skeb_vdof": SKEB_VDOF, - } - ) - - if DO_SPP or DO_LSM_SPP: - nam_stochy_dict.update({"new_lscale": NEW_LSCALE}) - - settings["nam_stochy"] = nam_stochy_dict - # - # Add the relevant SPP namelist variables to "settings" when running with - # SPP turned on. Otherwise only include an empty "nam_sppperts" stanza. - # - nam_sppperts_dict = {} - if DO_SPP: - nam_sppperts_dict = { - "iseed_spp": ISEED_SPP, - "spp_lscale": SPP_LSCALE, - "spp_prt_list": SPP_MAG_LIST, - "spp_sigtop1": SPP_SIGTOP1, - "spp_sigtop2": SPP_SIGTOP2, - "spp_stddev_cutoff": SPP_STDDEV_CUTOFF, - "spp_tau": SPP_TSCALE, - "spp_var_list": SPP_VAR_LIST, - } - - settings["nam_sppperts"] = nam_sppperts_dict - # - # Add the relevant LSM SPP namelist variables to "settings" when running with - # LSM SPP turned on. - # - nam_sfcperts_dict = {} - if DO_LSM_SPP: - nam_sfcperts_dict = { - "lndp_type": LNDP_TYPE, - "lndp_model_type": LNDP_MODEL_TYPE, - "lndp_tau": LSM_SPP_TSCALE, - "lndp_lscale": LSM_SPP_LSCALE, - "iseed_lndp": ISEED_LSM_SPP, - "lndp_var_list": LSM_SPP_VAR_LIST, - "lndp_prt_list": LSM_SPP_MAG_LIST, - } - - settings["nam_sfcperts"] = nam_sfcperts_dict - - fcst_nml_config.update_from(settings) - # # ----------------------------------------------------------------------- # diff --git a/ush/quilting.yaml b/ush/quilting.yaml index bf54aff728..ebac607352 100644 --- a/ush/quilting.yaml +++ b/ush/quilting.yaml @@ -2,7 +2,7 @@ no_quilting: task_run_fcst: - PE_MEMBER01: !int {{ task_run_fcst.LAYOUT_Y * task_run_fcst.LAYOUT_X }} + PE_MEMBER01: !int '{{ task_run_fcst.LAYOUT_Y * task_run_fcst.LAYOUT_X }}' model_configure: update_values: write_groups: !remove @@ -43,4 +43,4 @@ regional_latlon: dlat: '{{ task_run_fcst.WRTCMP_dlat }}' rotated_latlon: - <<: * latlon_reference + <<: *latlon_reference diff --git a/ush/setup.py b/ush/setup.py index fcabdc7b10..ada01eebe5 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -227,6 +227,14 @@ def add_jobname(tasks): ccpp_cfg = get_yaml_config(Path(ushdir, "ccpp_suites_defaults.yaml")).get(ccpp_suite, {}) update_dict(ccpp_cfg, cfg_d) + # Load stochastic physics params + stochastic_params = get_yaml_config(Path(ushdir, "stochastic_params.yaml")) + fcst_config = cfg_d["task_run_fcst"]["fv3"] + fcst_nml_config = get_yaml_config(fcst_config["namelist"]["update_values"]) + for switch_name in ("do_spp", "do_sppt", "do_shum", "do_skeb", "do_lsm_spp"): + if (cfg_d["global"][switch_name.upper()]): + fcst_nml_config.update_from(stochastic_params.get(switch_name) + # Set "Home" directory, the top-level ufs-srweather-app directory homedir = os.path.abspath(os.path.dirname(__file__) + os.sep + os.pardir) cfg_d["user"]["HOMEdir"] = homedir @@ -724,7 +732,7 @@ def get_location(xcs, fmt, expt_cfg): quilting = fcst_config["model_configure"]["update_values"]["quilting"] # Gather the pre-defined grid parameters, if needed - if workflow_config.get("PREDEF_GRID_NAME"): + if predef_grid_name := (workflow_config.get("PREDEF_GRID_NAME")): grid_params = set_predef_grid_params( USHdir, workflow_config["PREDEF_GRID_NAME"], @@ -763,6 +771,11 @@ def get_location(xcs, fmt, expt_cfg): else: grid_config[param] = value + # This logic belongs in predef_grid_params.yaml once merged with make_grid integration. + if predef_grid_name == "RRFS_NA_3km": + fv3_namelist = expt_config["task_run_fcst"]["fv3"]["namelist"] + fv3_namlelist["update_values"]["fms2_io_nml"]["netcdf_default_format"] = "netcdf4" + # Load model write component grid settings quilting_cfg = get_yaml_config(Path(USHdir, "quilting.yaml")) if not quilting: @@ -771,6 +784,8 @@ def get_location(xcs, fmt, expt_cfg): write_grid = expt_config["task_run_fcst"]["WRTCMP_output_grid"] update_dict(quilting_cfg[write_grid], expt_config) + + run_envir = expt_config["user"].get("RUN_ENVIR", "") fcst_len_hrs = workflow_config.get("FCST_LEN_HRS") @@ -910,116 +925,54 @@ def get_location(xcs, fmt, expt_cfg): # # ----------------------------------------------------------------------- # - # Set magnitude of stochastic ad-hoc schemes to -999.0 if they are not - # being used. This is required at the moment, since "do_shum/sppt/skeb" - # does not override the use of the scheme unless the magnitude is also - # specifically set to -999.0. If all "do_shum/sppt/skeb" are set to - # "false," then none will run, regardless of the magnitude values. - # - # ----------------------------------------------------------------------- - # - global_sect = expt_config["global"] - if not global_sect.get("DO_SHUM"): - global_sect["SHUM_MAG"] = -999.0 - if not global_sect.get("DO_SKEB"): - global_sect["SKEB_MAG"] = -999.0 - if not global_sect.get("DO_SPPT"): - global_sect["SPPT_MAG"] = -999.0 - # - # ----------------------------------------------------------------------- - # - # If running with SPP in MYNN PBL, MYNN SFC, GSL GWD, Thompson MP, or - # RRTMG, count the number of entries in SPP_VAR_LIST to correctly set - # N_VAR_SPP, otherwise set it to zero. - # - # ----------------------------------------------------------------------- - # - if global_sect.get("DO_SPP"): - global_sect["N_VAR_SPP"] = len(global_sect["SPP_VAR_LIST"]) - else: - global_sect["N_VAR_SPP"] = 0 - # - # ----------------------------------------------------------------------- - # - # If running with SPP, confirm that each SPP-related namelist value - # contains the same number of entries as N_VAR_SPP (set above to be equal - # to the number of entries in SPP_VAR_LIST). + # Update the FV3 namelist based on switches that turn on/off various + # stocastic schemes. # # ----------------------------------------------------------------------- - # - spp_vars = [ - "SPP_MAG_LIST", - "SPP_LSCALE", - "SPP_TSCALE", - "SPP_SIGTOP1", - "SPP_SIGTOP2", - "SPP_STDDEV_CUTOFF", - "ISEED_SPP", - ] + # Check to make sure all SPP and LSM_SPP lists are the same length. + stoch_config = fcst_config["namelist"]["update_values"]["nam_sppperts"] if global_sect.get("DO_SPP"): - for spp_var in spp_vars: - if len(global_sect[spp_var]) != global_sect["N_VAR_SPP"]: - raise Exception( - f""" - All MYNN PBL, MYNN SFC, GSL GWD, Thompson MP, or RRTMG SPP-related namelist - variables must be of equal length to SPP_VAR_LIST: - SPP_VAR_LIST (length {global_sect['N_VAR_SPP']}) - {spp_var} (length {len(global_sect[spp_var])}) - """ - ) - # - # ----------------------------------------------------------------------- - # - # If running with Noah or RUC-LSM SPP, count the number of entries in - # LSM_SPP_VAR_LIST to correctly set N_VAR_LNDP, otherwise set it to zero. - # Also set LNDP_TYPE to 2 for LSM SPP, otherwise set it to zero. Finally, - # initialize an "FHCYC_LSM_SPP" variable to 0 and set it to 999 if LSM SPP - # is turned on. This requirement is necessary since LSM SPP cannot run with - # FHCYC=0 at the moment, but FHCYC cannot be set to anything less than the - # length of the forecast either. A bug fix will be submitted to - # ufs-weather-model soon, at which point, this requirement can be removed - # from regional_workflow. - # - # ----------------------------------------------------------------------- - # - if global_sect.get("DO_LSM_SPP"): - global_sect["N_VAR_LNDP"] = len(global_sect["LSM_SPP_VAR_LIST"]) - global_sect["LNDP_TYPE"] = 2 - global_sect["LNDP_MODEL_TYPE"] = 2 - global_sect["FHCYC_LSM_SPP_OR_NOT"] = 999 - else: - global_sect["N_VAR_LNDP"] = 0 - global_sect["LNDP_TYPE"] = 0 - global_sect["LNDP_MODEL_TYPE"] = 0 - global_sect["FHCYC_LSM_SPP_OR_NOT"] = 0 - # - # ----------------------------------------------------------------------- - # - # If running with LSM SPP, confirm that each LSM SPP-related namelist - # value contains the same number of entries as N_VAR_LNDP (set above to - # be equal to the number of entries in LSM_SPP_VAR_LIST). - # - # ----------------------------------------------------------------------- - # - lsm_spp_vars = [ - "LSM_SPP_MAG_LIST", - "LSM_SPP_LSCALE", - "LSM_SPP_TSCALE", - ] + list_vars = ("iseed_spp", "spp_lscale", "spp_prt_list", + "spp_sigtop1", "spp_sigtop2", "spp_stddev_cutoff", "spp_tau", + "spp_var_list",) + list_len = fcst_config["namelist"]["update_values"]["n_var_spp"] + if any([len(stoch_config[v]) != list_len for v in list_vars ]): + report = "\n".join([f"{v}: {len(stoch_config[v])}" for v in list_vars]) + raise Exception( + f""" + All MYNN PBL, MYNN SFC, GSL GWD, Thompson MP, or RRTMG SPP-related namelist + variables must be of length equal to namelist setting + "n_var_spp". + n_var_spp: {list_len} + + Relevant namelist settings have counts: + + {report} + """ + ) + + stoch_config = fcst_config["namelist"]["update_values"]["nam_sfcperts"] if global_sect.get("DO_LSM_SPP"): - for lsm_spp_var in lsm_spp_vars: - if len(global_sect[lsm_spp_var]) != global_sect["N_VAR_LNDP"]: - raise Exception( - f""" - All MYNN PBL, MYNN SFC, GSL GWD, Thompson MP, or RRTMG SPP-related namelist - variables must be of equal length to SPP_VAR_LIST: - All Noah or RUC-LSM SPP-related namelist variables (except ISEED_LSM_SPP) - must be equal of equal length to LSM_SPP_VAR_LIST: - LSM_SPP_VAR_LIST (length {global_sect['N_VAR_LNDP']}) - {lsm_spp_var} (length {len(global_sect[lsm_spp_var])} - """ - ) + list_vars = ("lndp_tau", "lndp_lscale", "lndp_var_list", + "lndp_prt_list") + list_len = fcst_config["namelist"]["update_values"]["n_var_lndp"] + if any([len(stoch_config[v]) != list_len for v in list_vars ]): + report = "\n".join([f"{v}: {len(stoch_config[v])}" for v in list_vars]) + raise Exception( + f""" + All Noah or RUC-LSM SPP-related namelist variables (except ISEED_LSM_SPP) + must be equal of equal length to the n_var_lndp namelist + setting: + n_var_lndp: {list_len} + {lsm_spp_var} (length {len(global_sect[lsm_spp_var])} + + Relevant namelist settings have counts: + + {report} + """ + ) + # Check whether the forecast length (FCST_LEN_HRS) is evenly divisible # by the BC update interval (LBC_SPEC_INTVL_HRS). If so, generate an @@ -1037,10 +990,6 @@ def get_location(xcs, fmt, expt_cfg): rem = FCST_LEN_HRS%%LBC_SPEC_INTVL_HRS = {rem}""" ) - # Configure the model namelist - - - # # ----------------------------------------------------------------------- diff --git a/ush/stochastic_params.yaml b/ush/stochastic_params.yaml new file mode 100644 index 0000000000..5f64036118 --- /dev/null +++ b/ush/stochastic_params.yaml @@ -0,0 +1,112 @@ +# This file is loaded in setup.py to create an appropriate namelist when flags are turned on in the +# global section. +# +# For detailed documentation of these parameters, see: +# https://stochastic-physics.readthedocs.io/en/ufs_public_release/namelist_options.html +# +do_shum: + fv3: + namelist: + update_values: + gfs_physics_nml: + do_shum: true + nam_stochy: + iseed_shum: !int '{{ cdate.strftime("%Y%m%d%H")|int * 1000 * "MEM"|env|int * 10 + 2 }}' + new_lscale: true + shum: 0.006 + shum_lscale: 150000 + shum_tau: 21600 + shumint: 3600 +do_skeb: + fv3: + namelist: + update_values: + gfs_physics_nml: + do_skeb: true + nam_stochy: + iseed_skeb: !int '{{ cdate.strftime("%Y%m%d%H")|int * 1000 * "MEM"|env|int * 10 + 3 }}' + new_lscale: true + skeb: 0.5 + skeb_lscale: 150000 + skebnorm: 1 + skeb_tau: 21600 + skebint: 3600 + skeb_vdof: 10 + + +# LSM perturbations include SMC - soil moisture content (volume fraction), +# VGF - vegetation fraction, ALB - albedo, SAL - salinity, +# EMI - emissivity, ZOL - surface roughness (cm), and STC - soil temperature +# +# Only five perturbations at a time can be applied currently, but all seven +# are shown below. In addition, only one unique iseed value is allowed. +# at the moment, and is used for each pattern. +do_lsm_spp: + fv3: + namelist: + update_values: + gfs_physics_nml: + do_spp: true + n_var_lndp: 7 + lndp_type: 2 + fhcyc: 999 + nam_stochy: + new_lscale: true + nam_sfcperts: + iseed_lndp: + - !int '{{ cdate.strftime("%Y%m%d%H")|int * 1000 * "MEM"|env|int * 10 + 9 }}' + lndp_type: 2 + lndp_model_type: 2 + lndp_tau: [ 21600, 21600, 21600, 21600, 21600, 21600, 21600 ] + lndp_lscale: [ 150000, 150000, 150000, 150000, 150000, 150000, 150000 ] + lndp_var_list: [ "smc", "vgf", "alb", "sal", "emi", "zol", "stc" ] + lndp_prt_list: [ 0.017, 0.001, 0.001, 0.001, 0.001, 0.001, 0.2 ] + +# SPP perturbs specific tuning parameters within a physics parameterization (unlike SPPT, which +# multiplies overall physics tendencies by a random perturbation field *after* the call to the +# physics suite). Patterns evolve and are applied at each time step. Each SPP option is an array, +# applicable (in order) to the HRRR-based parameterization listed in spp_var_list. +# +# Note that SPP is currently only available for specific physics schemes used in the RAP/HRRR +# physics suite. Users need to be aware of which suite definition file is chosen when turning this +# option on. + +do_spp: + fv3: + namelist: + update_values: + gfs_physics_nml: + do_spp: true + n_var_spp: 5 + nam_stochy: + new_lscale: true + nam_sppperts: + iseed_spp: + - !int '{{ cdate.strftime("%Y%m%d%H")|int * 1000 * "MEM"|env|int * 10 + 4 }}' + - !int '{{ cdate.strftime("%Y%m%d%H")|int * 1000 * "MEM"|env|int * 10 + 5 }}' + - !int '{{ cdate.strftime("%Y%m%d%H")|int * 1000 * "MEM"|env|int * 10 + 6 }}' + - !int '{{ cdate.strftime("%Y%m%d%H")|int * 1000 * "MEM"|env|int * 10 + 7 }}' + - !int '{{ cdate.strftime("%Y%m%d%H")|int * 1000 * "MEM"|env|int * 10 + 8 }}' + spp_lscale: [ 150000.0, 150000.0, 150000.0, 150000.0, 150000.0 ] + spp_prt_list: [ 0.2, 0.2, 0.75, 0.2, 0.2 ] + spp_sigtop1: [ 0.1, 0.1, 0.1, 0.1, 0.1 ] + spp_sigtop2: [ 0.025, 0.025, 0.025, 0.025, 0.025 ] + spp_stddev_cutoff: [ 1.5, 1.5, 2.5, 1.5, 1.5 ] + spp_tau: [ 21600.0, 21600.0, 21600.0, 21600.0, 21600.0 ] + spp_var_list: [ "pbl", "sfc", "mp", "rad", "gwd" ] +do_sppt: + fv3: + namelist: + update_values: + gfs_physics_nml: + do_sppt: true + nam_stochy: + iseed_sppt: !int '{{ cdate.strftime("%Y%m%d%H")|int * 1000 * "MEM"|env|int * 10 + 1 }}' + new_lscale: true + sppt: 0.7 + sppt_logit: true + sppt_lscale: 150000 + sppt_sfclimit: true + sppt_tau: 21600 + spptint: 3600 + use_zmtnblck: false From aca7d7054297e7875dcfd03f7f7d57fe910e12bb Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Wed, 30 Oct 2024 22:30:10 +0000 Subject: [PATCH 13/18] Cleanup and applying black. --- parm/FV3.input.yml | 546 ----------------------------------- parm/model_configure | 120 -------- ush/generate_FV3LAM_wflow.py | 67 +++-- ush/setup.py | 301 +++++++++++-------- 4 files changed, 215 insertions(+), 819 deletions(-) delete mode 100644 parm/FV3.input.yml delete mode 100644 parm/model_configure diff --git a/parm/FV3.input.yml b/parm/FV3.input.yml deleted file mode 100644 index efb6c85f5b..0000000000 --- a/parm/FV3.input.yml +++ /dev/null @@ -1,546 +0,0 @@ -# This configuration file maintains the modifications that need to be -# made to the base FV3 namelist specified in -# -# parm/input.nml.FV3 -# -# to obtain the namelist for each physics suite that the SRW App can -# run with. - - -FV3_RRFS_v1beta: - gfs_physics_nml: &RRFS_v1beta_phys - do_deep: False - do_mynnsfclay: True - imfdeepcnv: -1 - imfshalcnv: -1 - iopt_alb: 2 - iopt_btr: 1 - iopt_crs: 1 - iopt_dveg: 2 - iopt_frz: 1 - iopt_inf: 1 - iopt_rad: 1 - iopt_run: 1 - iopt_sfc: 1 - iopt_snf: 4 - iopt_stc: 1 - iopt_tbot: 2 - iopt_trs: 2 - lsm: 2 - lsoil_lsm: 4 -FV3_WoFS_v0: - gfs_physics_nml: - do_deep: False - imfdeepcnv: 0 - imfshalcnv: 0 - iopt_alb: 2 - iopt_btr: 1 - iopt_crs: 1 - iopt_dveg: 2 - iopt_frz: 1 - iopt_inf: 1 - iopt_rad: 1 - iopt_run: 1 - iopt_sfc: 1 - iopt_snf: 4 - iopt_stc: 1 - iopt_tbot: 2 - do_mynnsfclay: True - imfdeepcnv: -1 - imfshalcnv: -1 - lsm: 1 - lsoil_lsm: 4 - imp_physics: 17 - nssl_cccn: 0.6e+9 - nssl_hail_on: True - nssl_ccn_on: True - fv_core_nml: - nwat: 7 - fv_diagnostics_nml: - do_hailcast: True - -FV3_HRRR: - fv_core_nml: &HRRR_fv_core - hord_dp: 6 - hord_mt: 6 - hord_tm: 6 - hord_vt: 6 - hord_tr: 8 - kord_mt: 9 - kord_tm: -9 - kord_tr: 9 - kord_wz: 9 - nord_tr: 0 - nrows_blend: 20 - d_con: 0.5 - n_sponge: 9 - gfs_physics_nml: - <<: *RRFS_v1beta_phys - cdmbgwd: [3.5, 1.0] - do_mynnsfclay: True - do_sfcperts: null - gwd_opt: 3 - do_gsl_drag_ss: True - do_gsl_drag_tofd: True - do_gsl_drag_ls_bl: True - iaer: 5111 - icliq_sw: 2 - iovr: 3 - lsm: 3 - lsoil_lsm: 9 - sfclay_compute_flux: True - diag_log: True - ialb: 2 - iems: 2 - isncond_opt: 2 - isncovr_opt: 3 - mosaic_lu: 0 - mosaic_soil: 0 - thsfc_loc: False - nst_anl: null - nstf_name: null - -FV3_RAP: - fv_core_nml: - <<: *HRRR_fv_core - gfs_physics_nml: - <<: *RRFS_v1beta_phys - cdmbgwd: [3.5, 1.0] - do_mynnsfclay: True - do_sfcperts: null - gwd_opt: 3 - do_gsl_drag_ss: True - do_gsl_drag_tofd: True - do_gsl_drag_ls_bl: True - iaer: 5111 - icliq_sw: 2 - iovr: 3 - lsm: 3 - lsoil_lsm: 9 - sfclay_compute_flux: False - do_deep: True - shal_cnv: True - imfdeepcnv: 3 - imfshalcnv: 3 - -FV3_GFS_2017_gfdlmp: - atmos_model_nml: - avg_max_length: 3600.0 - fv_core_nml: &gfs_2017_gfdlmp_fv_core - agrid_vel_rst: False - d4_bg: 0.15 - delt_max: 0.008 - do_sat_adj: True - fv_debug: False - k_split: 6 - n_split: 6 - nord: 2 - nord_zs_filter: null - range_warn: False - vtdm4: 0.075 - gfs_physics_nml: &gfs_2017_gfdlmp_phys - avg_max_length: 3600.0 - bl_mynn_tkeadvect: null - bl_mynn_edmf: null - bl_mynn_edmf_mom: null - cdmbgwd: [3.5, 0.01] - cplflx: null - do_deep: False - do_mynnedmf: null - do_mynnsfclay: null - fhcyc: 0.0 - fhlwr: 3600.0 - fhswr: 3600.0 - hybedmf: True - icloud_bl: null - imfdeepcnv: 2 - imfshalcnv: 2 - imp_physics: 11 - lgfdlmprad: True - lheatstrg: null - lndp_type: null - lsm: null - lsoil: null - lsoil_lsm: null - ltaerosol: null - n_var_lndp: null - oz_phys: True - oz_phys_2015: False - satmedmf: null - shal_cnv: True - ttendlim: null - gfdl_cloud_microphysics_nml: &gfs_gfdl_cloud_mp - c_cracw: 0.8 - c_paut: 0.5 - c_pgacs: 0.01 - c_psaci: 0.05 - ccn_l: 300.0 - ccn_o: 100.0 - const_vg: False - const_vi: False - const_vr: False - const_vs: False - de_ice: False - do_qa: True - do_sedi_heat: False - dw_land: 0.16 - dw_ocean: 0.1 - fast_sat_adj: True - fix_negative: True - icloud_f: 1 - mono_prof: True - mp_time: 90.0 - prog_ccn: False - qi0_crt: 8.0e-05 - qi_lim: 1.0 - ql_gen: 0.001 - ql_mlt: 0.001 - qs0_crt: 0.001 - rad_graupel: True - rad_rain: True - rad_snow: True - rh_inc: 0.3 - rh_inr: 0.3 - rh_ins: 0.3 - rthresh: 1.0e-05 - sedi_transport: False - tau_g2v: 900.0 - tau_i2s: 1000.0 - tau_l2v: 180.0 - tau_v2l: 90.0 - use_ccn: True - use_ppm: False - vg_max: 12.0 - vi_max: 1.0 - vr_max: 12.0 - vs_max: 2.0 - z_slope_ice: True - z_slope_liq: True - -FV3_GFS_2017_gfdlmp_regional: - atmos_model_nml: - avg_max_length: 3600.0 - fv_core_nml: - <<: *gfs_2017_gfdlmp_fv_core - k_split: 2 - gfs_physics_nml: - <<: *gfs_2017_gfdlmp_phys - cplflx: False - effr_in: False - iopt_alb: 2 - iopt_btr: 1 - iopt_crs: 1 - iopt_dveg: 2 - iopt_frz: 1 - iopt_inf: 1 - iopt_rad: 1 - iopt_run: 1 - iopt_sfc: 1 - iopt_snf: 4 - iopt_stc: 1 - iopt_tbot: 2 - iopt_trs: 2 - lgfdlmprad: True - lheatstrg: False - lndp_type: 0 - lsm: 1 - n_var_lndp: 0 - nstf_name: [2, 0, 0, 0, 0] - oz_phys: False - oz_phys_2015: True - satmedmf: False - gfdl_cloud_microphysics_nml: - <<: *gfs_gfdl_cloud_mp - -FV3_GFS_v15p2: - fv_core_nml: &gfs_v15_fv_core - agrid_vel_rst: False - d2_bg_k1: 0.15 - d2_bg_k2: 0.02 - do_sat_adj: True - fv_debug: False - fv_sg_adj: 600 - k_split: 1 - kord_mt: 9 - kord_tm: -9 - kord_tr: 9 - kord_wz: 9 - n_split: 8 - n_sponge: 30 - nord_zs_filter: null - nudge_qv: True - range_warn: False - rf_cutoff: 750.0 - rf_fast: False - gfdl_cloud_microphysics_nml: - <<: *gfs_gfdl_cloud_mp - sedi_transport: True - tau_l2v: 225.0 - tau_v2l: 150.0 - gfs_physics_nml: &gfs_v15_gfs_physics - bl_mynn_edmf: null - bl_mynn_edmf_mom: null - bl_mynn_tkeadvect: null - cnvcld: True - cnvgwd: True - cplflx: null - do_myjpbl: False - do_myjsfc: False - do_mynnedmf: null - do_mynnsfclay: null - do_tofd: False - do_ugwp: False - do_ysu: False - fhcyc: 0.0 - fhlwr: 3600.0 - fhswr: 3600.0 - hybedmf: True - iau_delthrs: null - iaufhrs: null - imfdeepcnv: 2 - imfshalcnv: 2 - imp_physics: 11 - icloud_bl: null - iopt_alb: 2 - iopt_btr: 1 - iopt_crs: 1 - iopt_dveg: 2 - iopt_frz: 1 - iopt_inf: 1 - iopt_rad: 1 - iopt_run: 1 - iopt_sfc: 1 - iopt_snf: 4 - iopt_stc: 1 - iopt_tbot: 2 - iopt_trs: 2 - ldiag_ugwp: False - lgfdlmprad: True - lradar: null - lsm: 1 - lsoil: null - lsoil_lsm: null - ltaerosol: null - shal_cnv: True - shinhong: False - ttendlim: null - xkzm_h: 1.0 - xkzm_m: 1.0 - xkzminv: 0.3 - namsfc: - landice: True - ldebug: False - surf_map_nml: null - -FV3_GFS_v15_thompson_mynn_lam3km: - atmos_model_nml: - avg_max_length: 3600.0 - fv_core_nml: - agrid_vel_rst: True - full_zs_filter: null - n_sponge: 9 - npz_type: '' - rf_fast: False - sg_cutoff: 10000.0 - vtdm4: 0.02 - gfs_physics_nml: - avg_max_length: 3600.0 - cdmbgwd: [0.88, 0.04] - debug: True - do_deep: False - do_gsl_drag_ls_bl: False - do_gsl_drag_ss: True - do_gsl_drag_tofd: True - do_mynnsfclay: True - do_tofd: False - do_ugwp: False - do_ugwp_v0: False - do_ugwp_v0_nst_only: False - do_ugwp_v0_orog_only: False - fhswr: 900.0 - fhlwr: 900.0 - gwd_opt: 2 - iaer: 1011 - iccn: 2 - icliq_sw: 2 - imfdeepcnv: 2 - imfshalcnv: 2 - iopt_alb: 2 - iopt_btr: 1 - iopt_crs: 1 - iopt_dveg: 2 - iopt_frz: 1 - iopt_inf: 1 - iopt_rad: 1 - iopt_run: 1 - iopt_sfc: 1 - iopt_snf: 4 - iopt_stc: 1 - iopt_tbot: 2 - iopt_trs: null - iovr: 3 - ldiag_ugwp: False - lgfdlmprad: False - lsm: 1 - lsoil: null - lsoil_lsm: null - ltaerosol: False - print_diff_pgr: True - sfclay_compute_flux: null - xkzminv: 0.3 - xkzm_m: 1.0 - xkzm_h: 1.0 - surf_map_nml: null - -FV3_GFS_v16: - cires_ugwp_nml: - launch_level: 27 - fv_core_nml: - <<: *gfs_v15_fv_core - agrid_vel_rst: False - d2_bg_k1: 0.2 - d2_bg_k2: 0.0 - delt_max: 0.002 - dz_min: 6 - fv_sg_adj: 450 - hord_dp: -5 - hord_mt: 5 - hord_tm: 5 - hord_vt: 5 - k_split: 6 - make_nh: False - n_split: 6 - n_sponge: 10 - na_init: 0 - nudge_dz: False - res_latlon_dynamics: '' - rf_fast: null - tau: 10.0 - gfdl_cloud_microphysics_nml: - <<: *gfs_gfdl_cloud_mp - mp_time: 150.0 - reiflag: 2 - sedi_transport: True - tau_l2v: 225.0 - tau_v2l: 150.0 - gfs_physics_nml: - <<: *gfs_v15_gfs_physics - cdmbgwd: [4.0, 0.15, 1.0, 1.0] - do_myjpbl: null - do_myjsfc: null - do_tofd: True - do_ysu: null - hybedmf: False - iaer: 5111 - icliq_sw: 2 - iopt_dveg: 1 - iovr: 3 - isatmedmf: 1 - lgfdlmprad: True - lheatstrg: True - lndp_type: null - lsoil: 4 - n_var_lndp: null - prautco: [0.00015, 0.00015] - psautco: [0.0008, 0.0005] - satmedmf: True - shinhong: null - xkzminv: null - xkzm_m: null - xkzm_h: null - mpp_io_nml: - deflate_level: 1 - shuffle: 1 - namsfc: - landice: True - ldebug: False - surf_map_nml: null - -FV3_GFS_v17_p8: - cires_ugwp_nml: - launch_level: 27 - fv_core_nml: - <<: *gfs_v15_fv_core - agrid_vel_rst: False - d2_bg_k1: 0.2 - d2_bg_k2: 0.0 - dnats: 0 - do_sat_adj: False - fv_sg_adj: 450 - hord_dp: -5 - hord_mt: 5 - hord_tm: 5 - hord_tr: 8 - hord_vt: 5 - k_split: 6 - make_nh: True - n_split: 6 - n_sponge: 10 - na_init: 1 - nord: 1 - nudge_dz: False - res_latlon_dynamics: '' - rf_fast: null - tau: 10.0 - gfs_physics_nml: - cdmbgwd: [4.0, 0.05, 1.0, 1.0] - cnvcld: True - cnvgwd: True - decfl: 10 - do_deep: True - do_gsl_drag_ls_bl: False - do_gsl_drag_ss: True - do_gsl_drag_tofd: False - do_mynnedmf: False - do_mynnsfclay: False - do_tofd: False - do_ugwp: False - do_ugwp_v0: True - do_ugwp_v0_orog_only: False - do_ugwp_v0_nst_only: False - do_ugwp_v1: False - do_ugwp_v1_orog_only: False - dt_inner: 150.0 - fhlwr: 1200.0 - fhswr: 1200.0 - frac_grid: False - gwd_opt: 2 - iaer: 1011 - ialb: 2 - icliq_sw: 2 - iems: 2 - imfdeepcnv: 2 - imfshalcnv: 2 - iopt_alb: 1 - iopt_btr: 1 - iopt_crs: 2 - iopt_dveg: 4 - iopt_frz: 1 - iopt_inf: 1 - iopt_rad: 3 - iopt_run: 1 - iopt_sfc: 3 - iopt_snf: 4 - iopt_stc: 3 - iopt_tbot: 2 - iovr: 3 - isatmedmf: 1 - ldiag_ugwp: False - lseaspray: True - lgfdlmprad: False - lheatstrg: False - lradar: False - lsm: 2 - lsoil_lsm: 4 - ltaerosol: False - min_lakeice: 0.15 - min_seaice: 0.15 - qdiag3d: False - ras: False - satmedmf: True - sedi_semi: True - shal_cnv: True - mpp_io_nml: - deflate_level: 1 - shuffle: 1 - surf_map_nml: null diff --git a/parm/model_configure b/parm/model_configure deleted file mode 100644 index aeb45f4719..0000000000 --- a/parm/model_configure +++ /dev/null @@ -1,120 +0,0 @@ -total_member: 1 -PE_MEMBER01: {{ PE_MEMBER01 }} -start_year: {{ start_year }} -start_month: {{ start_month }} -start_day: {{ start_day }} -start_hour: {{ start_hour }} -start_minute: 0 -start_second: 0 -nhours_fcst: {{ nhours_fcst }} -fhrot: {{ fhrot }} -RUN_CONTINUE: .false. -ENS_SPS: .false. -dt_atmos: {{ dt_atmos }} -calendar: 'julian' -memuse_verbose: .false. -atmos_nthreads: {{ atmos_nthreads }} -restart_interval: {{ restart_interval }} -output_1st_tstep_rst: .false. -write_dopost: {{ write_dopost }} -ideflate: 0 -nbits: 0 -ichunk2d: -1 -jchunk2d: -1 -ichunk3d: -1 -jchunk3d: -1 -kchunk3d: -1 -itasks: {{ itasks }} -quilting: {{ quilting }} -{% if quilting %} -# -# Write-component (quilting) computational parameters. -# -write_groups: {{ write_groups }} -write_tasks_per_group: {{ write_tasks_per_group }} -num_files: 2 -filename_base: 'dyn' 'phy' -output_file: 'netcdf' 'netcdf' -# -# Write-component output frequency parameter definitions: -# -# output_fh: Output frequency in hours. -# nsout: Output frequency in time steps (positive values override "output_fh"). -# -output_fh: {{ output_fh }} -1 -nsout: {{ nsout }} -# -# Coordinate system used by the output grid. -# -output_grid: '{{ output_grid }}' -# -# Parameter definitions for an output grid of type "{{ output_grid }}": -# - {%- if output_grid == "lambert_conformal" %} -# cen_lon: Longitude of center of grid (degrees). -# cen_lat: Latitude of center of grid (degrees). -# stdlat1: Latitude of first standard parallel (degrees). -# stdlat2: Latitude of second standard parallel (degrees). -# nx: Number of grid cells along x-axis in Lambert conformal (x,y) plane. -# ny: Number of grid cells along y-axis in Lambert conformal (x,y) plane. -# lon1: Longitude of center of grid cell at bottom-left corner of grid (degrees). -# lat1: Latitude of center of grid cell at bottom-left corner of grid (degrees). -# dx: Grid cell size in x direction (meters). -# dy: Grid cell size in y direction (meters). -# - {%- elif output_grid == "regional_latlon" %} -# cen_lon: Longitude of center of grid (degrees). -# cen_lat: Latitude of center of grid (degrees). -# lon1: Longitude of center of lower-left (southwest) grid cell (degrees). -# lat1: Latitude of center of lower-left (southwest) grid cell (degrees). -# lon2: Longitude of center of upper-right (northeast) grid cell (degrees). -# lat2: Latitude of center of upper-right (northeast) grid cell (degrees). -# dlon: Longitudinal grid size (degrees). -# dlat: Latitudinal grid size (degrees). -# - {%- elif output_grid == "rotated_latlon" %} -# cen_lon: Longitude of center of grid, expressed in the NON-ROTATED latlon coordinate -# system (degrees). This is also the longitude of the point at which the -# equator and prime meridian of the ROTATED coordinate system intersect (i.e. -# the point at which the longitude and latitude in the ROTATED latlon -# coordinate system are both 0). -# cen_lat: Latitude of center of grid, expressed in the NON-ROTATED latlon coordinate -# system (degrees). This is also the latitude of the point at which the -# equator and prime meridian of the ROTATED coordinate system intersect (i.e. -# the point at which the longitude and latitude in the ROTATED latlon -# coordinate system are both 0). -# lon1: Longitude of center of lower-left grid cell, expressed in the ROTATED latlon -# coordinate system (degrees). -# lat1: Latitude of center of lower-left grid cell, expressed in the ROTATED latlon -# coordinate system (degrees). -# lon2: Longitude of center of upper-right grid cell, expressed in the ROTATED latlon -# coordinate system (degrees). -# lat2: Latitude of center of upper-right grid cell, expressed in the ROTATED latlon -# coordinate system (degrees). -# dlon: Longitudinal grid size in the ROTATED latlon coordinate system (degrees). -# dlat: Latitudinal grid size in the ROTATED latlon coordinate system (degrees). -# - {%- endif %} - {%- if output_grid == "lambert_conformal" %} -cen_lon: {{ cen_lon }} -cen_lat: {{ cen_lat }} -stdlat1: {{ stdlat1 }} -stdlat2: {{ stdlat2 }} -nx: {{ nx }} -ny: {{ ny }} -lon1: {{ lon1 }} -lat1: {{ lat1 }} -dx: {{ dx }} -dy: {{ dy }} - {%- elif (output_grid == "regional_latlon") or (output_grid == "rotated_latlon") %} -cen_lon: {{ cen_lon }} -cen_lat: {{ cen_lat }} -lon1: {{ lon1 }} -lat1: {{ lat1 }} -lon2: {{ lon2 }} -lat2: {{ lat2 }} -dlon: {{ dlon }} -dlat: {{ dlat }} - {%- endif %} -{%- endif %} - diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 0ba32d320c..28808876d1 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -39,9 +39,8 @@ # pylint: disable=too-many-locals,too-many-branches, too-many-statements def generate_FV3LAM_wflow( - ushdir, - logfile: str = "log.generate_FV3LAM_wflow", - debug: bool = False) -> str: + ushdir, logfile: str = "log.generate_FV3LAM_wflow", debug: bool = False +) -> str: """Function to setup a forecast experiment and create a workflow (according to the parameters specified in the config file) @@ -69,7 +68,7 @@ def generate_FV3LAM_wflow( # The setup function reads the user configuration file and fills in # non-user-specified values from config_defaults.yaml - expt_config = setup(ushdir,debug=debug) + expt_config = setup(ushdir, debug=debug) # # ----------------------------------------------------------------------- @@ -114,10 +113,10 @@ def generate_FV3LAM_wflow( # rocoto_yaml_fp = expt_config["workflow"]["ROCOTO_YAML_FP"] render( - input_file = template_xml_fp, - output_file = wflow_xml_fp, - values_src = rocoto_yaml_fp, - ) + input_file=template_xml_fp, + output_file=wflow_xml_fp, + values_src=rocoto_yaml_fp, + ) # # ----------------------------------------------------------------------- # @@ -138,22 +137,27 @@ def generate_FV3LAM_wflow( verbose=debug, ) - with open(wflow_launch_script_fp, "r", encoding='utf-8') as launch_script_file: + with open(wflow_launch_script_fp, "r", encoding="utf-8") as launch_script_file: launch_script_content = launch_script_file.read() # Stage an experiment-specific launch file in the experiment directory template = Template(launch_script_content) # The script needs several variables from the workflow and user sections - template_variables = {**expt_config["user"], **expt_config["workflow"], - "valid_vals_BOOLEAN": list_to_str(expt_config["constants"]["valid_vals_BOOLEAN"])} - launch_content = template.safe_substitute(template_variables) + template_variables = { + **expt_config["user"], + **expt_config["workflow"], + "valid_vals_BOOLEAN": list_to_str( + expt_config["constants"]["valid_vals_BOOLEAN"] + ), + } + launch_content = template.safe_substitute(template_variables) launch_fp = os.path.join(exptdir, wflow_launch_script_fn) - with open(launch_fp, "w", encoding='utf-8') as expt_launch_fn: + with open(launch_fp, "w", encoding="utf-8") as expt_launch_fn: expt_launch_fn.write(launch_content) - os.chmod(launch_fp, os.stat(launch_fp).st_mode|S_IXUSR) + os.chmod(launch_fp, os.stat(launch_fp).st_mode | S_IXUSR) # # ----------------------------------------------------------------------- @@ -173,9 +177,13 @@ def generate_FV3LAM_wflow( # pylint: disable=undefined-variable if USE_CRON_TO_RELAUNCH: - add_crontab_line(called_from_cron=False,machine=expt_config["user"]["MACHINE"], - crontab_line=expt_config["workflow"]["CRONTAB_LINE"], - exptdir=exptdir,debug=debug) + add_crontab_line( + called_from_cron=False, + machine=expt_config["user"]["MACHINE"], + crontab_line=expt_config["workflow"]["CRONTAB_LINE"], + exptdir=exptdir, + debug=debug, + ) # # Copy or symlink fix files @@ -300,7 +308,7 @@ def generate_FV3LAM_wflow( # the C-resolution of the grid), and this parameter is in most workflow # configurations is not known until the grid is created. # - if not expt_config['rocoto']['tasks'].get('task_make_grid'): + if not expt_config["rocoto"]["tasks"].get("task_make_grid"): set_fv3nml_sfc_climo_filenames(flatten_dict(expt_config), debug) @@ -370,7 +378,9 @@ def generate_FV3LAM_wflow( return EXPTDIR -def setup_logging(logfile: str = "log.generate_FV3LAM_wflow", debug: bool = False) -> None: +def setup_logging( + logfile: str = "log.generate_FV3LAM_wflow", debug: bool = False +) -> None: """ Sets up logging, printing high-priority (INFO and higher) messages to screen, and printing all messages with detailed timing and routine info in the specified text file. @@ -381,7 +391,7 @@ def setup_logging(logfile: str = "log.generate_FV3LAM_wflow", debug: bool = Fals formatter = logging.Formatter("%(name)-22s %(levelname)-8s %(message)s") - fh = logging.FileHandler(logfile, mode='w') + fh = logging.FileHandler(logfile, mode="w") fh.setLevel(logging.DEBUG) fh.setFormatter(formatter) logging.getLogger().addHandler(fh) @@ -404,13 +414,18 @@ def setup_logging(logfile: str = "log.generate_FV3LAM_wflow", debug: bool = Fals if __name__ == "__main__": - #Parse arguments + # Parse arguments parser = argparse.ArgumentParser( - description="Script for setting up a forecast and creating a workflow"\ - "according to the parameters specified in the config file\n") + description="Script for setting up a forecast and creating a workflow" + "according to the parameters specified in the config file\n" + ) - parser.add_argument('-d', '--debug', action='store_true', - help='Script will be run in debug mode with more verbose output') + parser.add_argument( + "-d", + "--debug", + action="store_true", + help="Script will be run in debug mode with more verbose output", + ) pargs = parser.parse_args() USHdir = os.path.dirname(os.path.abspath(__file__)) @@ -420,7 +435,7 @@ def setup_logging(logfile: str = "log.generate_FV3LAM_wflow", debug: bool = Fals # experiment/workflow. try: expt_dir = generate_FV3LAM_wflow(USHdir, wflow_logfile, pargs.debug) - except: # pylint: disable=bare-except + except: # pylint: disable=bare-except logging.exception( dedent( f""" diff --git a/ush/setup.py b/ush/setup.py index ada01eebe5..9799460d13 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -47,6 +47,7 @@ from set_gridparams_GFDLgrid import set_gridparams_GFDLgrid from uwtools.api.config import get_yaml_config + def load_config_for_setup(ushdir, default_config, user_config): """Load in the default, machine, and user configuration files into Python dictionaries. Return the combined experiment dictionary. @@ -78,7 +79,9 @@ def load_config_for_setup(ushdir, default_config, user_config): try: cfg_u = load_config_file(user_config) - logging.debug(f"Read in the following values from YAML config file {user_config}:\n") + logging.debug( + f"Read in the following values from YAML config file {user_config}:\n" + ) logging.debug(cfg_u) except: errmsg = dedent( @@ -104,7 +107,9 @@ def load_config_for_setup(ushdir, default_config, user_config): errmsg = f"Invalid key(s) specified in {user_config}:\n" for entry in invalid: errmsg = errmsg + f"{entry} = {invalid[entry]}\n" - errmsg = errmsg + f"\nCheck {default_config} for allowed user-specified variables\n" + errmsg = ( + errmsg + f"\nCheck {default_config} for allowed user-specified variables\n" + ) raise Exception(errmsg) # Mandatory variables *must* be set in the user's config; the default value is invalid @@ -144,36 +149,36 @@ def load_config_for_setup(ushdir, default_config, user_config): # Load the constants file cfg_c = load_config_file(os.path.join(ushdir, "constants.yaml")) - # Load the rocoto workflow default file - cfg_wflow = load_config_file(os.path.join(ushdir, os.pardir, "parm", - "wflow", "default_workflow.yaml")) + cfg_wflow = load_config_file( + os.path.join(ushdir, os.pardir, "parm", "wflow", "default_workflow.yaml") + ) # Takes care of removing any potential "null" entries, i.e., # unsetting a default value from an anchored default_task update_dict(cfg_wflow, cfg_wflow) - # Take any user-specified taskgroups entry here. - taskgroups = cfg_u.get('rocoto', {}).get('tasks', {}).get('taskgroups') + taskgroups = cfg_u.get("rocoto", {}).get("tasks", {}).get("taskgroups") if taskgroups: - cfg_wflow['rocoto']['tasks']['taskgroups'] = taskgroups + cfg_wflow["rocoto"]["tasks"]["taskgroups"] = taskgroups # Extend yaml here on just the rocoto section to include the # appropriate groups of tasks extend_yaml(cfg_wflow) - # Put the entries expanded under taskgroups in tasks rocoto_tasks = cfg_wflow["rocoto"]["tasks"] - cfg_wflow["rocoto"]["tasks"] = yaml.load(rocoto_tasks.pop("taskgroups"),Loader=yaml.SafeLoader) + cfg_wflow["rocoto"]["tasks"] = yaml.load( + rocoto_tasks.pop("taskgroups"), Loader=yaml.SafeLoader + ) # Update wflow config from user one more time to make sure any of # the "null" settings are removed, i.e., tasks turned off. - update_dict(cfg_u.get('rocoto', {}), cfg_wflow["rocoto"]) + update_dict(cfg_u.get("rocoto", {}), cfg_wflow["rocoto"]) def add_jobname(tasks): - """ Add the jobname entry for all the tasks in the workflow """ + """Add the jobname entry for all the tasks in the workflow""" if not isinstance(tasks, dict): return @@ -182,13 +187,13 @@ def add_jobname(tasks): if task_type == "task": # Use the provided attribute if it is present, otherwise use # the name in the key - tasks[task]["jobname"] = \ - task_settings.get("attrs", {}).get("name") or \ - task.split("_", maxsplit=1)[1] + tasks[task]["jobname"] = ( + task_settings.get("attrs", {}).get("name") + or task.split("_", maxsplit=1)[1] + ) elif task_type == "metatask": add_jobname(task_settings) - # Add jobname entry to each remaining task add_jobname(cfg_wflow["rocoto"]["tasks"]) @@ -218,13 +223,15 @@ def add_jobname(tasks): update_dict(cfg_d, cfg_d) # Load one more if running Coupled AQM - if cfg_d['cpl_aqm_parm']['CPL_AQM']: + if cfg_d["cpl_aqm_parm"]["CPL_AQM"]: cfg_aqm = get_yaml_config(Path(ushdir, "config_defaults_aqm.yaml")) update_dict(cfg_aqm, cfg_d) # Load CCPP suite-specific settings - ccpp_suite = cfg_d['workflow']['CCPP_PHYS_SUITE'] - ccpp_cfg = get_yaml_config(Path(ushdir, "ccpp_suites_defaults.yaml")).get(ccpp_suite, {}) + ccpp_suite = cfg_d["workflow"]["CCPP_PHYS_SUITE"] + ccpp_cfg = get_yaml_config(Path(ushdir, "ccpp_suites_defaults.yaml")).get( + ccpp_suite, {} + ) update_dict(ccpp_cfg, cfg_d) # Load stochastic physics params @@ -232,8 +239,8 @@ def add_jobname(tasks): fcst_config = cfg_d["task_run_fcst"]["fv3"] fcst_nml_config = get_yaml_config(fcst_config["namelist"]["update_values"]) for switch_name in ("do_spp", "do_sppt", "do_shum", "do_skeb", "do_lsm_spp"): - if (cfg_d["global"][switch_name.upper()]): - fcst_nml_config.update_from(stochastic_params.get(switch_name) + if cfg_d["global"][switch_name.upper()]: + fcst_nml_config.update_from(stochastic_params.get(switch_name)) # Set "Home" directory, the top-level ufs-srweather-app directory homedir = os.path.abspath(os.path.dirname(__file__) + os.sep + os.pardir) @@ -298,7 +305,6 @@ def add_jobname(tasks): def set_srw_paths(ushdir, expt_config): - """ Generate a dictionary of directories that describe the SRW structure, i.e., where SRW is installed, and the paths to @@ -439,7 +445,6 @@ def setup(USHdir, user_config_fn="config.yaml", debug: bool = False): fcst_len_hrs_max = {fcst_len_hrs_max}""" ) - # # ----------------------------------------------------------------------- # @@ -503,11 +508,11 @@ def setup(USHdir, user_config_fn="config.yaml", debug: bool = False): # ----------------------------------------------------------------------- # - rocoto_config = expt_config.get('rocoto', {}) + rocoto_config = expt_config.get("rocoto", {}) rocoto_tasks = rocoto_config.get("tasks") - run_make_grid = rocoto_tasks.get('task_make_grid') is not None - run_make_orog = rocoto_tasks.get('task_make_orog') is not None - run_make_sfc_climo = rocoto_tasks.get('task_make_sfc_climo') is not None + run_make_grid = rocoto_tasks.get("task_make_grid") is not None + run_make_orog = rocoto_tasks.get("task_make_orog") is not None + run_make_sfc_climo = rocoto_tasks.get("task_make_sfc_climo") is not None # Necessary tasks are turned on pregen_basedir = expt_config["platform"].get("DOMAIN_PREGEN_BASEDIR") @@ -535,7 +540,7 @@ def setup(USHdir, user_config_fn="config.yaml", debug: bool = False): ) def remove_tag(tasks, tag): - """ Remove the tag for all the tasks in the workflow """ + """Remove the tag for all the tasks in the workflow""" if not isinstance(tasks, dict): return @@ -551,10 +556,10 @@ def remove_tag(tasks, tag): if remove_memory: remove_tag(rocoto_tasks, "memory") - for part in ['PARTITION_HPSS', 'PARTITION_DEFAULT', 'PARTITION_FCST']: + for part in ["PARTITION_HPSS", "PARTITION_DEFAULT", "PARTITION_FCST"]: partition = expt_config["platform"].get(part) if not partition: - remove_tag(rocoto_tasks, 'partition') + remove_tag(rocoto_tasks, "partition") # When not running subhourly post, remove those tasks, if they exist if not expt_config.get("task_run_post", {}).get("SUB_HOURLY_POST"): @@ -572,29 +577,37 @@ def remove_tag(tasks, tag): vx_metatasks_all = {} vx_fields_all["CCPA"] = ["APCP"] - vx_metatasks_all["CCPA"] = ["metatask_PcpCombine_obs", - "metatask_PcpCombine_fcst_APCP_all_accums_all_mems", - "metatask_GridStat_CCPA_all_accums_all_mems", - "metatask_GenEnsProd_EnsembleStat_CCPA", - "metatask_GridStat_CCPA_ensmeanprob_all_accums"] + vx_metatasks_all["CCPA"] = [ + "metatask_PcpCombine_obs", + "metatask_PcpCombine_fcst_APCP_all_accums_all_mems", + "metatask_GridStat_CCPA_all_accums_all_mems", + "metatask_GenEnsProd_EnsembleStat_CCPA", + "metatask_GridStat_CCPA_ensmeanprob_all_accums", + ] vx_fields_all["NOHRSC"] = ["ASNOW"] - vx_metatasks_all["NOHRSC"] = ["task_get_obs_nohrsc", - "metatask_PcpCombine_fcst_ASNOW_all_accums_all_mems", - "metatask_GridStat_NOHRSC_all_accums_all_mems", - "metatask_GenEnsProd_EnsembleStat_NOHRSC", - "metatask_GridStat_NOHRSC_ensmeanprob_all_accums"] + vx_metatasks_all["NOHRSC"] = [ + "task_get_obs_nohrsc", + "metatask_PcpCombine_fcst_ASNOW_all_accums_all_mems", + "metatask_GridStat_NOHRSC_all_accums_all_mems", + "metatask_GenEnsProd_EnsembleStat_NOHRSC", + "metatask_GridStat_NOHRSC_ensmeanprob_all_accums", + ] vx_fields_all["MRMS"] = ["REFC", "RETOP"] - vx_metatasks_all["MRMS"] = ["metatask_GridStat_MRMS_all_mems", - "metatask_GenEnsProd_EnsembleStat_MRMS", - "metatask_GridStat_MRMS_ensprob"] + vx_metatasks_all["MRMS"] = [ + "metatask_GridStat_MRMS_all_mems", + "metatask_GenEnsProd_EnsembleStat_MRMS", + "metatask_GridStat_MRMS_ensprob", + ] vx_fields_all["NDAS"] = ["ADPSFC", "ADPUPA"] - vx_metatasks_all["NDAS"] = ["task_run_MET_Pb2nc_obs", - "metatask_PointStat_NDAS_all_mems", - "metatask_GenEnsProd_EnsembleStat_NDAS", - "metatask_PointStat_NDAS_ensmeanprob"] + vx_metatasks_all["NDAS"] = [ + "task_run_MET_Pb2nc_obs", + "metatask_PointStat_NDAS_all_mems", + "metatask_GenEnsProd_EnsembleStat_NDAS", + "metatask_PointStat_NDAS_ensmeanprob", + ] # Get the vx fields specified in the experiment configuration. vx_fields_config = expt_config["verification"]["VX_FIELDS"] @@ -603,23 +616,27 @@ def remove_tag(tasks, tag): # for all observation types. if not vx_fields_config: metatask = "metatask_check_post_output_all_mems" - rocoto_config['tasks'].pop(metatask) + rocoto_config["tasks"].pop(metatask) # If for a given obstype no fields are specified, remove all vx metatasks # for that obstype. for obstype in vx_fields_all: - vx_fields_obstype = [field for field in vx_fields_config if field in vx_fields_all[obstype]] + vx_fields_obstype = [ + field for field in vx_fields_config if field in vx_fields_all[obstype] + ] if not vx_fields_obstype: for metatask in vx_metatasks_all[obstype]: - if metatask in rocoto_config['tasks']: - logging.info(dedent( - f""" + if metatask in rocoto_config["tasks"]: + logging.info( + dedent( + f""" Removing verification [meta]task "{metatask}" from workflow since no fields belonging to observation type "{obstype}" are specified for verification.""" - )) - rocoto_config['tasks'].pop(metatask) + ) + ) + rocoto_config["tasks"].pop(metatask) # # ----------------------------------------------------------------------- @@ -685,13 +702,13 @@ def get_location(xcs, fmt, expt_cfg): {data_key} = \"{basedir}\"''' ) - # Make sure the vertical coordinate file for both make_lbcs and # make_ics is the same. - if ics_vcoord := expt_config.get("task_make_ics", {}).get("VCOORD_FILE") != \ - (lbcs_vcoord := expt_config.get("task_make_lbcs", {}).get("VCOORD_FILE")): - raise ValueError( - f""" + if ics_vcoord := expt_config.get("task_make_ics", {}).get("VCOORD_FILE") != ( + lbcs_vcoord := expt_config.get("task_make_lbcs", {}).get("VCOORD_FILE") + ): + raise ValueError( + f""" The VCOORD_FILE must be set to the same value for both the make_ics task and the make_lbcs task. They are currently set to: @@ -702,7 +719,7 @@ def get_location(xcs, fmt, expt_cfg): make_lbcs: VCOORD_FILE: {lbcs_vcoord} """ - ) + ) # # ----------------------------------------------------------------------- @@ -721,14 +738,16 @@ def get_location(xcs, fmt, expt_cfg): dt = fcst_config.get("DT_ATMOS") if dt: if dt > 40: - logger.warning(dedent( - f""" + logger.warning( + dedent( + f""" WARNING: CCPP suite {workflow_config["CCPP_PHYS_SUITE"]} requires short time step regardless of grid resolution. The user-specified value DT_ATMOS = {fcst_config.get("DT_ATMOS")} may result in CFL violations or other errors! """ - )) + ) + ) quilting = fcst_config["model_configure"]["update_values"]["quilting"] # Gather the pre-defined grid parameters, if needed @@ -750,14 +769,19 @@ def get_location(xcs, fmt, expt_cfg): continue # DT_ATMOS needs special treatment based on CCPP suite elif param == "DT_ATMOS": - if workflow_config["CCPP_PHYS_SUITE"] in hires_ccpp_suites and grid_params[param] > 40: - logger.warning(dedent( - f""" + if ( + workflow_config["CCPP_PHYS_SUITE"] in hires_ccpp_suites + and grid_params[param] > 40 + ): + logger.warning( + dedent( + f""" WARNING: CCPP suite {workflow_config["CCPP_PHYS_SUITE"]} requires short time step regardless of grid resolution; setting DT_ATMOS to 40.\n This value can be overwritten in the user config file. """ - )) + ) + ) fcst_config[param] = 40 else: fcst_config[param] = value @@ -774,7 +798,9 @@ def get_location(xcs, fmt, expt_cfg): # This logic belongs in predef_grid_params.yaml once merged with make_grid integration. if predef_grid_name == "RRFS_NA_3km": fv3_namelist = expt_config["task_run_fcst"]["fv3"]["namelist"] - fv3_namlelist["update_values"]["fms2_io_nml"]["netcdf_default_format"] = "netcdf4" + fv3_namlelist["update_values"]["fms2_io_nml"][ + "netcdf_default_format" + ] = "netcdf4" # Load model write component grid settings quilting_cfg = get_yaml_config(Path(USHdir, "quilting.yaml")) @@ -784,8 +810,6 @@ def get_location(xcs, fmt, expt_cfg): write_grid = expt_config["task_run_fcst"]["WRTCMP_output_grid"] update_dict(quilting_cfg[write_grid], expt_config) - - run_envir = expt_config["user"].get("RUN_ENVIR", "") fcst_len_hrs = workflow_config.get("FCST_LEN_HRS") @@ -801,19 +825,19 @@ def get_location(xcs, fmt, expt_cfg): if 24 / incr_cycl_freq != len(fcst_len_cycl): # Also allow for the possibility that the user is running # cycles for less than a day: - num_cycles = len(set_cycle_dates( - date_first_cycl, - date_last_cycl, - incr_cycl_freq)) + num_cycles = len( + set_cycle_dates(date_first_cycl, date_last_cycl, incr_cycl_freq) + ) if num_cycles != len(fcst_len_cycl): - logger.error(f""" The number of entries in FCST_LEN_CYCL does + logger.error( + f""" The number of entries in FCST_LEN_CYCL does not divide evenly into a 24 hour day or the number of cycles in your experiment! FCST_LEN_CYCL = {fcst_len_cycl} """ - ) - raise ValueError + ) + raise ValueError # Build cycledefs entries for the long forecasts # Short forecast cycles will be relevant to all intended @@ -827,7 +851,7 @@ def get_location(xcs, fmt, expt_cfg): # Find the entries that match the long forecast, and map them to # their time of day. long_fcst_len = max(fcst_len_cycl) - long_indices = [i for i,x in enumerate(fcst_len_cycl) if x == long_fcst_len] + long_indices = [i for i, x in enumerate(fcst_len_cycl) if x == long_fcst_len] long_cycles = [i * incr_cycl_freq for i in long_indices] # add one forecast entry per cycle per day @@ -836,9 +860,9 @@ def get_location(xcs, fmt, expt_cfg): for hh in long_cycles: first = date_first_cycl.replace(hour=hh).strftime("%Y%m%d%H") last = date_last_cycl.replace(hour=hh).strftime("%Y%m%d%H") - fcst_cdef.append(f'{first}00 {last}00 24:00:00') + fcst_cdef.append(f"{first}00 {last}00 24:00:00") - rocoto_config['cycledefs']['long_forecast'] = fcst_cdef + rocoto_config["cycledefs"]["long_forecast"] = fcst_cdef # check the availability of restart intervals for restart capability of forecast do_fcst_restart = fcst_config.get("DO_FCST_RESTART") @@ -933,11 +957,18 @@ def get_location(xcs, fmt, expt_cfg): # Check to make sure all SPP and LSM_SPP lists are the same length. stoch_config = fcst_config["namelist"]["update_values"]["nam_sppperts"] if global_sect.get("DO_SPP"): - list_vars = ("iseed_spp", "spp_lscale", "spp_prt_list", - "spp_sigtop1", "spp_sigtop2", "spp_stddev_cutoff", "spp_tau", - "spp_var_list",) - list_len = fcst_config["namelist"]["update_values"]["n_var_spp"] - if any([len(stoch_config[v]) != list_len for v in list_vars ]): + list_vars = ( + "iseed_spp", + "spp_lscale", + "spp_prt_list", + "spp_sigtop1", + "spp_sigtop2", + "spp_stddev_cutoff", + "spp_tau", + "spp_var_list", + ) + list_len = fcst_config["namelist"]["update_values"]["n_var_spp"] + if any([len(stoch_config[v]) != list_len for v in list_vars]): report = "\n".join([f"{v}: {len(stoch_config[v])}" for v in list_vars]) raise Exception( f""" @@ -954,10 +985,9 @@ def get_location(xcs, fmt, expt_cfg): stoch_config = fcst_config["namelist"]["update_values"]["nam_sfcperts"] if global_sect.get("DO_LSM_SPP"): - list_vars = ("lndp_tau", "lndp_lscale", "lndp_var_list", - "lndp_prt_list") + list_vars = ("lndp_tau", "lndp_lscale", "lndp_var_list", "lndp_prt_list") list_len = fcst_config["namelist"]["update_values"]["n_var_lndp"] - if any([len(stoch_config[v]) != list_len for v in list_vars ]): + if any([len(stoch_config[v]) != list_len for v in list_vars]): report = "\n".join([f"{v}: {len(stoch_config[v])}" for v in list_vars]) raise Exception( f""" @@ -973,7 +1003,6 @@ def get_location(xcs, fmt, expt_cfg): """ ) - # Check whether the forecast length (FCST_LEN_HRS) is evenly divisible # by the BC update interval (LBC_SPEC_INTVL_HRS). If so, generate an # array of forecast hours at which the boundary values will be updated. @@ -990,7 +1019,6 @@ def get_location(xcs, fmt, expt_cfg): rem = FCST_LEN_HRS%%LBC_SPEC_INTVL_HRS = {rem}""" ) - # # ----------------------------------------------------------------------- # @@ -999,7 +1027,6 @@ def get_location(xcs, fmt, expt_cfg): # ----------------------------------------------------------------------- # - # If using external CRTM fix files to allow post-processing of synthetic # satellite products from the UPP, make sure the CRTM fix file directory exists. if global_sect.get("USE_CRTM"): @@ -1059,8 +1086,9 @@ def get_location(xcs, fmt, expt_cfg): # Update the rocoto string for the fcst output location if # running an ensemble in nco mode if global_sect["DO_ENSEMBLE"]: - rocoto_config["entities"]["FCST_DIR"] = \ - "{{ nco.PTMP }}/{{ nco.envir_default }}/tmp/run_fcst_mem#mem#.{{ workflow.WORKFLOW_ID }}_@Y@m@d@H" + rocoto_config["entities"][ + "FCST_DIR" + ] = "{{ nco.PTMP }}/{{ nco.envir_default }}/tmp/run_fcst_mem#mem#.{{ workflow.WORKFLOW_ID }}_@Y@m@d@H" # create experiment dir mkdir_vrfy(f' -p "{exptdir}"') @@ -1131,13 +1159,14 @@ def get_location(xcs, fmt, expt_cfg): # ----------------------------------------------------------------------- # # Get list of all top-level tasks and metatasks in the workflow. - task_defs = rocoto_config.get('tasks') + task_defs = rocoto_config.get("tasks") all_tasks = [task for task in task_defs] # Get list of all valid top-level tasks and metatasks pertaining to ensemble # verification. ens_vx_task_defns = load_config_file( - os.path.join(USHdir, os.pardir, "parm", "wflow", "verify_ens.yaml")) + os.path.join(USHdir, os.pardir, "parm", "wflow", "verify_ens.yaml") + ) ens_vx_valid_tasks = [task for task in ens_vx_task_defns] # Get list of all valid top-level tasks and metatasks in the workflow that @@ -1150,14 +1179,24 @@ def get_location(xcs, fmt, expt_cfg): do_ensemble = global_sect["DO_ENSEMBLE"] if (not do_ensemble) and ens_vx_tasks: task_str = " " + "\n ".join(ens_vx_tasks) - msg = dedent(f""" + msg = dedent( + f""" Ensemble verification can not be run unless running in ensemble mode: DO_ENSEMBLE = \"{do_ensemble}\" Ensemble verification tasks: - """) - msg = "".join([msg, task_str, dedent(f""" + """ + ) + msg = "".join( + [ + msg, + task_str, + dedent( + f""" Please set DO_ENSEMBLE to True or remove ensemble vx tasks from the - workflow.""")]) + workflow.""" + ), + ] + ) raise Exception(msg) # @@ -1195,18 +1234,14 @@ def dict_find(user_dict, substring): run_make_ics = dict_find(rocoto_tasks, "task_make_ics") run_make_lbcs = dict_find(rocoto_tasks, "task_make_lbcs") run_run_fcst = dict_find(rocoto_tasks, "task_run_fcst") - run_any_coldstart_task = run_make_ics or \ - run_make_lbcs or \ - run_run_fcst + run_any_coldstart_task = run_make_ics or run_make_lbcs or run_run_fcst # Flags for creating symlinks to pre-generated grid, orography, and sfc_climo files. # These consider dependencies of other tasks on each pre-processing task. create_symlinks_to_pregen_files = { - "GRID": (not run_make_grid) and \ - (run_make_orog or run_make_sfc_climo or run_any_coldstart_task), - "OROG": (not run_make_orog) and \ - (run_make_sfc_climo or run_any_coldstart_task), - "SFC_CLIMO": (not run_make_sfc_climo) and \ - (run_make_ics or run_make_lbcs), + "GRID": (not run_make_grid) + and (run_make_orog or run_make_sfc_climo or run_any_coldstart_task), + "OROG": (not run_make_orog) and (run_make_sfc_climo or run_any_coldstart_task), + "SFC_CLIMO": (not run_make_sfc_climo) and (run_make_ics or run_make_lbcs), } fixed_files = expt_config["fixed_files"] @@ -1295,7 +1330,7 @@ def dict_find(user_dict, substring): # if fcst_config["WRITE_DOPOST"]: # Turn off run_post - task_name = 'metatask_run_ens_post' + task_name = "metatask_run_ens_post" removed_task = task_defs.pop(task_name, None) if removed_task: logger.warning( @@ -1320,25 +1355,32 @@ def dict_find(user_dict, substring): ccpp_suite_xml = load_xml_file(workflow_config["CCPP_PHYS_SUITE_IN_CCPP_FP"]) # Need to track if we are using RUC LSM for the make_ics step - workflow_config["SDF_USES_RUC_LSM"] = has_tag_with_value(ccpp_suite_xml, "scheme", "lsm_ruc") + workflow_config["SDF_USES_RUC_LSM"] = has_tag_with_value( + ccpp_suite_xml, "scheme", "lsm_ruc" + ) # Thompson microphysics needs additional input files and namelist settings - workflow_config["SDF_USES_THOMPSON_MP"] = has_tag_with_value(ccpp_suite_xml, "scheme", "mp_thompson") + workflow_config["SDF_USES_THOMPSON_MP"] = has_tag_with_value( + ccpp_suite_xml, "scheme", "mp_thompson" + ) if workflow_config["SDF_USES_THOMPSON_MP"]: - logging.debug(f'Selected CCPP suite ({workflow_config["CCPP_PHYS_SUITE"]}) uses Thompson MP') - logging.debug(f'Setting up links for additional fix files') + logging.debug( + f'Selected CCPP suite ({workflow_config["CCPP_PHYS_SUITE"]}) uses Thompson MP' + ) + logging.debug(f"Setting up links for additional fix files") # If the model ICs or BCs are not from RAP or HRRR, they will not contain aerosol # climatology data needed by the Thompson scheme, so we need to provide a separate file thompson_files = fixed_files["THOMPSON_FIX_FILES"] - if (get_extrn_ics["EXTRN_MDL_NAME_ICS"] not in ["HRRR", "RAP"] or - get_extrn_lbcs["EXTRN_MDL_NAME_LBCS"] not in ["HRRR", "RAP"]): + if get_extrn_ics["EXTRN_MDL_NAME_ICS"] not in ["HRRR", "RAP"] or get_extrn_lbcs[ + "EXTRN_MDL_NAME_LBCS" + ] not in ["HRRR", "RAP"]: thompson_files.append(workflow_config["THOMPSON_MP_CLIMO_FN"]) # Add thompson-specific fix files to the FV3 configuration fixam = workflow_config["FIXam"] - thompson_fix_links = {fn: f"{fixam}/{fn}" in thompson_files} + thompson_fix_links = {fn: f"{fixam}/{fn}" in thompson_files} expt_config["task_run_fcst"]["fv3"]["files_to_link"].update(thompson_fix_links) # @@ -1388,7 +1430,6 @@ def dict_find(user_dict, substring): var_defns_cfg["workflow"][dates] = date_to_str(var_defns_cfg["workflow"][dates]) var_defns_cfg.dump(Path(global_var_defns_fp)) - # # ----------------------------------------------------------------------- # @@ -1403,28 +1444,33 @@ def dict_find(user_dict, substring): if v is None or v == "": continue vkey = "valid_vals_" + k - if (vkey in cfg_v): - if (type(v) == list): - if not(all(ele in cfg_v[vkey] for ele in v)): + if vkey in cfg_v: + if type(v) == list: + if not (all(ele in cfg_v[vkey] for ele in v)): raise Exception( - dedent(f""" + dedent( + f""" The variable {k} = {v} in the user's configuration has at least one invalid value. Possible values are: {k} = {cfg_v[vkey]}""" - )) + ) + ) else: if not (v in cfg_v[vkey]): raise Exception( - dedent(f""" + dedent( + f""" The variable {k} = {v} ({type(v)}) in the user's configuration does not have a valid value. Possible values are: {k} = {cfg_v[vkey]}""" - )) + ) + ) return expt_config + def clean_rocoto_dict(rocotodict): """Removes any invalid entries from rocotodict. Examples of invalid entries are: @@ -1438,7 +1484,9 @@ def clean_rocoto_dict(rocotodict): elif key.split("_", maxsplit=1)[0] in ["task"]: if not rocotodict[key].get("command"): popped = rocotodict.pop(key) - logging.warning(f"Invalid task {key} removed due to empty/unset run command") + logging.warning( + f"Invalid task {key} removed due to empty/unset run command" + ) logging.debug(f"Removed entry:\n{popped}") # Loop 2: search for metatasks with no tasks in them @@ -1448,7 +1496,7 @@ def clean_rocoto_dict(rocotodict): for key2 in list(rocotodict[key].keys()): if key2.split("_", maxsplit=1)[0] == "metatask": clean_rocoto_dict(rocotodict[key][key2]) - #After above recursion, any nested empty metatasks will have popped themselves + # After above recursion, any nested empty metatasks will have popped themselves if rocotodict[key].get(key2): valid = True elif key2.split("_", maxsplit=1)[0] == "task": @@ -1459,7 +1507,6 @@ def clean_rocoto_dict(rocotodict): logging.debug(f"Removed entry:\n{popped}") - # # ----------------------------------------------------------------------- # From 18ff14916759fcb6ff374ff8b5793d09419023d7 Mon Sep 17 00:00:00 2001 From: Naureen Date: Thu, 31 Oct 2024 13:40:04 +0000 Subject: [PATCH 14/18] Updates to config defaults and run script --- scripts/run_fcst.py | 2 +- ush/config_defaults.yaml | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/scripts/run_fcst.py b/scripts/run_fcst.py index 775ed6fb76..d94867348f 100755 --- a/scripts/run_fcst.py +++ b/scripts/run_fcst.py @@ -74,7 +74,7 @@ def run_fcst(config_file, cycle, key_path, member): # The experiment config will have {{ CRES | env }} and {{ MEMBER | env }} expressions in it that need to be # dereferenced during driver initialization os.environ["CRES"] = expt_config["workflow"]["CRES"] - os.environ["DOT_ENSMEM"] = dot_ensmem + os.environ["DOT_ENSMEM"] = f".mem{member}" os.environ["MEMBER"] = member # Run the FV3 program via UW driver diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index b1926a6796..81138d4db6 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1771,17 +1771,16 @@ task_run_fcst: threads: 1 field_table: base_file: '{{ workflow.FIELD_TABLE_FP }}' - files_to_copy: - INPUT/gfs_ctrl.nc: '{{ workflow.EXPTDIR }}/run_fcst/INPUT/{{ nco.NET_default }}.{{ cycle }}{{ "dot_ensmem" | env }}.gfs_ctrl.nc' - INPUT/gfs_data.nc: '{{ task_run_fcst.fv3.rundir }}/INPUT/{{ nco.NET_default }}.{{ cycle }}{{ "dot_ensmem" | env }}.gfs_data.tile{{ constants.TILE_RGNL }}.halo{{ constants.NH0 }}.nc' - INPUT/sfc_data.nc: '{{ task_run_fcst.fv3.rundir }}/INPUT/{{ nco.NET_default }}.{{ cycle }}{{ "dot_ensmem" | env }}.sfc_data.tile{{ constants.TILE_RGNL }}.halo{{ constants.NH0 }}.nc' files_to_link: - co2historicaldata_2010.txt: src/uwtools/drivers/global_co2historicaldata_2010.txt + co2historicaldata_2010.txt: '{{ fixed_files.CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING }}' co2historicaldata_2011.txt: src/uwtools/drivers/global_co2historicaldata_2011.txt + INPUT/gfs_ctrl.nc: '{{ workflow.EXPTDIR }}/run_fcst/INPUT/{{ nco.NET_default }}.{{ cycle }}{{ "DOT_ENSMEM" | env }}.gfs_ctrl.nc' + INPUT/gfs_data.nc: '{{ task_run_fcst.fv3.rundir }}/INPUT/{{ nco.NET_default }}.{{ cycle }}{{ "DOT_ENSMEM" | env }}.gfs_data.tile{{ constants.TILE_RGNL }}.halo{{ constants.NH0 }}.nc' + INPUT/sfc_data.nc: '{{ task_run_fcst.fv3.rundir }}/INPUT/{{ nco.NET_default }}.{{ cycle }}{{ "DOT_ENSMEM" | env }}.sfc_data.tile{{ constants.TILE_RGNL }}.halo{{ constants.NH0 }}.nc' lateral_boundary_conditions: interval_hours: 1 offset: 3 - path: '{{ task_run_fcst.fv3.rundir }}/INPUT/{{ nco.NET_default }}.{{ cycle }}{{ "dot_ensmem" | env }}.gfs_bndy.tile{{ constants.TILE_RGNL }}.f{{ task_run_fcst.fv3.lateral_boundary_conditions.offset }}.nc' + path: '{{ task_run_fcst.fv3.rundir }}/INPUT/{{ nco.NET_default }}.{{ cycle }}{{ "DOT_ENSMEM" | env }}.gfs_bndy.tile{{ constants.TILE_RGNL }}.f{{ task_run_fcst.fv3.lateral_boundary_conditions.offset }}.nc' length: 12 model_configure: base_file: '{{ user.USHdir }}/create_model_configure_file.py' From 2c080146d28deeaf473aa167af42fbfa9aad5a49 Mon Sep 17 00:00:00 2001 From: Naureen Date: Thu, 31 Oct 2024 14:28:50 +0000 Subject: [PATCH 15/18] Debugging --- ush/config_defaults.yaml | 2 ++ ush/setup.py | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index aff668dc62..b3543e0608 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1849,6 +1849,7 @@ task_run_fcst: gfs_physics_nml: kice: !int '{{ 9 if workflow.SDF_USES_RUC_LSM else 2 }}' print_diff_pgr: false + nam_sppperts: {} namsfc: # From sfc_climo_gen fnalbc: '{{ workflow.FIXlam }}/{{ "CRES" | env }}.snowfree_albedo.tileX.nc' @@ -2417,6 +2418,7 @@ global: # #----------------------------------------------------------------------- # + DO_SPP: false DO_SHUM: false DO_SPPT: false DO_SKEB: false diff --git a/ush/setup.py b/ush/setup.py index 9799460d13..458187dca5 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -749,7 +749,7 @@ def get_location(xcs, fmt, expt_cfg): ) ) - quilting = fcst_config["model_configure"]["update_values"]["quilting"] + quilting = fcst_config["fv3"]["model_configure"]["update_values"]["quilting"] # Gather the pre-defined grid parameters, if needed if predef_grid_name := (workflow_config.get("PREDEF_GRID_NAME")): grid_params = set_predef_grid_params( @@ -955,8 +955,8 @@ def get_location(xcs, fmt, expt_cfg): # ----------------------------------------------------------------------- # Check to make sure all SPP and LSM_SPP lists are the same length. - stoch_config = fcst_config["namelist"]["update_values"]["nam_sppperts"] - if global_sect.get("DO_SPP"): + stoch_config = fcst_config["fv3"]["namelist"]["update_values"]["nam_sppperts"] + if stoch_config[].get("DO_SPP"): list_vars = ( "iseed_spp", "spp_lscale", @@ -967,7 +967,7 @@ def get_location(xcs, fmt, expt_cfg): "spp_tau", "spp_var_list", ) - list_len = fcst_config["namelist"]["update_values"]["n_var_spp"] + list_len = fcst_config["fv3"]["namelist"]["update_values"]["n_var_spp"] if any([len(stoch_config[v]) != list_len for v in list_vars]): report = "\n".join([f"{v}: {len(stoch_config[v])}" for v in list_vars]) raise Exception( From 5cf2b789281f87adbb2840a18ea15d4de7a92c6a Mon Sep 17 00:00:00 2001 From: Naureen Date: Thu, 31 Oct 2024 15:30:35 +0000 Subject: [PATCH 16/18] Debugging setup.py --- ush/config_defaults.yaml | 1 + ush/config_defaults_aqm.yaml | 47 ++++++++++++++++++------------------ ush/setup.py | 13 +++++----- 3 files changed, 31 insertions(+), 30 deletions(-) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index b3543e0608..f350112c73 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1849,6 +1849,7 @@ task_run_fcst: gfs_physics_nml: kice: !int '{{ 9 if workflow.SDF_USES_RUC_LSM else 2 }}' print_diff_pgr: false + nam_sfcperts: {} nam_sppperts: {} namsfc: # From sfc_climo_gen diff --git a/ush/config_defaults_aqm.yaml b/ush/config_defaults_aqm.yaml index 8c1e36b493..d3f187bbbe 100644 --- a/ush/config_defaults_aqm.yaml +++ b/ush/config_defaults_aqm.yaml @@ -11,27 +11,26 @@ task_run_post: - cmaq task_run_fcst: - namelist: - update_values: - gfs_physics_nml: - cplaqm: true - cplocn2atm: false - fscav_aero: ["aacd:0.0", "acet:0.0", "acrolein:0.0", "acro_primary:0.0", "ald2:0.0", - "ald2_primary:0.0", "aldx:0.0", "benzene:0.0", "butadiene13:0.0", "cat1:0.0", - "cl2:0.0", "clno2:0.0", "co:0.0", "cres:0.0", "cron:0.0", - "ech4:0.0", "epox:0.0", "eth:0.0", "etha:0.0", "ethy:0.0", - "etoh:0.0", "facd:0.0", "fmcl:0.0", "form:0.0", "form_primary:0.0", - "gly:0.0", "glyd:0.0", "h2o2:0.0", "hcl:0.0", "hg:0.0", - "hgiigas:0.0", "hno3:0.0", "hocl:0.0", "hono:0.0", "hpld:0.0", - "intr:0.0", "iole:0.0", "isop:0.0", "ispd:0.0", "ispx:0.0", - "ket:0.0", "meoh:0.0", "mepx:0.0", "mgly:0.0", "n2o5:0.0", - "naph:0.0", "no:0.0", "no2:0.0", "no3:0.0", "ntr1:0.0", - "ntr2:0.0", "o3:0.0", "ole:0.0", "opan:0.0", "open:0.0", - "opo3:0.0", "pacd:0.0", "pan:0.0", "panx:0.0", "par:0.0", - "pcvoc:0.0", "pna:0.0", "prpa:0.0", "rooh:0.0", "sesq:0.0", - "so2:0.0", "soaalk:0.0", "sulf:0.0", "terp:0.0", "tol:0.0", - "tolu:0.0", "vivpo1:0.0", "vlvoo1:0.0", "vlvoo2:0.0", "vlvpo1:0.0", - "vsvoo1:0.0", "vsvoo2:0.0", "vsvoo3:0.0", "vsvpo1:0.0", "vsvpo2:0.0", - "vsvpo3:0.0", "xopn:0.0", "xylmn:0.0", "*:0.2" ] - - + fv3: + namelist: + update_values: + gfs_physics_nml: + cplaqm: true + cplocn2atm: false + fscav_aero: ["aacd:0.0", "acet:0.0", "acrolein:0.0", "acro_primary:0.0", "ald2:0.0", + "ald2_primary:0.0", "aldx:0.0", "benzene:0.0", "butadiene13:0.0", "cat1:0.0", + "cl2:0.0", "clno2:0.0", "co:0.0", "cres:0.0", "cron:0.0", + "ech4:0.0", "epox:0.0", "eth:0.0", "etha:0.0", "ethy:0.0", + "etoh:0.0", "facd:0.0", "fmcl:0.0", "form:0.0", "form_primary:0.0", + "gly:0.0", "glyd:0.0", "h2o2:0.0", "hcl:0.0", "hg:0.0", + "hgiigas:0.0", "hno3:0.0", "hocl:0.0", "hono:0.0", "hpld:0.0", + "intr:0.0", "iole:0.0", "isop:0.0", "ispd:0.0", "ispx:0.0", + "ket:0.0", "meoh:0.0", "mepx:0.0", "mgly:0.0", "n2o5:0.0", + "naph:0.0", "no:0.0", "no2:0.0", "no3:0.0", "ntr1:0.0", + "ntr2:0.0", "o3:0.0", "ole:0.0", "opan:0.0", "open:0.0", + "opo3:0.0", "pacd:0.0", "pan:0.0", "panx:0.0", "par:0.0", + "pcvoc:0.0", "pna:0.0", "prpa:0.0", "rooh:0.0", "sesq:0.0", + "so2:0.0", "soaalk:0.0", "sulf:0.0", "terp:0.0", "tol:0.0", + "tolu:0.0", "vivpo1:0.0", "vlvoo1:0.0", "vlvoo2:0.0", "vlvpo1:0.0", + "vsvoo1:0.0", "vsvoo2:0.0", "vsvoo3:0.0", "vsvpo1:0.0", "vsvpo2:0.0", + "vsvpo3:0.0", "xopn:0.0", "xylmn:0.0", "*:0.2" ] diff --git a/ush/setup.py b/ush/setup.py index 458187dca5..01d7d7ddb6 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -956,7 +956,8 @@ def get_location(xcs, fmt, expt_cfg): # Check to make sure all SPP and LSM_SPP lists are the same length. stoch_config = fcst_config["fv3"]["namelist"]["update_values"]["nam_sppperts"] - if stoch_config[].get("DO_SPP"): + global_sect = expt_config["global"] + if global_sect.get("DO_SPP"): list_vars = ( "iseed_spp", "spp_lscale", @@ -983,7 +984,7 @@ def get_location(xcs, fmt, expt_cfg): """ ) - stoch_config = fcst_config["namelist"]["update_values"]["nam_sfcperts"] + stoch_config = fcst_config["fv3"]["namelist"]["update_values"]["nam_sfcperts"] if global_sect.get("DO_LSM_SPP"): list_vars = ("lndp_tau", "lndp_lscale", "lndp_var_list", "lndp_prt_list") list_len = fcst_config["namelist"]["update_values"]["n_var_lndp"] @@ -1328,7 +1329,7 @@ def dict_find(user_dict, substring): # # ----------------------------------------------------------------------- # - if fcst_config["WRITE_DOPOST"]: + if fcst_config["fv3"]["model_configure"]["update_values"]["write_dopost"]: # Turn off run_post task_name = "metatask_run_ens_post" removed_task = task_defs.pop(task_name, None) @@ -1379,9 +1380,9 @@ def dict_find(user_dict, substring): thompson_files.append(workflow_config["THOMPSON_MP_CLIMO_FN"]) # Add thompson-specific fix files to the FV3 configuration - fixam = workflow_config["FIXam"] - thompson_fix_links = {fn: f"{fixam}/{fn}" in thompson_files} - expt_config["task_run_fcst"]["fv3"]["files_to_link"].update(thompson_fix_links) + fixed_files = expt_config["fixed_files"] + for fn in thompson_files: + fcst_config["fv3"]["namelist"]["update_values"]["namsfc"].update(fn) # # ----------------------------------------------------------------------- From d22af9573cdbc69b7c2a419659e1030e8defda8f Mon Sep 17 00:00:00 2001 From: Naureen Date: Thu, 31 Oct 2024 18:08:35 +0000 Subject: [PATCH 17/18] Add fv3nml sfc back --- ush/set_fv3nml_sfc_climo_filenames.py | 132 ++++++++++++++++++++++++++ ush/setup.py | 13 ++- 2 files changed, 142 insertions(+), 3 deletions(-) create mode 100644 ush/set_fv3nml_sfc_climo_filenames.py diff --git a/ush/set_fv3nml_sfc_climo_filenames.py b/ush/set_fv3nml_sfc_climo_filenames.py new file mode 100644 index 0000000000..de2edc077e --- /dev/null +++ b/ush/set_fv3nml_sfc_climo_filenames.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 + +""" +Update filenames for surface climotology files in the namelist. +""" + +import argparse +import os +import re +import sys +from textwrap import dedent + +from python_utils import ( + cfg_to_yaml_str, + check_var_valid_value, + flatten_dict, + import_vars, + print_info_msg, +) + +from uwtools.api.config import get_nml_config, get_yaml_config, realize + + +VERBOSE = os.environ.get("VERBOSE", "true") + +NEEDED_VARS = [ + "CRES", + "DO_ENSEMBLE", + "EXPTDIR", + "FIXlam", + "FV3_NML_FP", + "PARMdir", + "RUN_ENVIR", + ] + + +# pylint: disable=undefined-variable + +def set_fv3nml_sfc_climo_filenames(config, debug=False): + """ + This function sets the values of the variables in + the forecast model's namelist file that specify the paths to the surface + climatology files on the FV3LAM native grid (which are either pregenerated + or created by the TN_MAKE_SFC_CLIMO task). Note that the workflow + generation scripts create symlinks to these surface climatology files + in the FIXlam directory, and the values in the namelist file that get + set by this function are relative or full paths to these links. + + Args: + debug (bool): Enable extra output for debugging + Returns: + None + """ + + import_vars(dictionary=config, env_vars=NEEDED_VARS) + + fixed_cfg = get_yaml_config(os.path.join(PARMdir, "fixed_files_mapping.yaml"))["fixed_files"] + + # The regular expression regex_search set below will be used to extract + # from the elements of the array FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING + # the name of the namelist variable to set and the corresponding surface + # climatology field from which to form the name of the surface climatology file + regex_search = "^[ ]*([^| ]+)[ ]*[|][ ]*([^| ]+)[ ]*$" + + # Set the suffix of the surface climatology files. + suffix = "tileX.nc" + + # create yaml-compliant string + settings = {} + + dummy_run_dir = os.path.join(EXPTDIR, "any_cyc") + if DO_ENSEMBLE == "TRUE": + dummy_run_dir += os.sep + "any_ensmem" + + namsfc_dict = {} + for mapping in fixed_cfg["FIXgsm_FILES_TO_COPY_TO_FIXam"]: + nml_var_name, sfc_climo_field_name = re.search(regex_search, mapping).groups() + + check_var_valid_value(sfc_climo_field_name, fixed_cfg["SFC_CLIMO_FIELDS"]) + + file_path = os.path.join(FIXlam, f"{CRES}.{sfc_climo_field_name}.{suffix}") + if RUN_ENVIR != "nco": + file_path = os.path.relpath(os.path.realpath(file_path), start=dummy_run_dir) + + namsfc_dict[nml_var_name] = file_path + + settings["namsfc_dict"] = namsfc_dict + settings_str = cfg_to_yaml_str(settings) + + print_info_msg( + dedent( + f""" + The variable 'settings' specifying values of the namelist variables + has been set as follows:\n + settings = + + {settings_str} + """ + ), + verbose=debug, + ) + + realize( + input_config=FV3_NML_FP, + input_format="nml", + output_file=FV3_NML_FP, + output_format="nml", + update_config=get_nml_config(settings), + ) + +def parse_args(argv): + """Parse command line arguments""" + parser = argparse.ArgumentParser(description="Set surface climatology fields.") + + parser.add_argument( + "-p", + "--path-to-defns", + dest="path_to_defns", + required=True, + help="Path to var_defns file.", + ) + parser.add_argument('-d', '--debug', action='store_true', + help='Script will be run in debug mode with more verbose output') + + return parser.parse_args(argv) + + +if __name__ == "__main__": + args = parse_args(sys.argv[1:]) + cfg = get_yaml_config(args.path_to_defns) + cfg = flatten_dict(cfg) + set_fv3nml_sfc_climo_filenames(cfg, args.debug) diff --git a/ush/setup.py b/ush/setup.py index 01d7d7ddb6..9f12ba0353 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -1380,9 +1380,16 @@ def dict_find(user_dict, substring): thompson_files.append(workflow_config["THOMPSON_MP_CLIMO_FN"]) # Add thompson-specific fix files to the FV3 configuration - fixed_files = expt_config["fixed_files"] - for fn in thompson_files: - fcst_config["fv3"]["namelist"]["update_values"]["namsfc"].update(fn) + thompson_files = fixed_files["THOMPSON_FIX_FILES"] + if get_extrn_ics["EXTRN_MDL_NAME_ICS"] not in ["HRRR", "RAP"] or get_extrn_lbcs[ + "EXTRN_MDL_NAME_LBCS" + ] not in ["HRRR", "RAP"]: + thompson_files.append(workflow_config["THOMPSON_MP_CLIMO_FN"]) + + # Add thompson-specific fix files to the FV3 configuration + fixam = workflow_config["FIXam"] + thompson_fix_links = {fn: f"{fixam}/{fn}" for fn in thompson_files} + expt_config["task_run_fcst"]["fv3"]["files_to_link"].update(thompson_fix_links) # # ----------------------------------------------------------------------- From 5ecc61f45ebd6b44ad253a6d4088f286cf469a45 Mon Sep 17 00:00:00 2001 From: Naureen Date: Thu, 7 Nov 2024 15:25:25 +0000 Subject: [PATCH 18/18] Updates to make_grid --- scripts/exregional_make_grid.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index 0393eb07bb..0c80caeeea 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -736,11 +736,6 @@ failed." # #----------------------------------------------------------------------- # -python3 $USHdir/set_fv3nml_sfc_climo_filenames.py \ - --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ - || print_err_msg_exit "\ -Call to function to set surface climatology file names in the FV3 namelist -file failed." # #----------------------------------------------------------------------- #