From fdb996d4856885521f58904bd53ccfa14b8193e5 Mon Sep 17 00:00:00 2001 From: David Grumm Date: Wed, 27 Nov 2024 21:57:47 +0000 Subject: [PATCH 01/36] Stage_ic updates: GH2988 --- jobs/JGLOBAL_FETCH | 23 ++++++++ parm/config/gefs/config.fetch | 38 +++++++++++++ parm/config/gfs/config.fetch | 37 +++++++++++++ parm/fetch/stage_atm_cold.yaml | 19 +++++++ scripts/exglobal_fetch.py | 45 ++++++++++++++++ ush/python/pygfs/task/fetch.py | 98 ++++++++++++++++++++++++++++++++++ 6 files changed, 260 insertions(+) create mode 100755 jobs/JGLOBAL_FETCH create mode 100644 parm/config/gefs/config.fetch create mode 100644 parm/config/gfs/config.fetch create mode 100644 parm/fetch/stage_atm_cold.yaml create mode 100755 scripts/exglobal_fetch.py create mode 100755 ush/python/pygfs/task/fetch.py diff --git a/jobs/JGLOBAL_FETCH b/jobs/JGLOBAL_FETCH new file mode 100755 index 0000000000..e10049169a --- /dev/null +++ b/jobs/JGLOBAL_FETCH @@ -0,0 +1,23 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +source "${HOMEgfs}/ush/jjob_header.sh" -e "fetch" -c "base fetch" + +# Execute fetching +"${SCRgfs}/exglobal_fetch.py" +err=$? + +############################################################### +# Check for errors and exit if any of the above failed +if [[ "${err}" -ne 0 ]]; then + echo "FATAL ERROR: Unable to fetch ICs to ${ROTDIR}; ABORT!" + exit "${err}" +fi + +########################################## +# Remove the Temporary working directory +########################################## +cd "${DATAROOT}" || (echo "${DATAROOT} does not exist. ABORT!"; exit 1) +[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}" + +exit 0 diff --git a/parm/config/gefs/config.fetch b/parm/config/gefs/config.fetch new file mode 100644 index 0000000000..648ae52a54 --- /dev/null +++ b/parm/config/gefs/config.fetch @@ -0,0 +1,38 @@ +#! /usr/bin/env bash + +########## config.fetch ########## + +echo "BEGIN: config.fetch" + +# Get task specific resources +source "${EXPDIR}/config.resources" fetch + +export ICSDIR="@ICSDIR@" # User provided ICSDIR; blank if not provided +export BASE_IC="@BASE_IC@" # Platform home for staged ICs + +export FETCH_YAML_TMPL="${PARMgfs}/stage/master_gefs.yaml.j2" +export fetch_yaml="stage_atm_cold.yaml" + +# Set ICSDIR (if not defined) + +if [[ -z "${ICSDIR}" ]] ; then + + ic_ver="20240610" + + if (( NMEM_ENS > 0 )) ; then + ensic="${CASE_ENS}" + fi + + if [[ "${DO_OCN:-NO}" == "YES" ]] ; then + ocnic="mx${OCNRES}" + fi + + export ICSDIR="${BASE_IC}/${CASE}${ensic:-}${ocnic:-}/${ic_ver}" + +fi + +# Use of perturbations files for ensembles +export USE_OCN_ENS_PERTURB_FILES="NO" +export USE_ATM_ENS_PERTURB_FILES="NO" + +echo "END: config.fetch" diff --git a/parm/config/gfs/config.fetch b/parm/config/gfs/config.fetch new file mode 100644 index 0000000000..88fd784497 --- /dev/null +++ b/parm/config/gfs/config.fetch @@ -0,0 +1,37 @@ +#! /usr/bin/env bash + +########## config.fetch ########## + +echo "BEGIN: config.fetch" + +# Get task specific resources +source "${EXPDIR}/config.resources" fetch + +export ICSDIR="@ICSDIR@" # User provided ICSDIR; blank if not provided +export BASE_IC="@BASE_IC@" # Platform home for staged ICs + +export FETCH_YAML_TMPL="${PARMgfs}/stage/master_gfs.yaml.j2" +export fetch_yaml="stage_atm_cold.yaml" + +# Set ICSDIR (if not defined) +if [[ -z "${ICSDIR}" ]] ; then + + ic_ver="20240610" + + if (( NMEM_ENS > 0 )) ; then + ensic="${CASE_ENS}" + fi + + if [[ "${DO_OCN:-NO}" == "YES" ]] ; then + ocnic="mx${OCNRES}" + fi + + export ICSDIR="${BASE_IC}/${CASE}${ensic:-}${ocnic:-}/${ic_ver}" + +fi + +# Use of perturbations files for ensembles +export USE_OCN_ENS_PERTURB_FILES="NO" +export USE_ATM_ENS_PERTURB_FILES="NO" + +echo "END: config.fetch" diff --git a/parm/fetch/stage_atm_cold.yaml b/parm/fetch/stage_atm_cold.yaml new file mode 100644 index 0000000000..22cbe1d12b --- /dev/null +++ b/parm/fetch/stage_atm_cold.yaml @@ -0,0 +1,19 @@ +untar: + tarball : "{{ ATARDIR }}/{{ cycle_YMDH }}/atm_cold.tar" + on_hpss: True + contents: + - gfs_ctrl.nc + {% for ftype in ["gfs_data", "sfc_data"] %} + {% for ntile in range(1, ntiles + 1) %} + - {{ ftype }}.tile{{ ntile }}.nc + {% endfor %} # ntile + {% endfor %} # ftype + destination: "{{ DATA }}" +atmosphere_cold: + copy: + - ["{{ DATA }}/gfs_ctrl.nc", "{{ COMOUT_ATMOS_INPUT }}"] + {% for ftype in ["gfs_data", "sfc_data"] %} + {% for ntile in range(1, ntiles + 1) %} + - ["{{ DATA }}/{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_INPUT }}"] + {% endfor %} # ntile + {% endfor %} # ftype diff --git a/scripts/exglobal_fetch.py b/scripts/exglobal_fetch.py new file mode 100755 index 0000000000..c1446d14c7 --- /dev/null +++ b/scripts/exglobal_fetch.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 + +import os + +from fetch import Fetch +from wxflow import AttrDict, Logger, cast_strdict_as_dtypedict, logit + +# initialize root logger +logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=True) + + +@logit(logger) +def main(): + + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the Fetch object + fetch = Fetch(config) + + # Pull out all the configuration keys needed to run the fetch step + keys = ['current_cycle', 'RUN', 'PDY', 'PARMgfs', 'PSLOT', 'ROTDIR', 'fetch_yaml', 'ATARDIR', 'ntiles'] + + fetch_dict = AttrDict() + for key in keys: + fetch_dict[key] = fetch.task_config.get(key) + if fetch_dict[key] is None: + print(f"Warning: key ({key}) not found in task_config!") + + # Also import all COMOUT* directory and template variables + for key in fetch.task_config.keys(): + if key.startswith("COMOUT_"): + fetch_dict[key] = fetch.task_config.get(key) + if fetch_dict[key] is None: + print(f"Warning: key ({key}) not found in task_config!") + + # Determine which archives to retrieve from HPSS + # Read the input YAML file to get the list of tarballs on tape + atardir_set = fetch.configure(fetch_dict) + + # Pull the data from tape or locally and store the specified destination + fetch.execute_pull_data(atardir_set) + + +if __name__ == '__main__': + main() diff --git a/ush/python/pygfs/task/fetch.py b/ush/python/pygfs/task/fetch.py new file mode 100755 index 0000000000..657969f216 --- /dev/null +++ b/ush/python/pygfs/task/fetch.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 + +import os +from logging import getLogger +from typing import Any, Dict, List + +from wxflow import (AttrDict, FileHandler, Hsi, Task, + logit, parse_j2yaml) +from wxflow import htar as Htar +import tarfile + + +logger = getLogger(__name__.split('.')[-1]) + + +class Fetch(Task): + """Task to pull ROTDIR data from HPSS (or locally) + """ + + @logit(logger, name="Fetch") + def __init__(self, config: Dict[str, Any]) -> None: + """Constructor for the Fetch task + The constructor is responsible for collecting necessary yamls based on + the runtime options and RUN. + + Parameters + ---------- + config : Dict[str, Any] + Incoming configuration for the task from the environment + + Returns + ------- + None + """ + super().__init__(config) + + # Perhaps add other stuff to self. + + @logit(logger) + def configure(self, fetch_dict: Dict[str, Any]): + """Determine which tarballs will need to be extracted + + Parameters + ---------- + fetch_dict : Dict[str, Any] + Task specific keys, e.g. COM directories, etc + + Return + ------ + parsed_fetch: Dict[str, Any] + Dictionary derived from the yaml file with necessary HPSS info. + """ + + self.hsi = Hsi() + + fetch_yaml = fetch_dict.fetch_yaml + fetch_parm = os.path.join(fetch_dict.PARMgfs, "fetch") + + parsed_fetch = parse_j2yaml(os.path.join(fetch_parm, fetch_yaml), + fetch_dict) + + return parsed_fetch + + @logit(logger) + def execute_pull_data(self, atardir_set: Dict[str, Any]) -> None: + """Pull data from HPSS based on a yaml dictionary and store at the + specified destination. + + Parameters + ---------- + atardir_set: Dict[str, Any], + Dict defining set of tarballs to pull and where to put them. + + Return + None + """ + if len(f_names) <= 0: # Abort if no files + raise FileNotFoundError("FATAL ERROR: The tar ball has no files") # DG - add name + + f_names = atardir_set.untar.contents + on_hpss = atardir_set.untar.on_hpss + dest = atardir_set.untar.destination + + # Select action whether no_hpss is True or not, and pull these data from + # tape or locally and place where it needs to go + # DG - these need testing + if on_hpss is True: # htar all files in fnames + htar_obj = Htar.Htar() + htar_obj.cvf(dest, f_names) + + else: # tar all files in fnames + with tarfile.open(dest, "w") as tar: + for filename in f_names: + tar.add(filename) + + + # Other helper methods... + From a1f03ecd530d22758ffd746130d6faef6c05446a Mon Sep 17 00:00:00 2001 From: David Grumm Date: Tue, 10 Dec 2024 02:17:29 +0000 Subject: [PATCH 02/36] Removed white space --- ush/python/pygfs/task/fetch.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/ush/python/pygfs/task/fetch.py b/ush/python/pygfs/task/fetch.py index 657969f216..98b52e760a 100755 --- a/ush/python/pygfs/task/fetch.py +++ b/ush/python/pygfs/task/fetch.py @@ -5,9 +5,9 @@ from typing import Any, Dict, List from wxflow import (AttrDict, FileHandler, Hsi, Task, - logit, parse_j2yaml) + logit, parse_j2yaml) from wxflow import htar as Htar -import tarfile +import tarfile logger = getLogger(__name__.split('.')[-1]) @@ -33,7 +33,6 @@ def __init__(self, config: Dict[str, Any]) -> None: None """ super().__init__(config) - # Perhaps add other stuff to self. @logit(logger) @@ -50,7 +49,6 @@ def configure(self, fetch_dict: Dict[str, Any]): parsed_fetch: Dict[str, Any] Dictionary derived from the yaml file with necessary HPSS info. """ - self.hsi = Hsi() fetch_yaml = fetch_dict.fetch_yaml @@ -58,12 +56,11 @@ def configure(self, fetch_dict: Dict[str, Any]): parsed_fetch = parse_j2yaml(os.path.join(fetch_parm, fetch_yaml), fetch_dict) - return parsed_fetch @logit(logger) def execute_pull_data(self, atardir_set: Dict[str, Any]) -> None: - """Pull data from HPSS based on a yaml dictionary and store at the + """Pull data from HPSS based on a yaml dictionary and store at the specified destination. Parameters @@ -74,25 +71,20 @@ def execute_pull_data(self, atardir_set: Dict[str, Any]) -> None: Return None """ + if len(f_names) <= 0: # Abort if no files raise FileNotFoundError("FATAL ERROR: The tar ball has no files") # DG - add name - f_names = atardir_set.untar.contents on_hpss = atardir_set.untar.on_hpss dest = atardir_set.untar.destination - # Select action whether no_hpss is True or not, and pull these data from - # tape or locally and place where it needs to go + # tape or locally and place where it needs to go # DG - these need testing if on_hpss is True: # htar all files in fnames htar_obj = Htar.Htar() - htar_obj.cvf(dest, f_names) + htar_obj.cvf(dest, f_names) else: # tar all files in fnames with tarfile.open(dest, "w") as tar: for filename in f_names: tar.add(filename) - - - # Other helper methods... - From 17f7e6ca378e644d02773e08b07b1110e884cbcc Mon Sep 17 00:00:00 2001 From: David Grumm Date: Tue, 10 Dec 2024 17:25:07 +0000 Subject: [PATCH 03/36] remove more whitespace --- ush/python/pygfs/task/fetch.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ush/python/pygfs/task/fetch.py b/ush/python/pygfs/task/fetch.py index 98b52e760a..b6ef48a4c9 100755 --- a/ush/python/pygfs/task/fetch.py +++ b/ush/python/pygfs/task/fetch.py @@ -55,7 +55,7 @@ def configure(self, fetch_dict: Dict[str, Any]): fetch_parm = os.path.join(fetch_dict.PARMgfs, "fetch") parsed_fetch = parse_j2yaml(os.path.join(fetch_parm, fetch_yaml), - fetch_dict) + fetch_dict) return parsed_fetch @logit(logger) @@ -65,7 +65,7 @@ def execute_pull_data(self, atardir_set: Dict[str, Any]) -> None: Parameters ---------- - atardir_set: Dict[str, Any], + atardir_set: Dict[str, Any], Dict defining set of tarballs to pull and where to put them. Return @@ -73,18 +73,18 @@ def execute_pull_data(self, atardir_set: Dict[str, Any]) -> None: """ if len(f_names) <= 0: # Abort if no files - raise FileNotFoundError("FATAL ERROR: The tar ball has no files") # DG - add name + raise FileNotFoundError("FATAL ERROR: The tar ball has no files") f_names = atardir_set.untar.contents on_hpss = atardir_set.untar.on_hpss dest = atardir_set.untar.destination - # Select action whether no_hpss is True or not, and pull these data from - # tape or locally and place where it needs to go + # Select action whether no_hpss is True or not, and pull these + # data from tape or locally and place where it needs to go # DG - these need testing - if on_hpss is True: # htar all files in fnames + if on_hpss is True: # htar all files in fnames htar_obj = Htar.Htar() htar_obj.cvf(dest, f_names) - else: # tar all files in fnames + else: # tar all files in fnames with tarfile.open(dest, "w") as tar: for filename in f_names: tar.add(filename) From a9bd3e2829da6b081470cdaff014ad6f8f23341a Mon Sep 17 00:00:00 2001 From: David Grumm Date: Thu, 12 Dec 2024 21:40:25 +0000 Subject: [PATCH 04/36] Incorporate fetch into GW --- ci/cases/pr/ATM_cold.yaml | 11 +++++++++ jobs/rocoto/fetch.sh | 18 +++++++++++++++ parm/config/gfs/config.base | 8 +++++++ parm/config/gfs/config.resources | 4 ++-- workflow/applications/gfs_forecast_only.py | 8 +++++-- workflow/rocoto/gfs_tasks.py | 27 ++++++++++++++++++++++ workflow/rocoto/tasks.py | 2 +- 7 files changed, 73 insertions(+), 5 deletions(-) create mode 100644 ci/cases/pr/ATM_cold.yaml create mode 100755 jobs/rocoto/fetch.sh diff --git a/ci/cases/pr/ATM_cold.yaml b/ci/cases/pr/ATM_cold.yaml new file mode 100644 index 0000000000..0f55189bed --- /dev/null +++ b/ci/cases/pr/ATM_cold.yaml @@ -0,0 +1,11 @@ +untar: + tarball : "{{ ATARDIR }}/{{ cycle_YMDH }}/atm_cold.tar" + on_hpss: True + contents: + - gfs_ctrl.nc + {% for ftype in ["gfs_data", "sfc_data"] %} + {% for ntile in range(1, ntiles + 1) %} + - {{ ftype }}.tile{{ ntile }}.nc + {% endfor %} # ntile + {% endfor %} # ftype + destination: "{{ DATA }}" \ No newline at end of file diff --git a/jobs/rocoto/fetch.sh b/jobs/rocoto/fetch.sh new file mode 100755 index 0000000000..ee34f6bd92 --- /dev/null +++ b/jobs/rocoto/fetch.sh @@ -0,0 +1,18 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +# Source FV3GFS workflow modules +. "${HOMEgfs}/ush/load_fv3gfs_modules.sh" +status=$? +[[ "${status}" -ne 0 ]] && exit "${status}" + +export job="fetch" +export jobid="${job}.$$" + +# Execute the JJOB +"${HOMEgfs}/jobs/JGLOBAL_FETCH" +status=$? + + +exit "${status}" diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 4f702f9668..3255841e9d 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -457,6 +457,14 @@ export DO_VRFY_OCEANDA="@DO_VRFY_OCEANDA@" # Run SOCA Ocean and Seaice DA verif export FHMAX_FITS=132 [[ "${FHMAX_FITS}" -gt "${FHMAX_GFS}" ]] && export FHMAX_FITS=${FHMAX_GFS} +# Fetch options +export DO_FETCH_HPSS="@DO_FETCH_HPSS@" # pull from HPSS +export DO_FETCH_LOCAL="@DO_FETCH_LOCAL@" # pull from local +if [[ ${DO_FETCH_HPSS} = "YES" ]] && [[ ${DO_FETCH_LOCAL} = "YES" ]]; then + echo "Both HPSS and local fetch selected. Please choose one or the other." + exit 3 +fi + # Archiving options export HPSSARCH="@HPSSARCH@" # save data to HPSS archive export LOCALARCH="@LOCALARCH@" # save data to local archive diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index cddd1643fd..fccc1cd2f9 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -11,7 +11,7 @@ if (( $# != 1 )); then echo "Must specify an input task argument to set resource variables!" echo "argument can be any one of the following:" - echo "stage_ic aerosol_init" + echo "stage_ic aerosol_init fetch" echo "prep prepatmiodaobs" echo "atmanlinit atmanlvar atmanlfv3inc atmanlfinal" echo "atmensanlinit atmensanlobs atmensanlsol atmensanlletkf atmensanlfv3inc atmensanlfinal" @@ -1015,7 +1015,7 @@ case ${step} in export is_exclusive=True ;; - "arch" | "earc" | "getic") + "arch" | "earc" | "getic" | "fetch") walltime="06:00:00" ntasks=1 tasks_per_node=1 diff --git a/workflow/applications/gfs_forecast_only.py b/workflow/applications/gfs_forecast_only.py index fb1d2cdb8f..a92af0113a 100644 --- a/workflow/applications/gfs_forecast_only.py +++ b/workflow/applications/gfs_forecast_only.py @@ -20,7 +20,7 @@ def _get_app_configs(self): Returns the config_files that are involved in the forecast-only app """ - configs = ['stage_ic', 'fcst', 'arch', 'cleanup'] + configs = ['fetch', 'stage_ic', 'fcst', 'arch', 'cleanup'] if self.do_atm: @@ -89,7 +89,11 @@ def get_task_names(self): This is the place where that order is set. """ - tasks = ['stage_ic'] + tasks = [] + if self.do_fetch_hpss or self_fetch_local: + tasks += ['fetch'] + + tasks += ['stage_ic'] if self.do_aero: aero_fcst_run = self.aero_fcst_run diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 461241450e..1e6241839e 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -18,6 +18,13 @@ def _is_this_a_gdas_task(run, task_name): # Specific Tasks begin here def stage_ic(self): + deps = [] + dep_dict = { + 'type': 'task', 'name': f'fetch', + } + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + cycledef = 'gdas_half' if self.run in ['gdas', 'enkfgdas'] else self.run resources = self.get_resource('stage_ic') @@ -36,6 +43,26 @@ def stage_ic(self): return task + def fetch(self): + + cycledef = 'gdas_half' if self.run in ['gdas', 'enkfgdas'] else self.run + + resources = self.get_resource('fetch') + task_name = f'{self.run}_fetch' + task_dict = {'task_name': task_name, + 'resources': resources, + 'envars': self.envars, + 'cycledef': cycledef, + 'command': f'{self.HOMEgfs}/jobs/rocoto/fetch.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task + def prep(self): dump_suffix = self._base["DUMP_SUFFIX"] diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index df56f90718..51a1887a13 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -10,7 +10,7 @@ class Tasks: - SERVICE_TASKS = ['arch', 'earc'] + SERVICE_TASKS = ['arch', 'earc', 'fetch'] VALID_TASKS = ['aerosol_init', 'stage_ic', 'prep', 'anal', 'sfcanl', 'analcalc', 'analdiag', 'arch', "cleanup", 'prepatmiodaobs', 'atmanlinit', 'atmanlvar', 'atmanlfv3inc', 'atmanlfinal', From e62487273d39f49d802928b43e68fbedc0e5e378 Mon Sep 17 00:00:00 2001 From: David Grumm Date: Fri, 13 Dec 2024 18:24:34 +0000 Subject: [PATCH 05/36] Fix whitespace --- workflow/applications/gfs_forecast_only.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/workflow/applications/gfs_forecast_only.py b/workflow/applications/gfs_forecast_only.py index a92af0113a..ed6d45fab4 100644 --- a/workflow/applications/gfs_forecast_only.py +++ b/workflow/applications/gfs_forecast_only.py @@ -103,7 +103,8 @@ def get_task_names(self): if self.do_wave: tasks += ['waveinit'] - # tasks += ['waveprep'] # TODO - verify if waveprep is executed in forecast-only mode when APP=ATMW|S2SW + # tasks += ['waveprep'] # TODO - verify if waveprep is executed in ... + # ... forecast-only mode when APP=ATMW|S2SW tasks += ['fcst'] From f21f96b17eff54b3842ebd4dedb736fbab3a582b Mon Sep 17 00:00:00 2001 From: David Grumm Date: Mon, 16 Dec 2024 21:00:13 +0000 Subject: [PATCH 06/36] Move the fetch options to the run_options dict --- workflow/applications/applications.py | 3 +++ workflow/applications/gfs_forecast_only.py | 9 ++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index 22e299df20..13552eddb9 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -102,6 +102,9 @@ def _get_run_options(self, conf: Configuration) -> Dict[str, Any]: run_options[run]['do_hpssarch'] = run_base.get('HPSSARCH', False) run_options[run]['fcst_segments'] = run_base.get('FCST_SEGMENTS', None) + run_options[run]['fetch_hpss'] = run_base.get('FETCH_HPSS', False) + run_options[run]['fetch_local'] = run_base.get('FETCH_LOCAL', False) + if not AppConfig.is_monotonic(run_options[run]['fcst_segments']): raise ValueError(f'Forecast segments do not increase monotonically: {",".join(self.fcst_segments)}') diff --git a/workflow/applications/gfs_forecast_only.py b/workflow/applications/gfs_forecast_only.py index c17a8a4524..bd94979b10 100644 --- a/workflow/applications/gfs_forecast_only.py +++ b/workflow/applications/gfs_forecast_only.py @@ -96,12 +96,15 @@ def get_task_names(self): Note that the order of the task names matters in the XML. This is the place where that order is set. """ + + options = self.run_options[self.run] + tasks = [] - if self.do_fetch_hpss or self_fetch_local: + + if options['fetch_hpss'] or options['fetch_local']: tasks += ['fetch'] + tasks += ['stage_ic'] - - options = self.run_options[self.run] if options['do_aero_fcst'] and not options['exp_warm_start']: tasks += ['aerosol_init'] From 20f28620fc3b2ad4355c8a8a577c6c43ef558c2a Mon Sep 17 00:00:00 2001 From: David Grumm Date: Wed, 18 Dec 2024 06:29:08 +0000 Subject: [PATCH 07/36] Additonal code to incorporate fetch, including modifying some of the stage_ic and yaml code --- parm/config/gefs/config.base | 1 + parm/config/gefs/config.fetch | 25 ------------------- parm/config/gfs/config.base | 5 ++-- parm/config/gfs/config.fetch | 29 ++-------------------- parm/fetch/stage_atm_cold.yaml | 19 -------------- workflow/applications/gfs_forecast_only.py | 7 +++++- workflow/rocoto/gfs_tasks.py | 13 +++++----- workflow/rocoto/tasks.py | 2 +- 8 files changed, 20 insertions(+), 81 deletions(-) delete mode 100644 parm/fetch/stage_atm_cold.yaml diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index 8d5852a15b..e6eccd1a13 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -103,6 +103,7 @@ export ROTDIR="@COMROOT@/${PSLOT}" export ARCDIR="${NOSCRUB}/archive/${PSLOT}" export ATARDIR="@ATARDIR@" +export FETCHDIR="@FETCHDIR@" # DG adding 121724 # Commonly defined parameters in JJOBS export envir=${envir:-"prod"} diff --git a/parm/config/gefs/config.fetch b/parm/config/gefs/config.fetch index 648ae52a54..634cf3f54c 100644 --- a/parm/config/gefs/config.fetch +++ b/parm/config/gefs/config.fetch @@ -10,29 +10,4 @@ source "${EXPDIR}/config.resources" fetch export ICSDIR="@ICSDIR@" # User provided ICSDIR; blank if not provided export BASE_IC="@BASE_IC@" # Platform home for staged ICs -export FETCH_YAML_TMPL="${PARMgfs}/stage/master_gefs.yaml.j2" -export fetch_yaml="stage_atm_cold.yaml" - -# Set ICSDIR (if not defined) - -if [[ -z "${ICSDIR}" ]] ; then - - ic_ver="20240610" - - if (( NMEM_ENS > 0 )) ; then - ensic="${CASE_ENS}" - fi - - if [[ "${DO_OCN:-NO}" == "YES" ]] ; then - ocnic="mx${OCNRES}" - fi - - export ICSDIR="${BASE_IC}/${CASE}${ensic:-}${ocnic:-}/${ic_ver}" - -fi - -# Use of perturbations files for ensembles -export USE_OCN_ENS_PERTURB_FILES="NO" -export USE_ATM_ENS_PERTURB_FILES="NO" - echo "END: config.fetch" diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index a9a86af18e..39e252e349 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -132,6 +132,7 @@ if [[ "${PDY}${cyc}" -ge "2019092100" && "${PDY}${cyc}" -le "2019110700" ]]; the fi export ARCDIR="${NOSCRUB}/archive/${PSLOT}" export ATARDIR="@ATARDIR@" +export FETCHDIR="@FETCHDIR@" # DG adding 121724 # Commonly defined parameters in JJOBS export envir=${envir:-"prod"} @@ -475,8 +476,8 @@ export FHMAX_FITS=132 [[ "${FHMAX_FITS}" -gt "${FHMAX_GFS}" ]] && export FHMAX_FITS=${FHMAX_GFS} # Fetch options -export DO_FETCH_HPSS="@DO_FETCH_HPSS@" # pull from HPSS -export DO_FETCH_LOCAL="@DO_FETCH_LOCAL@" # pull from local +export DO_FETCH_HPSS="NO" # pull from HPSS +export DO_FETCH_LOCAL="NO" # pull from local if [[ ${DO_FETCH_HPSS} = "YES" ]] && [[ ${DO_FETCH_LOCAL} = "YES" ]]; then echo "Both HPSS and local fetch selected. Please choose one or the other." exit 3 diff --git a/parm/config/gfs/config.fetch b/parm/config/gfs/config.fetch index 88fd784497..bff7af2a82 100644 --- a/parm/config/gfs/config.fetch +++ b/parm/config/gfs/config.fetch @@ -4,34 +4,9 @@ echo "BEGIN: config.fetch" +export FETCH_YAML_TMPL="${PARMgfs}/fetch/C48_cold.yaml.j2" + # Get task specific resources source "${EXPDIR}/config.resources" fetch -export ICSDIR="@ICSDIR@" # User provided ICSDIR; blank if not provided -export BASE_IC="@BASE_IC@" # Platform home for staged ICs - -export FETCH_YAML_TMPL="${PARMgfs}/stage/master_gfs.yaml.j2" -export fetch_yaml="stage_atm_cold.yaml" - -# Set ICSDIR (if not defined) -if [[ -z "${ICSDIR}" ]] ; then - - ic_ver="20240610" - - if (( NMEM_ENS > 0 )) ; then - ensic="${CASE_ENS}" - fi - - if [[ "${DO_OCN:-NO}" == "YES" ]] ; then - ocnic="mx${OCNRES}" - fi - - export ICSDIR="${BASE_IC}/${CASE}${ensic:-}${ocnic:-}/${ic_ver}" - -fi - -# Use of perturbations files for ensembles -export USE_OCN_ENS_PERTURB_FILES="NO" -export USE_ATM_ENS_PERTURB_FILES="NO" - echo "END: config.fetch" diff --git a/parm/fetch/stage_atm_cold.yaml b/parm/fetch/stage_atm_cold.yaml deleted file mode 100644 index 22cbe1d12b..0000000000 --- a/parm/fetch/stage_atm_cold.yaml +++ /dev/null @@ -1,19 +0,0 @@ -untar: - tarball : "{{ ATARDIR }}/{{ cycle_YMDH }}/atm_cold.tar" - on_hpss: True - contents: - - gfs_ctrl.nc - {% for ftype in ["gfs_data", "sfc_data"] %} - {% for ntile in range(1, ntiles + 1) %} - - {{ ftype }}.tile{{ ntile }}.nc - {% endfor %} # ntile - {% endfor %} # ftype - destination: "{{ DATA }}" -atmosphere_cold: - copy: - - ["{{ DATA }}/gfs_ctrl.nc", "{{ COMOUT_ATMOS_INPUT }}"] - {% for ftype in ["gfs_data", "sfc_data"] %} - {% for ntile in range(1, ntiles + 1) %} - - ["{{ DATA }}/{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_INPUT }}"] - {% endfor %} # ntile - {% endfor %} # ftype diff --git a/workflow/applications/gfs_forecast_only.py b/workflow/applications/gfs_forecast_only.py index bd94979b10..565ae50812 100644 --- a/workflow/applications/gfs_forecast_only.py +++ b/workflow/applications/gfs_forecast_only.py @@ -27,8 +27,13 @@ def _get_app_configs(self, run): """ Returns the config_files that are involved in the forecast-only app """ - configs = ['fetch', 'stage_ic', 'fcst', 'arch', 'cleanup'] + + configs = [] options = self.run_options[run] + if options['fetch_hpss'] or options['fetch_local']: + configs += ['fetch'] + + configs += ['stage_ic', 'fcst', 'arch', 'cleanup'] if options['do_atm']: diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 2d8fe65c2d..c0d796f1a0 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -18,12 +18,13 @@ def _is_this_a_gdas_task(run, task_name): # Specific Tasks begin here def stage_ic(self): - deps = [] - dep_dict = { - 'type': 'task', 'name': f'fetch', - } - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) + if self.options['fetch_hpss'] or self.options['fetch_local']: + deps = [] + dep_dict = { + 'type': 'task', 'name': f'fetch', + } + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) cycledef = 'gdas_half' if self.run in ['gdas', 'enkfgdas'] else self.run diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 007413ab62..4f9ccee0f0 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -10,7 +10,7 @@ class Tasks: - SERVICE_TASKS = ['arch', 'earc', 'fetch', 'cleanup'] + SERVICE_TASKS = ['arch', 'earc', 'stage_ic', 'fetch', 'cleanup'] VALID_TASKS = ['aerosol_init', 'stage_ic', 'prep', 'anal', 'sfcanl', 'analcalc', 'analdiag', 'arch', "cleanup", 'prepatmiodaobs', 'atmanlinit', 'atmanlvar', 'atmanlfv3inc', 'atmanlfinal', From 5ca15310daf7a94a29a767409657c33a425311b5 Mon Sep 17 00:00:00 2001 From: David Grumm Date: Wed, 18 Dec 2024 20:40:00 +0000 Subject: [PATCH 08/36] Updated fetch code to use fetch directory FETCHDIR instead of ATARDIR, replaced gefs/config.fetch with a link to gfs/config.fetch --- ci/cases/pr/ATM_cold.yaml | 2 +- parm/config/gefs/config.base | 2 +- parm/config/gefs/config.fetch | 5 ++--- parm/fetch/ATM_cold.yaml.j2 | 11 +++++++++++ scripts/exglobal_fetch.py | 6 +++--- ush/python/pygfs/task/fetch.py | 10 +++++----- 6 files changed, 23 insertions(+), 13 deletions(-) create mode 100755 parm/fetch/ATM_cold.yaml.j2 diff --git a/ci/cases/pr/ATM_cold.yaml b/ci/cases/pr/ATM_cold.yaml index 0f55189bed..2c0f203416 100644 --- a/ci/cases/pr/ATM_cold.yaml +++ b/ci/cases/pr/ATM_cold.yaml @@ -1,5 +1,5 @@ untar: - tarball : "{{ ATARDIR }}/{{ cycle_YMDH }}/atm_cold.tar" + tarball : "{{ FETCHDIR }}/{{ cycle_YMDH }}/atm_cold.tar" on_hpss: True contents: - gfs_ctrl.nc diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index e6eccd1a13..a38e7a9346 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -103,7 +103,7 @@ export ROTDIR="@COMROOT@/${PSLOT}" export ARCDIR="${NOSCRUB}/archive/${PSLOT}" export ATARDIR="@ATARDIR@" -export FETCHDIR="@FETCHDIR@" # DG adding 121724 +export FETCHDIR="@FETCHDIR@" # HPSS or local directory where IC tarball(s) can be found. # Commonly defined parameters in JJOBS export envir=${envir:-"prod"} diff --git a/parm/config/gefs/config.fetch b/parm/config/gefs/config.fetch index 634cf3f54c..bff7af2a82 100644 --- a/parm/config/gefs/config.fetch +++ b/parm/config/gefs/config.fetch @@ -4,10 +4,9 @@ echo "BEGIN: config.fetch" +export FETCH_YAML_TMPL="${PARMgfs}/fetch/C48_cold.yaml.j2" + # Get task specific resources source "${EXPDIR}/config.resources" fetch -export ICSDIR="@ICSDIR@" # User provided ICSDIR; blank if not provided -export BASE_IC="@BASE_IC@" # Platform home for staged ICs - echo "END: config.fetch" diff --git a/parm/fetch/ATM_cold.yaml.j2 b/parm/fetch/ATM_cold.yaml.j2 new file mode 100755 index 0000000000..2c0f203416 --- /dev/null +++ b/parm/fetch/ATM_cold.yaml.j2 @@ -0,0 +1,11 @@ +untar: + tarball : "{{ FETCHDIR }}/{{ cycle_YMDH }}/atm_cold.tar" + on_hpss: True + contents: + - gfs_ctrl.nc + {% for ftype in ["gfs_data", "sfc_data"] %} + {% for ntile in range(1, ntiles + 1) %} + - {{ ftype }}.tile{{ ntile }}.nc + {% endfor %} # ntile + {% endfor %} # ftype + destination: "{{ DATA }}" \ No newline at end of file diff --git a/scripts/exglobal_fetch.py b/scripts/exglobal_fetch.py index c1446d14c7..8f23b60c7a 100755 --- a/scripts/exglobal_fetch.py +++ b/scripts/exglobal_fetch.py @@ -18,7 +18,7 @@ def main(): fetch = Fetch(config) # Pull out all the configuration keys needed to run the fetch step - keys = ['current_cycle', 'RUN', 'PDY', 'PARMgfs', 'PSLOT', 'ROTDIR', 'fetch_yaml', 'ATARDIR', 'ntiles'] + keys = ['current_cycle', 'RUN', 'PDY', 'PARMgfs', 'PSLOT', 'ROTDIR', 'fetch_yaml', 'FETCHDIR', 'ntiles'] fetch_dict = AttrDict() for key in keys: @@ -35,10 +35,10 @@ def main(): # Determine which archives to retrieve from HPSS # Read the input YAML file to get the list of tarballs on tape - atardir_set = fetch.configure(fetch_dict) + fetchdir_set = fetch.configure(fetch_dict) # Pull the data from tape or locally and store the specified destination - fetch.execute_pull_data(atardir_set) + fetch.execute_pull_data(fetchdir_set) if __name__ == '__main__': diff --git a/ush/python/pygfs/task/fetch.py b/ush/python/pygfs/task/fetch.py index b6ef48a4c9..b4fea91db2 100755 --- a/ush/python/pygfs/task/fetch.py +++ b/ush/python/pygfs/task/fetch.py @@ -59,13 +59,13 @@ def configure(self, fetch_dict: Dict[str, Any]): return parsed_fetch @logit(logger) - def execute_pull_data(self, atardir_set: Dict[str, Any]) -> None: + def execute_pull_data(self, fetchdir_set: Dict[str, Any]) -> None: """Pull data from HPSS based on a yaml dictionary and store at the specified destination. Parameters ---------- - atardir_set: Dict[str, Any], + fetchdir_set: Dict[str, Any], Dict defining set of tarballs to pull and where to put them. Return @@ -74,9 +74,9 @@ def execute_pull_data(self, atardir_set: Dict[str, Any]) -> None: if len(f_names) <= 0: # Abort if no files raise FileNotFoundError("FATAL ERROR: The tar ball has no files") - f_names = atardir_set.untar.contents - on_hpss = atardir_set.untar.on_hpss - dest = atardir_set.untar.destination + f_names = fetchdir_set.untar.contents + on_hpss = fetchdir_set.untar.on_hpss + dest = fetchdir_set.untar.destination # Select action whether no_hpss is True or not, and pull these # data from tape or locally and place where it needs to go # DG - these need testing From 09ce49c677d88a08d003704bac98098f88d98078 Mon Sep 17 00:00:00 2001 From: David Grumm Date: Thu, 19 Dec 2024 17:01:52 +0000 Subject: [PATCH 09/36] Deleted ci/cases/pr/ATM_cold.yaml --- ci/cases/pr/ATM_cold.yaml | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 ci/cases/pr/ATM_cold.yaml diff --git a/ci/cases/pr/ATM_cold.yaml b/ci/cases/pr/ATM_cold.yaml deleted file mode 100644 index 2c0f203416..0000000000 --- a/ci/cases/pr/ATM_cold.yaml +++ /dev/null @@ -1,11 +0,0 @@ -untar: - tarball : "{{ FETCHDIR }}/{{ cycle_YMDH }}/atm_cold.tar" - on_hpss: True - contents: - - gfs_ctrl.nc - {% for ftype in ["gfs_data", "sfc_data"] %} - {% for ntile in range(1, ntiles + 1) %} - - {{ ftype }}.tile{{ ntile }}.nc - {% endfor %} # ntile - {% endfor %} # ftype - destination: "{{ DATA }}" \ No newline at end of file From c71b340dd929b3428440f5f8f862572dd9bda8ba Mon Sep 17 00:00:00 2001 From: David Grumm Date: Fri, 20 Dec 2024 18:24:44 +0000 Subject: [PATCH 10/36] modifying stage_ic and fetch code to ensure that both function compatibly --- parm/config/gfs/config.stage_ic | 7 +++++- parm/fetch/ATM_cold.yaml.j2 | 2 +- scripts/exglobal_fetch.py | 4 ++-- ush/python/pygfs/__init__.py | 1 + ush/python/pygfs/task/fetch.py | 25 +++++++++++++--------- workflow/applications/applications.py | 4 ++-- workflow/applications/gfs_forecast_only.py | 4 ++-- workflow/rocoto/gfs_tasks.py | 6 ++++-- 8 files changed, 33 insertions(+), 20 deletions(-) diff --git a/parm/config/gfs/config.stage_ic b/parm/config/gfs/config.stage_ic index f30e402c93..d142669573 100644 --- a/parm/config/gfs/config.stage_ic +++ b/parm/config/gfs/config.stage_ic @@ -7,7 +7,12 @@ echo "BEGIN: config.stage_ic" # Get task specific resources source "${EXPDIR}/config.resources" stage_ic -export ICSDIR="@ICSDIR@" # User provided ICSDIR; blank if not provided +if [[ "${DO_FETCH_HPSS^^}" =~ "Y" || "${DO_FETCH_LOCAL^^}" =~ "Y" ]]; then + export ICSDIR="${DATAROOT}" # fetch untars data into DATAROOT +else + export ICSDIR="@ICSDIR@" # User provided ICSDIR; blank if not provided +fi + export BASE_IC="@BASE_IC@" # Platform home for staged ICs export STAGE_IC_YAML_TMPL="${PARMgfs}/stage/master_gfs.yaml.j2" diff --git a/parm/fetch/ATM_cold.yaml.j2 b/parm/fetch/ATM_cold.yaml.j2 index 2c0f203416..7914f20154 100755 --- a/parm/fetch/ATM_cold.yaml.j2 +++ b/parm/fetch/ATM_cold.yaml.j2 @@ -8,4 +8,4 @@ untar: - {{ ftype }}.tile{{ ntile }}.nc {% endfor %} # ntile {% endfor %} # ftype - destination: "{{ DATA }}" \ No newline at end of file + destination: "{{ DATAROOT }}" \ No newline at end of file diff --git a/scripts/exglobal_fetch.py b/scripts/exglobal_fetch.py index 8f23b60c7a..af2902b5d5 100755 --- a/scripts/exglobal_fetch.py +++ b/scripts/exglobal_fetch.py @@ -2,7 +2,7 @@ import os -from fetch import Fetch +from pygfs.task.fetch import Fetch from wxflow import AttrDict, Logger, cast_strdict_as_dtypedict, logit # initialize root logger @@ -18,7 +18,7 @@ def main(): fetch = Fetch(config) # Pull out all the configuration keys needed to run the fetch step - keys = ['current_cycle', 'RUN', 'PDY', 'PARMgfs', 'PSLOT', 'ROTDIR', 'fetch_yaml', 'FETCHDIR', 'ntiles'] + keys = ['current_cycle', 'RUN', 'PDY', 'PARMgfs', 'PSLOT', 'ROTDIR', 'fetch_yaml', 'FETCHDIR', 'ntiles', 'DATAROOT', 'cycle_YMDH'] fetch_dict = AttrDict() for key in keys: diff --git a/ush/python/pygfs/__init__.py b/ush/python/pygfs/__init__.py index 9f290fafd3..1f05cae517 100644 --- a/ush/python/pygfs/__init__.py +++ b/ush/python/pygfs/__init__.py @@ -15,6 +15,7 @@ from .task.oceanice_products import OceanIceProducts from .task.gfs_forecast import GFSForecast from .utils import marine_da_utils +from .task.fetch import Fetch __docformat__ = "restructuredtext" __version__ = "0.1.0" diff --git a/ush/python/pygfs/task/fetch.py b/ush/python/pygfs/task/fetch.py index b4fea91db2..9eeaf1f424 100755 --- a/ush/python/pygfs/task/fetch.py +++ b/ush/python/pygfs/task/fetch.py @@ -5,7 +5,7 @@ from typing import Any, Dict, List from wxflow import (AttrDict, FileHandler, Hsi, Task, - logit, parse_j2yaml) + logit, parse_j2yaml, chdir) from wxflow import htar as Htar import tarfile @@ -72,19 +72,24 @@ def execute_pull_data(self, fetchdir_set: Dict[str, Any]) -> None: None """ + f_names = fetchdir_set.untar.contents if len(f_names) <= 0: # Abort if no files raise FileNotFoundError("FATAL ERROR: The tar ball has no files") - f_names = fetchdir_set.untar.contents + on_hpss = fetchdir_set.untar.on_hpss dest = fetchdir_set.untar.destination + tarball = fetchdir_set.untar.tarball + # Select action whether no_hpss is True or not, and pull these # data from tape or locally and place where it needs to go # DG - these need testing - if on_hpss is True: # htar all files in fnames - htar_obj = Htar.Htar() - htar_obj.cvf(dest, f_names) - - else: # tar all files in fnames - with tarfile.open(dest, "w") as tar: - for filename in f_names: - tar.add(filename) + with chdir(dest): + if on_hpss is True: # htar all files in fnames + htar_obj = Htar.Htar() + htar_obj.xvf(tarball, f_names) + + else: # tar all files in fnames + pass # TODO +# with tarfile.open(dest, "w") as tar: +# for filename in f_names: +# tar.add(filename) diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index 13552eddb9..88be488b47 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -102,8 +102,8 @@ def _get_run_options(self, conf: Configuration) -> Dict[str, Any]: run_options[run]['do_hpssarch'] = run_base.get('HPSSARCH', False) run_options[run]['fcst_segments'] = run_base.get('FCST_SEGMENTS', None) - run_options[run]['fetch_hpss'] = run_base.get('FETCH_HPSS', False) - run_options[run]['fetch_local'] = run_base.get('FETCH_LOCAL', False) + run_options[run]['do_fetch_hpss'] = run_base.get('DO_FETCH_HPSS', False) + run_options[run]['do_fetch_local'] = run_base.get('DO_FETCH_LOCAL', False) if not AppConfig.is_monotonic(run_options[run]['fcst_segments']): raise ValueError(f'Forecast segments do not increase monotonically: {",".join(self.fcst_segments)}') diff --git a/workflow/applications/gfs_forecast_only.py b/workflow/applications/gfs_forecast_only.py index 565ae50812..1004188c8d 100644 --- a/workflow/applications/gfs_forecast_only.py +++ b/workflow/applications/gfs_forecast_only.py @@ -30,7 +30,7 @@ def _get_app_configs(self, run): configs = [] options = self.run_options[run] - if options['fetch_hpss'] or options['fetch_local']: + if options['do_fetch_hpss'] or options['do_fetch_local']: configs += ['fetch'] configs += ['stage_ic', 'fcst', 'arch', 'cleanup'] @@ -106,7 +106,7 @@ def get_task_names(self): tasks = [] - if options['fetch_hpss'] or options['fetch_local']: + if options['do_fetch_hpss'] or options['do_fetch_local']: tasks += ['fetch'] tasks += ['stage_ic'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index c0d796f1a0..ad874d7cd5 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -18,7 +18,8 @@ def _is_this_a_gdas_task(run, task_name): # Specific Tasks begin here def stage_ic(self): - if self.options['fetch_hpss'] or self.options['fetch_local']: + dependencies = None + if self.options['do_fetch_hpss'] or self.options['do_fetch_local']: deps = [] dep_dict = { 'type': 'task', 'name': f'fetch', @@ -37,7 +38,8 @@ def stage_ic(self): 'command': f'{self.HOMEgfs}/jobs/rocoto/stage_ic.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', - 'maxtries': '&MAXTRIES;' + 'maxtries': '&MAXTRIES;', + 'dependency': dependencies } task = rocoto.create_task(task_dict) From a6dade0cde51dc6ea92f2cfababa9ca6bd6473aa Mon Sep 17 00:00:00 2001 From: David Grumm Date: Mon, 23 Dec 2024 18:48:13 +0000 Subject: [PATCH 11/36] Moved setting of cycle_YMDH --- parm/fetch/ATM_cold.yaml.j2 | 1 + scripts/exglobal_fetch.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/parm/fetch/ATM_cold.yaml.j2 b/parm/fetch/ATM_cold.yaml.j2 index 7914f20154..f4e9cae92e 100755 --- a/parm/fetch/ATM_cold.yaml.j2 +++ b/parm/fetch/ATM_cold.yaml.j2 @@ -1,3 +1,4 @@ +{% set cycle_YMDH = current_cycle | to_YMDH %} untar: tarball : "{{ FETCHDIR }}/{{ cycle_YMDH }}/atm_cold.tar" on_hpss: True diff --git a/scripts/exglobal_fetch.py b/scripts/exglobal_fetch.py index af2902b5d5..af4678b956 100755 --- a/scripts/exglobal_fetch.py +++ b/scripts/exglobal_fetch.py @@ -18,7 +18,7 @@ def main(): fetch = Fetch(config) # Pull out all the configuration keys needed to run the fetch step - keys = ['current_cycle', 'RUN', 'PDY', 'PARMgfs', 'PSLOT', 'ROTDIR', 'fetch_yaml', 'FETCHDIR', 'ntiles', 'DATAROOT', 'cycle_YMDH'] + keys = ['current_cycle', 'RUN', 'PDY', 'PARMgfs', 'PSLOT', 'ROTDIR', 'fetch_yaml', 'FETCHDIR', 'ntiles', 'DATAROOT'] fetch_dict = AttrDict() for key in keys: From a6a84d4e6a8e875bc5c3789d72e291420042b3a3 Mon Sep 17 00:00:00 2001 From: David Grumm Date: Sat, 4 Jan 2025 02:32:01 +0000 Subject: [PATCH 12/36] Updates for the fetch and yaml code, and keys. --- parm/config/gfs/config.base | 6 +++--- parm/config/gfs/config.fetch | 2 +- scripts/exglobal_fetch.py | 9 +-------- ush/python/pygfs/task/fetch.py | 3 +-- workflow/rocoto/gfs_tasks.py | 2 +- workflow/rocoto/tasks.py | 2 +- 6 files changed, 8 insertions(+), 16 deletions(-) diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 39e252e349..55522a6565 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -132,7 +132,7 @@ if [[ "${PDY}${cyc}" -ge "2019092100" && "${PDY}${cyc}" -le "2019110700" ]]; the fi export ARCDIR="${NOSCRUB}/archive/${PSLOT}" export ATARDIR="@ATARDIR@" -export FETCHDIR="@FETCHDIR@" # DG adding 121724 +export FETCHDIR="@FETCHDIR@" # Commonly defined parameters in JJOBS export envir=${envir:-"prod"} @@ -476,8 +476,8 @@ export FHMAX_FITS=132 [[ "${FHMAX_FITS}" -gt "${FHMAX_GFS}" ]] && export FHMAX_FITS=${FHMAX_GFS} # Fetch options -export DO_FETCH_HPSS="NO" # pull from HPSS -export DO_FETCH_LOCAL="NO" # pull from local +export DO_FETCH_HPSS="NO" # pull from HPSS ## 123124 DG change to YES, or next +export DO_FETCH_LOCAL="YES" # pull from local if [[ ${DO_FETCH_HPSS} = "YES" ]] && [[ ${DO_FETCH_LOCAL} = "YES" ]]; then echo "Both HPSS and local fetch selected. Please choose one or the other." exit 3 diff --git a/parm/config/gfs/config.fetch b/parm/config/gfs/config.fetch index bff7af2a82..b348f07d73 100644 --- a/parm/config/gfs/config.fetch +++ b/parm/config/gfs/config.fetch @@ -4,7 +4,7 @@ echo "BEGIN: config.fetch" -export FETCH_YAML_TMPL="${PARMgfs}/fetch/C48_cold.yaml.j2" +export FETCH_YAML_TMPL="${PARMgfs}/fetch/ATM_cold.yaml.j2" # Get task specific resources source "${EXPDIR}/config.resources" fetch diff --git a/scripts/exglobal_fetch.py b/scripts/exglobal_fetch.py index af4678b956..d630cdd3ae 100755 --- a/scripts/exglobal_fetch.py +++ b/scripts/exglobal_fetch.py @@ -18,7 +18,7 @@ def main(): fetch = Fetch(config) # Pull out all the configuration keys needed to run the fetch step - keys = ['current_cycle', 'RUN', 'PDY', 'PARMgfs', 'PSLOT', 'ROTDIR', 'fetch_yaml', 'FETCHDIR', 'ntiles', 'DATAROOT'] + keys = ['current_cycle', 'RUN', 'PDY', 'PARMgfs', 'PSLOT', 'ROTDIR', 'FETCH_YAML_DICT', 'FETCHDIR', 'ntiles', 'DATAROOT'] fetch_dict = AttrDict() for key in keys: @@ -26,13 +26,6 @@ def main(): if fetch_dict[key] is None: print(f"Warning: key ({key}) not found in task_config!") - # Also import all COMOUT* directory and template variables - for key in fetch.task_config.keys(): - if key.startswith("COMOUT_"): - fetch_dict[key] = fetch.task_config.get(key) - if fetch_dict[key] is None: - print(f"Warning: key ({key}) not found in task_config!") - # Determine which archives to retrieve from HPSS # Read the input YAML file to get the list of tarballs on tape fetchdir_set = fetch.configure(fetch_dict) diff --git a/ush/python/pygfs/task/fetch.py b/ush/python/pygfs/task/fetch.py index 9eeaf1f424..f8cc556425 100755 --- a/ush/python/pygfs/task/fetch.py +++ b/ush/python/pygfs/task/fetch.py @@ -51,7 +51,7 @@ def configure(self, fetch_dict: Dict[str, Any]): """ self.hsi = Hsi() - fetch_yaml = fetch_dict.fetch_yaml + fetch_yaml = fetch_dict.FETCH_YAML_TMPL fetch_parm = os.path.join(fetch_dict.PARMgfs, "fetch") parsed_fetch = parse_j2yaml(os.path.join(fetch_parm, fetch_yaml), @@ -87,7 +87,6 @@ def execute_pull_data(self, fetchdir_set: Dict[str, Any]) -> None: if on_hpss is True: # htar all files in fnames htar_obj = Htar.Htar() htar_obj.xvf(tarball, f_names) - else: # tar all files in fnames pass # TODO # with tarfile.open(dest, "w") as tar: diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index ad874d7cd5..ee355e7e04 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -22,7 +22,7 @@ def stage_ic(self): if self.options['do_fetch_hpss'] or self.options['do_fetch_local']: deps = [] dep_dict = { - 'type': 'task', 'name': f'fetch', + 'type': 'task', 'name': f'{self.run}_fetch', } deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 4f9ccee0f0..934adae0bb 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -11,7 +11,7 @@ class Tasks: SERVICE_TASKS = ['arch', 'earc', 'stage_ic', 'fetch', 'cleanup'] - VALID_TASKS = ['aerosol_init', 'stage_ic', + VALID_TASKS = ['aerosol_init', 'stage_ic', 'fetch', 'prep', 'anal', 'sfcanl', 'analcalc', 'analdiag', 'arch', "cleanup", 'prepatmiodaobs', 'atmanlinit', 'atmanlvar', 'atmanlfv3inc', 'atmanlfinal', 'prepoceanobs', From 8aead592a32e07f424992c408fbdc47998c5af8f Mon Sep 17 00:00:00 2001 From: David Grumm Date: Tue, 7 Jan 2025 19:18:16 +0000 Subject: [PATCH 13/36] Corrected typo --- scripts/exglobal_fetch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_fetch.py b/scripts/exglobal_fetch.py index d630cdd3ae..802e1cd8bc 100755 --- a/scripts/exglobal_fetch.py +++ b/scripts/exglobal_fetch.py @@ -18,7 +18,7 @@ def main(): fetch = Fetch(config) # Pull out all the configuration keys needed to run the fetch step - keys = ['current_cycle', 'RUN', 'PDY', 'PARMgfs', 'PSLOT', 'ROTDIR', 'FETCH_YAML_DICT', 'FETCHDIR', 'ntiles', 'DATAROOT'] + keys = ['current_cycle', 'RUN', 'PDY', 'PARMgfs', 'PSLOT', 'ROTDIR', 'FETCH_YAML_TMPL', 'FETCHDIR', 'ntiles', 'DATAROOT'] fetch_dict = AttrDict() for key in keys: From ecc1b48694b763c87ae1792ad23ce134586c2bdc Mon Sep 17 00:00:00 2001 From: David Grumm Date: Fri, 10 Jan 2025 00:54:59 +0000 Subject: [PATCH 14/36] Add specification of fetch directory, corrrected name of yaml file. --- ci/cases/yamls/gfs_defaults_ci.yaml | 9 +++++++++ parm/config/gefs/config.fetch | 2 +- parm/config/gfs/config.base | 5 +++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/ci/cases/yamls/gfs_defaults_ci.yaml b/ci/cases/yamls/gfs_defaults_ci.yaml index 65d440ac93..84e20e0598 100644 --- a/ci/cases/yamls/gfs_defaults_ci.yaml +++ b/ci/cases/yamls/gfs_defaults_ci.yaml @@ -3,3 +3,12 @@ defaults: base: ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} DO_TEST_MODE: "YES" + + +defaults: + !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml +base: + ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} + DO_TEST_MODE: "NO" + FETCHDIR: "/NCEPDEV/emc-global/1year/David.Grumm/test_data" + DO_FETCH_HPSS: "YES" \ No newline at end of file diff --git a/parm/config/gefs/config.fetch b/parm/config/gefs/config.fetch index bff7af2a82..b348f07d73 100644 --- a/parm/config/gefs/config.fetch +++ b/parm/config/gefs/config.fetch @@ -4,7 +4,7 @@ echo "BEGIN: config.fetch" -export FETCH_YAML_TMPL="${PARMgfs}/fetch/C48_cold.yaml.j2" +export FETCH_YAML_TMPL="${PARMgfs}/fetch/ATM_cold.yaml.j2" # Get task specific resources source "${EXPDIR}/config.resources" fetch diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 55522a6565..811c1ad040 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -476,8 +476,9 @@ export FHMAX_FITS=132 [[ "${FHMAX_FITS}" -gt "${FHMAX_GFS}" ]] && export FHMAX_FITS=${FHMAX_GFS} # Fetch options -export DO_FETCH_HPSS="NO" # pull from HPSS ## 123124 DG change to YES, or next -export DO_FETCH_LOCAL="YES" # pull from local +export DO_FETCH_HPSS="@DO_FETCH_HPSS@" # pull from HPSS ## 123124 DG change to YES, or next +export DO_FETCH_LOCAL="NO" # pull from local + if [[ ${DO_FETCH_HPSS} = "YES" ]] && [[ ${DO_FETCH_LOCAL} = "YES" ]]; then echo "Both HPSS and local fetch selected. Please choose one or the other." exit 3 From dd645eb7bfd2cfbb0f434445d19580cfe9a3a70f Mon Sep 17 00:00:00 2001 From: David Grumm Date: Fri, 10 Jan 2025 03:55:45 +0000 Subject: [PATCH 15/36] Add DO_FETCH_HPSS setting to all non-hera files in workflow/hosts --- workflow/hosts/awspw.yaml | 1 + workflow/hosts/azurepw.yaml | 1 + workflow/hosts/container.yaml | 1 + workflow/hosts/gaea.yaml | 1 + workflow/hosts/googlepw.yaml | 1 + workflow/hosts/hercules.yaml | 1 + workflow/hosts/jet.yaml | 1 + workflow/hosts/orion.yaml | 1 + workflow/hosts/s4.yaml | 1 + workflow/hosts/wcoss2.yaml | 1 + 10 files changed, 10 insertions(+) diff --git a/workflow/hosts/awspw.yaml b/workflow/hosts/awspw.yaml index b98c838faa..b31acf3885 100644 --- a/workflow/hosts/awspw.yaml +++ b/workflow/hosts/awspw.yaml @@ -27,5 +27,6 @@ MAKE_ACFTBUFR: 'NO' DO_TRACKER: 'NO' DO_GENESIS: 'NO' DO_METP: 'NO' +DO_FETCH_HPSS: 'NO' SUPPORT_WAVES: 'NO' SUPPORTED_RESOLUTIONS: ['C48', 'C96'] # TODO: Test and support all cubed-sphere resolutions. diff --git a/workflow/hosts/azurepw.yaml b/workflow/hosts/azurepw.yaml index 4725e28962..2a23932c25 100644 --- a/workflow/hosts/azurepw.yaml +++ b/workflow/hosts/azurepw.yaml @@ -24,5 +24,6 @@ LOCALARCH: 'NO' ATARDIR: '' # TODO: This will not yet work from AZURE. MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' +DO_FETCH_HPSS: 'NO' SUPPORT_WAVES: 'NO' SUPPORTED_RESOLUTIONS: ['C48', 'C96'] # TODO: Test and support all cubed-sphere resolutions. diff --git a/workflow/hosts/container.yaml b/workflow/hosts/container.yaml index d7924724ae..f2ca5931cb 100644 --- a/workflow/hosts/container.yaml +++ b/workflow/hosts/container.yaml @@ -23,3 +23,4 @@ ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' SUPPORTED_RESOLUTIONS: ['C96', 'C48'] +DO_FETCH_HPSS: 'NO' \ No newline at end of file diff --git a/workflow/hosts/gaea.yaml b/workflow/hosts/gaea.yaml index 23ac75ca03..87487a3f7b 100644 --- a/workflow/hosts/gaea.yaml +++ b/workflow/hosts/gaea.yaml @@ -25,5 +25,6 @@ LOCALARCH: 'NO' ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' +DO_FETCH_HPSS: 'NO' SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] AERO_INPUTS_DIR: /gpfs/f5/epic/proj-shared/global/glopara/data/gocart_emissions diff --git a/workflow/hosts/googlepw.yaml b/workflow/hosts/googlepw.yaml index 1b979b6bc9..5adcb442c5 100644 --- a/workflow/hosts/googlepw.yaml +++ b/workflow/hosts/googlepw.yaml @@ -24,5 +24,6 @@ LOCALARCH: 'NO' ATARDIR: '' # TODO: This will not yet work from GOOGLE. MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' +DO_FETCH_HPSS: 'NO' SUPPORT_WAVES: 'NO' SUPPORTED_RESOLUTIONS: ['C48', 'C96'] # TODO: Test and support all cubed-sphere resolutions. diff --git a/workflow/hosts/hercules.yaml b/workflow/hosts/hercules.yaml index f528761cf1..c2730e6d1b 100644 --- a/workflow/hosts/hercules.yaml +++ b/workflow/hosts/hercules.yaml @@ -26,6 +26,7 @@ MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' DO_TRACKER: 'NO' DO_GENESIS: 'NO' +DO_FETCH_HPSS: 'NO' SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] COMINecmwf: /work/noaa/global/glopara/data/external_gempak/ecmwf COMINnam: /work/noaa/global/glopara/data/external_gempak/nam diff --git a/workflow/hosts/jet.yaml b/workflow/hosts/jet.yaml index 737e2e7f94..257a88ed18 100644 --- a/workflow/hosts/jet.yaml +++ b/workflow/hosts/jet.yaml @@ -24,6 +24,7 @@ LOCALARCH: 'NO' ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' +DO_FETCH_HPSS: 'NO' SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48'] COMINecmwf: /mnt/lfs5/HFIP/hfv3gfs/glopara/data/external_gempak/ecmwf COMINnam: /mnt/lfs5/HFIP/hfv3gfs/glopara/data/external_gempak/nam diff --git a/workflow/hosts/orion.yaml b/workflow/hosts/orion.yaml index 985c24c6fb..6bf8e695c1 100644 --- a/workflow/hosts/orion.yaml +++ b/workflow/hosts/orion.yaml @@ -26,6 +26,7 @@ MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' DO_TRACKER: 'NO' DO_GENESIS: 'NO' +DO_FETCH_HPSS: 'NO' SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] COMINecmwf: /work/noaa/global/glopara/data/external_gempak/ecmwf COMINnam: /work/noaa/global/glopara/data/external_gempak/nam diff --git a/workflow/hosts/s4.yaml b/workflow/hosts/s4.yaml index 2e77c112b1..b919be39ef 100644 --- a/workflow/hosts/s4.yaml +++ b/workflow/hosts/s4.yaml @@ -24,5 +24,6 @@ LOCALARCH: 'NO' ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' MAKE_NSSTBUFR: 'YES' MAKE_ACFTBUFR: 'YES' +DO_FETCH_HPSS: 'NO' SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48'] AERO_INPUTS_DIR: /data/prod/glopara/gocart_emissions diff --git a/workflow/hosts/wcoss2.yaml b/workflow/hosts/wcoss2.yaml index 4fb4b1d64a..64ef5a60a2 100644 --- a/workflow/hosts/wcoss2.yaml +++ b/workflow/hosts/wcoss2.yaml @@ -24,6 +24,7 @@ LOCALARCH: 'NO' ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' +DO_FETCH_HPSS: 'NO' SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] COMINecmwf: /lfs/h2/emc/global/noscrub/emc.global/data/external_gempak/ecmwf COMINnam: /lfs/h2/emc/global/noscrub/emc.global/data/external_gempak/nam From 4ca9d49f94987e25e411691dfb725bb7af8b5be5 Mon Sep 17 00:00:00 2001 From: David Grumm Date: Fri, 10 Jan 2025 18:42:09 +0000 Subject: [PATCH 16/36] Change pass to raise an error --- ush/python/pygfs/task/fetch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/fetch.py b/ush/python/pygfs/task/fetch.py index f8cc556425..4078697c2a 100755 --- a/ush/python/pygfs/task/fetch.py +++ b/ush/python/pygfs/task/fetch.py @@ -88,7 +88,7 @@ def execute_pull_data(self, fetchdir_set: Dict[str, Any]) -> None: htar_obj = Htar.Htar() htar_obj.xvf(tarball, f_names) else: # tar all files in fnames - pass # TODO + raise(NotImplementedError) # with tarfile.open(dest, "w") as tar: # for filename in f_names: # tar.add(filename) From df88c2931ab2d22a5cdf353cf3022ab60c2a9ab6 Mon Sep 17 00:00:00 2001 From: David Grumm Date: Fri, 10 Jan 2025 18:46:56 +0000 Subject: [PATCH 17/36] whitespace repair --- ush/python/pygfs/task/fetch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/fetch.py b/ush/python/pygfs/task/fetch.py index 4078697c2a..b34346c28c 100755 --- a/ush/python/pygfs/task/fetch.py +++ b/ush/python/pygfs/task/fetch.py @@ -88,7 +88,7 @@ def execute_pull_data(self, fetchdir_set: Dict[str, Any]) -> None: htar_obj = Htar.Htar() htar_obj.xvf(tarball, f_names) else: # tar all files in fnames - raise(NotImplementedError) + raise (NotImplementedError) # with tarfile.open(dest, "w") as tar: # for filename in f_names: # tar.add(filename) From 89c42ef2b41293b23152a9517cd7d0fe8ea316b6 Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 13 Jan 2025 15:47:49 +0000 Subject: [PATCH 18/36] Add S2SW_cold fetch template --- parm/fetch/S2SW_cold.yaml.j2 | 37 ++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100755 parm/fetch/S2SW_cold.yaml.j2 diff --git a/parm/fetch/S2SW_cold.yaml.j2 b/parm/fetch/S2SW_cold.yaml.j2 new file mode 100755 index 0000000000..fdefa39ce0 --- /dev/null +++ b/parm/fetch/S2SW_cold.yaml.j2 @@ -0,0 +1,37 @@ +{% set cycle_YMD = current_cycle | to_YMD %} +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set p_cycle_YMD = previous_cycle | to_YMD %} +{% set p_cycle_HH = previous_cycle | strftime("%H") %} +# For cold starts, the ATM component is in the current cycle RUN.YYYYMMDD/HH +# For ocean/ice, some files are in the current cyle, some in the previous +# For waves, all files are in the previous cycle +# Previous cycles are always gdas (gdas.YYYYMMDD/HH) +{% set atm_dir = RUN + "." ~ cycle_YMD ~ "/" ~ cycle_HH ~ "/model/atmos/input" %} +{% set ocean_dir = RUN + "." ~ cycle_YMD ~ "/" ~ cycle_HH ~ "/model/ocean/restart" %} +{% set ice_dir = RUN + "." ~ cycle_YMD ~ "/" ~ cycle_HH ~ "/model/ice/restart" %} +{% set p_ocean_dir = "gdas." ~ p_cycle_YMD ~ "/" ~ p_cycle_HH ~ "/model/ocean/restart" %} +{% set p_ice_dir = "gdas." ~ p_cycle_YMD ~ "/" ~ p_cycle_HH ~ "/model/ice/restart" %} +{% set p_wave_dir = "gdas." ~ p_cycle_YMD ~ "/" ~ cycle_HH ~ "/model/wave/restart" %} +{% set restart_prefix = cycle_YMD ~ "." ~ cycle_HH %} +untar: + tarball : "{{ FETCHDIR }}/{{ cycle_YMDH }}/s2sw_cold.tar" + on_hpss: True + contents: + # ATM + - {{atm_dir}}/gfs_ctrl.nc + {% for ftype in ["gfs_data", "sfc_data"] %} + {% for ntile in range(1, ntiles + 1) %} + - {{atm_dir}}/{{ ftype }}.tile{{ ntile }}.nc + {% endfor %} # ntile + {% endfor %} # ftype + # Ocean + - {{ocean_dir}}/{{restart_prefix}}.MOM.res.nc + - {{p_ocean_dir}}/{{restart_prefix}}.MOM.res.nc + # Ice + - {{ice_dir}}/cice5_model_{{ + - {{ice_dir}}/{{restart_prefix}}.cice_model.res.nc + - {{p_ice_dir}}/{{restart_prefix}}.cice_model.res.nc + # Wave + - {{p_wave_dir}}/{{restart_prefix}}.restart.ww3 + - {{p_wave_dir}}/{{restart_prefix}}.restart.{{waveGRD}} + destination: "{{ DATAROOT }}" From d0e6f9b73b684e8af6511255cde42460a6c1e1bc Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 13 Jan 2025 15:48:02 +0000 Subject: [PATCH 19/36] Update ATM_cold.yaml.j2 --- parm/fetch/ATM_cold.yaml.j2 | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/parm/fetch/ATM_cold.yaml.j2 b/parm/fetch/ATM_cold.yaml.j2 index f4e9cae92e..71e16f06ea 100755 --- a/parm/fetch/ATM_cold.yaml.j2 +++ b/parm/fetch/ATM_cold.yaml.j2 @@ -1,12 +1,15 @@ -{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set cycle_YMD = current_cycle | to_YMD %} +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set atm_dir = RUN + "." ~ cycle_YMD ~ "/" ~ cycle_HH ~ "/model/atmos/input" %} untar: tarball : "{{ FETCHDIR }}/{{ cycle_YMDH }}/atm_cold.tar" on_hpss: True contents: - - gfs_ctrl.nc + # ATM + - {{atm_dir}}/gfs_ctrl.nc {% for ftype in ["gfs_data", "sfc_data"] %} {% for ntile in range(1, ntiles + 1) %} - - {{ ftype }}.tile{{ ntile }}.nc + - {{atm_dir}}/{{ ftype }}.tile{{ ntile }}.nc {% endfor %} # ntile {% endfor %} # ftype - destination: "{{ DATAROOT }}" \ No newline at end of file + destination: "{{ DATAROOT }}" From 701093cc7a689efa69206f7e89369b08b283382b Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 13 Jan 2025 17:01:36 +0000 Subject: [PATCH 20/36] Add YMDH to fetch_yamls --- parm/fetch/ATM_cold.yaml.j2 | 1 + parm/fetch/S2SW_cold.yaml.j2 | 1 + 2 files changed, 2 insertions(+) diff --git a/parm/fetch/ATM_cold.yaml.j2 b/parm/fetch/ATM_cold.yaml.j2 index 71e16f06ea..2940d6a956 100755 --- a/parm/fetch/ATM_cold.yaml.j2 +++ b/parm/fetch/ATM_cold.yaml.j2 @@ -1,3 +1,4 @@ +{% set cycle_YMDH = current_cycle | to_YMDH %} {% set cycle_YMD = current_cycle | to_YMD %} {% set cycle_HH = current_cycle | strftime("%H") %} {% set atm_dir = RUN + "." ~ cycle_YMD ~ "/" ~ cycle_HH ~ "/model/atmos/input" %} diff --git a/parm/fetch/S2SW_cold.yaml.j2 b/parm/fetch/S2SW_cold.yaml.j2 index fdefa39ce0..7e30c46e19 100755 --- a/parm/fetch/S2SW_cold.yaml.j2 +++ b/parm/fetch/S2SW_cold.yaml.j2 @@ -1,3 +1,4 @@ +{% set cycle_YMDH = current_cycle | to_YMDH %} {% set cycle_YMD = current_cycle | to_YMD %} {% set cycle_HH = current_cycle | strftime("%H") %} {% set p_cycle_YMD = previous_cycle | to_YMD %} From 9433473e9548c201b3451e5c54aaee0eac78ffd8 Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 13 Jan 2025 17:41:09 +0000 Subject: [PATCH 21/36] Add previous_cycle to fetch dict --- .../{ATM_cold.yaml.j2 => gfs_ATM_cold_forecast-only.yaml.j2} | 0 scripts/exglobal_fetch.py | 3 ++- 2 files changed, 2 insertions(+), 1 deletion(-) rename parm/fetch/{ATM_cold.yaml.j2 => gfs_ATM_cold_forecast-only.yaml.j2} (100%) diff --git a/parm/fetch/ATM_cold.yaml.j2 b/parm/fetch/gfs_ATM_cold_forecast-only.yaml.j2 similarity index 100% rename from parm/fetch/ATM_cold.yaml.j2 rename to parm/fetch/gfs_ATM_cold_forecast-only.yaml.j2 diff --git a/scripts/exglobal_fetch.py b/scripts/exglobal_fetch.py index 802e1cd8bc..d9efe24d7f 100755 --- a/scripts/exglobal_fetch.py +++ b/scripts/exglobal_fetch.py @@ -18,7 +18,8 @@ def main(): fetch = Fetch(config) # Pull out all the configuration keys needed to run the fetch step - keys = ['current_cycle', 'RUN', 'PDY', 'PARMgfs', 'PSLOT', 'ROTDIR', 'FETCH_YAML_TMPL', 'FETCHDIR', 'ntiles', 'DATAROOT'] + keys = ['current_cycle', 'previous_cycle', 'RUN', 'PDY', 'PARMgfs', 'PSLOT', 'ROTDIR', + 'FETCH_YAML_TMPL', 'FETCHDIR', 'ntiles', 'DATAROOT', 'waveGRD'] fetch_dict = AttrDict() for key in keys: From 04f8da90b51157159f5cdb504b97e50858976317 Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 13 Jan 2025 17:41:48 +0000 Subject: [PATCH 22/36] Check for missing files after untarring, add logging --- ush/python/pygfs/task/fetch.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/ush/python/pygfs/task/fetch.py b/ush/python/pygfs/task/fetch.py index b34346c28c..13f3d89d58 100755 --- a/ush/python/pygfs/task/fetch.py +++ b/ush/python/pygfs/task/fetch.py @@ -2,12 +2,11 @@ import os from logging import getLogger -from typing import Any, Dict, List +from typing import Any, Dict -from wxflow import (AttrDict, FileHandler, Hsi, Task, +from wxflow import (Hsi, Task, htar, logit, parse_j2yaml, chdir) -from wxflow import htar as Htar -import tarfile +# import tarfile logger = getLogger(__name__.split('.')[-1]) @@ -84,11 +83,23 @@ def execute_pull_data(self, fetchdir_set: Dict[str, Any]) -> None: # data from tape or locally and place where it needs to go # DG - these need testing with chdir(dest): + logger.info(f"Changed working directory to {dest}") if on_hpss is True: # htar all files in fnames - htar_obj = Htar.Htar() + htar_obj = htar.Htar() htar_obj.xvf(tarball, f_names) else: # tar all files in fnames raise (NotImplementedError) # with tarfile.open(dest, "w") as tar: # for filename in f_names: # tar.add(filename) + # Verify all data files were extracted + missing_files = [] + for f in f_names: + if not os.path.exists(f): + missing_files.append(f) + if len(missing_files) > 0: + message = "Failed to extract all required files. Missing files:\n" + for f in missing_files: + message += f"{f}\n" + + raise FileNotFoundError(message) From d1f2a00224d691e1a184c30ae35b2004cec7c068 Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 13 Jan 2025 17:43:22 +0000 Subject: [PATCH 23/36] Renamed fetch configs --- parm/config/gfs/config.fetch | 9 ++++++++- ..._cold.yaml.j2 => gfs_S2SW_cold_forecast-only.yaml.j2} | 5 ++--- 2 files changed, 10 insertions(+), 4 deletions(-) rename parm/fetch/{S2SW_cold.yaml.j2 => gfs_S2SW_cold_forecast-only.yaml.j2} (90%) diff --git a/parm/config/gfs/config.fetch b/parm/config/gfs/config.fetch index b348f07d73..8dd3d262ee 100644 --- a/parm/config/gfs/config.fetch +++ b/parm/config/gfs/config.fetch @@ -4,7 +4,14 @@ echo "BEGIN: config.fetch" -export FETCH_YAML_TMPL="${PARMgfs}/fetch/ATM_cold.yaml.j2" +# Determine start type +if [[ "${EXP_WARM_START}" == ".false." ]]; then + ic_type="cold" +else + ic_type="warm" +fi + +export FETCH_YAML_TMPL="${PARMgfs}/fetch/${NET}_${APP}_${ic_type}_${MODE}.yaml.j2" # Get task specific resources source "${EXPDIR}/config.resources" fetch diff --git a/parm/fetch/S2SW_cold.yaml.j2 b/parm/fetch/gfs_S2SW_cold_forecast-only.yaml.j2 similarity index 90% rename from parm/fetch/S2SW_cold.yaml.j2 rename to parm/fetch/gfs_S2SW_cold_forecast-only.yaml.j2 index 7e30c46e19..bbb5165ecc 100755 --- a/parm/fetch/S2SW_cold.yaml.j2 +++ b/parm/fetch/gfs_S2SW_cold_forecast-only.yaml.j2 @@ -12,8 +12,8 @@ {% set ice_dir = RUN + "." ~ cycle_YMD ~ "/" ~ cycle_HH ~ "/model/ice/restart" %} {% set p_ocean_dir = "gdas." ~ p_cycle_YMD ~ "/" ~ p_cycle_HH ~ "/model/ocean/restart" %} {% set p_ice_dir = "gdas." ~ p_cycle_YMD ~ "/" ~ p_cycle_HH ~ "/model/ice/restart" %} -{% set p_wave_dir = "gdas." ~ p_cycle_YMD ~ "/" ~ cycle_HH ~ "/model/wave/restart" %} -{% set restart_prefix = cycle_YMD ~ "." ~ cycle_HH %} +{% set p_wave_dir = "gdas." ~ p_cycle_YMD ~ "/" ~ p_cycle_HH ~ "/model/wave/restart" %} +{% set restart_prefix = cycle_YMD ~ "." ~ cycle_HH ~ "0000" %} untar: tarball : "{{ FETCHDIR }}/{{ cycle_YMDH }}/s2sw_cold.tar" on_hpss: True @@ -29,7 +29,6 @@ untar: - {{ocean_dir}}/{{restart_prefix}}.MOM.res.nc - {{p_ocean_dir}}/{{restart_prefix}}.MOM.res.nc # Ice - - {{ice_dir}}/cice5_model_{{ - {{ice_dir}}/{{restart_prefix}}.cice_model.res.nc - {{p_ice_dir}}/{{restart_prefix}}.cice_model.res.nc # Wave From 34ae4a8cf2c283f80dc85e45821ef48105635ee4 Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 13 Jan 2025 17:44:37 +0000 Subject: [PATCH 24/36] Replace gefs config.fetch with link to gfs --- parm/config/gefs/config.fetch | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) mode change 100644 => 120000 parm/config/gefs/config.fetch diff --git a/parm/config/gefs/config.fetch b/parm/config/gefs/config.fetch deleted file mode 100644 index b348f07d73..0000000000 --- a/parm/config/gefs/config.fetch +++ /dev/null @@ -1,12 +0,0 @@ -#! /usr/bin/env bash - -########## config.fetch ########## - -echo "BEGIN: config.fetch" - -export FETCH_YAML_TMPL="${PARMgfs}/fetch/ATM_cold.yaml.j2" - -# Get task specific resources -source "${EXPDIR}/config.resources" fetch - -echo "END: config.fetch" diff --git a/parm/config/gefs/config.fetch b/parm/config/gefs/config.fetch new file mode 120000 index 0000000000..21e571a2fd --- /dev/null +++ b/parm/config/gefs/config.fetch @@ -0,0 +1 @@ +../gfs/config.fetch \ No newline at end of file From 5c25f2615f8674f903ed02ee47e68e1b6b9c6a9b Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 13 Jan 2025 17:45:45 +0000 Subject: [PATCH 25/36] Update fetch directory --- ci/cases/yamls/gfs_defaults_ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cases/yamls/gfs_defaults_ci.yaml b/ci/cases/yamls/gfs_defaults_ci.yaml index abcf41eaba..29967baaff 100644 --- a/ci/cases/yamls/gfs_defaults_ci.yaml +++ b/ci/cases/yamls/gfs_defaults_ci.yaml @@ -9,6 +9,6 @@ defaults: base: ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} DO_TEST_MODE: "NO" - FETCHDIR: "/NCEPDEV/emc-global/1year/David.Grumm/test_data" + FETCHDIR: "/NCEPDEV/emc-global/1year/David.Huber/test_data" DO_FETCH_HPSS: "YES" From 35f2454b0a7099046de7e3f9aa119a9ec12ae39e Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 13 Jan 2025 17:58:10 +0000 Subject: [PATCH 26/36] Restrict fetch cases to C48_S2SW and C48_ATM --- parm/config/gfs/config.base | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 811c1ad040..0f4793d5cc 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -476,8 +476,12 @@ export FHMAX_FITS=132 [[ "${FHMAX_FITS}" -gt "${FHMAX_GFS}" ]] && export FHMAX_FITS=${FHMAX_GFS} # Fetch options -export DO_FETCH_HPSS="@DO_FETCH_HPSS@" # pull from HPSS ## 123124 DG change to YES, or next -export DO_FETCH_LOCAL="NO" # pull from local +# TODO once tarballs are available for CI at other resolutions/start types, etc, remove this if block +if [[ "${APP}" == "ATM" || "${APP}" == "S2SW" ]] && \ + [[ "${EXP_WARM_START}" == ".false." && "${CASE}" == "C48" ]]; then + export DO_FETCH_HPSS="@DO_FETCH_HPSS@" # pull from HPSS ## 123124 DG change to YES, or next + export DO_FETCH_LOCAL="NO" # pull from local +fi if [[ ${DO_FETCH_HPSS} = "YES" ]] && [[ ${DO_FETCH_LOCAL} = "YES" ]]; then echo "Both HPSS and local fetch selected. Please choose one or the other." From b92920053150abcfd80cc9d1c7629bf6e16fcef4 Mon Sep 17 00:00:00 2001 From: David Grumm Date: Fri, 17 Jan 2025 19:42:45 +0000 Subject: [PATCH 27/36] Cleanup of env files --- ci/cases/yamls/gfs_defaults_ci.yaml | 3 +-- workflow/hosts/awspw.yaml | 4 ++-- workflow/hosts/azurepw.yaml | 5 ++++- workflow/hosts/googlepw.yaml | 5 ++++- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/ci/cases/yamls/gfs_defaults_ci.yaml b/ci/cases/yamls/gfs_defaults_ci.yaml index 29967baaff..88eca7b7e0 100644 --- a/ci/cases/yamls/gfs_defaults_ci.yaml +++ b/ci/cases/yamls/gfs_defaults_ci.yaml @@ -9,6 +9,5 @@ defaults: base: ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} DO_TEST_MODE: "NO" - FETCHDIR: "/NCEPDEV/emc-global/1year/David.Huber/test_data" + FETCHDIR: "/NCEPDEV/emc-global/1year/David.Grumm/test_data" DO_FETCH_HPSS: "YES" - diff --git a/workflow/hosts/awspw.yaml b/workflow/hosts/awspw.yaml index efbe2d0d7a..f523e976b0 100644 --- a/workflow/hosts/awspw.yaml +++ b/workflow/hosts/awspw.yaml @@ -28,6 +28,6 @@ DO_TRACKER: 'NO' DO_GENESIS: 'NO' DO_METP: 'NO' DO_FETCH_HPSS: 'NO' -SUPPORT_WAVES: 'NO' -SUPPORTED_RESOLUTIONS: ['C48', 'C96'] # TODO: Test and support all cubed-sphere resolutions. +SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C192', 'C384', 'C768'] # TODO: Test and support all cubed-sphere resolutions. +AERO_INPUTS_DIR: /contrib/global-workflow-shared-data/data/gocart_emissions diff --git a/workflow/hosts/azurepw.yaml b/workflow/hosts/azurepw.yaml index acfea95cfa..6d1927faa8 100644 --- a/workflow/hosts/azurepw.yaml +++ b/workflow/hosts/azurepw.yaml @@ -24,7 +24,10 @@ LOCALARCH: 'NO' ATARDIR: '' # TODO: This will not yet work from AZURE. MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' +DO_TRACKER: 'NO' +DO_GENESIS: 'NO' +DO_METP: 'NO' DO_FETCH_HPSS: 'NO' SUPPORT_WAVES: 'NO' -SUPPORTED_RESOLUTIONS: ['C48', 'C96'] # TODO: Test and support all cubed-sphere resolutions. +SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C384', 'C768'] # TODO: Test and support all cubed-sphere resolutions. diff --git a/workflow/hosts/googlepw.yaml b/workflow/hosts/googlepw.yaml index f4eef3bde5..82cf41bf43 100644 --- a/workflow/hosts/googlepw.yaml +++ b/workflow/hosts/googlepw.yaml @@ -24,8 +24,11 @@ LOCALARCH: 'NO' ATARDIR: '' # TODO: This will not yet work from GOOGLE. MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' +DO_TRACKER: 'NO' +DO_GENESIS: 'NO' +DO_METP: 'NO' DO_FETCH_HPSS: 'NO' SUPPORT_WAVES: 'NO' -SUPPORTED_RESOLUTIONS: ['C48', 'C96'] # TODO: Test and support all cubed-sphere resolutions. +SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C384'] # TODO: Test and support all cubed-sphere resolutions. From 23b2057e0f5a0f3216778dca77080ad95adc5979 Mon Sep 17 00:00:00 2001 From: David Grumm Date: Fri, 17 Jan 2025 21:30:09 +0000 Subject: [PATCH 28/36] Renamed variable, removed unused code --- parm/fetch/gfs_ATM_cold_forecast-only.yaml.j2 | 2 +- ush/python/pygfs/task/fetch.py | 12 ++++++------ workflow/hosts/azurepw.yaml | 1 - workflow/hosts/googlepw.yaml | 1 - 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/parm/fetch/gfs_ATM_cold_forecast-only.yaml.j2 b/parm/fetch/gfs_ATM_cold_forecast-only.yaml.j2 index 2940d6a956..927527c760 100755 --- a/parm/fetch/gfs_ATM_cold_forecast-only.yaml.j2 +++ b/parm/fetch/gfs_ATM_cold_forecast-only.yaml.j2 @@ -2,7 +2,7 @@ {% set cycle_YMD = current_cycle | to_YMD %} {% set cycle_HH = current_cycle | strftime("%H") %} {% set atm_dir = RUN + "." ~ cycle_YMD ~ "/" ~ cycle_HH ~ "/model/atmos/input" %} -untar: +target: tarball : "{{ FETCHDIR }}/{{ cycle_YMDH }}/atm_cold.tar" on_hpss: True contents: diff --git a/ush/python/pygfs/task/fetch.py b/ush/python/pygfs/task/fetch.py index 13f3d89d58..19db8746c1 100755 --- a/ush/python/pygfs/task/fetch.py +++ b/ush/python/pygfs/task/fetch.py @@ -32,7 +32,6 @@ def __init__(self, config: Dict[str, Any]) -> None: None """ super().__init__(config) - # Perhaps add other stuff to self. @logit(logger) def configure(self, fetch_dict: Dict[str, Any]): @@ -71,13 +70,13 @@ def execute_pull_data(self, fetchdir_set: Dict[str, Any]) -> None: None """ - f_names = fetchdir_set.untar.contents + f_names = fetchdir_set.target.contents if len(f_names) <= 0: # Abort if no files raise FileNotFoundError("FATAL ERROR: The tar ball has no files") - on_hpss = fetchdir_set.untar.on_hpss - dest = fetchdir_set.untar.destination - tarball = fetchdir_set.untar.tarball + on_hpss = fetchdir_set.target.on_hpss + dest = fetchdir_set.target.destination + tarball = fetchdir_set.targettarball # Select action whether no_hpss is True or not, and pull these # data from tape or locally and place where it needs to go @@ -88,7 +87,8 @@ def execute_pull_data(self, fetchdir_set: Dict[str, Any]) -> None: htar_obj = htar.Htar() htar_obj.xvf(tarball, f_names) else: # tar all files in fnames - raise (NotImplementedError) + raise NotImplementedError("The fetch job does not yet support pulling from local archives") + # with tarfile.open(dest, "w") as tar: # for filename in f_names: # tar.add(filename) diff --git a/workflow/hosts/azurepw.yaml b/workflow/hosts/azurepw.yaml index 6d1927faa8..f642b76543 100644 --- a/workflow/hosts/azurepw.yaml +++ b/workflow/hosts/azurepw.yaml @@ -28,6 +28,5 @@ DO_TRACKER: 'NO' DO_GENESIS: 'NO' DO_METP: 'NO' DO_FETCH_HPSS: 'NO' -SUPPORT_WAVES: 'NO' SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C384', 'C768'] # TODO: Test and support all cubed-sphere resolutions. diff --git a/workflow/hosts/googlepw.yaml b/workflow/hosts/googlepw.yaml index 82cf41bf43..4b580c3215 100644 --- a/workflow/hosts/googlepw.yaml +++ b/workflow/hosts/googlepw.yaml @@ -28,7 +28,6 @@ DO_TRACKER: 'NO' DO_GENESIS: 'NO' DO_METP: 'NO' DO_FETCH_HPSS: 'NO' -SUPPORT_WAVES: 'NO' SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C384'] # TODO: Test and support all cubed-sphere resolutions. From 60241bd69a5746b4e622e2fc0b2534e56e31fa61 Mon Sep 17 00:00:00 2001 From: David Grumm Date: Thu, 23 Jan 2025 17:48:08 +0000 Subject: [PATCH 29/36] Merged in develop --- workflow/hosts/gaeac6.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/workflow/hosts/gaeac6.yaml b/workflow/hosts/gaeac6.yaml index fed31634bf..c5082c4a88 100644 --- a/workflow/hosts/gaeac6.yaml +++ b/workflow/hosts/gaeac6.yaml @@ -24,4 +24,5 @@ LOCALARCH: 'NO' ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' +DO_FETCH_HPSS: 'NO' SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48'] From 00e3d620ff0b76933958bf4e67dd9e0974b5a9f9 Mon Sep 17 00:00:00 2001 From: David Grumm Date: Thu, 23 Jan 2025 18:50:21 +0000 Subject: [PATCH 30/36] Undo merge mangling --- workflow/hosts/gaeac5.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/hosts/gaeac5.yaml b/workflow/hosts/gaeac5.yaml index 6b1281b7c2..32452dd699 100644 --- a/workflow/hosts/gaeac5.yaml +++ b/workflow/hosts/gaeac5.yaml @@ -25,6 +25,6 @@ ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' DO_FETCH_HPSS: 'NO' -SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] +SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48'] AERO_INPUTS_DIR: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/gocart_emissions' From e7396bc1dca02f31c5b38282a143ea4bc0955065 Mon Sep 17 00:00:00 2001 From: David Grumm Date: Thu, 23 Jan 2025 19:13:02 +0000 Subject: [PATCH 31/36] Adding higher resolutions back --- workflow/hosts/gaeac5.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/hosts/gaeac5.yaml b/workflow/hosts/gaeac5.yaml index 32452dd699..6b1281b7c2 100644 --- a/workflow/hosts/gaeac5.yaml +++ b/workflow/hosts/gaeac5.yaml @@ -25,6 +25,6 @@ ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' DO_FETCH_HPSS: 'NO' -SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48'] +SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] AERO_INPUTS_DIR: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/gocart_emissions' From eba78413bd1d3dc83a875ffac2d86a21da21721b Mon Sep 17 00:00:00 2001 From: David Grumm Date: Thu, 23 Jan 2025 19:21:30 +0000 Subject: [PATCH 32/36] Removing higher resolutions (they will be added in a different PR) --- workflow/hosts/gaeac5.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/hosts/gaeac5.yaml b/workflow/hosts/gaeac5.yaml index 6b1281b7c2..32452dd699 100644 --- a/workflow/hosts/gaeac5.yaml +++ b/workflow/hosts/gaeac5.yaml @@ -25,6 +25,6 @@ ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' DO_FETCH_HPSS: 'NO' -SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] +SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48'] AERO_INPUTS_DIR: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/gocart_emissions' From 86444499f1835141d8a1dfb453ae56cfc8de1fee Mon Sep 17 00:00:00 2001 From: David Grumm Date: Thu, 23 Jan 2025 20:17:27 +0000 Subject: [PATCH 33/36] Address additional reviewer comments --- ci/cases/yamls/gfs_defaults_ci.yaml | 2 +- parm/config/gfs/config.base | 8 ---- parm/config/gfs/config.fetch | 6 +-- .../fetch/gfs_S2SW_cold_forecast-only.yaml.j2 | 18 ++++----- workflow/rocoto/gfs_tasks.py | 40 +++++++++---------- 5 files changed, 33 insertions(+), 41 deletions(-) diff --git a/ci/cases/yamls/gfs_defaults_ci.yaml b/ci/cases/yamls/gfs_defaults_ci.yaml index adc59ae87f..ab4114553a 100644 --- a/ci/cases/yamls/gfs_defaults_ci.yaml +++ b/ci/cases/yamls/gfs_defaults_ci.yaml @@ -10,6 +10,6 @@ base: ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} DO_TEST_MODE: "NO" FETCHDIR: "/NCEPDEV/emc-global/1year/David.Grumm/test_data" - DO_FETCH_HPSS: "YES" + DO_FETCH_HPSS: "NO" DO_METP: "NO" diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 13ab2033df..d4ebaa7667 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -475,14 +475,6 @@ export DO_VRFY_OCEANDA="@DO_VRFY_OCEANDA@" # Run SOCA Ocean and Seaice DA verif export FHMAX_FITS=132 [[ "${FHMAX_FITS}" -gt "${FHMAX_GFS}" ]] && export FHMAX_FITS=${FHMAX_GFS} -# Fetch options -# TODO once tarballs are available for CI at other resolutions/start types, etc, remove this if block -if [[ "${APP}" == "ATM" || "${APP}" == "S2SW" ]] && \ - [[ "${EXP_WARM_START}" == ".false." && "${CASE}" == "C48" ]]; then - export DO_FETCH_HPSS="@DO_FETCH_HPSS@" # pull from HPSS ## 123124 DG change to YES, or next - export DO_FETCH_LOCAL="NO" # pull from local -fi - if [[ ${DO_FETCH_HPSS} = "YES" ]] && [[ ${DO_FETCH_LOCAL} = "YES" ]]; then echo "Both HPSS and local fetch selected. Please choose one or the other." exit 3 diff --git a/parm/config/gfs/config.fetch b/parm/config/gfs/config.fetch index 8dd3d262ee..86ab5e3e2f 100644 --- a/parm/config/gfs/config.fetch +++ b/parm/config/gfs/config.fetch @@ -4,6 +4,9 @@ echo "BEGIN: config.fetch" +# Get task specific resources +source "${EXPDIR}/config.resources" fetch + # Determine start type if [[ "${EXP_WARM_START}" == ".false." ]]; then ic_type="cold" @@ -13,7 +16,4 @@ fi export FETCH_YAML_TMPL="${PARMgfs}/fetch/${NET}_${APP}_${ic_type}_${MODE}.yaml.j2" -# Get task specific resources -source "${EXPDIR}/config.resources" fetch - echo "END: config.fetch" diff --git a/parm/fetch/gfs_S2SW_cold_forecast-only.yaml.j2 b/parm/fetch/gfs_S2SW_cold_forecast-only.yaml.j2 index bbb5165ecc..2588b85117 100755 --- a/parm/fetch/gfs_S2SW_cold_forecast-only.yaml.j2 +++ b/parm/fetch/gfs_S2SW_cold_forecast-only.yaml.j2 @@ -1,8 +1,8 @@ {% set cycle_YMDH = current_cycle | to_YMDH %} {% set cycle_YMD = current_cycle | to_YMD %} {% set cycle_HH = current_cycle | strftime("%H") %} -{% set p_cycle_YMD = previous_cycle | to_YMD %} -{% set p_cycle_HH = previous_cycle | strftime("%H") %} +{% set prev_cycle_YMD = previous_cycle | to_YMD %} +{% set prev_cycle_HH = previous_cycle | strftime("%H") %} # For cold starts, the ATM component is in the current cycle RUN.YYYYMMDD/HH # For ocean/ice, some files are in the current cyle, some in the previous # For waves, all files are in the previous cycle @@ -10,9 +10,9 @@ {% set atm_dir = RUN + "." ~ cycle_YMD ~ "/" ~ cycle_HH ~ "/model/atmos/input" %} {% set ocean_dir = RUN + "." ~ cycle_YMD ~ "/" ~ cycle_HH ~ "/model/ocean/restart" %} {% set ice_dir = RUN + "." ~ cycle_YMD ~ "/" ~ cycle_HH ~ "/model/ice/restart" %} -{% set p_ocean_dir = "gdas." ~ p_cycle_YMD ~ "/" ~ p_cycle_HH ~ "/model/ocean/restart" %} -{% set p_ice_dir = "gdas." ~ p_cycle_YMD ~ "/" ~ p_cycle_HH ~ "/model/ice/restart" %} -{% set p_wave_dir = "gdas." ~ p_cycle_YMD ~ "/" ~ p_cycle_HH ~ "/model/wave/restart" %} +{% set prev_ocean_dir = "gdas." ~ prev_cycle_YMD ~ "/" ~ prev_cycle_HH ~ "/model/ocean/restart" %} +{% set prev_ice_dir = "gdas." ~ prev_cycle_YMD ~ "/" ~ prev_cycle_HH ~ "/model/ice/restart" %} +{% set prev_wave_dir = "gdas." ~ prev_cycle_YMD ~ "/" ~ prev_cycle_HH ~ "/model/wave/restart" %} {% set restart_prefix = cycle_YMD ~ "." ~ cycle_HH ~ "0000" %} untar: tarball : "{{ FETCHDIR }}/{{ cycle_YMDH }}/s2sw_cold.tar" @@ -27,11 +27,11 @@ untar: {% endfor %} # ftype # Ocean - {{ocean_dir}}/{{restart_prefix}}.MOM.res.nc - - {{p_ocean_dir}}/{{restart_prefix}}.MOM.res.nc + - {{prev_ocean_dir}}/{{restart_prefix}}.MOM.res.nc # Ice - {{ice_dir}}/{{restart_prefix}}.cice_model.res.nc - - {{p_ice_dir}}/{{restart_prefix}}.cice_model.res.nc + - {{prev_ice_dir}}/{{restart_prefix}}.cice_model.res.nc # Wave - - {{p_wave_dir}}/{{restart_prefix}}.restart.ww3 - - {{p_wave_dir}}/{{restart_prefix}}.restart.{{waveGRD}} + - {{prev_wave_dir}}/{{restart_prefix}}.restart.ww3 + - {{prev_wave_dir}}/{{restart_prefix}}.restart.{{waveGRD}} destination: "{{ DATAROOT }}" diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 8963dd747d..f2010b5378 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -16,50 +16,50 @@ def _is_this_a_gdas_task(run, task_name): raise TypeError(f'{task_name} must be part of the "enkfgdas" cycle and not {run}') # Specific Tasks begin here - def stage_ic(self): - - dependencies = None - if self.options['do_fetch_hpss'] or self.options['do_fetch_local']: - deps = [] - dep_dict = { - 'type': 'task', 'name': f'{self.run}_fetch', - } - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) + def fetch(self): cycledef = 'gdas_half' if self.run in ['gdas', 'enkfgdas'] else self.run - resources = self.get_resource('stage_ic') - task_name = f'{self.run}_stage_ic' + resources = self.get_resource('fetch') + task_name = f'{self.run}_fetch' task_dict = {'task_name': task_name, 'resources': resources, 'envars': self.envars, 'cycledef': cycledef, - 'command': f'{self.HOMEgfs}/jobs/rocoto/stage_ic.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/fetch.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', - 'maxtries': '&MAXTRIES;', - 'dependency': dependencies + 'maxtries': '&MAXTRIES;' } task = rocoto.create_task(task_dict) return task - def fetch(self): + def stage_ic(self): + + dependencies = None + if self.options['do_fetch_hpss'] or self.options['do_fetch_local']: + deps = [] + dep_dict = { + 'type': 'task', 'name': f'{self.run}_fetch', + } + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) cycledef = 'gdas_half' if self.run in ['gdas', 'enkfgdas'] else self.run - resources = self.get_resource('fetch') - task_name = f'{self.run}_fetch' + resources = self.get_resource('stage_ic') + task_name = f'{self.run}_stage_ic' task_dict = {'task_name': task_name, 'resources': resources, 'envars': self.envars, 'cycledef': cycledef, - 'command': f'{self.HOMEgfs}/jobs/rocoto/fetch.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/stage_ic.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', - 'maxtries': '&MAXTRIES;' + 'maxtries': '&MAXTRIES;', + 'dependency': dependencies } task = rocoto.create_task(task_dict) From b7c919e424c0a2cfdba9e205004037b2c0f3571d Mon Sep 17 00:00:00 2001 From: David Grumm Date: Thu, 23 Jan 2025 21:32:39 +0000 Subject: [PATCH 34/36] Remove some fetch options for now --- ci/cases/yamls/gfs_defaults_ci.yaml | 1 - parm/config/gfs/config.base | 7 +++---- workflow/hosts/awspw.yaml | 1 - workflow/hosts/azurepw.yaml | 1 - workflow/hosts/container.yaml | 1 - workflow/hosts/gaeac5.yaml | 1 - workflow/hosts/gaeac6.yaml | 1 - workflow/hosts/googlepw.yaml | 1 - workflow/hosts/hercules.yaml | 1 - workflow/hosts/jet.yaml | 1 - workflow/hosts/orion.yaml | 1 - workflow/hosts/s4.yaml | 1 - workflow/hosts/wcoss2.yaml | 1 - 13 files changed, 3 insertions(+), 16 deletions(-) diff --git a/ci/cases/yamls/gfs_defaults_ci.yaml b/ci/cases/yamls/gfs_defaults_ci.yaml index ab4114553a..1515d03a5c 100644 --- a/ci/cases/yamls/gfs_defaults_ci.yaml +++ b/ci/cases/yamls/gfs_defaults_ci.yaml @@ -10,6 +10,5 @@ base: ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} DO_TEST_MODE: "NO" FETCHDIR: "/NCEPDEV/emc-global/1year/David.Grumm/test_data" - DO_FETCH_HPSS: "NO" DO_METP: "NO" diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index d4ebaa7667..d60affb911 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -475,10 +475,9 @@ export DO_VRFY_OCEANDA="@DO_VRFY_OCEANDA@" # Run SOCA Ocean and Seaice DA verif export FHMAX_FITS=132 [[ "${FHMAX_FITS}" -gt "${FHMAX_GFS}" ]] && export FHMAX_FITS=${FHMAX_GFS} -if [[ ${DO_FETCH_HPSS} = "YES" ]] && [[ ${DO_FETCH_LOCAL} = "YES" ]]; then - echo "Both HPSS and local fetch selected. Please choose one or the other." - exit 3 -fi +# User may choose to reset these at experiment setup time +export DO_FETCH_HPSS="NO" +export DO_FETCH_LOCAL="NO" # Archiving options export HPSSARCH="@HPSSARCH@" # save data to HPSS archive diff --git a/workflow/hosts/awspw.yaml b/workflow/hosts/awspw.yaml index f523e976b0..c27a22ab7e 100644 --- a/workflow/hosts/awspw.yaml +++ b/workflow/hosts/awspw.yaml @@ -27,7 +27,6 @@ MAKE_ACFTBUFR: 'NO' DO_TRACKER: 'NO' DO_GENESIS: 'NO' DO_METP: 'NO' -DO_FETCH_HPSS: 'NO' SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C192', 'C384', 'C768'] # TODO: Test and support all cubed-sphere resolutions. AERO_INPUTS_DIR: /contrib/global-workflow-shared-data/data/gocart_emissions diff --git a/workflow/hosts/azurepw.yaml b/workflow/hosts/azurepw.yaml index f642b76543..abab09b414 100644 --- a/workflow/hosts/azurepw.yaml +++ b/workflow/hosts/azurepw.yaml @@ -27,6 +27,5 @@ MAKE_ACFTBUFR: 'NO' DO_TRACKER: 'NO' DO_GENESIS: 'NO' DO_METP: 'NO' -DO_FETCH_HPSS: 'NO' SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C384', 'C768'] # TODO: Test and support all cubed-sphere resolutions. diff --git a/workflow/hosts/container.yaml b/workflow/hosts/container.yaml index f2ca5931cb..d7924724ae 100644 --- a/workflow/hosts/container.yaml +++ b/workflow/hosts/container.yaml @@ -23,4 +23,3 @@ ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' SUPPORTED_RESOLUTIONS: ['C96', 'C48'] -DO_FETCH_HPSS: 'NO' \ No newline at end of file diff --git a/workflow/hosts/gaeac5.yaml b/workflow/hosts/gaeac5.yaml index 32452dd699..110a187ae1 100644 --- a/workflow/hosts/gaeac5.yaml +++ b/workflow/hosts/gaeac5.yaml @@ -24,7 +24,6 @@ LOCALARCH: 'NO' ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' -DO_FETCH_HPSS: 'NO' SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48'] AERO_INPUTS_DIR: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/gocart_emissions' diff --git a/workflow/hosts/gaeac6.yaml b/workflow/hosts/gaeac6.yaml index c5082c4a88..fed31634bf 100644 --- a/workflow/hosts/gaeac6.yaml +++ b/workflow/hosts/gaeac6.yaml @@ -24,5 +24,4 @@ LOCALARCH: 'NO' ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' -DO_FETCH_HPSS: 'NO' SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48'] diff --git a/workflow/hosts/googlepw.yaml b/workflow/hosts/googlepw.yaml index 4b580c3215..4c0bb02c07 100644 --- a/workflow/hosts/googlepw.yaml +++ b/workflow/hosts/googlepw.yaml @@ -27,7 +27,6 @@ MAKE_ACFTBUFR: 'NO' DO_TRACKER: 'NO' DO_GENESIS: 'NO' DO_METP: 'NO' -DO_FETCH_HPSS: 'NO' SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C384'] # TODO: Test and support all cubed-sphere resolutions. diff --git a/workflow/hosts/hercules.yaml b/workflow/hosts/hercules.yaml index bd6e685754..a2974377dd 100644 --- a/workflow/hosts/hercules.yaml +++ b/workflow/hosts/hercules.yaml @@ -26,7 +26,6 @@ MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' DO_TRACKER: 'NO' DO_GENESIS: 'NO' -DO_FETCH_HPSS: 'NO' DO_AWIPS: 'NO' SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] COMINecmwf: /work/noaa/global/glopara/data/external_gempak/ecmwf diff --git a/workflow/hosts/jet.yaml b/workflow/hosts/jet.yaml index 257a88ed18..737e2e7f94 100644 --- a/workflow/hosts/jet.yaml +++ b/workflow/hosts/jet.yaml @@ -24,7 +24,6 @@ LOCALARCH: 'NO' ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' -DO_FETCH_HPSS: 'NO' SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48'] COMINecmwf: /mnt/lfs5/HFIP/hfv3gfs/glopara/data/external_gempak/ecmwf COMINnam: /mnt/lfs5/HFIP/hfv3gfs/glopara/data/external_gempak/nam diff --git a/workflow/hosts/orion.yaml b/workflow/hosts/orion.yaml index 4f1f1374ab..9b1a908e2c 100644 --- a/workflow/hosts/orion.yaml +++ b/workflow/hosts/orion.yaml @@ -26,7 +26,6 @@ MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' DO_TRACKER: 'NO' DO_GENESIS: 'NO' -DO_FETCH_HPSS: 'NO' DO_AWIPS: 'NO' SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] COMINecmwf: /work/noaa/global/glopara/data/external_gempak/ecmwf diff --git a/workflow/hosts/s4.yaml b/workflow/hosts/s4.yaml index b919be39ef..2e77c112b1 100644 --- a/workflow/hosts/s4.yaml +++ b/workflow/hosts/s4.yaml @@ -24,6 +24,5 @@ LOCALARCH: 'NO' ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' MAKE_NSSTBUFR: 'YES' MAKE_ACFTBUFR: 'YES' -DO_FETCH_HPSS: 'NO' SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48'] AERO_INPUTS_DIR: /data/prod/glopara/gocart_emissions diff --git a/workflow/hosts/wcoss2.yaml b/workflow/hosts/wcoss2.yaml index 64ef5a60a2..4fb4b1d64a 100644 --- a/workflow/hosts/wcoss2.yaml +++ b/workflow/hosts/wcoss2.yaml @@ -24,7 +24,6 @@ LOCALARCH: 'NO' ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' -DO_FETCH_HPSS: 'NO' SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] COMINecmwf: /lfs/h2/emc/global/noscrub/emc.global/data/external_gempak/ecmwf COMINnam: /lfs/h2/emc/global/noscrub/emc.global/data/external_gempak/nam From aae3e6e04cc2b72d78b201ac179ac92d29e219fc Mon Sep 17 00:00:00 2001 From: David Grumm Date: Fri, 24 Jan 2025 17:29:08 +0000 Subject: [PATCH 35/36] Address reviewer comments(.venv) [David.Grumm@hfe10 G_WF_2988]$ git add ci/cases/yamls/gfs_defaults_ci.yaml parm/config/gfs/config.base --- ci/cases/yamls/gfs_defaults_ci.yaml | 6 ------ parm/config/gfs/config.base | 4 ++-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/ci/cases/yamls/gfs_defaults_ci.yaml b/ci/cases/yamls/gfs_defaults_ci.yaml index 1515d03a5c..9fcfa7d604 100644 --- a/ci/cases/yamls/gfs_defaults_ci.yaml +++ b/ci/cases/yamls/gfs_defaults_ci.yaml @@ -3,12 +3,6 @@ defaults: base: ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} DO_TEST_MODE: "YES" - -defaults: - !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml -base: - ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} - DO_TEST_MODE: "NO" FETCHDIR: "/NCEPDEV/emc-global/1year/David.Grumm/test_data" DO_METP: "NO" diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index d60affb911..96954b4acb 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -476,8 +476,8 @@ export FHMAX_FITS=132 [[ "${FHMAX_FITS}" -gt "${FHMAX_GFS}" ]] && export FHMAX_FITS=${FHMAX_GFS} # User may choose to reset these at experiment setup time -export DO_FETCH_HPSS="NO" -export DO_FETCH_LOCAL="NO" +export DO_FETCH_HPSS="NO" # Copy from HPSS (on HPSS-accessible machines) onto COM +export DO_FETCH_LOCAL="NO" # Copy from local disk onto COM # Archiving options export HPSSARCH="@HPSSARCH@" # save data to HPSS archive From 2eec8da724d491c70aac956c521ec0480cbfbf64 Mon Sep 17 00:00:00 2001 From: David Grumm Date: Fri, 24 Jan 2025 20:13:13 +0000 Subject: [PATCH 36/36] Remove extraneous new lines --- ci/cases/yamls/gfs_defaults_ci.yaml | 3 +-- workflow/hosts/awspw.yaml | 3 +-- workflow/hosts/azurepw.yaml | 3 +-- workflow/hosts/container.yaml | 2 +- workflow/hosts/gaeac5.yaml | 3 +-- workflow/hosts/gaeac6.yaml | 2 +- workflow/hosts/googlepw.yaml | 4 +--- workflow/hosts/hercules.yaml | 2 +- workflow/hosts/jet.yaml | 2 +- workflow/hosts/orion.yaml | 2 +- workflow/hosts/s4.yaml | 2 +- workflow/hosts/wcoss2.yaml | 2 +- 12 files changed, 12 insertions(+), 18 deletions(-) diff --git a/ci/cases/yamls/gfs_defaults_ci.yaml b/ci/cases/yamls/gfs_defaults_ci.yaml index 9fcfa7d604..bf5622845c 100644 --- a/ci/cases/yamls/gfs_defaults_ci.yaml +++ b/ci/cases/yamls/gfs_defaults_ci.yaml @@ -4,5 +4,4 @@ base: ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} DO_TEST_MODE: "YES" FETCHDIR: "/NCEPDEV/emc-global/1year/David.Grumm/test_data" - DO_METP: "NO" - + DO_METP: "NO" \ No newline at end of file diff --git a/workflow/hosts/awspw.yaml b/workflow/hosts/awspw.yaml index c27a22ab7e..6c73c36be5 100644 --- a/workflow/hosts/awspw.yaml +++ b/workflow/hosts/awspw.yaml @@ -28,5 +28,4 @@ DO_TRACKER: 'NO' DO_GENESIS: 'NO' DO_METP: 'NO' SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C192', 'C384', 'C768'] # TODO: Test and support all cubed-sphere resolutions. -AERO_INPUTS_DIR: /contrib/global-workflow-shared-data/data/gocart_emissions - +AERO_INPUTS_DIR: /contrib/global-workflow-shared-data/data/gocart_emissions \ No newline at end of file diff --git a/workflow/hosts/azurepw.yaml b/workflow/hosts/azurepw.yaml index abab09b414..39ea09d41d 100644 --- a/workflow/hosts/azurepw.yaml +++ b/workflow/hosts/azurepw.yaml @@ -27,5 +27,4 @@ MAKE_ACFTBUFR: 'NO' DO_TRACKER: 'NO' DO_GENESIS: 'NO' DO_METP: 'NO' -SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C384', 'C768'] # TODO: Test and support all cubed-sphere resolutions. - +SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C384', 'C768'] # TODO: Test and support all cubed-sphere resolutions. \ No newline at end of file diff --git a/workflow/hosts/container.yaml b/workflow/hosts/container.yaml index d7924724ae..256e461027 100644 --- a/workflow/hosts/container.yaml +++ b/workflow/hosts/container.yaml @@ -22,4 +22,4 @@ LOCALARCH: 'NO' ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' -SUPPORTED_RESOLUTIONS: ['C96', 'C48'] +SUPPORTED_RESOLUTIONS: ['C96', 'C48'] \ No newline at end of file diff --git a/workflow/hosts/gaeac5.yaml b/workflow/hosts/gaeac5.yaml index 110a187ae1..2db7f4629f 100644 --- a/workflow/hosts/gaeac5.yaml +++ b/workflow/hosts/gaeac5.yaml @@ -25,5 +25,4 @@ ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48'] -AERO_INPUTS_DIR: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/gocart_emissions' - +AERO_INPUTS_DIR: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/gocart_emissions' \ No newline at end of file diff --git a/workflow/hosts/gaeac6.yaml b/workflow/hosts/gaeac6.yaml index fed31634bf..0bf8b2ef0e 100644 --- a/workflow/hosts/gaeac6.yaml +++ b/workflow/hosts/gaeac6.yaml @@ -24,4 +24,4 @@ LOCALARCH: 'NO' ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' -SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48'] +SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48'] \ No newline at end of file diff --git a/workflow/hosts/googlepw.yaml b/workflow/hosts/googlepw.yaml index 4c0bb02c07..7778961e7e 100644 --- a/workflow/hosts/googlepw.yaml +++ b/workflow/hosts/googlepw.yaml @@ -27,6 +27,4 @@ MAKE_ACFTBUFR: 'NO' DO_TRACKER: 'NO' DO_GENESIS: 'NO' DO_METP: 'NO' -SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C384'] # TODO: Test and support all cubed-sphere resolutions. - - +SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C384'] # TODO: Test and support all cubed-sphere resolutions. \ No newline at end of file diff --git a/workflow/hosts/hercules.yaml b/workflow/hosts/hercules.yaml index a2974377dd..86577b0849 100644 --- a/workflow/hosts/hercules.yaml +++ b/workflow/hosts/hercules.yaml @@ -31,4 +31,4 @@ SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] COMINecmwf: /work/noaa/global/glopara/data/external_gempak/ecmwf COMINnam: /work/noaa/global/glopara/data/external_gempak/nam COMINukmet: /work/noaa/global/glopara/data/external_gempak/ukmet -AERO_INPUTS_DIR: /work2/noaa/global/wkolczyn/noscrub/global-workflow/gocart_emissions +AERO_INPUTS_DIR: /work2/noaa/global/wkolczyn/noscrub/global-workflow/gocart_emissions \ No newline at end of file diff --git a/workflow/hosts/jet.yaml b/workflow/hosts/jet.yaml index 737e2e7f94..43f960de2f 100644 --- a/workflow/hosts/jet.yaml +++ b/workflow/hosts/jet.yaml @@ -28,4 +28,4 @@ SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48'] COMINecmwf: /mnt/lfs5/HFIP/hfv3gfs/glopara/data/external_gempak/ecmwf COMINnam: /mnt/lfs5/HFIP/hfv3gfs/glopara/data/external_gempak/nam COMINukmet: /mnt/lfs5/HFIP/hfv3gfs/glopara/data/external_gempak/ukmet -AERO_INPUTS_DIR: /lfs5/HFIP/hfv3gfs/glopara/data/gocart_emissions +AERO_INPUTS_DIR: /lfs5/HFIP/hfv3gfs/glopara/data/gocart_emissions \ No newline at end of file diff --git a/workflow/hosts/orion.yaml b/workflow/hosts/orion.yaml index 9b1a908e2c..60147d74a7 100644 --- a/workflow/hosts/orion.yaml +++ b/workflow/hosts/orion.yaml @@ -31,4 +31,4 @@ SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] COMINecmwf: /work/noaa/global/glopara/data/external_gempak/ecmwf COMINnam: /work/noaa/global/glopara/data/external_gempak/nam COMINukmet: /work/noaa/global/glopara/data/external_gempak/ukmet -AERO_INPUTS_DIR: /work2/noaa/global/wkolczyn/noscrub/global-workflow/gocart_emissions +AERO_INPUTS_DIR: /work2/noaa/global/wkolczyn/noscrub/global-workflow/gocart_emissions \ No newline at end of file diff --git a/workflow/hosts/s4.yaml b/workflow/hosts/s4.yaml index 2e77c112b1..3ae31adc2d 100644 --- a/workflow/hosts/s4.yaml +++ b/workflow/hosts/s4.yaml @@ -25,4 +25,4 @@ ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' MAKE_NSSTBUFR: 'YES' MAKE_ACFTBUFR: 'YES' SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48'] -AERO_INPUTS_DIR: /data/prod/glopara/gocart_emissions +AERO_INPUTS_DIR: /data/prod/glopara/gocart_emissions \ No newline at end of file diff --git a/workflow/hosts/wcoss2.yaml b/workflow/hosts/wcoss2.yaml index 4fb4b1d64a..172b001edf 100644 --- a/workflow/hosts/wcoss2.yaml +++ b/workflow/hosts/wcoss2.yaml @@ -28,4 +28,4 @@ SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] COMINecmwf: /lfs/h2/emc/global/noscrub/emc.global/data/external_gempak/ecmwf COMINnam: /lfs/h2/emc/global/noscrub/emc.global/data/external_gempak/nam COMINukmet: /lfs/h2/emc/global/noscrub/emc.global/data/external_gempak/ukmet -AERO_INPUTS_DIR: /lfs/h2/emc/global/noscrub/emc.global/data/gocart_emissions +AERO_INPUTS_DIR: /lfs/h2/emc/global/noscrub/emc.global/data/gocart_emissions \ No newline at end of file