Script to clone, build workflow and generate experiment for prototypes (

#470)

* Commit to test on Hera

* Update to clone/checkout properly

* Incremental commit

* incremental commit

* fix for CI with workflow change

* actually fix CI

* another attempt to fix CI

* Small change

* end of day commit

* Changes to be consistent with g-w

* typo fix

* Change hashes

* added gfs arg (#471)

* Run rocotorun and rocotostat

* comment out bad test

* Fix issues russ pointed out in issue

* replace NMEM_ENKF with NMEM_ENS in jjobs_ens_init.sh (#467)

* modify run_jjobs

---------

Co-authored-by: Guillaume Vernieres <[email protected]>
Co-authored-by: RussTreadon-NOAA <[email protected]>

modulefiles/GDAS/hera.lua

@@ -64,6 +64,7 @@ load("netcdf-cxx4/4.3.1")
 load("json/3.10.5")
 load("py-pybind11/2.8.1")
 --load("crtm/v2.4_jedi")
+load("rocoto/1.3.3")
 
 load("hpc/1.2.0")
 load("miniconda3/4.6.14")

modulefiles/GDAS/orion.lua

@@ -58,6 +58,9 @@ load("ncview/2.1.8")
 load("netcdf-cxx4/4.3.1")
 load("py-pybind11/2.8.1")
 --load("crtm/v2.4_jedi")
+load("contrib/0.1")
+load("noaatools/2.0")
+load("rocoto/1.3.3")
 
 load("hpc/1.2.0")
 load("miniconda3/4.6.14")

parm/atm/berror/hybvar_bump.yaml

@@ -105,7 +105,7 @@ components:
         filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc'
         filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res'
       pattern: '%mem%'
-      nmembers: $(NMEM_ENKF)
+      nmembers: $(NMEM_ENS)
       zero padding: 3
 #    localization:
 #      localization method: SABER

parm/atm/berror/hybvar_gsibec.yaml

@@ -42,7 +42,7 @@ components:
         filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc'
         filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res'
       pattern: '%mem%'
-      nmembers: $(NMEM_ENKF)
+      nmembers: $(NMEM_ENS)
       zero padding: 3
     localization:
       localization method: SABER

parm/atm/lgetkf/lgetkf.yaml

@@ -30,7 +30,7 @@ background:
        filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc'
        filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res'
      pattern: '%mem%'
-     nmembers: $(NMEM_ENKF)
+     nmembers: $(NMEM_ENS)
      zero padding: 3
 
 observations: !INC ${OBS_LIST}

prototypes/configs/cp0.sh

@@ -0,0 +1,25 @@
+# top level options
+GWHASH=255d99b
+GDASHASH=ab6f06d
+BUILD="NO"
+PSLOT=cp0
+PROTOROOT=/scratch1/NCEPDEV/da/Cory.R.Martin/GDAS_prototypes
+GWDIR=$PROTOROOT/$PSLOT
+DUMPDIR=/scratch1/NCEPDEV/da/role.jedipara/dump/
+ICSDIR=/scratch1/NCEPDEV/stmp4/Cory.R.Martin/tmp_ics
+expdir=$PROTOROOT/expdirs
+comrot=$PROTOROOT/rotdirs
+idate=2021073118
+edate=2021080118
+app="ATM"
+starttype="cold"
+gfscyc=0
+resdet=96
+resens=96
+nens=0
+# config.* options
+DO_JEDIATMVAR="YES"
+DO_JEDIATMENS="NO"
+DO_JEDIOCNVAR="NO"
+DO_JEDILANDDA="NO"
+DO_MERGENSST="NO"

prototypes/gen_prototype.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+set -e
+
+# ==============================================================================
+usage() {
+  set +x
+  echo
+  echo "Usage: $0 -c <config> -t <target> -h"
+  echo
+  echo "  -c  Configuration for prototype defined in shell script <config>"
+  echo "  -t  Supported platform script is running on <Hera|Orion>"
+  echo "  -h  display this message and quit"
+  echo
+  exit 1
+}
+
+# ==============================================================================
+while getopts "c:t:h" opt; do
+  case $opt in
+    c)
+      config=$OPTARG
+      ;;
+    t)
+      MACHINE=${MACHINE:-$OPTARG}
+      ;;
+    h|\?|:)
+      usage
+      ;;
+  esac
+done
+
+dir_root="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../ >/dev/null 2>&1 && pwd )"
+
+# source the input configuration
+source $config
+
+# create directories
+mkdir -p $PROTOROOT/$PSLOT
+
+# clone/build/link workflow and GDASApp
+if [[ $BUILD == 'YES' ]]; then
+    cd $GWDIR
+    git clone https://github.com/NOAA-EMC/global-workflow.git
+    cd global-workflow/sorc
+    git checkout $GWHASH
+    ./checkout.sh -ug
+    cd gdas.cd
+    git checkout $GDASHASH
+    cd ../
+    ./build_all.sh
+    ./link_workflow.sh
+fi
+
+# load modules to then generate experiment directory and rocoto XML
+module use $dir_root/modulefiles
+module load GDAS/$MACHINE
+
+# move expdir if it exists, delete backup if it exists
+if [[ -d $expdir/$PSLOT ]]; then
+  [[ -d $expdir/${PSLOT}.bak ]] && rm -rf $expdir/${PSLOT}.bak
+  mv $expdir/$PSLOT $expdir/${PSLOT}.bak
+fi
+
+# move rotdir if it exists, delete backup if it exists
+if [[ -d $comrot/$PSLOT ]]; then
+  [[ -d $comrot/${PSLOT}.bak ]] && rm -rf $comrot/${PSLOT}.bak
+  mv $comrot/$PSLOT $comrot/${PSLOT}.bak
+fi
+
+# create YAML to override workflow config defaults
+mkdir -p $expdir
+cat > $expdir/config_${PSLOT}.yaml << EOF
+base:
+  ACCOUNT: "da-cpu"
+  HPSS_PROJECT: "emc-da"
+  HOMEDIR: "/scratch1/NCEPDEV/da/${USER}"
+  DMPDIR: "${DUMPDIR}"
+  DO_JEDIATMVAR: "${DO_JEDIATMVAR}"
+  DO_JEDIATMENS: "${DO_JEDIATMENS}"
+  DO_JEDIOCNVAR: "${DO_JEDIOCNVAR}"
+  DO_JEDILANDDA: "${DO_JEDILANDDA}"
+  DO_MERGENSST: "${DO_MERGENSST}"
+EOF
+
+# setup experiment
+cd $GWDIR/global-workflow/workflow
+./setup_expt.py gfs cycled --idate $idate  \
+                           --edate $edate \
+                           --app $app \
+                           --start $starttype \
+                           --gfs_cyc $gfscyc \
+                           --resdet $resdet \
+                           --resens $resens \
+                           --nens $nens \
+                           --pslot $PSLOT \
+                           --configdir $GWDIR/global-workflow/parm/config/gfs \
+                           --comrot $comrot \
+                           --expdir $expdir \
+                           --icsdir $ICSDIR \
+                           --yaml $expdir/config_${PSLOT}.yaml
+
+# setup XML for workflow
+./setup_xml.py $expdir/$PSLOT
+
+# run rocotorun one time
+rocotorun -w $expdir/$PSLOT/${PSLOT}.xml -d $expdir/$PSLOT/${PSLOT}.db
+
+# run rocotostat on the first cycle to see if things were submitted
+rocotostat -w $expdir/$PSLOT/${PSLOT}.xml -d $expdir/$PSLOT/${PSLOT}.db -c ${idate}00

test/CMakeLists.txt

@@ -110,9 +110,10 @@ add_test(NAME test_gdasapp_check_yaml_keys
          WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/)
 
 # test to ensure all YAML in repo is valid YAML
-add_test(NAME test_gdasapp_check_valid_yaml
-         COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/test/check_valid_yaml.py ${PROJECT_SOURCE_DIR}
-         WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/)
+#add_test(NAME test_gdasapp_check_valid_yaml
+#         COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/test/check_valid_yaml.py ${PROJECT_SOURCE_DIR}
+#         WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/)
+# commented out above test because it does not really do as advertised with all these templates which are not valid yaml
 
 # test for ush/jediinc2fv3.py
 add_test(NAME test_gdasapp_jedi_increment_to_fv3

test/atm/global-workflow/config.base.emc.dyn

@@ -162,7 +162,7 @@ export OPS_RES="C768" # Do not change
 export LEVS=128
 export CASE="@CASECTL@"
 export CASE_ANL="$CASE"
-export CASE_ENKF="@CASEENS@"
+export CASE_ENS="@CASEENS@"
 
 # TODO: This should not depend on $CASE or $CASE_ENKF
 # These are the currently available grid-combinations
@@ -302,15 +302,15 @@ export imp_physics=@IMP_PHYSICS@
 
 # Shared parameters
 # DA engine
-export DO_JEDIVAR="@DO_JEDIVAR@"
-export DO_JEDIENS="@DO_JEDIENS@"
+export DO_JEDIATMVAR="@DO_JEDIATMVAR@"
+export DO_JEDIATMENS="@DO_JEDIATMENS@"
 export DO_JEDIOCNVAR="NO"
 export DO_JEDILANDDA="NO"
 export DO_MERGENSST="NO"
 
 # Hybrid related
 export DOHYBVAR="@DOHYBVAR@"
-export NMEM_ENKF=@NMEM_ENKF@
+export NMEM_ENS=@NMEM_ENS@
 export NMEM_EFCS=30
 export SMOOTH_ENKF="NO"
 export l4densvar=".true."

test/atm/global-workflow/config.yaml

@@ -1,7 +1,7 @@
 base:
   NCDUMP: $CONDA_PREFIX/ncdump
-  DO_JEDIVAR: "YES"
-  DO_JEDIENS: "YES"
+  DO_JEDIATMVAR: "YES"
+  DO_JEDIATMENS: "YES"
   UTILROOT: /apps/contrib/NCEP/libs/hpc-stack/intel-2018.4/prod_util/1.2.2
   DATAPATH: "@bindir@/test/atm/global-workflow/testrun"
   DUMPDIR: "@dumpdir@"

test/atm/global-workflow/jjob_ens_final.sh

@@ -19,7 +19,7 @@ export COMIN_GES=${bindir}/test/atm/bkg
 export pid=${pid:-$$}
 export jobid=$pid
 export COMROOT=$DATAROOT
-export NMEM_ENKF=3
+export NMEM_ENS=3
 export ACCOUNT=da-cpu
 export DO_JEDIENS="YES"
 

test/atm/global-workflow/jjob_ens_init.sh

@@ -20,13 +20,13 @@ export COMIN_GES=${bindir}/test/atm/bkg
 export pid=${pid:-$$}
 export jobid=$pid
 export COMROOT=$DATAROOT
-export NMEM_ENKF=3
+export NMEM_ENS=3
 export ACCOUNT=da-cpu
 export COM_TOP=$ROTDIR
 
 # Set GFS COM paths
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/parm/config/config.com"
+source "${HOMEgfs}/parm/config/gfs/config.com"
 
 # Set python path for workflow utilities and tasks
 pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src"
@@ -70,7 +70,7 @@ done
 
 # Link tiled ges and atmf006 files to ROTDIR
 dpath=enkfgdas.$gPDY/$gcyc
-for imem in $(seq 1 $NMEM_ENKF); do
+for imem in $(seq 1 $NMEM_ENS); do
     memchar="mem"$(printf %03i $imem)
 
     MEMDIR=${memchar} RUN=${RUN} YMD=${gPDY} HH=${gcyc} generate_com -x \

test/atm/global-workflow/jjob_ens_run.sh

@@ -22,7 +22,7 @@ export COMIN_GES=${bindir}/test/atm/bkg
 export pid=${pid:-$$}
 export jobid=$pid
 export COMROOT=$DATAROOT
-export NMEM_ENKF=3
+export NMEM_ENS=3
 export ACCOUNT=da-cpu
 export DO_JEDIENS="YES"
 

test/atm/global-workflow/jjob_var_init.sh

@@ -26,7 +26,7 @@ export COM_TOP=$ROTDIR
 
 # Set GFS COM paths
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/parm/config/config.com"
+source "${HOMEgfs}/parm/config/gfs/config.com"
 
 # Set python path for workflow utilities and tasks
 pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src"

test/atm/global-workflow/setup_workflow_exp.sh

@@ -14,7 +14,7 @@ resdet='48'
 resens='48'
 nens=3
 pslot='gdas_test'
-configdir=$srcdir/../../parm/config
+configdir=$srcdir/../../parm/config/gfs
 comrot=$bindir/test/atm/global-workflow/testrun/ROTDIRS
 expdir=$bindir/test/atm/global-workflow/testrun/experiments
 
@@ -37,7 +37,7 @@ ln -sf $srcdir/../../workflow/setup_expt.py .
 
 
 echo "Running global-workflow experiment generation script"
-$srcdir/../../workflow/setup_expt.py cycled --idate $idate  \
+$srcdir/../../workflow/setup_expt.py gfs cycled --idate $idate  \
                        --edate $edate \
                        --app $app \
                        --start $starttype \

test/setup_workflow_exp.sh

@@ -13,7 +13,7 @@ resdet='48'
 resens='24'
 nens=0
 pslot='gdas_test'
-configdir=$srcdir/../../parm/config
+configdir=$srcdir/../../parm/config/gfs
 comrot=$bindir/test/testrun/ROTDIRS
 expdir=$bindir/test/testrun/experiments
 
@@ -24,6 +24,6 @@ rm -rf "${comrot}" "${expdir}"
 cd $srcdir/../../workflow
 
 echo "Running global-workflow experiment generation script"
-./setup_expt.py cycled --idate $idate --edate $edate --app $app --start $starttype --gfs_cyc $gfscyc --resdet $resdet --resens $resens --nens $nens --pslot $pslot --configdir $configdir --comrot $comrot --expdir $expdir
+./setup_expt.py gfs cycled --idate $idate --edate $edate --app $app --start $starttype --gfs_cyc $gfscyc --resdet $resdet --resens $resens --nens $nens --pslot $pslot --configdir $configdir --comrot $comrot --expdir $expdir
 
 exit $?

test/soca/gw/run_jjobs.yaml.test

@@ -30,6 +30,12 @@ gw environement:
     OMP_NUM_THREADS: 1
 
 setup_expt config:
+  base:
+    DO_JEDIATMVAR: "NO"
+    DO_JEDIATMENS: "NO"
+    DO_JEDIOCNVAR: "YES"
+    DO_JEDILANDDA: "NO"
+    DO_MERGENSST: "NO"
   ocnanal:
     SOCA_INPUT_FIX_DIR: @HOMEgfs@/sorc/gdas.cd/build/soca_static
     CASE_ANL: C48

ush/run_jedi_exe.py

@@ -124,10 +124,10 @@ def run_jedi_exe(yamlconfig):
         'COMOUT': os.path.join(workdir, 'obs'),
         'CASE': executable_subconfig['case'],
         'CASE_ANL': executable_subconfig.get('case_anl', executable_subconfig['case']),
-        'CASE_ENKF': executable_subconfig.get('case_enkf', executable_subconfig['case']),
+        'CASE_ENS': executable_subconfig.get('case_enkf', executable_subconfig['case']),
         'DOHYBVAR': executable_subconfig.get('dohybvar', False),
         'LEVS': str(executable_subconfig['levs']),
-        'NMEM_ENKF': executable_subconfig.get('nmem', 0),
+        'NMEM_ENS': executable_subconfig.get('nmem', 0),
         'COMIN_GES_ENS': f"{comin_ges_ens}",
         'forecast_steps': calc_fcst_steps(executable_subconfig.get('forecast_step', 'PT6H'),
                                           executable_subconfig['atm_window_length']),
@@ -153,7 +153,7 @@ def run_jedi_exe(yamlconfig):
         'npx_anl': f"{int(os.environ['CASE_ANL'][1:]) + 1}",
         'npy_anl': f"{int(os.environ['CASE_ANL'][1:]) + 1}",
         'npz_anl': f"{int(os.environ['LEVS']) - 1}",
-        'NMEM_ENKF': f"{int(os.environ['NMEM_ENKF'])}",
+        'NMEM_ENS': f"{int(os.environ['NMEM_ENS'])}",
         'ATM_WINDOW_BEGIN': window_begin,
         'ATM_WINDOW_LENGTH': f"PT{assim_freq}H",
         'BKG_TSTEP': executable_subconfig.get('forecast_step', 'PT6H'),

ush/soca/run_jjobs.py

@@ -89,10 +89,11 @@ def setupexpt(self):
         """
 
         # Make a copy of the configs
-        origconfig = "${HOMEgfs}/parm/config"
+        origconfig = "${HOMEgfs}/parm/config/gfs"
         self.f.write("\n")
         self.f.write("# Make a copy of config\n")
-        self.f.write(f"cp -r {origconfig} .\n")
+        self.f.write(f"mkdir -p config\n")
+        self.f.write(f"cp -r {origconfig} config/\n")
 
         # Dump the configs in a separate yaml file
         with open("overwrite_defaults.yaml", "w") as f:
@@ -102,7 +103,7 @@ def setupexpt(self):
         self.f.write("\n")
         self.f.write("# Setup the experiment\n")
 
-        setupexpt = "${HOMEgfs}/workflow/setup_expt.py cycled "
+        setupexpt = "${HOMEgfs}/workflow/setup_expt.py gfs cycled "
         # Most of the args keys are not used to run the jjobs but are needed to run setup_expt.py
         args = {
             "idate": "${PDY}${cyc}",
@@ -114,7 +115,7 @@ def setupexpt(self):
             "resens": "24",
             "nens": "0",
             "pslot": "${PSLOT}",
-            "configdir": "${PWD}/config",
+            "configdir": "${PWD}/config/gfs",
             "comrot": self.rotdir,
             "expdir": "${EXPDIRS}",
             "yaml": "overwrite_defaults.yaml"}
@@ -155,7 +156,7 @@ def copy_bkgs(self):
         print(f"RUN: {self.RUN}")
 
         # setup COM variables
-        self.f.write("source ${HOMEgfs}/parm/config/config.com\n")
+        self.f.write("source ${HOMEgfs}/parm/config/gfs/config.com\n")
         self.f.write("source ${HOMEgfs}/ush/preamble.sh\n")
         self.precom('COM_OCEAN_HISTORY_PREV', 'COM_OCEAN_HISTORY_TMPL')
         self.precom('COM_ICE_HISTORY_PREV', 'COM_ICE_HISTORY_TMPL')