Merge branch 'text/us-215' of github.com:gspetro-NOAA/land-DA_workflo…

…w into text/us-215

.github/pull_request_template.md

@@ -27,6 +27,8 @@ EXAMPLE: Closes ufs-community/land-DA/issues/<issue_number>
     - [ ] Hercules
 - CI
   - [ ] Completed
+- WE2E
+  - [ ] Completed
 - PW-Clouds
   - [ ] AWS
   - [ ] AZURE

modulefiles/build_hercules_intel.lua

@@ -0,0 +1,46 @@
+help([[
+loads modules necessary for building the land-DA workflow on Hercules using Intel
+]])
+
+whatis([===[Loads modules necessary for building the land-DA workflow on Hercules]===])
+
+prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack"))
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
+load(pathJoin("stack-python", stack_python_ver))
+
+load(pathJoin("cmake", cmake_ver))
+load(pathJoin("ecbuild", ecbuild_ver))
+
+load(pathJoin("jasper", jasper_ver))
+load(pathJoin("zlib", zlib_ver))
+load(pathJoin("libpng", libpng_ver))
+load(pathJoin("hdf5", hdf5_ver))
+load(pathJoin("netcdf-c", netcdf_c_ver))
+load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
+load(pathJoin("parallelio", parallelio_ver))
+load(pathJoin("esmf", esmf_ver))
+load(pathJoin("fms",fms_ver))
+load(pathJoin("bacio", bacio_ver))
+load(pathJoin("crtm", crtm_ver))
+load(pathJoin("g2", g2_ver))
+load(pathJoin("g2tmpl", g2tmpl_ver))
+load(pathJoin("ip", ip_ver))
+load(pathJoin("sp", sp_ver))
+load(pathJoin("w3emc", w3emc_ver))
+load(pathJoin("gftl-shared", gftl_shared_ver))
+load(pathJoin("mapl", mapl_ver))
+load(pathJoin("prod_util", prod_util_ver))
+load(pathJoin("ufs-pyenv", ufs_pyenv_ver))
+
+setenv("CFLAGS","-diag-disable=10441")
+setenv("FFLAGS","-diag-disable=10441")
+
+setenv("CC", "mpiicc")
+setenv("CXX", "mpiicpc")
+setenv("FC", "mpiifort")
+setenv("CMAKE_Platform", "hercules.intel")
+
+setenv("EPICHOME", "/work/noaa/epic/UFS_Land-DA_Dev")
+setenv("JEDI_INSTALL", "/work2/noaa/epic/UFS_Land-DA_Dev/jedi_v7_hercules")

modulefiles/tasks/hera/task.analysis.lua

@@ -5,4 +5,6 @@ load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
 load(pathJoin("stack-python", stack_python_ver))
 
 load(pathJoin("prod_util", prod_util_ver))
-load(pathJoin("ufs-pyenv", ufs_pyenv_ver))
+load(pathJoin("py-netcdf4", py_netcdf4_ver))
+load(pathJoin("py-numpy", py_numpy_ver))
+

modulefiles/tasks/hera/task.forecast.lua

@@ -23,4 +23,6 @@ load(pathJoin("w3emc", w3emc_ver))
 load(pathJoin("gftl-shared", gftl_shared_ver))
 load(pathJoin("mapl", mapl_ver))
 load(pathJoin("prod_util", prod_util_ver))
-load(pathJoin("ufs-pyenv", ufs_pyenv_ver))
+load(pathJoin("py-netcdf4", py_netcdf4_ver))
+load(pathJoin("py-numpy", py_numpy_ver))
+

modulefiles/tasks/hera/task.post_anal.lua

@@ -8,3 +8,6 @@ load(pathJoin("hdf5", hdf5_ver))
 load(pathJoin("netcdf-c", netcdf_c_ver))
 load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
 load(pathJoin("prod_util", prod_util_ver))
+load(pathJoin("py-netcdf4", py_netcdf4_ver))
+load(pathJoin("py-numpy", py_numpy_ver))
+

modulefiles/tasks/hercules/task.analysis.lua

@@ -0,0 +1,10 @@
+prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack"))
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
+load(pathJoin("stack-python", stack_python_ver))
+
+load(pathJoin("prod_util", prod_util_ver))
+load(pathJoin("py-netcdf4", py_netcdf4_ver))
+load(pathJoin("py-numpy", py_numpy_ver))
+

modulefiles/tasks/hercules/task.forecast.lua

@@ -0,0 +1,28 @@
+prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack"))
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
+load(pathJoin("stack-python", stack_python_ver))
+
+load(pathJoin("jasper", jasper_ver))
+load(pathJoin("zlib", zlib_ver))
+load(pathJoin("libpng", libpng_ver))
+load(pathJoin("hdf5", hdf5_ver))
+load(pathJoin("netcdf-c", netcdf_c_ver))
+load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
+load(pathJoin("parallelio", parallelio_ver))
+load(pathJoin("esmf", esmf_ver))
+load(pathJoin("fms",fms_ver))
+load(pathJoin("bacio", bacio_ver))
+load(pathJoin("crtm", crtm_ver))
+load(pathJoin("g2", g2_ver))
+load(pathJoin("g2tmpl", g2tmpl_ver))
+load(pathJoin("ip", ip_ver))
+load(pathJoin("sp", sp_ver))
+load(pathJoin("w3emc", w3emc_ver))
+load(pathJoin("gftl-shared", gftl_shared_ver))
+load(pathJoin("mapl", mapl_ver))
+load(pathJoin("prod_util", prod_util_ver))
+load(pathJoin("py-netcdf4", py_netcdf4_ver))
+load(pathJoin("py-numpy", py_numpy_ver))
+

modulefiles/tasks/hercules/task.plot_stats.lua

@@ -0,0 +1,15 @@
+prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack"))
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
+load(pathJoin("stack-python", stack_python_ver))
+
+load(pathJoin("prod_util", prod_util_ver))
+load(pathJoin("py-cartopy", py_cartopy_ver))
+load(pathJoin("py-matplotlib", py_matplotlib_ver))
+load(pathJoin("py-netcdf4", py_netcdf4_ver))
+load(pathJoin("py-numpy", py_numpy_ver))
+load(pathJoin("py-pyyaml", py_pyyaml_ver))
+load(pathJoin("py-scipy", py_scipy_ver))
+load(pathJoin("py-xarray", py_xarray_ver))
+

modulefiles/tasks/hercules/task.post_anal.lua

@@ -0,0 +1,13 @@
+prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack"))
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
+load(pathJoin("stack-python", stack_python_ver))
+
+load(pathJoin("hdf5", hdf5_ver))
+load(pathJoin("netcdf-c", netcdf_c_ver))
+load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
+load(pathJoin("prod_util", prod_util_ver))
+load(pathJoin("py-netcdf4", py_netcdf4_ver))
+load(pathJoin("py-numpy", py_numpy_ver))
+

modulefiles/tasks/hercules/task.pre_anal.lua

@@ -0,0 +1,10 @@
+prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack"))
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
+load(pathJoin("stack-python", stack_python_ver))
+
+load(pathJoin("hdf5", hdf5_ver))
+load(pathJoin("netcdf-c", netcdf_c_ver))
+load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
+load(pathJoin("prod_util", prod_util_ver))

modulefiles/tasks/hercules/task.prep_obs.lua

@@ -0,0 +1,7 @@
+prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack"))
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
+load(pathJoin("stack-python", stack_python_ver))
+
+load(pathJoin("prod_util", prod_util_ver))

modulefiles/tasks/orion/task.analysis.lua

@@ -5,4 +5,6 @@ load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
 load(pathJoin("stack-python", stack_python_ver))
 
 load(pathJoin("prod_util", prod_util_ver))
-load(pathJoin("ufs-pyenv", ufs_pyenv_ver))
+load(pathJoin("py-netcdf4", py_netcdf4_ver))
+load(pathJoin("py-numpy", py_numpy_ver))
+

modulefiles/tasks/orion/task.forecast.lua

@@ -23,4 +23,6 @@ load(pathJoin("w3emc", w3emc_ver))
 load(pathJoin("gftl-shared", gftl_shared_ver))
 load(pathJoin("mapl", mapl_ver))
 load(pathJoin("prod_util", prod_util_ver))
-load(pathJoin("ufs-pyenv", ufs_pyenv_ver))
+load(pathJoin("py-netcdf4", py_netcdf4_ver))
+load(pathJoin("py-numpy", py_numpy_ver))
+

modulefiles/tasks/orion/task.post_anal.lua

@@ -8,3 +8,6 @@ load(pathJoin("hdf5", hdf5_ver))
 load(pathJoin("netcdf-c", netcdf_c_ver))
 load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
 load(pathJoin("prod_util", prod_util_ver))
+load(pathJoin("py-netcdf4", py_netcdf4_ver))
+load(pathJoin("py-numpy", py_numpy_ver))
+

modulefiles/wflow_hercules.lua

@@ -0,0 +1,18 @@
+help([[
+This module loads python environement for running the land-DA workflow on
+the MSU machine Hercules
+]])
+
+whatis([===[Loads libraries needed for running the land-DA workflow on Hercules ]===])
+
+load("contrib")
+load("rocoto")
+
+unload("python")
+load("conda")
+
+if mode() == "load" then
+   LmodMsgRaw([===[Please do the following to activate conda:
+       > conda activate land_da
+]===])
+end

parm/land_analysis_hera.yaml

@@ -7,8 +7,7 @@ workflow:
   cycledef:
     - attrs:
         group: cycled
-      spec: 200001030000 200001040000 24:00:00   # for gswp3
-#      spec: 201912210000 201912220000 24:00:00   # for era5
+      spec: 200001030000 200001040000 24:00:00
   entities:
     MACHINE: "hera"
     SCHED: "slurm"
@@ -25,11 +24,13 @@ workflow:
     OBSDIR_SUBDIR: ""
     OBS_TYPES: "GHCN"
     DAtype: "letkfoi_snow"
-    SNOWDEPTHVAR: "snwdph"
     TSTUB: "oro_C96.mx100"
+    WE2E_VAV: "YES"
+    WE2E_ATOL: "1e-7"
+    WE2E_LOG_FN: "we2e.log"
     NET: "landda"
     envir: "test"
-    model_ver: "v1.2.1"
+    model_ver: "v2.0.0"
     RUN: "landda"
     HOMElandda: "&EXP_BASEDIR;/land-DA_workflow"
     PTMP: "&EXP_BASEDIR;/ptmp"
@@ -127,6 +128,10 @@ workflow:
         ACCOUNT: "&ACCOUNT;"
         RES: "&RES;"
         TSTUB: "&TSTUB;"
+        WE2E_VAV: "&WE2E_VAV;"
+        WE2E_ATOL: "&WE2E_ATOL;"
+        WE2E_LOG_FN: "&WE2E_LOG_FN;"
+        LOGDIR: "&LOGDIR;"
         model_ver: "&model_ver;"
         HOMElandda: "&HOMElandda;"
         COMROOT: "&COMROOT;"
@@ -135,7 +140,6 @@ workflow:
         PDY: "&PDY;"
         cyc: "&cyc;"
         DAtype: "&DAtype;"
-        SNOWDEPTHVAR: "&SNOWDEPTHVAR;"
         NPROCS_ANALYSIS: "&NPROCS_ANALYSIS;"
         JEDI_INSTALL: "&JEDI_INSTALL;"
       account: "&ACCOUNT;"
@@ -159,6 +163,10 @@ workflow:
         ACCOUNT: "&ACCOUNT;"
         RES: "&RES;"
         TSTUB: "&TSTUB;"
+        WE2E_VAV: "&WE2E_VAV;"
+        WE2E_ATOL: "&WE2E_ATOL;"
+        WE2E_LOG_FN: "&WE2E_LOG_FN;"
+        LOGDIR: "&LOGDIR;"
         model_ver: "&model_ver;"
         RUN: "&RUN;"
         HOMElandda: "&HOMElandda;"
@@ -218,6 +226,10 @@ workflow:
         ATMOS_FORC: "&ATMOS_FORC;"
         RES: "&RES;"
         WARMSTART_DIR: "&WARMSTART_DIR;"
+        WE2E_VAV: "&WE2E_VAV;"
+        WE2E_ATOL: "&WE2E_ATOL;"
+        WE2E_LOG_FN: "&WE2E_LOG_FN;"
+        LOGDIR: "&LOGDIR;"
         model_ver: "&model_ver;"
         HOMElandda: "&HOMElandda;"
         COMROOT: "&COMROOT;"