diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 9b930f20..754864a7 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -7,7 +7,7 @@ Provide a detailed description of what this PR does. What bug does it fix, or wh - [ ] Is baseline change expected ? ## Subcomponents involved: -- [ ] DA_update (ufs-community/land-DA) +- [ ] apply_incr.fd (NOAA-PSL/land-apply_jedi_incr) - [ ] ufsLand.fd (NOAA-EPIC/ufs-land-driver-emc-dev) - [ ] ufs_model.fd (ufs-community/ufs-weather-model) - [ ] vector2tile_converter.fd (NOAA-PSL/land-vector2tile) diff --git a/.gitmodules b/.gitmodules index d75662be..24058bb6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,3 @@ -[submodule "sorc/DA_update"] - path = sorc/DA_update - url = https://github.com/ufs-community/land-DA.git - ignore = dirty [submodule "sorc/ufsLand.fd"] path = sorc/ufsLand.fd url = https://github.com/NOAA-EPIC/ufs-land-driver-emc-dev.git @@ -14,3 +10,8 @@ path = sorc/ufs_model.fd url = https://github.com/ufs-community/ufs-weather-model.git ignore = dirty +[submodule "sorc/apply_incr.fd"] + path = sorc/apply_incr.fd + url = https://github.com/NOAA-PSL/land-apply_jedi_incr.git + ignore = dirty + diff --git a/modulefiles/build_hera_intel.lua b/modulefiles/build_hera_intel.lua index 760c025f..e24a5584 100644 --- a/modulefiles/build_hera_intel.lua +++ b/modulefiles/build_hera_intel.lua @@ -8,7 +8,7 @@ prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack_jedi")) load(pathJoin("stack-intel", stack_intel_ver)) -load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) load(pathJoin("stack-python", stack_python_ver)) load(pathJoin("cmake", cmake_ver)) @@ -43,5 +43,4 @@ setenv("FC", "mpiifort") setenv("CMAKE_Platform", "hera.intel") setenv("EPICHOME", "/scratch2/NAGAPE/epic/UFS_Land-DA") -setenv("JEDI_INSTALL", pathJoin(os.getenv("EPICHOME"),"jedi")) - +setenv("JEDI_INSTALL", pathJoin(os.getenv("EPICHOME"),"jedi_skylabv7.0")) diff --git a/modulefiles/build_orion_intel.lua b/modulefiles/build_orion_intel.lua index 656e9e87..39f47062 100644 --- a/modulefiles/build_orion_intel.lua +++ b/modulefiles/build_orion_intel.lua @@ -8,7 +8,7 @@ prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack_jedi")) load(pathJoin("stack-intel", stack_intel_ver)) -load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) load(pathJoin("stack-python", stack_python_ver)) load(pathJoin("cmake", cmake_ver)) @@ -42,5 +42,4 @@ setenv("FC", "mpiifort") setenv("CMAKE_Platform", "orion.intel") setenv("EPICHOME", "/work/noaa/epic/UFS_Land-DA") -setenv("JEDI_INSTALL", pathJoin(os.getenv("EPICHOME"),"jedi")) - +setenv("JEDI_INSTALL", pathJoin(os.getenv("EPICHOME"),"jedi_skylabv7.0")) diff --git a/modulefiles/build_singularity_intel.lua b/modulefiles/build_singularity_intel.lua index a8a9d61e..80843f2e 100644 --- a/modulefiles/build_singularity_intel.lua +++ b/modulefiles/build_singularity_intel.lua @@ -10,8 +10,8 @@ stack_intel_ver=os.getenv("stack_intel_ver") or "2021.8.0" load(pathJoin("stack-intel", stack_intel_ver)) load("intel-oneapi-mpi/2021.8.0") -stack_impi_ver=os.getenv("stack_impi_ver") or "2021.8.0" -load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) +stack_intel_oneapi_mpi_ver=os.getenv("stack_intel_oneapi_mpi_ver") or "2021.8.0" +load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) stack_python_ver=os.getenv("stack_python_ver") or "3.8.10" load(pathJoin("stack-python", stack_python_ver)) diff --git a/parm/jedi/GHCN.yaml b/parm/jedi/GHCN.yaml new file mode 100644 index 00000000..2b8c725d --- /dev/null +++ b/parm/jedi/GHCN.yaml @@ -0,0 +1,55 @@ + - obs space: + name: SnowDepthGHCN + distribution: + name: Halo + halo size: 250e3 + simulated variables: [totalSnowDepth] + obsdatain: + engine: + type: H5File + obsfile: GHCN_XXYYYYXXMMXXDDXXHH.nc + obsdataout: + engine: + type: H5File + obsfile: output/DA/hofx/letkf_hofx_ghcn_XXYYYYXXMMXXDDXXHH.nc + obs operator: + name: Identity + obs error: + covariance model: diagonal + obs localizations: + - localization method: Horizontal SOAR + lengthscale: 250e3 + soar horizontal decay: 0.000021 + max nobs: 50 + - localization method: Vertical Brasnett + vertical lengthscale: 700 + obs filters: + - filter: Bounds Check # negative / missing snow + filter variables: + - name: totalSnowDepth + minvalue: 0.0 + maxvalue: 10000.0 + - filter: Domain Check # missing station elevation (-999.9) + where: + - variable: + name: MetaData/stationElevation + minvalue: -999.0 + maxvalue: 10000.0 + - filter: Domain Check # land only + where: + - variable: + name: GeoVaLs/slmsk + minvalue: 0.5 + maxvalue: 1.5 + - filter: RejectList # no land-ice + where: + - variable: + name: GeoVaLs/vtype + minvalue: 14.5 + maxvalue: 15.5 + - filter: Background Check # gross error check + filter variables: + - name: totalSnowDepth + threshold: 6.25 + action: + name: reject diff --git a/parm/jedi/gfs-land.yaml b/parm/jedi/gfs-land.yaml new file mode 100644 index 00000000..df0dcb81 --- /dev/null +++ b/parm/jedi/gfs-land.yaml @@ -0,0 +1,10 @@ +field metadata: + +- long name: slmsk + interpolation type: nearest + +- long name: totalSnowDepth + interpolation type: nearest + +- long name: vtype + interpolation type: nearest diff --git a/parm/jedi/letkfoi_snow.yaml b/parm/jedi/letkfoi_snow.yaml new file mode 100644 index 00000000..330bde46 --- /dev/null +++ b/parm/jedi/letkfoi_snow.yaml @@ -0,0 +1,58 @@ +geometry: + fms initialization: + namelist filename: Data/fv3files/fmsmpp.nml + field table filename: Data/fv3files/field_table + akbk: Data/fv3files/akbk64.nc4 + npx: XXREP + npy: XXREP + npz: 64 + field metadata override: gfs-land.yaml + time invariant fields: + state fields: + datetime: XXYYYP-XXMP-XXDPTXXHP:00:00Z + filetype: fms restart + skip coupler file: true + state variables: [orog_filt] + datapath: XXTPATH + filename_orog: XXTSTUB.nc + derived fields: [nominal_surface_pressure] + +time window: + begin: XXYYYP-XXMP-XXDPTXXHP:00:00Z + length: PT24H + +background: + date: &date XXYYYY-XXMM-XXDDTXXHH:00:00Z + members: + - datetime: XXYYYY-XXMM-XXDDTXXHH:00:00Z + filetype: fms restart + state variables: [snwdph,vtype,slmsk] + datapath: mem_pos/ + filename_sfcd: XXYYYYXXMMXXDD.XXHH0000.sfc_data.nc + filename_cplr: XXYYYYXXMMXXDD.XXHH0000.coupler.res + - datetime: XXYYYY-XXMM-XXDDTXXHH:00:00Z + filetype: fms restart + state variables: [snwdph,vtype,slmsk] + datapath: mem_neg/ + filename_sfcd: XXYYYYXXMMXXDD.XXHH0000.sfc_data.nc + filename_cplr: XXYYYYXXMMXXDD.XXHH0000.coupler.res + +driver: + save posterior mean: false + save posterior mean increment: true + save posterior ensemble: false + run as observer only: XXHOFX + +local ensemble DA: + solver: LETKF + inflation: + rtps: 0.0 + rtpp: 0.0 + mult: 1.0 + +output increment: + filetype: fms restart + filename_sfcd: xainc.sfc_data.nc + +observations: + observers: diff --git a/parm/land_analysis_era5_hera.yaml b/parm/land_analysis_era5_hera.yaml index 0a4df89f..214861d2 100644 --- a/parm/land_analysis_era5_hera.yaml +++ b/parm/land_analysis_era5_hera.yaml @@ -12,7 +12,7 @@ workflow: ACCOUNT: "nems" EXP_NAME: "LETKF" EXP_BASEDIR: "/scratch2/NAGAPE/epic/{USER}/landda_test" - JEDI_INSTALL: "/scratch2/NAGAPE/epic/UFS_Land-DA/jedi" + JEDI_INSTALL: "/scratch2/NAGAPE/epic/UFS_Land-DA/jedi_skylabv7.0" LANDDA_INPUTS: "/scratch2/NAGAPE/epic/UFS_Land-DA/inputs" FORCING: "era5" RES: "96" @@ -20,7 +20,6 @@ workflow: NPROCS_ANALYSIS: "6" NPROCS_FORECAST: "6" OBS_TYPES: "GHCN" - fv3bundle_vn: "psl_develop" DAtype: "letkfoi_snow" SNOWDEPTHVAR: "snwdph" TSTUB: "oro_C96.mx100" @@ -117,7 +116,6 @@ workflow: cyc: "&cyc;" SLASH_ENSMEM_SUBDIR: "&SLASH_ENSMEM_SUBDIR;" PTIME: "&PTIME;" - fv3bundle_vn: "&fv3bundle_vn;" DAtype: "&DAtype;" SNOWDEPTHVAR: "&SNOWDEPTHVAR;" account: "&ACCOUNT;" @@ -153,7 +151,6 @@ workflow: SLASH_ENSMEM_SUBDIR: "&SLASH_ENSMEM_SUBDIR;" PTIME: "&PTIME;" NTIME: "&NTIME;" - fv3bundle_vn: "&fv3bundle_vn;" DAtype: "&DAtype;" SNOWDEPTHVAR: "&SNOWDEPTHVAR;" NPROC_JEDI: "&NPROCS_ANALYSIS;" @@ -192,7 +189,6 @@ workflow: SLASH_ENSMEM_SUBDIR: "&SLASH_ENSMEM_SUBDIR;" PTIME: "&PTIME;" NTIME: "&NTIME;" - fv3bundle_vn: "&fv3bundle_vn;" DAtype: "&DAtype;" SNOWDEPTHVAR: "&SNOWDEPTHVAR;" JEDI_INSTALL: "&JEDI_INSTALL;" diff --git a/parm/land_analysis_era5_orion.yaml b/parm/land_analysis_era5_orion.yaml index 5b049c86..4f1ed4af 100644 --- a/parm/land_analysis_era5_orion.yaml +++ b/parm/land_analysis_era5_orion.yaml @@ -12,7 +12,7 @@ workflow: ACCOUNT: "epic" EXP_NAME: "LETKF" EXP_BASEDIR: "/work/noaa/epic/{USER}/landda_test" - JEDI_INSTALL: "/work/noaa/epic/UFS_Land-DA/jedi" + JEDI_INSTALL: "/work/noaa/epic/UFS_Land-DA/jedi_skylabv7.0" LANDDA_INPUTS: "/work/noaa/epic/UFS_Land-DA/inputs" FORCING: "era5" RES: "96" @@ -20,7 +20,6 @@ workflow: NPROCS_ANALYSIS: "6" NPROCS_FORECAST: "6" OBS_TYPES: "GHCN" - fv3bundle_vn: "psl_develop" DAtype: "letkfoi_snow" SNOWDEPTHVAR: "snwdph" TSTUB: "oro_C96.mx100" @@ -117,7 +116,6 @@ workflow: cyc: "&cyc;" SLASH_ENSMEM_SUBDIR: "&SLASH_ENSMEM_SUBDIR;" PTIME: "&PTIME;" - fv3bundle_vn: "&fv3bundle_vn;" DAtype: "&DAtype;" SNOWDEPTHVAR: "&SNOWDEPTHVAR;" account: "&ACCOUNT;" @@ -153,7 +151,6 @@ workflow: SLASH_ENSMEM_SUBDIR: "&SLASH_ENSMEM_SUBDIR;" PTIME: "&PTIME;" NTIME: "&NTIME;" - fv3bundle_vn: "&fv3bundle_vn;" DAtype: "&DAtype;" SNOWDEPTHVAR: "&SNOWDEPTHVAR;" NPROC_JEDI: "&NPROCS_ANALYSIS;" @@ -192,7 +189,6 @@ workflow: SLASH_ENSMEM_SUBDIR: "&SLASH_ENSMEM_SUBDIR;" PTIME: "&PTIME;" NTIME: "&NTIME;" - fv3bundle_vn: "&fv3bundle_vn;" DAtype: "&DAtype;" SNOWDEPTHVAR: "&SNOWDEPTHVAR;" JEDI_INSTALL: "&JEDI_INSTALL;" diff --git a/parm/land_analysis_gswp3_hera.yaml b/parm/land_analysis_gswp3_hera.yaml index c4d42f57..2119b85d 100644 --- a/parm/land_analysis_gswp3_hera.yaml +++ b/parm/land_analysis_gswp3_hera.yaml @@ -12,7 +12,7 @@ workflow: ACCOUNT: "nems" EXP_NAME: "LETKF" EXP_BASEDIR: "/scratch2/NAGAPE/epic/{USER}/landda_test" - JEDI_INSTALL: "/scratch2/NAGAPE/epic/UFS_Land-DA/jedi" + JEDI_INSTALL: "/scratch2/NAGAPE/epic/UFS_Land-DA/jedi_skylabv7.0" LANDDA_INPUTS: "/scratch2/NAGAPE/epic/UFS_Land-DA/inputs" FORCING: "gswp3" RES: "96" @@ -20,7 +20,6 @@ workflow: NPROCS_ANALYSIS: "6" NPROCS_FORECAST: "6" OBS_TYPES: "GHCN" - fv3bundle_vn: "psl_develop" DAtype: "letkfoi_snow" SNOWDEPTHVAR: "snwdph" TSTUB: "oro_C96.mx100" @@ -117,7 +116,6 @@ workflow: cyc: "&cyc;" SLASH_ENSMEM_SUBDIR: "&SLASH_ENSMEM_SUBDIR;" PTIME: "&PTIME;" - fv3bundle_vn: "&fv3bundle_vn;" DAtype: "&DAtype;" SNOWDEPTHVAR: "&SNOWDEPTHVAR;" account: "&ACCOUNT;" @@ -153,7 +151,6 @@ workflow: SLASH_ENSMEM_SUBDIR: "&SLASH_ENSMEM_SUBDIR;" PTIME: "&PTIME;" NTIME: "&NTIME;" - fv3bundle_vn: "&fv3bundle_vn;" DAtype: "&DAtype;" SNOWDEPTHVAR: "&SNOWDEPTHVAR;" NPROC_JEDI: "&NPROCS_ANALYSIS;" @@ -192,7 +189,6 @@ workflow: SLASH_ENSMEM_SUBDIR: "&SLASH_ENSMEM_SUBDIR;" PTIME: "&PTIME;" NTIME: "&NTIME;" - fv3bundle_vn: "&fv3bundle_vn;" DAtype: "&DAtype;" SNOWDEPTHVAR: "&SNOWDEPTHVAR;" JEDI_INSTALL: "&JEDI_INSTALL;" diff --git a/parm/land_analysis_gswp3_orion.yaml b/parm/land_analysis_gswp3_orion.yaml index 51c09e43..b2d0dd7b 100644 --- a/parm/land_analysis_gswp3_orion.yaml +++ b/parm/land_analysis_gswp3_orion.yaml @@ -12,7 +12,7 @@ workflow: ACCOUNT: "epic" EXP_NAME: "LETKF" EXP_BASEDIR: "/work/noaa/epic/{USER}/landda_test" - JEDI_INSTALL: "/work/noaa/epic/UFS_Land-DA/jedi" + JEDI_INSTALL: "/work/noaa/epic/UFS_Land-DA/jedi_skylabv7.0" LANDDA_INPUTS: "/work/noaa/epic/UFS_Land-DA/inputs" FORCING: "gswp3" RES: "96" @@ -20,7 +20,6 @@ workflow: NPROCS_ANALYSIS: "6" NPROCS_FORECAST: "6" OBS_TYPES: "GHCN" - fv3bundle_vn: "psl_develop" DAtype: "letkfoi_snow" SNOWDEPTHVAR: "snwdph" TSTUB: "oro_C96.mx100" @@ -117,7 +116,6 @@ workflow: cyc: "&cyc;" SLASH_ENSMEM_SUBDIR: "&SLASH_ENSMEM_SUBDIR;" PTIME: "&PTIME;" - fv3bundle_vn: "&fv3bundle_vn;" DAtype: "&DAtype;" SNOWDEPTHVAR: "&SNOWDEPTHVAR;" account: "&ACCOUNT;" @@ -153,7 +151,6 @@ workflow: SLASH_ENSMEM_SUBDIR: "&SLASH_ENSMEM_SUBDIR;" PTIME: "&PTIME;" NTIME: "&NTIME;" - fv3bundle_vn: "&fv3bundle_vn;" DAtype: "&DAtype;" SNOWDEPTHVAR: "&SNOWDEPTHVAR;" NPROC_JEDI: "&NPROCS_ANALYSIS;" @@ -192,7 +189,6 @@ workflow: SLASH_ENSMEM_SUBDIR: "&SLASH_ENSMEM_SUBDIR;" PTIME: "&PTIME;" NTIME: "&NTIME;" - fv3bundle_vn: "&fv3bundle_vn;" DAtype: "&DAtype;" SNOWDEPTHVAR: "&SNOWDEPTHVAR;" JEDI_INSTALL: "&JEDI_INSTALL;" @@ -201,7 +197,7 @@ workflow: command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "forecast" "&HOMElandda;" "&MACHINE;"' jobname: forecast nodes: "1:ppn=&NPROCS_FORECAST;" - walltime: 00:30:00 + walltime: 00:40:00 queue: batch join: "&LOGDIR;/forecast.log" dependency: diff --git a/parm/run_without_rocoto.sh b/parm/run_without_rocoto.sh index 87c22ef7..69528430 100755 --- a/parm/run_without_rocoto.sh +++ b/parm/run_without_rocoto.sh @@ -17,11 +17,11 @@ export FORCING="era5" if [ "${MACHINE}" = "hera" ]; then export EXP_BASEDIR="/scratch2/NAGAPE/epic/{USER}/landda_test" - export JEDI_INSTALL="/scratch2/NAGAPE/epic/UFS_Land-DA/jedi" + export JEDI_INSTALL="/scratch2/NAGAPE/epic/UFS_Land-DA/jedi_skylabv7.0" export LANDDA_INPUTS="/scratch2/NAGAPE/epic/UFS_Land-DA/inputs" elif [ "${MACHINE}" = "orion" ]; then export EXP_BASEDIR="/work/noaa/epic/chjeon/landda_test" - export JEDI_INSTALL="/work/noaa/epic/UFS_Land-DA/jedi" + export JEDI_INSTALL="/work/noaa/epic/UFS_Land-DA/jedi_skylabv7.0" export LANDDA_INPUTS="/work/noaa/epic/UFS_Land-DA/inputs" fi @@ -30,7 +30,6 @@ export FCSTHR="24" export NPROCS_ANALYSIS="6" export NPROCS_FORECAST="6" export OBS_TYPES="GHCN" -export fv3bundle_vn="psl_develop" export DAtype="letkfoi_snow" export SNOWDEPTHVAR="snwdph" export TSTUB="oro_C96.mx100" diff --git a/parm/templates/template.coupler.res b/parm/templates/template.coupler.res new file mode 100644 index 00000000..a3dc1afe --- /dev/null +++ b/parm/templates/template.coupler.res @@ -0,0 +1,4 @@ + 2 (Calendar: no_calendar=0, thirty_day_months=1, julian=2, gregorian=3, noleap=4) + XXYYYP XXMP XXDP XXHP 0 0 Model start time: year, month, day, hour, minute, second + XXYYYY XXMM XXDD XXHH 0 0 Current model time: year, month, day, hour, minute, second + diff --git a/scripts/exlandda_analysis.sh b/scripts/exlandda_analysis.sh index 53ec35ed..9abf5c19 100755 --- a/scripts/exlandda_analysis.sh +++ b/scripts/exlandda_analysis.sh @@ -10,7 +10,6 @@ if [[ ${EXP_NAME} == "openloop" ]]; then else do_jedi="YES" SAVE_TILE="YES" - LANDDADIR=${HOMElandda}/sorc/DA_update fi TPATH=${LANDDA_INPUTS}/forcing/${ATMOS_FORC}/orog_files/ @@ -32,9 +31,6 @@ JEDIWORKDIR=${WORKDIR}/mem000/jedi FILEDATE=${YYYY}${MM}${DD}.${HH}0000 JEDI_STATICDIR=${JEDI_INSTALL}/jedi-bundle/fv3-jedi/test/Data JEDI_EXECDIR=${JEDI_INSTALL}/build/bin -JEDI_EXEC=$JEDI_EXECDIR/fv3jedi_letkf.x -LOGDIR=${COMOUT}/DA/logs -apply_incr_EXEC=${EXEClandda}/apply_incr.exe SAVE_INCR="YES" KEEPJEDIDIR="YES" @@ -46,11 +42,8 @@ if [ -e ${BUILD_VERSION_FILE} ]; then . ${BUILD_VERSION_FILE} fi module use modulefiles; module load modules.landda -PYTHON=$(/usr/bin/which python) MPIEXEC=`which mpiexec` -#fv3bundle_vn=psl_develop -#DAtype=letkfoi_snow #SNOWDEPTHVAR=snwdph YAML_DA=construct GFSv17="NO" @@ -73,17 +66,14 @@ fi if [[ $do_DA == "YES" ]]; then if [[ $YAML_DA == "construct" ]];then # construct the yaml - - cp ${LANDDADIR}/jedi/fv3-jedi/yaml_files/${fv3bundle_vn}/${DAtype}.yaml ${JEDIWORKDIR}/letkf_land.yaml - - for obs in "${OBS_TYPES[@]}"; - do - cat ${LANDDADIR}/jedi/fv3-jedi/yaml_files/${fv3bundle_vn}/${obs}.yaml >> letkf_land.yaml - done - + cp ${PARMlandda}/jedi/${DAtype}.yaml ${JEDIWORKDIR}/letkf_land.yaml + for obs in "${OBS_TYPES[@]}"; + do + cat ${PARMlandda}/jedi/${obs}.yaml >> letkf_land.yaml + done else # use specified yaml - echo "Using user specified YAML: ${YAML_DA}" - cp ${LANDDADIR}/jedi/fv3-jedi/yaml_files/${fv3bundle_vn}/${YAML_DA} ${JEDIWORKDIR}/letkf_land.yaml + echo "Using user specified YAML: ${YAML_DA}" + cp ${PARMlandda}/jedi/${YAML_DA} ${JEDIWORKDIR}/letkf_land.yaml fi sed -i -e "s/XXYYYY/${YYYY}/g" letkf_land.yaml @@ -108,16 +98,14 @@ fi if [[ $do_HOFX == "YES" ]]; then if [[ $YAML_HOFX == "construct" ]];then # construct the yaml - - cp ${LANDDADIR}/jedi/fv3-jedi/yaml_files/${fv3bundle_vn}/${DAtype}.yaml ${JEDIWORKDIR}/hofx_land.yaml - - for obs in "${OBS_TYPES[@]}"; - do - cat ${LANDDADIR}/jedi/fv3-jedi/yaml_files/${fv3bundle_vn}/${obs}.yaml >> hofx_land.yaml - done + cp ${PARMlandda}/jedi/${DAtype}.yaml ${JEDIWORKDIR}/hofx_land.yaml + for obs in "${OBS_TYPES[@]}"; + do + cat ${PARMlandda}/jedi/${obs}.yaml >> hofx_land.yaml + done else # use specified yaml - echo "Using user specified YAML: ${YAML_HOFX}" - cp ${LANDDADIR}/jedi/fv3-jedi/yaml_files/${fv3bundle_vn}/${YAML_HOFX} ${JEDIWORKDIR}/hofx_land.yaml + echo "Using user specified YAML: ${YAML_HOFX}" + cp ${PARMlandda}/jedi/${YAML_HOFX} ${JEDIWORKDIR}/hofx_land.yaml fi sed -i -e "s/XXYYYY/${YYYY}/g" hofx_land.yaml @@ -140,6 +128,12 @@ if [[ $do_HOFX == "YES" ]]; then fi +if [[ "$GFSv17" == "NO" ]]; then + cp ${PARMlandda}/jedi/gfs-land.yaml ${JEDIWORKDIR}/gfs-land.yaml +else + cp ${JEDI_INSTALL}/jedi-bundle/fv3-jedi/test/Data/fieldmetadata/gfs_v17-land.yaml ${JEDIWORKDIR}/gfs-land.yaml +fi + ################################################ # 4. CREATE BACKGROUND ENSEMBLE (LETKFOI) ################################################ @@ -154,18 +148,26 @@ fi echo 'do_landDA: calling fv3-jedi' if [[ $do_DA == "YES" ]]; then - ${MPIEXEC} -n $NPROC_JEDI ${JEDI_EXEC} letkf_land.yaml ${LOGDIR}/jedi_letkf.log - if [[ $? != 0 ]]; then - echo "JEDI DA failed" - exit 10 - fi + export pgm="fv3jedi_letkf.x" + . prep_step + ${MPIEXEC} -n $NPROC_JEDI ${JEDI_EXECDIR}/$pgm letkf_land.yaml >>$pgmout 2>errfile + export err=$?; err_chk + cp errfile errfile_jedi_letkf + if [[ $err != 0 ]]; then + echo "JEDI DA failed" + exit 10 + fi fi if [[ $do_HOFX == "YES" ]]; then - ${MPIEXEC} -n $NPROC_JEDI ${JEDI_EXEC} hofx_land.yaml ${LOGDIR}/jedi_hofx.log - if [[ $? != 0 ]]; then - echo "JEDI hofx failed" - exit 10 - fi + export pgm="fv3jedi_letkf.x" + . prep_step + ${MPIEXEC} -n $NPROC_JEDI ${JEDI_EXEC} hofx_land.yaml >>$pgmout 2>errfile + export err=$?; err_chk + cp errfile errfile_jedi_hofx + if [[ $err != 0 ]]; then + echo "JEDI hofx failed" + exit 10 + fi fi ################################################ @@ -189,11 +191,15 @@ EOF echo 'do_landDA: calling apply snow increment' + export pgm="apply_incr.exe" + . prep_step # (n=6) -> this is fixed, at one task per tile (with minor code change, could run on a single proc). - ${MPIEXEC} -n 6 ${apply_incr_EXEC} ${LOGDIR}/apply_incr.log - if [[ $? != 0 ]]; then - echo "apply snow increment failed" - exit 10 + ${MPIEXEC} -n 6 ${EXEClandda}/$pgm >>$pgmout 2>errfile + export err=$?; err_chk + cp errfile errfile_apply_incr + if [[ $err != 0 ]]; then + echo "apply snow increment failed" + exit 10 fi fi diff --git a/scripts/exlandda_forecast.sh b/scripts/exlandda_forecast.sh index e78c7b70..a016ec08 100755 --- a/scripts/exlandda_forecast.sh +++ b/scripts/exlandda_forecast.sh @@ -10,7 +10,6 @@ if [[ ${EXP_NAME} == "openloop" ]]; then else do_jedi="YES" SAVE_TILE="YES" - LANDDADIR=${HOMElandda}/sorc/DA_update fi MACHINE_ID=${MACHINE} @@ -32,15 +31,8 @@ mem_ens="mem000" MEM_WORKDIR=${WORKDIR}/${mem_ens} MEM_MODL_OUTDIR=${COMOUT}/${mem_ens} RSTRDIR=${MEM_WORKDIR} -JEDIWORKDIR=${WORKDIR}/mem000/jedi FILEDATE=${YYYY}${MM}${DD}.${HH}0000 -JEDI_STATICDIR=${JEDI_INSTALL}/jedi-bundle/fv3-jedi/test/Data -JEDI_EXECDIR=${JEDI_INSTALL}/build/bin -JEDI_EXEC=$JEDI_EXECDIR/fv3jedi_letkf.x -LOGDIR=${COMOUT}/DA/logs -apply_incr_EXEC=${EXEClandda}/apply_incr.exe SAVE_INCR="YES" -KEEPJEDIDIR="YES" FREQ=$((${FCSTHR}*3600)) RDD=$((${FCSTHR}/24)) RHH=$((${FCSTHR}%24)) @@ -53,12 +45,9 @@ if [ -e ${BUILD_VERSION_FILE} ]; then . ${BUILD_VERSION_FILE} fi module use modulefiles; module load modules.landda -PYTHON=$(/usr/bin/which python) MPIEXEC=`which mpiexec` MPIRUN=${MPIRUN:-`which mpiexec`} -#fv3bundle_vn=psl_develop -#DAtype=letkfoi_snow #SNOWDEPTHVAR=snwdph cd $MEM_WORKDIR @@ -68,7 +57,7 @@ if [[ $do_jedi == "YES" && ${ATMOS_FORC} == "era5" ]]; then echo '************************************************' echo 'calling tile2vector' - cp ${HOMElandda}/parm/templates/template.tile2vector tile2vector.namelist + cp ${PARMlandda}/templates/template.tile2vector tile2vector.namelist sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" tile2vector.namelist sed -i -e "s/XXYYYY/${YYYY}/g" tile2vector.namelist @@ -80,10 +69,14 @@ if [[ $do_jedi == "YES" && ${ATMOS_FORC} == "era5" ]]; then sed -i -e "s/XXTSTUB/${TSTUB}/g" tile2vector.namelist sed -i -e "s#XXTPATH#${TPATH}#g" tile2vector.namelist - ${EXEClandda}/vector2tile_converter.exe tile2vector.namelist - if [[ $? != 0 ]]; then - echo "tile2vector failed" - exit + export pgm="vector2tile_converter.exe" + . prep_step + ${EXEClandda}/$pgm tile2vector.namelist >>$pgmout 2>errfile + export err=$?; err_chk + cp errfile errfile_tile2vector + if [[ $err != 0 ]]; then + echo "tile2vector failed" + exit 10 fi # save analysis restart @@ -96,7 +89,7 @@ if [[ $do_jedi == "YES" && ${ATMOS_FORC} == "gswp3" ]]; then echo '************************************************' echo 'calling tile2tile' - cp ${HOMElandda}/parm/templates/template.jedi2ufs jedi2ufs.namelist + cp ${PARMlandda}/templates/template.jedi2ufs jedi2ufs.namelist sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" jedi2ufs.namelist sed -i -e "s/XXYYYY/${YYYY}/g" jedi2ufs.namelist @@ -108,10 +101,14 @@ if [[ $do_jedi == "YES" && ${ATMOS_FORC} == "gswp3" ]]; then sed -i -e "s/XXTSTUB/${TSTUB}/g" jedi2ufs.namelist sed -i -e "s#XXTPATH#${TPATH}#g" jedi2ufs.namelist - ${EXEClandda}/tile2tile_converter.exe jedi2ufs.namelist - if [[ $? != 0 ]]; then - echo "tile2tile failed" - exit + export pgm="tile2tile_converter.exe" + . prep_step + ${EXEClandda}/$pgm jedi2ufs.namelist >>$pgmout 2>errfile + export err=$?; err_chk + cp errfile errfile_tile2tile + if [[ $err != 0 ]]; then + echo "tile2tile failed" + exit 10 fi # save analysis restart @@ -133,7 +130,7 @@ if [[ $do_jedi == "YES" && ${ATMOS_FORC} == "era5" ]]; then echo 'running the forecast model' # update model namelist - cp ${HOMElandda}/parm/templates/template.ufs-noahMP.namelist.${ATMOS_FORC} ufs-land.namelist + cp ${PARMlandda}/templates/template.ufs-noahMP.namelist.${ATMOS_FORC} ufs-land.namelist sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" ufs-land.namelist sed -i -e "s/XXYYYY/${YYYY}/g" ufs-land.namelist @@ -149,7 +146,15 @@ if [[ $do_jedi == "YES" && ${ATMOS_FORC} == "era5" ]]; then nt=$SLURM_NTASKS - ${MPIEXEC} -n 1 ${EXEClandda}/ufsLand.exe + export pgm="ufsLand.exe" + . prep_step + ${MPIEXEC} -n 1 ${EXEClandda}/$pgm >>$pgmout 2>errfile + export err=$?; err_chk + cp errfile errfile_ufsLand + if [[ $err != 0 ]]; then + echo "ufsLand failed" + exit 10 + fi fi # no error codes on exit from model, check for restart below instead @@ -198,7 +203,7 @@ if [[ $do_jedi == "YES" && ${ATMOS_FORC} == "gswp3" ]]; then export layout_y=1 # FV3 executable: - cp ${EXEClandda}/ufs_model ./ufs_model +# cp ${EXEClandda}/ufs_model ./ufs_model cp ${HOMElandda}/fv3_run ./fv3_run if [[ $DATM_CDEPS = 'true' ]] || [[ $FV3 = 'true' ]] || [[ $S2S = 'true' ]]; then @@ -239,7 +244,15 @@ if [[ $do_jedi == "YES" && ${ATMOS_FORC} == "gswp3" ]]; then # start runs echo "Start ufs-cdeps-land model run with TASKS: ${TASKS}" - ${MPIRUN} -n ${TASKS} ./ufs_model + export pgm="ufs_model" + . prep_step + ${MPIRUN} -n ${TASKS} ${EXEClandda}/$pgm >>$pgmout 2>errfile + export err=$?; err_chk + cp errfile errfile_ufs_model + if [[ $err != 0 ]]; then + echo "ufs_model failed" + exit 10 + fi fi # no error codes on exit from model, check for restart below instead diff --git a/scripts/exlandda_prep_bmat.sh b/scripts/exlandda_prep_bmat.sh index cfc85ebf..4573cfc4 100755 --- a/scripts/exlandda_prep_bmat.sh +++ b/scripts/exlandda_prep_bmat.sh @@ -6,11 +6,10 @@ set -ex # copy restarts to workdir, convert to UFS tile for DA (all members) if [[ ${EXP_NAME} == "openloop" ]]; then - do_jedi="NO" + do_jedi="NO" else - do_jedi="YES" - SAVE_TILE="YES" - LANDDADIR=${HOMElandda}/sorc/DA_update + do_jedi="YES" + SAVE_TILE="YES" fi TPATH=${LANDDA_INPUTS}/forcing/${ATMOS_FORC}/orog_files/ @@ -35,10 +34,7 @@ if [ -e ${BUILD_VERSION_FILE} ]; then . ${BUILD_VERSION_FILE} fi module use modulefiles; module load modules.landda -PYTHON=$(/usr/bin/which python) -#fv3bundle_vn=psl_develop -#DAtype=letkfoi_snow #SNOWDEPTHVAR=snwdph YAML_DA=construct GFSv17="NO" @@ -49,22 +45,20 @@ cd $JEDIWORKDIR # 4. CREATE BACKGROUND ENSEMBLE (LETKFOI) ################################################ -if [[ ${DAtype} == 'letkfoi_snow' ]]; then - - JEDI_EXEC="fv3jedi_letkf.x" +if [[ ${DAtype} == "letkfoi_snow" ]]; then if [ $GFSv17 == "YES" ]; then - SNOWDEPTHVAR="snodl" - # field overwrite file with GFSv17 variables. - cp ${LANDDADIR}/jedi/fv3-jedi/yaml_files/${fv3bundle_vn}/gfs-land-v17.yaml ${JEDIWORKDIR}/gfs-land-v17.yaml + SNOWDEPTHVAR="snodl" else - SNOWDEPTHVAR="snwdph" + SNOWDEPTHVAR="snwdph" + # replace field overwrite file + cp ${PARMlandda}/jedi/gfs-land.yaml ${JEDIWORKDIR}/gfs-land.yaml fi # FOR LETKFOI, CREATE THE PSEUDO-ENSEMBLE for ens in pos neg do if [ -e $JEDIWORKDIR/mem_${ens} ]; then - rm -r $JEDIWORKDIR/mem_${ens} + rm -r $JEDIWORKDIR/mem_${ens} fi mkdir -p $JEDIWORKDIR/mem_${ens} for tile in 1 2 3 4 5 6 @@ -77,7 +71,7 @@ if [[ ${DAtype} == 'letkfoi_snow' ]]; then echo 'do_landDA: calling create ensemble' # using ioda mods to get a python version with netCDF4 - ${PYTHON} ${LANDDADIR}/letkf_create_ens.py $FILEDATE $SNOWDEPTHVAR $B + ${USHlandda}/letkf_create_ens.py $FILEDATE $SNOWDEPTHVAR $B if [[ $? != 0 ]]; then echo "letkf create failed" exit 10 diff --git a/scripts/exlandda_prep_exp.sh b/scripts/exlandda_prep_exp.sh index 39cc41df..7e532ce2 100755 --- a/scripts/exlandda_prep_exp.sh +++ b/scripts/exlandda_prep_exp.sh @@ -10,7 +10,6 @@ if [[ ${EXP_NAME} == "openloop" ]]; then else do_jedi="YES" SAVE_TILE="YES" - LANDDADIR=${HOMElandda}/sorc/DA_update fi echo ${LANDDA_INPUTS}, ${ATMOS_FORC} @@ -65,7 +64,7 @@ if [[ $do_jedi == "YES" && $ATMOS_FORC == "era5" ]]; then export MEM_WORKDIR # update vec2tile and tile2vec namelists - cp ${HOMElandda}/parm/templates/template.vector2tile vector2tile.namelist + cp ${PARMlandda}/templates/template.vector2tile vector2tile.namelist sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" vector2tile.namelist sed -i -e "s/XXYYYY/${YYYY}/g" vector2tile.namelist @@ -82,33 +81,37 @@ if [[ $do_jedi == "YES" && $ATMOS_FORC == "era5" ]]; then echo '************************************************' echo 'calling vector2tile' - ${EXEClandda}/vector2tile_converter.exe vector2tile.namelist - if [[ $? != 0 ]]; then - echo "vec2tile failed" - exit + export pgm="vector2tile_converter.exe" + . prep_step + ${EXEClandda}/$pgm vector2tile.namelist >>$pgmout 2>errfile + cp errfile errfile_vector2tile + export err=$?; err_chk + if [[ $err != 0 ]]; then + echo "vec2tile failed" + exit fi fi # vector2tile for DA if [[ $do_jedi == "YES" && $ATMOS_FORC == "gswp3" ]]; then - echo '************************************************' - echo 'calling tile2tile' + echo '************************************************' + echo 'calling tile2tile' - export MEM_WORKDIR + export MEM_WORKDIR - # copy restarts into work directory - for tile in 1 2 3 4 5 6 - do - rst_in=${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_back.${YYYY}-${MM}-${DD}_${HH}-00-00.nc - if [[ ! -e ${rst_in} ]]; then - rst_in=${LANDDA_INPUTS}/restarts/${ATMOS_FORC}/ufs.cpld.lnd.out.${YYYY}-${MM}-${DD}-00000.tile${tile}.nc - fi - rst_out=${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc - cp ${rst_in} ${rst_out} - done + # copy restarts into work directory + for tile in 1 2 3 4 5 6 + do + rst_in=${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_back.${YYYY}-${MM}-${DD}_${HH}-00-00.nc + if [[ ! -e ${rst_in} ]]; then + rst_in=${LANDDA_INPUTS}/restarts/${ATMOS_FORC}/ufs.cpld.lnd.out.${YYYY}-${MM}-${DD}-00000.tile${tile}.nc + fi + rst_out=${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc + cp ${rst_in} ${rst_out} + done # update tile2tile namelist - cp ${HOMElandda}/parm/templates/template.ufs2jedi ufs2jedi.namelist + cp ${PARMlandda}/templates/template.ufs2jedi ufs2jedi.namelist sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" ufs2jedi.namelist sed -i -e "s/XXYYYY/${YYYY}/g" ufs2jedi.namelist @@ -121,11 +124,15 @@ if [[ $do_jedi == "YES" && $ATMOS_FORC == "gswp3" ]]; then sed -i -e "s/XXTSTUB/${TSTUB}/g" ufs2jedi.namelist sed -i -e "s#XXTPATH#${TPATH}#g" ufs2jedi.namelist - # submit tile2tile - ${EXEClandda}/tile2tile_converter.exe ufs2jedi.namelist - if [[ $? != 0 ]]; then - echo "tile2tile failed" - exit + # submit tile2tile + export pgm="tile2tile_converter.exe" + . prep_step + ${EXEClandda}/$pgm ufs2jedi.namelist >>$pgmout 2>errfile + cp errfile errfile_tile2tile + export err=$?; err_chk + if [[ $err != 0 ]]; then + echo "tile2tile failed" + exit fi fi # tile2tile for DA @@ -162,7 +169,7 @@ if [[ $do_jedi == "YES" ]]; then if [[ -e ${RSTRDIR}/${FILEDATE}.coupler.res ]]; then ln -sf ${RSTRDIR}/${FILEDATE}.coupler.res $cres_file else # if not present, need to create coupler.res for JEDI - cp ${LANDDADIR}/template.coupler.res $cres_file + cp ${PARMlandda}/templates/template.coupler.res $cres_file sed -i -e "s/XXYYYY/${YYYY}/g" $cres_file sed -i -e "s/XXMM/${MM}/g" $cres_file diff --git a/scripts/exlandda_prep_obs.sh b/scripts/exlandda_prep_obs.sh index dad81427..bb5b201e 100755 --- a/scripts/exlandda_prep_obs.sh +++ b/scripts/exlandda_prep_obs.sh @@ -10,7 +10,6 @@ if [[ ${EXP_NAME} == "openloop" ]]; then else do_jedi="YES" SAVE_TILE="YES" - LANDDADIR=${HOMElandda}/sorc/DA_update fi TPATH=${LANDDA_INPUTS}/forcing/${ATMOS_FORC}/orog_files/ @@ -33,36 +32,32 @@ FILEDATE=${YYYY}${MM}${DD}.${HH}0000 cd $JEDIWORKDIR -# load modulefiles -#module use modulefiles; module load modules.landda - OBSDIR=${LANDDA_INPUTS}/DA ################################################ # 2. PREPARE OBS FILES ################################################ for obs in "${OBS_TYPES[@]}"; do - # get the. obs file name - if [ ${obs} == "GTS" ]; then - obsfile=$OBSDIR/snow_depth/GTS/data_proc/${YYYY}${MM}/adpsfc_snow_${YYYY}${MM}${DD}${HH}.nc4 - # GHCN are time-stamped at 18. If assimilating at 00, need to use previous day's obs, so that - # obs are within DA window. - elif [ $ATMOS_FORC == "era5" ] && [ ${obs} == "GHCN" ]; then - obsfile=$OBSDIR/snow_depth/GHCN/data_proc/v3/${YYYY}/ghcn_snwd_ioda_${YYYP}${MP}${DP}.nc - elif [ $ATMOS_FORC == "gswp3" ] && [ ${obs} == "GHCN" ]; then - obsfile=$OBSDIR/snow_depth/GHCN/data_proc/v3/${YYYY}/fake_ghcn_snwd_ioda_${YYYP}${MP}${DP}.nc - elif [ ${obs} == "SYNTH" ]; then - obsfile=$OBSDIR/synthetic_noahmp/IODA.synthetic_gswp_obs.${YYYY}${MM}${DD}${HH}.nc - else - echo "do_landDA: Unknown obs type requested ${obs}, exiting" - exit 1 - fi - - # check obs are available - if [[ -e $obsfile ]]; then - echo "do_landDA: $i observations found: $obsfile" - ln -fs $obsfile ${obs}_${YYYY}${MM}${DD}${HH}.nc - else - echo "${obs} observations not found: $obsfile" - # JEDI_TYPES[$ii]="SKIP" - fi + # get the obs file name + if [ ${obs} == "GTS" ]; then + obsfile=$OBSDIR/snow_depth/GTS/data_proc/${YYYY}${MM}/adpsfc_snow_${YYYY}${MM}${DD}${HH}.nc4 + # GHCN are time-stamped at 18. If assimilating at 00, need to use previous day's obs, so that + # obs are within DA window. + elif [ $ATMOS_FORC == "era5" ] && [ ${obs} == "GHCN" ]; then + obsfile=$OBSDIR/snow_depth/GHCN/data_proc/v3/${YYYY}/ghcn_snwd_ioda_${YYYP}${MP}${DP}_jediv7.nc + elif [ $ATMOS_FORC == "gswp3" ] && [ ${obs} == "GHCN" ]; then + obsfile=$OBSDIR/snow_depth/GHCN/data_proc/v3/${YYYY}/fake_ghcn_snwd_ioda_${YYYP}${MP}${DP}_jediv7.nc + elif [ ${obs} == "SYNTH" ]; then + obsfile=$OBSDIR/synthetic_noahmp/IODA.synthetic_gswp_obs.${YYYY}${MM}${DD}${HH}.nc + else + echo "do_landDA: Unknown obs type requested ${obs}, exiting" + exit 1 + fi + + # check obs are available + if [[ -e $obsfile ]]; then + echo "do_landDA: $i observations found: $obsfile" + ln -fs $obsfile ${obs}_${YYYY}${MM}${DD}${HH}.nc + else + echo "${obs} observations not found: $obsfile" + fi done diff --git a/sorc/CMakeLists.txt b/sorc/CMakeLists.txt index d11c472f..835f0b77 100755 --- a/sorc/CMakeLists.txt +++ b/sorc/CMakeLists.txt @@ -69,18 +69,14 @@ ExternalProject_Add(ufs_model.fd add_subdirectory(tile2tile_converter.fd) add_subdirectory(vector2tile_converter.fd) add_subdirectory(ufsLand.fd) -add_subdirectory(DA_update/add_jedi_incr) - +add_subdirectory(apply_incr.fd) # Include testing. add_subdirectory(test) if(ECBUILD_INSTALL_FORTRAN_MODULES) install(DIRECTORY ${CMAKE_Fortran_MODULE_DIRECTORY}/${CMAKE_CFG_INTDIR} DESTINATION ${INSTALL_INCLUDE_DIR} ) endif() -execute_process( - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/DA_update/jedi/fv3-jedi/Data/make_links.sh ${JEDI_INSTALL} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/DA_update/jedi/fv3-jedi/Data -) + ################################################################################ # Finalise configuration ################################################################################ diff --git a/sorc/DA_update b/sorc/DA_update deleted file mode 160000 index 3f87503c..00000000 --- a/sorc/DA_update +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3f87503c2653fde4367cc614dccc1c44c57c4b53 diff --git a/sorc/app_build.sh b/sorc/app_build.sh index 6853c3c8..993f4243 100755 --- a/sorc/app_build.sh +++ b/sorc/app_build.sh @@ -194,9 +194,9 @@ if [ "${REMOVE}" = true ]; then rm -rf "${HOME_DIR}/lib64" fi printf "Remove submodules\n" - if [ -d "${SORC_DIR}/DA_update" ]; then - printf "... Remove DA_update ...\n" - rm -rf "${SORC_DIR}/DA_update" + if [ -d "${SORC_DIR}/apply_incr.fd" ]; then + printf "... Remove apply_incr.fd ...\n" + rm -rf "${SORC_DIR}/apply_incr.fd" fi if [ -d "${SORC_DIR}/ufsLand.fd" ]; then printf "... Remove ufsLand.fd ...\n" diff --git a/sorc/apply_incr.fd b/sorc/apply_incr.fd new file mode 160000 index 00000000..c28cc0cb --- /dev/null +++ b/sorc/apply_incr.fd @@ -0,0 +1 @@ +Subproject commit c28cc0cb7936ccc74d86dcde76829fc0faf79f82 diff --git a/sorc/test/ci/Dockerfile b/sorc/test/ci/Dockerfile index d64621c7..ce233b50 100644 --- a/sorc/test/ci/Dockerfile +++ b/sorc/test/ci/Dockerfile @@ -12,7 +12,7 @@ COPY . $HOME/land-offline_workflow # set env vars ENV LANDDA_INPUTS=$HOME/inputs ENV EPICHOME=/opt -ENV JEDI_INSTALL=${EPICHOME}/jedi +ENV JEDI_INSTALL=${EPICHOME}/jedi_skylabv7.0 ENV TEST_BASEDIR=${EPICHOME}/test_base/mem000/restarts/vector"} #build & unit testing diff --git a/sorc/test/create_bkg_ens.sh b/sorc/test/create_bkg_ens.sh index 8c7af393..eb38485a 100755 --- a/sorc/test/create_bkg_ens.sh +++ b/sorc/test/create_bkg_ens.sh @@ -12,7 +12,7 @@ source ${project_source_dir}/test/runtime_vars.sh ${project_binary_dir} ${projec RSTDIR=$project_binary_dir/test/bkg/restarts/tile # set executables -TEST_EXEC="${project_source_dir}/DA_update/letkf_create_ens.py" +TEST_EXEC="${project_source_dir}/../ush/letkf_create_ens.py" NPROC=1 # move to work directory @@ -36,7 +36,7 @@ if [[ ${DAtype} == 'letkfoi_snow' ]]; then done cres_file=$WORKDIR/mem_${ens}/${FILEDATE}.coupler.res - cp ${project_source_dir}/DA_update/template.coupler.res $cres_file + cp ${project_source_dir}/../parm/templates/template.coupler.res $cres_file sed -i -e "s/XXYYYY/${YY}/g" $cres_file sed -i -e "s/XXMM/${MM}/g" $cres_file sed -i -e "s/XXDD/${DD}/g" $cres_file diff --git a/sorc/test/runtime_vars.sh b/sorc/test/runtime_vars.sh index 9e0ad329..12183e32 100755 --- a/sorc/test/runtime_vars.sh +++ b/sorc/test/runtime_vars.sh @@ -56,7 +56,6 @@ export TSTUB="oro_C${RES}.mx100" export GFSv17=NO export OBS_TYPES=("GHCN") export DAtype=letkfoi_snow -export fv3bundle_vn=psl_develop export B=30 # background error std for LETKFOI if [ $GFSv17 == "YES" ]; then export SNOWDEPTHVAR="snodl" diff --git a/sorc/test/test_letkfoi_snowda.sh b/sorc/test/test_letkfoi_snowda.sh index 655a9836..be197cc9 100755 --- a/sorc/test/test_letkfoi_snowda.sh +++ b/sorc/test/test_letkfoi_snowda.sh @@ -28,17 +28,17 @@ do done # prepare yaml files -cp $project_source_dir/DA_update/jedi/fv3-jedi/yaml_files/${fv3bundle_vn}/${DAtype}.yaml letkf_land.yaml +cp $project_source_dir/../parm/jedi/${DAtype}.yaml letkf_land.yaml for ii in "${!OBS_TYPES[@]}"; do echo "============================= ${OBS_TYPES[$ii]}" - cat $project_source_dir/DA_update/jedi/fv3-jedi/yaml_files/${fv3bundle_vn}/${OBS_TYPES[$ii]}.yaml >> letkf_land.yaml + cat $project_source_dir/../parm/jedi/${OBS_TYPES[$ii]}.yaml >> letkf_land.yaml # link ioda obs file # GHCN are time-stamped at 18. If assimilating at 00, need to use previous day's obs, so that # obs are within DA window. [[ -e ${OBS_TYPES[$ii]}_${YY}${MP}${DP}${HP}.nc ]] && rm ${OBS_TYPES[$ii]}_${YY}${MP}${DP}${HP}.nc - obs_file=${OBSDIR}/snow_depth/${OBS_TYPES[$ii]}/data_proc/v3/${YY}/${OBS_TYPES[$ii],,}_snwd_ioda_${YY}${MP}${DP}.nc + obs_file=${OBSDIR}/snow_depth/${OBS_TYPES[$ii]}/data_proc/v3/${YY}/${OBS_TYPES[$ii],,}_snwd_ioda_${YY}${MP}${DP}_jediv7.nc if [[ -e $obs_file ]]; then echo "${OBS_TYPES[$ii]} observations found: $obs_file" else @@ -70,6 +70,9 @@ mkdir -p ./output/DA/hofx # link jedi static files ln -fs $JEDI_STATICDIR ./ +# copy gfs-land.yaml +cp $project_source_dir/../parm/jedi/gfs-land.yaml . + # echo "============================= calling ${JEDI_EXEC}" ${MPIRUN} -n $NPROC ${JEDI_EXEC} letkf_land.yaml diff --git a/ush/letkf_create_ens.py b/ush/letkf_create_ens.py new file mode 100755 index 00000000..1be62b08 --- /dev/null +++ b/ush/letkf_create_ens.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +import numpy as np +from netCDF4 import Dataset +import sys +# routine to create pseudo-ensemble for use in LETKF-OI for snow depth +# reads in sfc_data restarts, and sets snowdepth to +/- +# perturbation, given stdev(ensemble) of B +# Clara Draper, October, 2021. + +if (len(sys.argv) != 4): + print('argument error, usage: letkf_create file_stub variable back_error' ) + +fstub=sys.argv[1] +vname=sys.argv[2] +b = float(sys.argv[3]) + +# 2 ens members +offset=b/np.sqrt(2) + +print('adjusting '+fstub+'* by '+str(offset)) + +sign = [1,-1] +ens_dirs=['mem_pos','mem_neg'] + +for ens in range(2): + for tt in range(6): + # open file + out_netcdf = ens_dirs[ens]+'/'+fstub+'.sfc_data.tile'+str(tt+1)+'.nc' + #print (out_netcdf) + ncOut = Dataset(out_netcdf, "r+") + # get land mask + slmsk_array = ncOut.variables['slmsk'][:] + vtype_array = ncOut.variables['vtype'][:] + slmsk_array[vtype_array==15]=0 # remove glacier locations + # add offset to the snow + var_array = ncOut.variables[vname][:] + var_array[slmsk_array==1] = var_array[slmsk_array==1] + sign[ens]*offset + ncOut.variables[vname][0,:,:] = var_array[:] + ncOut.close() diff --git a/versions/build.ver_hera b/versions/build.ver_hera index c0065431..ca953bd7 100644 --- a/versions/build.ver_hera +++ b/versions/build.ver_hera @@ -22,7 +22,7 @@ export netcdf_fortran_ver="4.6.0" export parallelio_ver="2.5.10" export prod_util_ver="1.2.2" export sp_ver="2.3.3" -export stack_impi_ver="2021.5.1" +export stack_intel_oneapi_mpi_ver="2021.5.1" export stack_intel_ver="2021.5.0" export stack_python_ver="3.10.8" export w3emc_ver="2.10.0" diff --git a/versions/build.ver_orion b/versions/build.ver_orion index e7953bd6..cf2c2286 100644 --- a/versions/build.ver_orion +++ b/versions/build.ver_orion @@ -22,7 +22,7 @@ export netcdf_fortran_ver="4.6.0" export parallelio_ver="2.5.10" export prod_util_ver="1.2.2" export sp_ver="2.3.3" -export stack_impi_ver="2021.5.1" +export stack_intel_oneapi_mpi_ver="2021.5.1" export stack_intel_ver="2022.0.2" export stack_python_ver="3.10.8" export w3emc_ver="2.10.0"