From ec8bd2f66801c1ca5aaf39d648de24f643b3ab1a Mon Sep 17 00:00:00 2001 From: "Chan-hoo.Jeon" Date: Mon, 20 May 2024 19:35:32 +0000 Subject: [PATCH 1/7] update modules with spack-stack 1.6.0 --- modulefiles/build_hera_intel.lua | 3 +-- versions/build.ver_hera | 19 +++++++++---------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/modulefiles/build_hera_intel.lua b/modulefiles/build_hera_intel.lua index e24a5584..23b723a0 100644 --- a/modulefiles/build_hera_intel.lua +++ b/modulefiles/build_hera_intel.lua @@ -5,7 +5,6 @@ loads modules necessary for building the land-DA workflow on Hera using Intel whatis([===[Loads modules necessary for building the land-DA workflow on Hera]===]) prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) -prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack_jedi")) load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) @@ -43,4 +42,4 @@ setenv("FC", "mpiifort") setenv("CMAKE_Platform", "hera.intel") setenv("EPICHOME", "/scratch2/NAGAPE/epic/UFS_Land-DA") -setenv("JEDI_INSTALL", pathJoin(os.getenv("EPICHOME"),"jedi_skylabv7.0")) +setenv("JEDI_INSTALL", "/scratch2/NAGAPE/epic/UFS_Land-DA_Dev/jedi_v7") diff --git a/versions/build.ver_hera b/versions/build.ver_hera index ca953bd7..2ac07d9f 100644 --- a/versions/build.ver_hera +++ b/versions/build.ver_hera @@ -1,14 +1,13 @@ #!/bin/bash -export modulepath_spack_stack="/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env-rocky8/install/modulefiles/Core" -export modulepath_spack_stack_jedi="/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles" +export modulepath_spack_stack="/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/modulefiles/Core" export bacio_ver="2.4.1" export cmake_ver="3.23.1" -export crtm_ver="2.4.0" +export crtm_ver="2.4.0.1" export ecbuild_ver="3.7.2" -export esmf_ver="8.5.0" -export fms_ver="2023.02.01" +export esmf_ver="8.6.0" +export fms_ver="2023.04" export gftl_shared_ver="1.6.1" export g2_ver="3.4.5" export g2tmpl_ver="1.10.2" @@ -16,15 +15,15 @@ export hdf5_ver="1.14.0" export ip_ver="4.3.0" export jasper_ver="2.0.32" export libpng_ver="1.6.37" -export mapl_ver="2.40.3-esmf-8.5.0" +export mapl_ver="2.40.3-esmf-8.6.0" export netcdf_c_ver="4.9.2" -export netcdf_fortran_ver="4.6.0" +export netcdf_fortran_ver="4.6.1" export parallelio_ver="2.5.10" -export prod_util_ver="1.2.2" -export sp_ver="2.3.3" +export prod_util_ver="2.1.1" +export sp_ver="2.5.0" export stack_intel_oneapi_mpi_ver="2021.5.1" export stack_intel_ver="2021.5.0" -export stack_python_ver="3.10.8" +export stack_python_ver="3.10.13" export w3emc_ver="2.10.0" export zlib_ver="1.2.13" From 69316f62d53b2228cb0bcbe6ead4de32ae4ae44f Mon Sep 17 00:00:00 2001 From: "Chan-hoo.Jeon" Date: Mon, 20 May 2024 19:52:15 +0000 Subject: [PATCH 2/7] remove jedi module load from task module files --- modulefiles/tasks/hera/task.analysis.lua | 1 - modulefiles/tasks/hera/task.forecast.lua | 1 - modulefiles/tasks/hera/task.post_anal.lua | 1 - modulefiles/tasks/hera/task.pre_anal.lua | 1 - modulefiles/tasks/hera/task.prep_obs.lua | 1 - modulefiles/tasks/orion/task.analysis.lua | 1 - modulefiles/tasks/orion/task.forecast.lua | 1 - modulefiles/tasks/orion/task.post_anal.lua | 1 - modulefiles/tasks/orion/task.pre_anal.lua | 1 - modulefiles/tasks/orion/task.prep_obs.lua | 1 - 10 files changed, 10 deletions(-) diff --git a/modulefiles/tasks/hera/task.analysis.lua b/modulefiles/tasks/hera/task.analysis.lua index d7439d6d..4f16978f 100644 --- a/modulefiles/tasks/hera/task.analysis.lua +++ b/modulefiles/tasks/hera/task.analysis.lua @@ -1,5 +1,4 @@ prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) -prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack_jedi")) load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) diff --git a/modulefiles/tasks/hera/task.forecast.lua b/modulefiles/tasks/hera/task.forecast.lua index d7439d6d..4f16978f 100644 --- a/modulefiles/tasks/hera/task.forecast.lua +++ b/modulefiles/tasks/hera/task.forecast.lua @@ -1,5 +1,4 @@ prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) -prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack_jedi")) load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) diff --git a/modulefiles/tasks/hera/task.post_anal.lua b/modulefiles/tasks/hera/task.post_anal.lua index d7439d6d..4f16978f 100644 --- a/modulefiles/tasks/hera/task.post_anal.lua +++ b/modulefiles/tasks/hera/task.post_anal.lua @@ -1,5 +1,4 @@ prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) -prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack_jedi")) load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) diff --git a/modulefiles/tasks/hera/task.pre_anal.lua b/modulefiles/tasks/hera/task.pre_anal.lua index d7439d6d..4f16978f 100644 --- a/modulefiles/tasks/hera/task.pre_anal.lua +++ b/modulefiles/tasks/hera/task.pre_anal.lua @@ -1,5 +1,4 @@ prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) -prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack_jedi")) load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) diff --git a/modulefiles/tasks/hera/task.prep_obs.lua b/modulefiles/tasks/hera/task.prep_obs.lua index d7439d6d..4f16978f 100644 --- a/modulefiles/tasks/hera/task.prep_obs.lua +++ b/modulefiles/tasks/hera/task.prep_obs.lua @@ -1,5 +1,4 @@ prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) -prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack_jedi")) load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) diff --git a/modulefiles/tasks/orion/task.analysis.lua b/modulefiles/tasks/orion/task.analysis.lua index d7439d6d..4f16978f 100644 --- a/modulefiles/tasks/orion/task.analysis.lua +++ b/modulefiles/tasks/orion/task.analysis.lua @@ -1,5 +1,4 @@ prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) -prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack_jedi")) load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) diff --git a/modulefiles/tasks/orion/task.forecast.lua b/modulefiles/tasks/orion/task.forecast.lua index d7439d6d..4f16978f 100644 --- a/modulefiles/tasks/orion/task.forecast.lua +++ b/modulefiles/tasks/orion/task.forecast.lua @@ -1,5 +1,4 @@ prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) -prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack_jedi")) load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) diff --git a/modulefiles/tasks/orion/task.post_anal.lua b/modulefiles/tasks/orion/task.post_anal.lua index d7439d6d..4f16978f 100644 --- a/modulefiles/tasks/orion/task.post_anal.lua +++ b/modulefiles/tasks/orion/task.post_anal.lua @@ -1,5 +1,4 @@ prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) -prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack_jedi")) load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) diff --git a/modulefiles/tasks/orion/task.pre_anal.lua b/modulefiles/tasks/orion/task.pre_anal.lua index d7439d6d..4f16978f 100644 --- a/modulefiles/tasks/orion/task.pre_anal.lua +++ b/modulefiles/tasks/orion/task.pre_anal.lua @@ -1,5 +1,4 @@ prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) -prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack_jedi")) load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) diff --git a/modulefiles/tasks/orion/task.prep_obs.lua b/modulefiles/tasks/orion/task.prep_obs.lua index d7439d6d..4f16978f 100644 --- a/modulefiles/tasks/orion/task.prep_obs.lua +++ b/modulefiles/tasks/orion/task.prep_obs.lua @@ -1,5 +1,4 @@ prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) -prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack_jedi")) load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) From 6816a80922e2fca6bd2e128549ecb1913ebd0ffe Mon Sep 17 00:00:00 2001 From: Chan-Hoo Jeon Date: Mon, 20 May 2024 19:33:26 -0500 Subject: [PATCH 3/7] update module files with stack-stack 1.6.0 --- modulefiles/build_orion_intel.lua | 1 - versions/build.ver_orion | 21 ++++++++++----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/modulefiles/build_orion_intel.lua b/modulefiles/build_orion_intel.lua index cd570a8d..62480013 100644 --- a/modulefiles/build_orion_intel.lua +++ b/modulefiles/build_orion_intel.lua @@ -5,7 +5,6 @@ loads modules necessary for building the land-DA workflow on Orion using Intel whatis([===[Loads modules necessary for building the land-DA workflow on Orion]===]) prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) -prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack_jedi")) load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) diff --git a/versions/build.ver_orion b/versions/build.ver_orion index cf2c2286..bf41438b 100644 --- a/versions/build.ver_orion +++ b/versions/build.ver_orion @@ -1,14 +1,13 @@ #!/bin/bash -export modulepath_spack_stack="/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core" -export modulepath_spack_stack_jedi="/work/noaa/da/role-da/spack-stack/modulefiles" +export modulepath_spack_stack="/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core" export bacio_ver="2.4.1" -export cmake_ver="3.22.1" -export crtm_ver="2.4.0" +export cmake_ver="3.23.1" +export crtm_ver="2.4.0.1" export ecbuild_ver="3.7.2" -export esmf_ver="8.5.0" -export fms_ver="2023.02.01" +export esmf_ver="8.6.0" +export fms_ver="2023.04" export gftl_shared_ver="1.6.1" export g2_ver="3.4.5" export g2tmpl_ver="1.10.2" @@ -16,15 +15,15 @@ export hdf5_ver="1.14.0" export ip_ver="4.3.0" export jasper_ver="2.0.32" export libpng_ver="1.6.37" -export mapl_ver="2.40.3-esmf-8.5.0" +export mapl_ver="2.40.3-esmf-8.6.0" export netcdf_c_ver="4.9.2" -export netcdf_fortran_ver="4.6.0" +export netcdf_fortran_ver="4.6.1" export parallelio_ver="2.5.10" -export prod_util_ver="1.2.2" -export sp_ver="2.3.3" +export prod_util_ver="2.1.1" +export sp_ver="2.5.0" export stack_intel_oneapi_mpi_ver="2021.5.1" export stack_intel_ver="2022.0.2" -export stack_python_ver="3.10.8" +export stack_python_ver="3.10.13" export w3emc_ver="2.10.0" export zlib_ver="1.2.13" From 0d4f20f3e09967a061152f1e2d888edefdc16036 Mon Sep 17 00:00:00 2001 From: Chan-Hoo Jeon Date: Tue, 21 May 2024 07:23:18 -0500 Subject: [PATCH 4/7] change path to jedi on orion --- modulefiles/build_orion_intel.lua | 2 +- parm/land_analysis_hera.yaml | 5 +++++ parm/land_analysis_orion.yaml | 7 ++++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/modulefiles/build_orion_intel.lua b/modulefiles/build_orion_intel.lua index 62480013..77f69ac4 100644 --- a/modulefiles/build_orion_intel.lua +++ b/modulefiles/build_orion_intel.lua @@ -41,4 +41,4 @@ setenv("FC", "mpiifort") setenv("CMAKE_Platform", "orion.intel") setenv("EPICHOME", "/work/noaa/epic/UFS_Land-DA") -setenv("JEDI_INSTALL", "/work/noaa/epic/UFS_Land-DA_Dev/jedi_v7") +setenv("JEDI_INSTALL", "/work/noaa/epic/UFS_Land-DA_Dev/jedi_v7_stack1.6") diff --git a/parm/land_analysis_hera.yaml b/parm/land_analysis_hera.yaml index c62e9f9a..154522e7 100644 --- a/parm/land_analysis_hera.yaml +++ b/parm/land_analysis_hera.yaml @@ -53,6 +53,7 @@ workflow: task_prep_obs: attrs: cycledefs: cycled + maxtries: 2 envars: OBSDIR: "&OBSDIR;" OBSDIR_SUBDIR: "&OBSDIR_SUBDIR;" @@ -80,6 +81,7 @@ workflow: task_pre_anal: attrs: cycledefs: cycled + maxtries: 2 envars: MACHINE: "&MACHINE;" SCHED: "&SCHED;" @@ -126,6 +128,7 @@ workflow: task_analysis: attrs: cycledefs: cycled + maxtries: 2 envars: OBS_TYPES: "&OBS_TYPES;" MACHINE: "&MACHINE;" @@ -162,6 +165,7 @@ workflow: task_post_anal: attrs: cycledefs: cycled + maxtries: 2 envars: MACHINE: "&MACHINE;" SCHED: "&SCHED;" @@ -195,6 +199,7 @@ workflow: task_forecast: attrs: cycledefs: cycled + maxtries: 2 envars: OBS_TYPES: "&OBS_TYPES;" MACHINE: "&MACHINE;" diff --git a/parm/land_analysis_orion.yaml b/parm/land_analysis_orion.yaml index 4642e7f8..cfd343bf 100644 --- a/parm/land_analysis_orion.yaml +++ b/parm/land_analysis_orion.yaml @@ -15,7 +15,7 @@ workflow: ACCOUNT: "epic" EXP_NAME: "LETKF" EXP_BASEDIR: "/work/noaa/epic/{USER}/landda_test" - JEDI_INSTALL: "/work/noaa/epic/UFS_Land-DA_Dev/jedi_v7" + JEDI_INSTALL: "/work/noaa/epic/UFS_Land-DA_Dev/jedi_v7_stack1.6" WARMSTART_DIR: "/work/noaa/epic/UFS_Land-DA_Dev/DATA_RESTART" FORCING: "gswp3" # "gswp3" or "era5" RES: "96" @@ -53,6 +53,7 @@ workflow: task_prep_obs: attrs: cycledefs: cycled + maxtries: 2 envars: OBSDIR: "&OBSDIR;" OBSDIR_SUBDIR: "&OBSDIR_SUBDIR;" @@ -80,6 +81,7 @@ workflow: task_pre_anal: attrs: cycledefs: cycled + maxtries: 2 envars: MACHINE: "&MACHINE;" SCHED: "&SCHED;" @@ -126,6 +128,7 @@ workflow: task_analysis: attrs: cycledefs: cycled + maxtries: 2 envars: OBS_TYPES: "&OBS_TYPES;" MACHINE: "&MACHINE;" @@ -162,6 +165,7 @@ workflow: task_post_anal: attrs: cycledefs: cycled + maxtries: 2 envars: MACHINE: "&MACHINE;" SCHED: "&SCHED;" @@ -195,6 +199,7 @@ workflow: task_forecast: attrs: cycledefs: cycled + maxtries: 2 envars: OBS_TYPES: "&OBS_TYPES;" MACHINE: "&MACHINE;" From a1a4e267cc3d884d7b06bfd54920878812a1b157 Mon Sep 17 00:00:00 2001 From: "Chan-hoo.Jeon" Date: Tue, 21 May 2024 21:11:45 +0000 Subject: [PATCH 5/7] fix ctest failure on Hera --- modulefiles/build_hera_intel.lua | 2 +- sorc/test/run_ufs_datm_lnd.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modulefiles/build_hera_intel.lua b/modulefiles/build_hera_intel.lua index 23b723a0..7ad63028 100644 --- a/modulefiles/build_hera_intel.lua +++ b/modulefiles/build_hera_intel.lua @@ -41,5 +41,5 @@ setenv("CXX", "mpiicpc") setenv("FC", "mpiifort") setenv("CMAKE_Platform", "hera.intel") -setenv("EPICHOME", "/scratch2/NAGAPE/epic/UFS_Land-DA") +setenv("EPICHOME", "/scratch2/NAGAPE/epic/UFS_Land-DA_Dev") setenv("JEDI_INSTALL", "/scratch2/NAGAPE/epic/UFS_Land-DA_Dev/jedi_v7") diff --git a/sorc/test/run_ufs_datm_lnd.sh b/sorc/test/run_ufs_datm_lnd.sh index 274f067a..e673ae2b 100755 --- a/sorc/test/run_ufs_datm_lnd.sh +++ b/sorc/test/run_ufs_datm_lnd.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -ex ################################################ # pass arguments project_binary_dir=$1 @@ -37,8 +37,8 @@ RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${TEST_NAME}_${RT_COMPILER} INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/NEMSfv3gfs/input-data-20221101} if [[ ! -d ${INPUTDATA_ROOT} ]] || [[ ! -d ${RTPWD} ]]; then -echo "Error: cannot find either folder for INPUTDATA_ROOT or RTPWD, please check!" -exit 1 + echo "Error: cannot find either folder for INPUTDATA_ROOT or RTPWD, please check!" + exit 1 fi # create test folder From e31644ee8738f3c7dbe2995c84ac3826637a8e36 Mon Sep 17 00:00:00 2001 From: "Chan-hoo.Jeon" Date: Tue, 21 May 2024 21:24:51 +0000 Subject: [PATCH 6/7] fix typo in sample yaml for hera --- parm/land_analysis_hera.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/land_analysis_hera.yaml b/parm/land_analysis_hera.yaml index 154522e7..ddc27ac6 100644 --- a/parm/land_analysis_hera.yaml +++ b/parm/land_analysis_hera.yaml @@ -10,7 +10,7 @@ workflow: spec: 200001030000 200001040000 24:00:00 # for gswp3 # spec: 201912210000 201912220000 24:00:00 # for era5 entities: - MACHINE: "orion" + MACHINE: "hera" SCHED: "slurm" ACCOUNT: "epic" EXP_NAME: "LETKF" From c87b5b7acb4e8623035f561c5baca3fd8256c36a Mon Sep 17 00:00:00 2001 From: "Chan-hoo.Jeon" Date: Tue, 21 May 2024 21:42:16 +0000 Subject: [PATCH 7/7] change path to fix inputs on Hera --- sorc/app_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/app_build.sh b/sorc/app_build.sh index f5f10dad..ff141bfa 100755 --- a/sorc/app_build.sh +++ b/sorc/app_build.sh @@ -394,7 +394,7 @@ fi # Link land-DA input files to FIXlandda directory if [ "${PLATFORM}" = "hera" ]; then - landda_fix_orig="/scratch2/NAGAPE/epic/UFS_Land-DA/inputs" + landda_fix_orig="/scratch2/NAGAPE/epic/UFS_Land-DA_Dev/inputs" elif [ "${PLATFORM}" = "orion" ]; then landda_fix_orig="/work/noaa/epic/UFS_Land-DA/inputs" fi