diff --git a/tests/integration/generated/test_min_case_add_dependencies_chrysalis.cfg b/tests/integration/generated/test_min_case_add_dependencies_chrysalis.cfg index dc36251b..0ee81cdc 100644 --- a/tests/integration/generated/test_min_case_add_dependencies_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_add_dependencies_chrysalis.cfg @@ -25,7 +25,6 @@ years = "1985:1995:10", [ts] active = True -e3sm_to_cmip_environment_commands = "" walltime = "00:50:00" # ts is in 5 year increments years = "1985:1995:5" @@ -35,7 +34,6 @@ years = "1985:1995:5" frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" - ts_fmt = "cmip" vars = "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,ICEFRAC,LANDFRAC,OCNFRAC,PS,CLDICE,CLDLIQ,T,AODDUST" [[ land_monthly ]] @@ -45,7 +43,6 @@ years = "1985:1995:5" input_files = "elm.h0" input_subdir = "archive/lnd/hist" mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc" - ts_fmt = "cmip" vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" [[ rof_monthly ]] @@ -79,6 +76,21 @@ years = "1985:1995:5" mapping_file = "glb" vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" +[e3sm_to_cmip] +active = True +frequency = "monthly" +walltime = "00:50:00" +ts_num_years = 5 +years = "1985:1995:5" + + [[ atm_monthly_180x360_aave ]] + # for ilamb + input_files = "eam.h0" + + [[ land_monthly ]] + # for ilamb + input_files = "elm.h0" + [e3sm_diags] active = True environment_commands = "source /conda.sh; conda activate " diff --git a/tests/integration/generated/test_min_case_carryover_dependencies_chrysalis.cfg b/tests/integration/generated/test_min_case_carryover_dependencies_chrysalis.cfg index a380f43f..cc0fd50e 100644 --- a/tests/integration/generated/test_min_case_carryover_dependencies_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_carryover_dependencies_chrysalis.cfg @@ -60,14 +60,12 @@ walltime = "00:30:00" [ts] active = True -e3sm_to_cmip_environment_commands = "" walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" - ts_fmt = "cmip" [[ rof_monthly ]] extra_vars = 'areatotal2' @@ -99,9 +97,20 @@ walltime = "00:30:00" input_files = "elm.h0" input_subdir = "archive/lnd/hist" mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc" - ts_fmt = "cmip" vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" +[e3sm_to_cmip] +active = True +frequency = "monthly" +ts_num_years = 2 +walltime = "00:30:00" + + [[ atm_monthly_180x360_aave ]] + input_files = "eam.h0" + + [[ land_monthly ]] + input_files = "elm.h0" + # TODO: Add "tc_analysis" back in after empty dat is resolved. # [tc_analysis] # # The second run should run in parallel with the first run. diff --git a/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_chrysalis.cfg index 6678e040..68bd25af 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_chrysalis.cfg @@ -14,14 +14,12 @@ years = "1985:1989:4", [ts] active = True -e3sm_to_cmip_environment_commands = "" walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" - #ts_fmt = "cmip" [e3sm_diags] active = True diff --git a/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_mvm_1_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_mvm_1_chrysalis.cfg index fdb3025a..03147e49 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_mvm_1_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_mvm_1_chrysalis.cfg @@ -14,11 +14,9 @@ years = "1985:1989:4", [ts] active = True -e3sm_to_cmip_environment_commands = "" walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" - #ts_fmt = "cmip" diff --git a/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_mvm_2_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_mvm_2_chrysalis.cfg index bb57d3a1..703179a0 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_mvm_2_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_mvm_2_chrysalis.cfg @@ -14,14 +14,12 @@ years = "1995:1999:4", [ts] active = True -e3sm_to_cmip_environment_commands = "" walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" - #ts_fmt = "cmip" [e3sm_diags] active = True diff --git a/tests/integration/generated/test_min_case_e3sm_diags_streamflow_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_streamflow_chrysalis.cfg index 5bfa9302..de3942ca 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_streamflow_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_streamflow_chrysalis.cfg @@ -14,7 +14,6 @@ years = "1985:1989:4", [ts] active = True -e3sm_to_cmip_environment_commands = "" walltime = "00:30:00" [[ rof_monthly ]] diff --git a/tests/integration/generated/test_min_case_e3sm_diags_streamflow_mvm_1_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_streamflow_mvm_1_chrysalis.cfg index bb8de9cf..37ab3c9f 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_streamflow_mvm_1_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_streamflow_mvm_1_chrysalis.cfg @@ -14,7 +14,6 @@ years = "1985:1989:4", [ts] active = True -e3sm_to_cmip_environment_commands = "" walltime = "00:30:00" [[ rof_monthly ]] diff --git a/tests/integration/generated/test_min_case_e3sm_diags_streamflow_mvm_2_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_streamflow_mvm_2_chrysalis.cfg index 4f2f099e..d1705ec3 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_streamflow_mvm_2_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_streamflow_mvm_2_chrysalis.cfg @@ -14,7 +14,6 @@ years = "1995:1999:4", [ts] active = True -e3sm_to_cmip_environment_commands = "" walltime = "00:30:00" [[ rof_monthly ]] diff --git a/tests/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_chrysalis.cfg index feaa8fe1..5cd57105 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_chrysalis.cfg @@ -14,7 +14,6 @@ years = "1985:1989:4", [ts] active = True -e3sm_to_cmip_environment_commands = "" walltime = "00:30:00" [[ atm_daily_180x360_aave ]] diff --git a/tests/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_mvm_1_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_mvm_1_chrysalis.cfg index 7e9a2d31..90919f53 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_mvm_1_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_mvm_1_chrysalis.cfg @@ -14,7 +14,6 @@ years = "1985:1989:4", [ts] active = True -e3sm_to_cmip_environment_commands = "" walltime = "00:30:00" [[ atm_daily_180x360_aave ]] diff --git a/tests/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_mvm_2_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_mvm_2_chrysalis.cfg index 27a64582..72f6ad58 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_mvm_2_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_mvm_2_chrysalis.cfg @@ -14,7 +14,6 @@ years = "1995:1999:4", [ts] active = True -e3sm_to_cmip_environment_commands = "" walltime = "00:30:00" [[ atm_daily_180x360_aave ]] diff --git a/tests/integration/generated/test_min_case_global_time_series_comprehensive_v3_setup_only_chrysalis.cfg b/tests/integration/generated/test_min_case_global_time_series_comprehensive_v3_setup_only_chrysalis.cfg index 51d80069..ad7b5de1 100644 --- a/tests/integration/generated/test_min_case_global_time_series_comprehensive_v3_setup_only_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_global_time_series_comprehensive_v3_setup_only_chrysalis.cfg @@ -17,7 +17,6 @@ years = "1985:1989:2", [ts] active = True -e3sm_to_cmip_environment_commands = "" walltime = "00:30:00" [[ atm_monthly_glb ]] @@ -51,14 +50,3 @@ walltime = "00:30:00" # (This cfg is the setup portion only) # [global_time_series] -# active = True -# climo_years = "1985-1989", "1990-1995", -# environment_commands = "source /conda.sh; conda activate " -# experiment_name = "v3.LR.historical_0051" -# figstr = "v3.LR.historical_0051" -# moc_file=mocTimeSeries_1985-1995.nc -# plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" -# ts_num_years = 5 -# ts_years = "1985-1989", "1985-1995", -# walltime = "00:30:00" -# years = "1985-1995", diff --git a/tests/integration/generated/test_min_case_global_time_series_custom_chrysalis.cfg b/tests/integration/generated/test_min_case_global_time_series_custom_chrysalis.cfg index e29f01bb..895bcb53 100644 --- a/tests/integration/generated/test_min_case_global_time_series_custom_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_global_time_series_custom_chrysalis.cfg @@ -13,7 +13,6 @@ www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_ [ts] active = True -e3sm_to_cmip_environment_commands = "" walltime = "00:30:00" years = "1985:1995:5", diff --git a/tests/integration/generated/test_min_case_global_time_series_original_8_chrysalis.cfg b/tests/integration/generated/test_min_case_global_time_series_original_8_chrysalis.cfg index f9346e33..907b851f 100644 --- a/tests/integration/generated/test_min_case_global_time_series_original_8_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_global_time_series_original_8_chrysalis.cfg @@ -13,7 +13,6 @@ www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_ [ts] active = True -e3sm_to_cmip_environment_commands = "" walltime = "00:30:00" [[ atm_monthly_glb ]] diff --git a/tests/integration/generated/test_min_case_global_time_series_original_8_no_ocn_chrysalis.cfg b/tests/integration/generated/test_min_case_global_time_series_original_8_no_ocn_chrysalis.cfg index f3bea41f..47337ead 100644 --- a/tests/integration/generated/test_min_case_global_time_series_original_8_no_ocn_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_global_time_series_original_8_no_ocn_chrysalis.cfg @@ -13,7 +13,6 @@ www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_ [ts] active = True -e3sm_to_cmip_environment_commands = "" walltime = "00:30:00" [[ atm_monthly_glb ]] diff --git a/tests/integration/generated/test_min_case_ilamb_chrysalis.cfg b/tests/integration/generated/test_min_case_ilamb_chrysalis.cfg index 2c361140..28ffa23b 100644 --- a/tests/integration/generated/test_min_case_ilamb_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_ilamb_chrysalis.cfg @@ -9,13 +9,11 @@ mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" output = "/lcrc/group/e3sm/ac.forsyth2/zppy_min_case_ilamb_output/unique_id/v3.LR.historical_0051" partition = "debug" qos = "regular" +walltime = "00:30:00" www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_ilamb_www/unique_id" [ts] active = True -e3sm_to_cmip_environment_commands = "" -walltime = "00:50:00" -ts_fmt = "cmip" years = "1985:2014:30" [[ atm_monthly_180x360_aave ]] @@ -32,11 +30,21 @@ years = "1985:2014:30" mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc" vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" +[e3sm_to_cmip] +active = True +frequency = "monthly" +ts_num_years = 30 +years = "1985:2014:30" + + [[ atm_monthly_180x360_aave ]] + input_files = "eam.h0" + + [[ land_monthly ]] + input_files = "elm.h0" + [ilamb] active = True nodes = 8 -partition = "compute" short_name = "v3.LR.historical_0051" ts_num_years = 30 -walltime = "2:00:00" years = "1985:2014:30" diff --git a/tests/integration/generated/test_min_case_ilamb_diff_years_chrysalis.cfg b/tests/integration/generated/test_min_case_ilamb_diff_years_chrysalis.cfg new file mode 100644 index 00000000..dd2a3169 --- /dev/null +++ b/tests/integration/generated/test_min_case_ilamb_diff_years_chrysalis.cfg @@ -0,0 +1,51 @@ +[default] +case = "v3.LR.historical_0051" +constraint = "" +dry_run = "False" +environment_commands = "" +input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/v3.LR.historical_0051 +input_subdir = archive/atm/hist +mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_min_case_ilamb_diff_years_output/unique_id/v3.LR.historical_0051" +partition = "debug" +qos = "regular" +walltime = "00:30:00" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_ilamb_diff_years_www/unique_id" + +[ts] +active = True +# 15 year increments for ts and e3sm_to_cmip, 30 for ilamb +years = "1985:2014:15" + + [[ atm_monthly_180x360_aave ]] + frequency = "monthly" + input_files = "eam.h0" + input_subdir = "archive/atm/hist" + vars = "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,ICEFRAC,LANDFRAC,OCNFRAC,PS,CLDICE,CLDLIQ,T,AODDUST" + + [[ land_monthly ]] + extra_vars = "landfrac" + frequency = "monthly" + input_files = "elm.h0" + input_subdir = "archive/lnd/hist" + mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc" + vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" + +[e3sm_to_cmip] +active = True +frequency = "monthly" +ts_num_years = 15 +years = "1985:2014:15" + + [[ atm_monthly_180x360_aave ]] + input_files = "eam.h0" + + [[ land_monthly ]] + input_files = "elm.h0" + +[ilamb] +active = True +nodes = 8 +short_name = "v3.LR.historical_0051" +ts_num_years = 15 +years = "1985:2014:30" diff --git a/tests/integration/generated/test_min_case_ilamb_land_only_chrysalis.cfg b/tests/integration/generated/test_min_case_ilamb_land_only_chrysalis.cfg index 89b08f57..b78d5445 100644 --- a/tests/integration/generated/test_min_case_ilamb_land_only_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_ilamb_land_only_chrysalis.cfg @@ -13,9 +13,7 @@ www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_ [ts] active = True -e3sm_to_cmip_environment_commands = "" walltime = "00:50:00" -ts_fmt = "cmip" years = "1985:2014:30" [[ land_monthly ]] @@ -26,6 +24,16 @@ years = "1985:2014:30" mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc" vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" +[e3sm_to_cmip] +active = True +frequency = "monthly" +ts_num_years = 30 +walltime = "00:50:00" +years = "1985:2014:30" + + [[ land_monthly ]] + input_files = "elm.h0" + [ilamb] active = True land_only = True diff --git a/tests/integration/generated/test_min_case_nco_chrysalis.cfg b/tests/integration/generated/test_min_case_nco_chrysalis.cfg index 89b5b810..7e8ec09f 100644 --- a/tests/integration/generated/test_min_case_nco_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_nco_chrysalis.cfg @@ -40,14 +40,12 @@ walltime = "00:30:00" [ts] active = True -e3sm_to_cmip_environment_commands = "" walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" - ts_fmt = "cmip" [[ atm_daily_180x360_aave ]] frequency = "daily" @@ -85,5 +83,4 @@ walltime = "00:30:00" input_files = "elm.h0" input_subdir = "archive/lnd/hist" mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc" - ts_fmt = "cmip" vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" diff --git a/tests/integration/generated/test_weekly_bundles_chrysalis.cfg b/tests/integration/generated/test_weekly_bundles_chrysalis.cfg index 068899be..e731a6c5 100644 --- a/tests/integration/generated/test_weekly_bundles_chrysalis.cfg +++ b/tests/integration/generated/test_weekly_bundles_chrysalis.cfg @@ -8,6 +8,8 @@ # > atm_monthly_diurnal_8xdaily_180x360_aave # ts > atm_monthly_180x360_aave # > land_monthly +# e3sm_to_cmip > atm_monthly_180x360_aave +# > land_monthly # e3sm_diags > atm_monthly_180x360_aave # bundle 2: # ts > atm_monthly_glb @@ -59,14 +61,12 @@ years = "1985:1989:2", [ts] active = True bundle = "bundle1" -e3sm_to_cmip_environment_commands = "" years = "1985:1989:2", [[ atm_monthly_180x360_aave ]] frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" - ts_fmt = "cmip" [[ atm_monthly_glb ]] bundle = "bundle2" # Override bundle1 @@ -77,14 +77,12 @@ years = "1985:1989:2", years = "1985:1995:5", [[ land_monthly ]] - e3sm_to_cmip_environment_commands = "" extra_vars = "landfrac" frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc" vars = "LAISHA,LAISUN" - ts_fmt = "cmip" [[ rof_monthly ]] bundle = "bundle3" # Override bundle1, let bundle1 finish first because "e3sm_diags: atm_monthly_180x360_aave_mvm" requires "ts: atm_monthly_180x360_aave" @@ -95,6 +93,19 @@ years = "1985:1989:2", mapping_file = "" vars = "RIVER_DISCHARGE_OVER_LAND_LIQ" +[e3sm_to_cmip] +active = True +bundle = "bundle1" +frequency = "monthly" +ts_num_years = 2 +years = "1985:1989:2", + + [[ atm_monthly_180x360_aave ]] + input_files = "eam.h0" + + [[ land_monthly ]] + input_files = "elm.h0" + # TODO: Add "tc_analysis" back in after empty dat is resolved. # [tc_analysis] # active = True diff --git a/tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg b/tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg index 3a2e641b..255eccb9 100644 --- a/tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg +++ b/tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg @@ -37,14 +37,12 @@ walltime = "00:30:00" [ts] active = True -e3sm_to_cmip_environment_commands = "" walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" - ts_fmt = "cmip" [[ atm_daily_180x360_aave ]] frequency = "daily" @@ -81,9 +79,20 @@ walltime = "00:30:00" frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" - ts_fmt = "cmip" vars = "LAISHA,LAISUN" +[e3sm_to_cmip] +active = True +frequency = "monthly" +ts_num_years=2 +walltime = "00:30:00" + + [[ atm_monthly_180x360_aave ]] + input_files = "eam.h0" + + [[ land_monthly ]] + input_files = "elm.h0" + [tc_analysis] active = True walltime = "00:30:00" @@ -180,7 +189,7 @@ years = "1980-1990", [ilamb] active = True nodes = 8 -partition = "compute" +partition = "debug" short_name = "v2.LR.historical_0201" ts_num_years = 2 walltime = "2:00:00" diff --git a/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg b/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg index 0722bb10..d09e8113 100644 --- a/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg +++ b/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg @@ -40,14 +40,12 @@ walltime = "00:30:00" [ts] active = True -e3sm_to_cmip_environment_commands = "" walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" - ts_fmt = "cmip" [[ atm_daily_180x360_aave ]] frequency = "daily" @@ -85,9 +83,22 @@ walltime = "00:30:00" input_files = "elm.h0" input_subdir = "archive/lnd/hist" mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc" - ts_fmt = "cmip" vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" +[e3sm_to_cmip] +active = True +frequency = "monthly" +ts_num_years=2 +walltime = "00:30:00" + + [[ atm_monthly_180x360_aave ]] + input_files = "eam.h0" + ts_subsection = "atm_monthly_180x360_aave" + + [[ land_monthly ]] + input_files = "elm.h0" + ts_subsection = "land_monthly" + # TODO: Add "tc_analysis" back in after empty dat is resolved. # [tc_analysis] # active = True @@ -207,9 +218,11 @@ years = "1985-1995", [ilamb] active = True +e3sm_to_cmip_atm_subsection = "atm_monthly_180x360_aave" +e3sm_to_cmip_land_subsection = "land_monthly" ilamb_obs = "/lcrc/group/e3sm/diagnostics/ilamb_data" nodes = 8 -partition = "compute" +partition = "debug" short_name = "v3.LR.historical_0051" ts_atm_subsection = "atm_monthly_180x360_aave" ts_land_subsection = "land_monthly" diff --git a/tests/integration/template_min_case_add_dependencies.cfg b/tests/integration/template_min_case_add_dependencies.cfg index f4b796d9..df16ef97 100644 --- a/tests/integration/template_min_case_add_dependencies.cfg +++ b/tests/integration/template_min_case_add_dependencies.cfg @@ -25,7 +25,6 @@ years = "1985:1995:10", [ts] active = True -e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" walltime = "00:50:00" # ts is in 5 year increments years = "1985:1995:5" @@ -35,7 +34,6 @@ years = "1985:1995:5" frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" - ts_fmt = "cmip" vars = "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,ICEFRAC,LANDFRAC,OCNFRAC,PS,CLDICE,CLDLIQ,T,AODDUST" [[ land_monthly ]] @@ -45,7 +43,6 @@ years = "1985:1995:5" input_files = "elm.h0" input_subdir = "archive/lnd/hist" mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc" - ts_fmt = "cmip" vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" [[ rof_monthly ]] @@ -79,6 +76,21 @@ years = "1985:1995:5" mapping_file = "glb" vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" +[e3sm_to_cmip] +active = True +frequency = "monthly" +walltime = "00:50:00" +ts_num_years = 5 +years = "1985:1995:5" + + [[ atm_monthly_180x360_aave ]] + # for ilamb + input_files = "eam.h0" + + [[ land_monthly ]] + # for ilamb + input_files = "elm.h0" + [e3sm_diags] active = True environment_commands = "#expand diags_environment_commands#" diff --git a/tests/integration/template_min_case_carryover_dependencies.cfg b/tests/integration/template_min_case_carryover_dependencies.cfg index 8981b443..c789ef00 100644 --- a/tests/integration/template_min_case_carryover_dependencies.cfg +++ b/tests/integration/template_min_case_carryover_dependencies.cfg @@ -60,14 +60,12 @@ walltime = "00:30:00" [ts] active = True -e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" - ts_fmt = "cmip" [[ rof_monthly ]] extra_vars = 'areatotal2' @@ -99,9 +97,20 @@ walltime = "00:30:00" input_files = "elm.h0" input_subdir = "archive/lnd/hist" mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc" - ts_fmt = "cmip" vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" +[e3sm_to_cmip] +active = True +frequency = "monthly" +ts_num_years = 2 +walltime = "00:30:00" + + [[ atm_monthly_180x360_aave ]] + input_files = "eam.h0" + + [[ land_monthly ]] + input_files = "elm.h0" + # TODO: Add "tc_analysis" back in after empty dat is resolved. # [tc_analysis] # # The second run should run in parallel with the first run. diff --git a/tests/integration/template_min_case_e3sm_diags_depend_on_ts.cfg b/tests/integration/template_min_case_e3sm_diags_depend_on_ts.cfg index 81c369ea..66cd4c8b 100644 --- a/tests/integration/template_min_case_e3sm_diags_depend_on_ts.cfg +++ b/tests/integration/template_min_case_e3sm_diags_depend_on_ts.cfg @@ -14,14 +14,12 @@ years = "1985:1989:4", [ts] active = True -e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" - #ts_fmt = "cmip" [e3sm_diags] active = True diff --git a/tests/integration/template_min_case_e3sm_diags_depend_on_ts_mvm_1.cfg b/tests/integration/template_min_case_e3sm_diags_depend_on_ts_mvm_1.cfg index e39c25f4..c840c724 100644 --- a/tests/integration/template_min_case_e3sm_diags_depend_on_ts_mvm_1.cfg +++ b/tests/integration/template_min_case_e3sm_diags_depend_on_ts_mvm_1.cfg @@ -14,11 +14,9 @@ years = "1985:1989:4", [ts] active = True -e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" - #ts_fmt = "cmip" diff --git a/tests/integration/template_min_case_e3sm_diags_depend_on_ts_mvm_2.cfg b/tests/integration/template_min_case_e3sm_diags_depend_on_ts_mvm_2.cfg index 2c03d520..747afc20 100644 --- a/tests/integration/template_min_case_e3sm_diags_depend_on_ts_mvm_2.cfg +++ b/tests/integration/template_min_case_e3sm_diags_depend_on_ts_mvm_2.cfg @@ -14,14 +14,12 @@ years = "1995:1999:4", [ts] active = True -e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" - #ts_fmt = "cmip" [e3sm_diags] active = True diff --git a/tests/integration/template_min_case_e3sm_diags_streamflow.cfg b/tests/integration/template_min_case_e3sm_diags_streamflow.cfg index 06540ee5..2712f094 100644 --- a/tests/integration/template_min_case_e3sm_diags_streamflow.cfg +++ b/tests/integration/template_min_case_e3sm_diags_streamflow.cfg @@ -14,7 +14,6 @@ years = "1985:1989:4", [ts] active = True -e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" walltime = "00:30:00" [[ rof_monthly ]] diff --git a/tests/integration/template_min_case_e3sm_diags_streamflow_mvm_1.cfg b/tests/integration/template_min_case_e3sm_diags_streamflow_mvm_1.cfg index 2977e6b0..76200561 100644 --- a/tests/integration/template_min_case_e3sm_diags_streamflow_mvm_1.cfg +++ b/tests/integration/template_min_case_e3sm_diags_streamflow_mvm_1.cfg @@ -14,7 +14,6 @@ years = "1985:1989:4", [ts] active = True -e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" walltime = "00:30:00" [[ rof_monthly ]] diff --git a/tests/integration/template_min_case_e3sm_diags_streamflow_mvm_2.cfg b/tests/integration/template_min_case_e3sm_diags_streamflow_mvm_2.cfg index 75ed5d85..3fe7c451 100644 --- a/tests/integration/template_min_case_e3sm_diags_streamflow_mvm_2.cfg +++ b/tests/integration/template_min_case_e3sm_diags_streamflow_mvm_2.cfg @@ -14,7 +14,6 @@ years = "1995:1999:4", [ts] active = True -e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" walltime = "00:30:00" [[ rof_monthly ]] diff --git a/tests/integration/template_min_case_e3sm_diags_tropical_subseasonal.cfg b/tests/integration/template_min_case_e3sm_diags_tropical_subseasonal.cfg index 35602291..788bd1da 100644 --- a/tests/integration/template_min_case_e3sm_diags_tropical_subseasonal.cfg +++ b/tests/integration/template_min_case_e3sm_diags_tropical_subseasonal.cfg @@ -14,7 +14,6 @@ years = "1985:1989:4", [ts] active = True -e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" walltime = "00:30:00" [[ atm_daily_180x360_aave ]] diff --git a/tests/integration/template_min_case_e3sm_diags_tropical_subseasonal_mvm_1.cfg b/tests/integration/template_min_case_e3sm_diags_tropical_subseasonal_mvm_1.cfg index 1540069e..0c190691 100644 --- a/tests/integration/template_min_case_e3sm_diags_tropical_subseasonal_mvm_1.cfg +++ b/tests/integration/template_min_case_e3sm_diags_tropical_subseasonal_mvm_1.cfg @@ -14,7 +14,6 @@ years = "1985:1989:4", [ts] active = True -e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" walltime = "00:30:00" [[ atm_daily_180x360_aave ]] diff --git a/tests/integration/template_min_case_e3sm_diags_tropical_subseasonal_mvm_2.cfg b/tests/integration/template_min_case_e3sm_diags_tropical_subseasonal_mvm_2.cfg index c409f953..66d46f7c 100644 --- a/tests/integration/template_min_case_e3sm_diags_tropical_subseasonal_mvm_2.cfg +++ b/tests/integration/template_min_case_e3sm_diags_tropical_subseasonal_mvm_2.cfg @@ -14,7 +14,6 @@ years = "1995:1999:4", [ts] active = True -e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" walltime = "00:30:00" [[ atm_daily_180x360_aave ]] diff --git a/tests/integration/template_min_case_global_time_series_comprehensive_v3_setup_only.cfg b/tests/integration/template_min_case_global_time_series_comprehensive_v3_setup_only.cfg index 87ee6999..3c743694 100644 --- a/tests/integration/template_min_case_global_time_series_comprehensive_v3_setup_only.cfg +++ b/tests/integration/template_min_case_global_time_series_comprehensive_v3_setup_only.cfg @@ -17,7 +17,6 @@ years = "1985:1989:2", [ts] active = True -e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" walltime = "00:30:00" [[ atm_monthly_glb ]] @@ -51,14 +50,3 @@ walltime = "#expand mpas_analysis_walltime#" # (This cfg is the setup portion only) # [global_time_series] -# active = True -# climo_years = "1985-1989", "1990-1995", -# environment_commands = "#expand global_time_series_environment_commands#" -# experiment_name = "#expand case_name#" -# figstr = "#expand case_name#" -# moc_file=mocTimeSeries_1985-1995.nc -# plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" -# ts_num_years = 5 -# ts_years = "1985-1989", "1985-1995", -# walltime = "00:30:00" -# years = "1985-1995", diff --git a/tests/integration/template_min_case_global_time_series_custom.cfg b/tests/integration/template_min_case_global_time_series_custom.cfg index 99aa67ef..3b16536d 100644 --- a/tests/integration/template_min_case_global_time_series_custom.cfg +++ b/tests/integration/template_min_case_global_time_series_custom.cfg @@ -13,7 +13,6 @@ www = "#expand user_www#zppy_min_case_global_time_series_custom_www/#expand uniq [ts] active = True -e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" walltime = "00:30:00" years = "1985:1995:5", diff --git a/tests/integration/template_min_case_global_time_series_original_8.cfg b/tests/integration/template_min_case_global_time_series_original_8.cfg index 8f2d2a74..fd676996 100644 --- a/tests/integration/template_min_case_global_time_series_original_8.cfg +++ b/tests/integration/template_min_case_global_time_series_original_8.cfg @@ -13,7 +13,6 @@ www = "#expand user_www#zppy_min_case_global_time_series_original_8_www/#expand [ts] active = True -e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" walltime = "00:30:00" [[ atm_monthly_glb ]] diff --git a/tests/integration/template_min_case_global_time_series_original_8_no_ocn.cfg b/tests/integration/template_min_case_global_time_series_original_8_no_ocn.cfg index a8adecb6..b22072c0 100644 --- a/tests/integration/template_min_case_global_time_series_original_8_no_ocn.cfg +++ b/tests/integration/template_min_case_global_time_series_original_8_no_ocn.cfg @@ -13,7 +13,6 @@ www = "#expand user_www#zppy_min_case_global_time_series_original_8_no_ocn_www/# [ts] active = True -e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" walltime = "00:30:00" [[ atm_monthly_glb ]] diff --git a/tests/integration/template_min_case_ilamb.cfg b/tests/integration/template_min_case_ilamb.cfg index 33ae0a2f..5bc64dd8 100644 --- a/tests/integration/template_min_case_ilamb.cfg +++ b/tests/integration/template_min_case_ilamb.cfg @@ -9,13 +9,11 @@ mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" output = "#expand user_output#zppy_min_case_ilamb_output/#expand unique_id#/#expand case_name#" partition = "#expand partition_short#" qos = "#expand qos_short#" +walltime = "00:30:00" www = "#expand user_www#zppy_min_case_ilamb_www/#expand unique_id#" [ts] active = True -e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" -walltime = "00:50:00" -ts_fmt = "cmip" years = "1985:2014:30" [[ atm_monthly_180x360_aave ]] @@ -32,11 +30,21 @@ years = "1985:2014:30" mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc" vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" +[e3sm_to_cmip] +active = True +frequency = "monthly" +ts_num_years = 30 +years = "1985:2014:30" + + [[ atm_monthly_180x360_aave ]] + input_files = "eam.h0" + + [[ land_monthly ]] + input_files = "elm.h0" + [ilamb] active = True nodes = 8 -partition = "#expand partition_long#" short_name = "#expand case_name#" ts_num_years = 30 -walltime = "2:00:00" years = "1985:2014:30" diff --git a/tests/integration/template_min_case_ilamb_diff_years.cfg b/tests/integration/template_min_case_ilamb_diff_years.cfg new file mode 100644 index 00000000..2f37ff87 --- /dev/null +++ b/tests/integration/template_min_case_ilamb_diff_years.cfg @@ -0,0 +1,51 @@ +[default] +case = "#expand case_name#" +constraint = "#expand constraint#" +dry_run = "#expand dry_run#" +environment_commands = "#expand environment_commands#" +input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/#expand case_name# +input_subdir = archive/atm/hist +mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" +output = "#expand user_output#zppy_min_case_ilamb_diff_years_output/#expand unique_id#/#expand case_name#" +partition = "#expand partition_short#" +qos = "#expand qos_short#" +walltime = "00:30:00" +www = "#expand user_www#zppy_min_case_ilamb_diff_years_www/#expand unique_id#" + +[ts] +active = True +# 15 year increments for ts and e3sm_to_cmip, 30 for ilamb +years = "1985:2014:15" + + [[ atm_monthly_180x360_aave ]] + frequency = "monthly" + input_files = "eam.h0" + input_subdir = "archive/atm/hist" + vars = "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,ICEFRAC,LANDFRAC,OCNFRAC,PS,CLDICE,CLDLIQ,T,AODDUST" + + [[ land_monthly ]] + extra_vars = "landfrac" + frequency = "monthly" + input_files = "elm.h0" + input_subdir = "archive/lnd/hist" + mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc" + vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" + +[e3sm_to_cmip] +active = True +frequency = "monthly" +ts_num_years = 15 +years = "1985:2014:15" + + [[ atm_monthly_180x360_aave ]] + input_files = "eam.h0" + + [[ land_monthly ]] + input_files = "elm.h0" + +[ilamb] +active = True +nodes = 8 +short_name = "#expand case_name#" +ts_num_years = 15 +years = "1985:2014:30" diff --git a/tests/integration/template_min_case_ilamb_land_only.cfg b/tests/integration/template_min_case_ilamb_land_only.cfg index f737d946..add58685 100644 --- a/tests/integration/template_min_case_ilamb_land_only.cfg +++ b/tests/integration/template_min_case_ilamb_land_only.cfg @@ -13,9 +13,7 @@ www = "#expand user_www#zppy_min_case_ilamb_land_only_www/#expand unique_id#" [ts] active = True -e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" walltime = "00:50:00" -ts_fmt = "cmip" years = "1985:2014:30" [[ land_monthly ]] @@ -26,6 +24,16 @@ years = "1985:2014:30" mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc" vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" +[e3sm_to_cmip] +active = True +frequency = "monthly" +ts_num_years = 30 +walltime = "00:50:00" +years = "1985:2014:30" + + [[ land_monthly ]] + input_files = "elm.h0" + [ilamb] active = True land_only = True diff --git a/tests/integration/template_min_case_nco.cfg b/tests/integration/template_min_case_nco.cfg index ec830d2f..9b9ea993 100644 --- a/tests/integration/template_min_case_nco.cfg +++ b/tests/integration/template_min_case_nco.cfg @@ -40,14 +40,12 @@ walltime = "00:30:00" [ts] active = True -e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" - ts_fmt = "cmip" [[ atm_daily_180x360_aave ]] frequency = "daily" @@ -85,5 +83,4 @@ walltime = "00:30:00" input_files = "elm.h0" input_subdir = "archive/lnd/hist" mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc" - ts_fmt = "cmip" vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" diff --git a/tests/integration/template_weekly_bundles.cfg b/tests/integration/template_weekly_bundles.cfg index 4ab9b60d..b5aeb8d6 100644 --- a/tests/integration/template_weekly_bundles.cfg +++ b/tests/integration/template_weekly_bundles.cfg @@ -8,6 +8,8 @@ # > atm_monthly_diurnal_8xdaily_180x360_aave # ts > atm_monthly_180x360_aave # > land_monthly +# e3sm_to_cmip > atm_monthly_180x360_aave +# > land_monthly # e3sm_diags > atm_monthly_180x360_aave # bundle 2: # ts > atm_monthly_glb @@ -59,14 +61,12 @@ years = "1985:1989:2", [ts] active = True bundle = "bundle1" -e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" years = "1985:1989:2", [[ atm_monthly_180x360_aave ]] frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" - ts_fmt = "cmip" [[ atm_monthly_glb ]] bundle = "bundle2" # Override bundle1 @@ -77,14 +77,12 @@ years = "1985:1989:2", years = "1985:1995:5", [[ land_monthly ]] - e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" extra_vars = "landfrac" frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc" vars = "LAISHA,LAISUN" - ts_fmt = "cmip" [[ rof_monthly ]] bundle = "bundle3" # Override bundle1, let bundle1 finish first because "e3sm_diags: atm_monthly_180x360_aave_mvm" requires "ts: atm_monthly_180x360_aave" @@ -95,6 +93,19 @@ years = "1985:1989:2", mapping_file = "" vars = "RIVER_DISCHARGE_OVER_LAND_LIQ" +[e3sm_to_cmip] +active = True +bundle = "bundle1" +frequency = "monthly" +ts_num_years = 2 +years = "1985:1989:2", + + [[ atm_monthly_180x360_aave ]] + input_files = "eam.h0" + + [[ land_monthly ]] + input_files = "elm.h0" + # TODO: Add "tc_analysis" back in after empty dat is resolved. # [tc_analysis] # active = True diff --git a/tests/integration/template_weekly_comprehensive_v2.cfg b/tests/integration/template_weekly_comprehensive_v2.cfg index 7bffe3e5..db3d592a 100644 --- a/tests/integration/template_weekly_comprehensive_v2.cfg +++ b/tests/integration/template_weekly_comprehensive_v2.cfg @@ -37,14 +37,12 @@ walltime = "00:30:00" [ts] active = True -e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" - ts_fmt = "cmip" [[ atm_daily_180x360_aave ]] frequency = "daily" @@ -81,9 +79,20 @@ walltime = "00:30:00" frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" - ts_fmt = "cmip" vars = "LAISHA,LAISUN" +[e3sm_to_cmip] +active = True +frequency = "monthly" +ts_num_years=2 +walltime = "00:30:00" + + [[ atm_monthly_180x360_aave ]] + input_files = "eam.h0" + + [[ land_monthly ]] + input_files = "elm.h0" + [tc_analysis] active = True walltime = "00:30:00" @@ -180,7 +189,7 @@ years = "1980-1990", [ilamb] active = True nodes = 8 -partition = "#expand partition_long#" +partition = "#expand partition_short#" short_name = "#expand case_name_v2#" ts_num_years = 2 walltime = "2:00:00" diff --git a/tests/integration/template_weekly_comprehensive_v3.cfg b/tests/integration/template_weekly_comprehensive_v3.cfg index bf66c268..7872c923 100644 --- a/tests/integration/template_weekly_comprehensive_v3.cfg +++ b/tests/integration/template_weekly_comprehensive_v3.cfg @@ -40,14 +40,12 @@ walltime = "00:30:00" [ts] active = True -e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" walltime = "00:30:00" [[ atm_monthly_180x360_aave ]] frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" - ts_fmt = "cmip" [[ atm_daily_180x360_aave ]] frequency = "daily" @@ -85,9 +83,22 @@ walltime = "00:30:00" input_files = "elm.h0" input_subdir = "archive/lnd/hist" mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc" - ts_fmt = "cmip" vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" +[e3sm_to_cmip] +active = True +frequency = "monthly" +ts_num_years=2 +walltime = "00:30:00" + + [[ atm_monthly_180x360_aave ]] + input_files = "eam.h0" + ts_subsection = "atm_monthly_180x360_aave" + + [[ land_monthly ]] + input_files = "elm.h0" + ts_subsection = "land_monthly" + # TODO: Add "tc_analysis" back in after empty dat is resolved. # [tc_analysis] # active = True @@ -207,9 +218,11 @@ years = "1985-1995", [ilamb] active = True +e3sm_to_cmip_atm_subsection = "atm_monthly_180x360_aave" +e3sm_to_cmip_land_subsection = "land_monthly" ilamb_obs = "#expand diagnostics_base_path#/ilamb_data" nodes = 8 -partition = "#expand partition_long#" +partition = "#expand partition_short#" short_name = "#expand case_name#" ts_atm_subsection = "atm_monthly_180x360_aave" ts_land_subsection = "land_monthly" diff --git a/tests/integration/test_weekly.py b/tests/integration/test_weekly.py index 5ee8f827..5a5710cb 100644 --- a/tests/integration/test_weekly.py +++ b/tests/integration/test_weekly.py @@ -23,25 +23,41 @@ def check_images(test_name, case_name): # The directory to place differences in. diff_dir = f"{actual_images_dir}image_check_failures_{test_name}" + # To run a subset of the image-check test: + # 1) comment out sections you don't want to run in the `cfg` + # 2) set `subdirs_to_check` to the subset of tasks you want to run + subdirs_to_check = ["mpas_analysis", "e3sm_diags", "global_time_series", "ilamb"] + check_mismatched_images( - actual_images_dir, expected_images_file, expected_images_dir, diff_dir + actual_images_dir, + expected_images_file, + expected_images_dir, + diff_dir, + subdirs_to_check, ) # Run with: # pytest tests/integration/test_weekly.py +# Comment/uncomment `skip` declarations to run specific tests. + + +# @pytest.mark.skip(reason="Not testing") def test_comprehensive_v2_images(): check_images("comprehensive_v2", V2_CASE_NAME) +# @pytest.mark.skip(reason="Not testing") def test_comprehensive_v3_images(): check_images("comprehensive_v3", V3_CASE_NAME) +# @pytest.mark.skip(reason="Not testing") def test_bundles_images(): check_images("bundles", V3_CASE_NAME) +# @pytest.mark.skip(reason="Not testing") def test_bundles_bash_file_list(): # Check that the correct bash files are generated expansions = get_expansions() @@ -67,6 +83,10 @@ def test_bundles_bash_file_list(): "e3sm_diags_atm_monthly_180x360_aave_model_vs_obs_1985-1986.bash", "e3sm_diags_atm_monthly_180x360_aave_model_vs_obs_1987-1988.bash", "e3sm_diags_atm_monthly_180x360_aave_mvm_model_vs_model_1987-1988_vs_1985-1986.bash", + "e3sm_to_cmip_atm_monthly_180x360_aave_1985-1986-0002.bash", + "e3sm_to_cmip_atm_monthly_180x360_aave_1987-1988-0002.bash", + "e3sm_to_cmip_land_monthly_1985-1986-0002.bash", + "e3sm_to_cmip_land_monthly_1987-1988-0002.bash", "global_time_series_1985-1995.bash", "ilamb_1985-1986.bash", # "tc_analysis_1985-1986.bash", @@ -84,6 +104,7 @@ def test_bundles_bash_file_list(): assert actual_bash_files == expected_bash_files +# @pytest.mark.skip(reason="Not testing") def test_bundles_bash_file_content(): expansions = get_expansions() user_output = expansions["user_output"] diff --git a/tests/integration/utils.py b/tests/integration/utils.py index d008fd41..ae7753fe 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -53,11 +53,13 @@ def compare_images( fraction = num_nonzero_pixels / num_pixels # Fraction of mismatched pixels should be less than 0.02% if fraction >= 0.0002: - print("\npath_to_actual_png={}".format(path_to_actual_png)) - print("path_to_expected_png={}".format(path_to_expected_png)) - print("diff has {} nonzero pixels.".format(num_nonzero_pixels)) - print("total number of pixels={}".format(num_pixels)) - print("num_nonzero_pixels/num_pixels fraction={}".format(fraction)) + verbose = False + if verbose: + print("\npath_to_actual_png={}".format(path_to_actual_png)) + print("path_to_expected_png={}".format(path_to_expected_png)) + print("diff has {} nonzero pixels.".format(num_nonzero_pixels)) + print("total number of pixels={}".format(num_pixels)) + print("num_nonzero_pixels/num_pixels fraction={}".format(fraction)) mismatched_images.append(image_name) @@ -90,7 +92,11 @@ def compare_images( def check_mismatched_images( - actual_images_dir, expected_images_file, expected_images_dir, diff_dir + actual_images_dir, + expected_images_file, + expected_images_dir, + diff_dir, + subdirs_to_check, ): missing_images: List[str] = [] mismatched_images: List[str] = [] @@ -98,33 +104,45 @@ def check_mismatched_images( counter = 0 with open(expected_images_file) as f: for line in f: - counter += 1 - if counter % 250 == 0: - print("On line #", counter) image_name = line.strip("./").strip("\n") - path_to_actual_png = os.path.join(actual_images_dir, image_name) - path_to_expected_png = os.path.join(expected_images_dir, image_name) - - compare_images( - missing_images, - mismatched_images, - image_name, - path_to_actual_png, - path_to_expected_png, - diff_dir, - ) + proceed = False + for subdir in subdirs_to_check: + if image_name.startswith(subdir): + proceed = True + break + if proceed: + counter += 1 + if counter % 250 == 0: + print("On line #", counter) + path_to_actual_png = os.path.join(actual_images_dir, image_name) + path_to_expected_png = os.path.join(expected_images_dir, image_name) + + compare_images( + missing_images, + mismatched_images, + image_name, + path_to_actual_png, + path_to_expected_png, + diff_dir, + ) + print(f"Total number of images checked: {counter}") if missing_images: - print("Missing images:") + print(f"Missing images: {len(missing_images)}") for i in missing_images: print(i) if mismatched_images: - print("Mismatched images:") + print(f"Mismatched images: {len(mismatched_images)}") for i in mismatched_images: print(i) # Make diff_dir readable - os.system(f"chmod -R 755 {diff_dir}") + if os.path.exists(diff_dir): + os.system(f"chmod -R 755 {diff_dir}") + else: + # diff_dir won't exist if all the expected images are missing + # That is, if we're in this case, we expect the following: + assert len(missing_images) == counter assert missing_images == [] assert mismatched_images == [] @@ -148,7 +166,6 @@ def get_chyrsalis_expansions(config): # To use default environment_commands, set to "" "diags_environment_commands": "source /conda.sh; conda activate ", "diags_walltime": "5:00:00", - "e3sm_to_cmip_environment_commands": "", "environment_commands_test": "", "expected_dir": "/lcrc/group/e3sm/public_html/zppy_test_resources/", "global_time_series_environment_commands": "source /conda.sh; conda activate ", @@ -178,7 +195,6 @@ def get_compy_expansions(config): # To use default environment_commands, set to "" "diags_environment_commands": "source /conda.sh; conda activate ", "diags_walltime": "03:00:00", - "e3sm_to_cmip_environment_commands": "", "environment_commands_test": "", "expected_dir": "/compyfs/www/zppy_test_resources/", "global_time_series_environment_commands": "source /conda.sh; conda activate ", @@ -208,7 +224,6 @@ def get_perlmutter_expansions(config): # To use default environment_commands, set to "" "diags_environment_commands": "source /conda.sh; conda activate ", "diags_walltime": "6:00:00", - "e3sm_to_cmip_environment_commands": "", "environment_commands_test": "", "expected_dir": "/global/cfs/cdirs/e3sm/www/zppy_test_resources/", "global_time_series_environment_commands": "source /conda.sh; conda activate ", @@ -317,6 +332,7 @@ def generate_cfgs(unified_testing=False, dry_run=False): "min_case_global_time_series_custom", "min_case_global_time_series_original_8_no_ocn", "min_case_global_time_series_original_8", + "min_case_ilamb_diff_years", "min_case_ilamb_land_only", "min_case_ilamb", "min_case_mpas_analysis", @@ -356,13 +372,7 @@ def generate_cfgs(unified_testing=False, dry_run=False): print( f"global_time_series_environment_commands={expansions['global_time_series_environment_commands']}" ) - print( - f"e3sm_to_cmip_environment_commands={expansions['e3sm_to_cmip_environment_commands']}" - ) print(f"environment_commands={expansions['environment_commands']}") - print( - "Reminder: `e3sm_to_cmip_environment_commands=''` => the environment of the `ts` task will be used" - ) print( "Reminder: `environment_commands=''` => the latest E3SM Unified environment will be used" ) diff --git a/tests/test_sections.py b/tests/test_sections.py index ab862d80..df49f4a9 100644 --- a/tests/test_sections.py +++ b/tests/test_sections.py @@ -72,7 +72,6 @@ def test_sections(): "constraint": "", "debug": False, "dry_run": False, - "e3sm_to_cmip_environment_commands": "", "environment_commands": "", "fail_on_dependency_skip": False, "frequency": "monthly", @@ -105,13 +104,11 @@ def test_sections(): expected_section = { "active": "True", "area_nm": "area", - "cmip_metadata": "inclusions/e3sm_to_cmip/default_metadata.json", "dpf": 30, "extra_vars": "", "input_component": "", "mapping_file": "MAPPING_FILE_TS", "tpd": 1, - "ts_fmt": "ts_only", "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT", "years": ["0001:0020:5"], } @@ -126,12 +123,10 @@ def test_sections(): "area_nm": "area", "campaign": "none", "case": "CASE", - "cmip_metadata": "inclusions/e3sm_to_cmip/default_metadata.json", "constraint": "", "debug": False, "dpf": 30, "dry_run": False, - "e3sm_to_cmip_environment_commands": "", "environment_commands": "", "extra_vars": "", "fail_on_dependency_skip": False, @@ -154,7 +149,6 @@ def test_sections(): "subsection": None, "templateDir": "zppy/templates", "tpd": 1, - "ts_fmt": "ts_only", "ts_num_years": 5, "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT", "walltime": "02:00:00", @@ -189,7 +183,6 @@ def test_sections(): "constraint": "", "debug": False, "dry_run": False, - "e3sm_to_cmip_environment_commands": "", "environment_commands": "", "exclude": False, "fail_on_dependency_skip": False, @@ -248,7 +241,6 @@ def test_subsections(): "constraint": "", "debug": False, "dry_run": False, - "e3sm_to_cmip_environment_commands": "", "environment_commands": "", "fail_on_dependency_skip": False, "frequency": "monthly", @@ -281,32 +273,26 @@ def test_subsections(): expected_section = { "active": "True", "area_nm": "area", - "cmip_metadata": "inclusions/e3sm_to_cmip/default_metadata.json", "dpf": 30, "extra_vars": "", "input_component": "", "tpd": 1, - "ts_fmt": "ts_only", "ts_grid1": { "area_nm": None, - "cmip_metadata": None, "dpf": None, "extra_vars": None, "input_component": None, "mapping_file": "MAPPING_FILE_TS_GRID1", "tpd": None, - "ts_fmt": None, "years": ["0001:0020:5"], }, "ts_grid2": { "area_nm": None, - "cmip_metadata": None, "dpf": None, "extra_vars": None, "input_component": None, "mapping_file": "MAPPING_FILE_TS_GRID2", "tpd": None, - "ts_fmt": None, "years": ["0001:0020:10"], }, "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT", @@ -322,12 +308,10 @@ def test_subsections(): "bundle": "", "campaign": "none", "case": "CASE", - "cmip_metadata": "inclusions/e3sm_to_cmip/default_metadata.json", "constraint": "", "debug": False, "dpf": 30, "dry_run": False, - "e3sm_to_cmip_environment_commands": "", "environment_commands": "", "extra_vars": "", "fail_on_dependency_skip": False, @@ -350,7 +334,6 @@ def test_subsections(): "subsection": "ts_grid1", "templateDir": "zppy/templates", "tpd": 1, - "ts_fmt": "ts_only", "ts_num_years": 5, "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT", "walltime": "02:00:00", @@ -366,12 +349,10 @@ def test_subsections(): "bundle": "", "campaign": "none", "case": "CASE", - "cmip_metadata": "inclusions/e3sm_to_cmip/default_metadata.json", "constraint": "", "debug": False, "dpf": 30, "dry_run": False, - "e3sm_to_cmip_environment_commands": "", "environment_commands": "", "extra_vars": "", "fail_on_dependency_skip": False, @@ -394,7 +375,6 @@ def test_subsections(): "subsection": "ts_grid2", "templateDir": "zppy/templates", "tpd": 1, - "ts_fmt": "ts_only", "ts_num_years": 5, "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT", "walltime": "02:00:00", @@ -445,7 +425,6 @@ def test_subsections(): "constraint": "", "debug": False, "dry_run": False, - "e3sm_to_cmip_environment_commands": "", "environment_commands": "", "exclude": False, "fail_on_dependency_skip": False, @@ -484,7 +463,6 @@ def test_subsections(): "constraint": "", "debug": False, "dry_run": False, - "e3sm_to_cmip_environment_commands": "", "environment_commands": "", "exclude": False, "fail_on_dependency_skip": False, diff --git a/tests/test_zppy_e3sm_diags.py b/tests/test_zppy_e3sm_diags.py index 7114d7cb..61216a86 100644 --- a/tests/test_zppy_e3sm_diags.py +++ b/tests/test_zppy_e3sm_diags.py @@ -546,7 +546,9 @@ def test_add_ts_dependencies(): c.update(base) dependencies: List[str] = [] add_ts_dependencies(c, dependencies, "script_dir", 1980) - assert dependencies == ["script_dir/ts_sub_1980-1984-0005.status"] + assert dependencies == [ + "script_dir/ts_sub_1980-1984-0005.status", + ] c = {"sets": ["streamflow"]} c.update(base) diff --git a/tests/test_zppy_ilamb.py b/tests/test_zppy_ilamb.py index 0f828305..a130ba7b 100644 --- a/tests/test_zppy_ilamb.py +++ b/tests/test_zppy_ilamb.py @@ -5,6 +5,7 @@ def test_determine_and_add_dependencies(): c = { + "e3sm_to_cmip_land_subsection": "land_monthly", "land_only": True, "ts_land_subsection": "land_monthly", "year1": 1980, @@ -16,11 +17,15 @@ def test_determine_and_add_dependencies(): expected = [ "script_dir/ts_land_monthly_1980-1984-0005.status", "script_dir/ts_land_monthly_1985-1989-0005.status", + "script_dir/e3sm_to_cmip_land_monthly_1980-1984-0005.status", + "script_dir/e3sm_to_cmip_land_monthly_1985-1989-0005.status", ] assert dependencies == expected # Have zppy guess the subsection names c = { + "e3sm_to_cmip_atm_subsection": "", + "e3sm_to_cmip_land_subsection": "", "land_only": False, "ts_land_subsection": "", "ts_atm_subsection": "", @@ -35,7 +40,11 @@ def test_determine_and_add_dependencies(): expected = [ "script_dir/ts_land_monthly_1980-1984-0005.status", "script_dir/ts_land_monthly_1985-1989-0005.status", + "script_dir/e3sm_to_cmip_land_monthly_1980-1984-0005.status", + "script_dir/e3sm_to_cmip_land_monthly_1985-1989-0005.status", "script_dir/ts_atm_monthly_180x360_aave_1980-1984-0005.status", "script_dir/ts_atm_monthly_180x360_aave_1985-1989-0005.status", + "script_dir/e3sm_to_cmip_atm_monthly_180x360_aave_1980-1984-0005.status", + "script_dir/e3sm_to_cmip_atm_monthly_180x360_aave_1985-1989-0005.status", ] assert dependencies == expected diff --git a/zppy/__main__.py b/zppy/__main__.py index 50e6d572..3685ace4 100644 --- a/zppy/__main__.py +++ b/zppy/__main__.py @@ -12,6 +12,7 @@ from zppy.bundle import Bundle, predefined_bundles from zppy.climo import climo from zppy.e3sm_diags import e3sm_diags +from zppy.e3sm_to_cmip import e3sm_to_cmip from zppy.global_time_series import global_time_series from zppy.ilamb import ilamb from zppy.logger import _setup_custom_logger @@ -226,6 +227,9 @@ def _launch_scripts(config: ConfigObj, script_dir, job_ids_file, plugins) -> Non # time series tasks existing_bundles = ts(config, script_dir, existing_bundles, job_ids_file) + # e3sm_to_cmip tasks + existing_bundles = e3sm_to_cmip(config, script_dir, existing_bundles, job_ids_file) + # tc_analysis tasks existing_bundles = tc_analysis(config, script_dir, existing_bundles, job_ids_file) diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index 72060bb5..25d2e834 100644 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -17,11 +17,6 @@ debug = boolean(default=False) # diagnostics_base_path -- NOTE: this parameter is created internally # This should be set to True if you don't want the batch jobs to be submitted dry_run = boolean(default=False) -# Set up the environment for e3sm_to_cmip. -# Because e3sm_to_cmip is used internally in the `ts` task, -# the `environment_commands` parameter below is insufficient -# and this parameter must be used instead. -e3sm_to_cmip_environment_commands = string(default="") # Set up the environment -- this is where you can tell zppy to use a custom conda environment. # To use a custom conda environment, you can set `environment_commands="source ; conda activate "`. environment_commands = string(default="") @@ -106,28 +101,44 @@ input_component = string(default="") [ts] area_nm = string(default="area") -cmip_metadata = string(default="inclusions/e3sm_to_cmip/default_metadata.json") # Days per file dpf = integer(default=30) extra_vars = string(default="") # Time-steps per day tpd = integer(default=1) -ts_fmt = string(default="ts_only") # Model component having generated input files (eam, eamxx, elm, mosart, ...) input_component = string(default="") [[__many__]] area_nm = string(default=None) - cmip_metadata = string(default=None) dpf = integer(default=None) extra_vars = string(default=None) tpd = integer(default=None) - ts_fmt = string(default=None) input_component = string(default=None) +[e3sm_to_cmip] +# Metadata json file +cmip_metadata = string(default="inclusions/e3sm_to_cmip/default_metadata.json") +# variables to run e3sm_to_cmip on +cmip_vars = string(default="") +# Model component having generated input files (eam, eamxx, elm, mosart, ...) +input_component = string(default="") +# Name of the grid used by the relevant `[ts]` task +ts_grid = string(default="180x360_aave") +# Name of the `[ts]` subtask to depend on +ts_subsection = string(default="") + + [[__many__]] + cmip_metadata = string(default=None) + cmip_vars = string(default=None) + input_component = string(default=None) + ts_grid = string(default=None) + ts_subsection = string(default=None) + [tc_analysis] # NOTE: always overrides value in [default] input_files = string(default="eam.h2") +# TODO for v3.0.0: Remove this parameter # DEPRECATED. No longer used. # The scratch directory scratch = string(default="") @@ -289,6 +300,7 @@ stream_ocn = string(default="streams.ocean") walltime = string(default="06:00:00") [global_time_series] +# TODO for v3.0.0: Remove this parameter # Deprecated; kept for backwards compatibility # Set to True to skip figures requiring ocean data; only affects original plots atmosphere_only = boolean(default=False) @@ -302,6 +314,7 @@ figstr = string(default="") # NOTE: always overrides value in [default] input_subdir = string(default="archive/ocn/hist") moc_file = string(default="") +# TODO for v3.0.0: Remove this parameter # Deprecated; legacy name for plots_original; kept for backwards compatibility # plots_original replaces it with the same default. # So, if a cfg used the default value before, the behavior will remain the same. @@ -329,13 +342,19 @@ ts_years = string_list(default=list("")) [ilamb] cfg = string(default="inclusions/ilamb/cmip.cfg") +# Name of the `[e3sm_to_cmip]` atm subtask to depend on +e3sm_to_cmip_atm_subsection = string(default="") +# Name of the `[e3sm_to_cmip]` land subtask to depend on +e3sm_to_cmip_land_subsection = string(default="") # observational data path, default uses mache's "diagnostics_base_path" ilamb_obs = string(default="") # for land_only run land_only = boolean(default=False) -ts_atm_subsection = string(default="") -ts_land_subsection = string(default="") -# Name of the grid used by the relevant `[ts]` `atm` task +# Name of the grid used by the relevant `[ts]` atm subtask ts_atm_grid = string(default="180x360_aave") -# Name of the grid used by the relevant `[ts]` `land` task +# Name of the `[ts]` atm subtask to depend on +ts_atm_subsection = string(default="") +# Name of the grid used by the relevant `[ts]` land subtask ts_land_grid = string(default="180x360_aave") +# Name of the `[ts]` land subtask to depend on +ts_land_subsection = string(default="") diff --git a/zppy/e3sm_diags.py b/zppy/e3sm_diags.py index bbe93b57..2c77f5b9 100644 --- a/zppy/e3sm_diags.py +++ b/zppy/e3sm_diags.py @@ -285,6 +285,7 @@ def add_ts_dependencies( ) depend_on_ts: Set[str] = set(["enso_diags", "qbo", "area_mean_time_series"]) if depend_on_ts & set(c["sets"]): + # ts task add_dependencies( dependencies, script_dir, diff --git a/zppy/e3sm_to_cmip.py b/zppy/e3sm_to_cmip.py new file mode 100644 index 00000000..38decbe5 --- /dev/null +++ b/zppy/e3sm_to_cmip.py @@ -0,0 +1,108 @@ +from typing import Any, Dict, List, Tuple + +from configobj import ConfigObj + +from zppy.bundle import handle_bundles +from zppy.utils import ( + ParameterGuessType, + add_dependencies, + check_status, + define_or_guess, + define_or_guess2, + get_file_names, + get_tasks, + get_years, + initialize_template, + make_executable, + set_component_and_prc_typ, + submit_script, + write_settings_file, +) + + +# ----------------------------------------------------------------------------- +def e3sm_to_cmip(config: ConfigObj, script_dir: str, existing_bundles, job_ids_file): + + template, _ = initialize_template(config, "e3sm_to_cmip.bash") + + # --- List of tasks --- + tasks: List[Dict[str, Any]] = get_tasks(config, "e3sm_to_cmip") + if len(tasks) == 0: + return existing_bundles + + # --- Generate and submit e3sm_to_cmip scripts --- + for c in tasks: + dependencies: List[str] = [] + set_component_and_prc_typ(c) + c["cmor_tables_prefix"] = c["diagnostics_base_path"] + year_sets: List[Tuple[int, int]] = get_years(c["years"]) + # Loop over year sets + for s in year_sets: + c["yr_start"] = s[0] + c["yr_end"] = s[1] + if ("last_year" in c.keys()) and (c["yr_end"] > c["last_year"]): + continue # Skip this year set + c["ypf"] = s[1] - s[0] + 1 + c["scriptDir"] = script_dir + if "ts_num_years" in c.keys(): + c["ts_num_years"] = int(c["ts_num_years"]) + sub: str = define_or_guess( + c, "subsection", "grid", ParameterGuessType.SECTION_GUESS + ) + # Run default variables if none are specified + if c["cmip_vars"] == "": + if c["component"] == "atm": + c["cmip_vars"] = "pr, tas, rsds, rlds, rsus" + elif c["component"] == "lnd": + c["cmip_vars"] = ( + "mrsos, mrso, mrfso, mrros, mrro, prveg, evspsblveg, evspsblsoi, tran, tsl, lai, cLitter, cProduct, cSoilFast, cSoilMedium, cSoilSlow, fFire, fHarvest, cVeg, nbp, gpp, ra, rh" + ) + prefix = f"e3sm_to_cmip_{sub}_{c['yr_start']:04d}-{c['yr_end']:04d}-{c['ypf']:04d}" + print(prefix) + c["prefix"] = prefix + bash_file, settings_file, status_file = get_file_names(script_dir, prefix) + skip: bool = check_status(status_file) + if skip: + continue + # Create script + with open(bash_file, "w") as f: + f.write(template.render(**c)) + make_executable(bash_file) + # Default to the name of this task if ts_subsection is not defined + define_or_guess2(c, "ts_subsection", sub, ParameterGuessType.SECTION_GUESS) + add_dependencies( + dependencies, + script_dir, + "ts", + c["ts_subsection"], + c["yr_start"], + c["yr_end"], + c["ts_num_years"], + ) + c["dependencies"] = dependencies + write_settings_file(settings_file, c, s) + export = "ALL" + existing_bundles = handle_bundles( + c, + bash_file, + export, + dependFiles=dependencies, + existing_bundles=existing_bundles, + ) + if not c["dry_run"]: + if c["bundle"] == "": + # Submit job + submit_script( + bash_file, + status_file, + export, + job_ids_file, + dependFiles=dependencies, + fail_on_dependency_skip=c["fail_on_dependency_skip"], + ) + else: + print(f"...adding to bundle {c['bundle']}") + + print(f" environment_commands={c['environment_commands']}") + + return existing_bundles diff --git a/zppy/ilamb.py b/zppy/ilamb.py index f3a44a26..c4a159d7 100644 --- a/zppy/ilamb.py +++ b/zppy/ilamb.py @@ -109,6 +109,21 @@ def determine_and_add_dependencies( c["year2"], c["ts_num_years"], ) + define_or_guess2( + c, + "e3sm_to_cmip_land_subsection", + "land_monthly", + ParameterGuessType.SECTION_GUESS, + ) + add_dependencies( + dependencies, + script_dir, + "e3sm_to_cmip", + c["e3sm_to_cmip_land_subsection"], + c["year1"], + c["year2"], + c["ts_num_years"], + ) if not c["land_only"]: define_or_guess2( c, @@ -125,3 +140,18 @@ def determine_and_add_dependencies( c["year2"], c["ts_num_years"], ) + define_or_guess2( + c, + "e3sm_to_cmip_atm_subsection", + "atm_monthly_180x360_aave", + ParameterGuessType.SECTION_GUESS, + ) + add_dependencies( + dependencies, + script_dir, + "e3sm_to_cmip", + c["e3sm_to_cmip_atm_subsection"], + c["year1"], + c["year2"], + c["ts_num_years"], + ) diff --git a/zppy/templates/bundle.bash b/zppy/templates/bundle.bash index 7ac01532..5a20a496 100644 --- a/zppy/templates/bundle.bash +++ b/zppy/templates/bundle.bash @@ -1,21 +1,6 @@ #!/bin/bash -{% include 'inclusions/slurm_header.sh' %} - -# Turn on debug output if needed -debug={{ debug }} -if [[ "${debug,,}" == "true" ]]; then - set -x -fi - -# Script dir -cd {{ scriptDir }} - -# Get jobid -id=${SLURM_JOBID} - -# Update status file -STARTTIME=$(date +%s) -echo "RUNNING ${id}" > {{ prefix }}.status +{% include 'inclusions/slurm_header.bash' %} +{% include 'inclusions/boilerplate.bash' %} # Loop over all tasks error=false diff --git a/zppy/templates/climo.bash b/zppy/templates/climo.bash index b01af510..333de34c 100644 --- a/zppy/templates/climo.bash +++ b/zppy/templates/climo.bash @@ -1,23 +1,8 @@ #!/bin/bash -{% include 'inclusions/slurm_header.sh' %} +{% include 'inclusions/slurm_header.bash' %} +{% include 'inclusions/boilerplate.bash' %} {{ environment_commands }} -# Turn on debug output if needed -debug={{ debug }} -if [[ "${debug,,}" == "true" ]]; then - set -x -fi - -# Script dir -cd {{ scriptDir }} - -# Get jobid -id=${SLURM_JOBID} - -# Update status file -STARTTIME=$(date +%s) -echo "RUNNING ${id}" > {{ prefix }}.status - # Create temporary workdir hash=`mktemp --dry-run -d XXXX` workdir=tmp.{{ prefix }}.${id}.${hash} diff --git a/zppy/templates/e3sm_diags.bash b/zppy/templates/e3sm_diags.bash index f94cca41..304161c3 100644 --- a/zppy/templates/e3sm_diags.bash +++ b/zppy/templates/e3sm_diags.bash @@ -1,27 +1,11 @@ #!/bin/bash -{% include 'inclusions/slurm_header.sh' %} - +{% include 'inclusions/slurm_header.bash' %} +{% include 'inclusions/boilerplate.bash' %} {{ environment_commands }} -# Turn on debug output if needed -debug={{ debug }} -if [[ "${debug,,}" == "true" ]]; then - set -x -fi - # Make sure UVCDAT doesn't prompt us about anonymous logging export UVCDAT_ANONYMOUS_LOG=False -# Script dir -cd {{ scriptDir }} - -# Get jobid -id=${SLURM_JOBID} - -# Update status file -STARTTIME=$(date +%s) -echo "RUNNING ${id}" > {{ prefix }}.status - # Basic definitions case="{{ case }}" short="{{ short_name }}" diff --git a/zppy/templates/e3sm_to_cmip.bash b/zppy/templates/e3sm_to_cmip.bash new file mode 100644 index 00000000..ac30d992 --- /dev/null +++ b/zppy/templates/e3sm_to_cmip.bash @@ -0,0 +1,85 @@ +#!/bin/bash +{% include 'inclusions/slurm_header.bash' %} +{% include 'inclusions/boilerplate.bash' %} +{{ environment_commands }} + +# Create temporary workdir +hash=`mktemp --dry-run -d XXXX` +workdir=tmp.{{ prefix }}.${id}.${hash} +mkdir ${workdir} +cd ${workdir} + +# From the ts task +dest={{ output }}/post/{{ component }}/{{ ts_grid }}/ts/{{ frequency }}/{{ '%dyr' % (ypf) }} + +tmp_dir=tmp_{{ prefix }} + +# Generate CMIP ts +cat > default_metadata.json << EOF +{% include cmip_metadata %} +EOF +{ + export cmortables_dir={{ cmor_tables_prefix }}/e3sm_to_cmip_data/cmip6-cmor-tables/Tables + input_dir=${dest}/{{ '%04d' % (yr_start) }}_{{ '%04d' % (yr_end) }} + mkdir -p $input_dir + + cp -s $dest/*_{{ '%04d' % (yr_start) }}??_{{ '%04d' % (yr_end) }}??.nc $input_dir + dest_cmip={{ output }}/post/{{ component }}/{{ ts_grid }}/cmip_ts/{{ frequency }} + mkdir -p ${dest_cmip} + + srun -N 1 e3sm_to_cmip \ + --output-path \ + ${dest_cmip}/${tmp_dir} \ + --var-list \ + {{ cmip_vars }} \ + --realm \ + {{ component }} \ + --input-path \ + ${input_dir} \ + --user-metadata \ + {{ scriptDir }}/${workdir}/default_metadata.json \ + --num-proc \ + 12 \ + --tables-path \ + ${cmortables_dir} + + if [ $? != 0 ]; then + cd {{ scriptDir }} + echo 'ERROR (1)' > {{ prefix }}.status + exit 1 + fi + + # Move output ts files to final destination + mv ${dest_cmip}/${tmp_dir}/CMIP6/CMIP/*/*/*/*/*/*/*/*/*.nc ${dest_cmip} + if [ $? != 0 ]; then + cd {{ scriptDir }} + echo 'ERROR (2)' > {{ prefix }}.status + exit 2 + fi + + rm -r ${dest_cmip}/${tmp_dir} + +} +if [ $? != 0 ]; then + cd {{ scriptDir }} + echo 'ERROR (3)' > {{ prefix }}.status + exit 3 +fi + +# Delete temporary workdir +cd .. +if [[ "${debug,,}" != "true" ]]; then + rm -rf ${workdir} +fi + +# Update status file and exit +{% raw %} +ENDTIME=$(date +%s) +ELAPSEDTIME=$(($ENDTIME - $STARTTIME)) +{% endraw %} +echo ============================================== +echo "Elapsed time: $ELAPSEDTIME seconds" +echo ============================================== +rm -f {{ prefix }}.status +echo 'OK' > {{ prefix }}.status +exit 0 diff --git a/zppy/templates/global_time_series.bash b/zppy/templates/global_time_series.bash index 8c367528..4a3183eb 100644 --- a/zppy/templates/global_time_series.bash +++ b/zppy/templates/global_time_series.bash @@ -1,21 +1,8 @@ #!/bin/bash -{% include 'inclusions/slurm_header.sh' %} +{% include 'inclusions/slurm_header.bash' %} +{% include 'inclusions/boilerplate.bash' %} {{ environment_commands }} -# Turn on debug output if needed -debug={{ debug }} -if [[ "${debug,,}" == "true" ]]; then - set -x -fi - -# Get jobid -id=${SLURM_JOBID} - -# Update status file -STARTTIME=$(date +%s) -cd {{ scriptDir }} -echo "RUNNING ${id}" > {{ prefix }}.status - # Generate global time series plots ################################################################################ results_dir={{ prefix }}_results diff --git a/zppy/templates/ilamb.bash b/zppy/templates/ilamb.bash index 155c9b76..a43c23e6 100644 --- a/zppy/templates/ilamb.bash +++ b/zppy/templates/ilamb.bash @@ -1,27 +1,11 @@ #!/bin/bash -{% include 'inclusions/slurm_header.sh' %} - +{% include 'inclusions/slurm_header.bash' %} +{% include 'inclusions/boilerplate.bash' %} {{ environment_commands }} -# Turn on debug output if needed -debug={{ debug }} -if [[ "${debug,,}" == "true" ]]; then - set -x -fi - # Point to observation data export ILAMB_ROOT={{ ilamb_obs }} -# Script dir -cd {{ scriptDir }} - -# Get jobid -id=${SLURM_JOBID} - -# Update status file -STARTTIME=$(date +%s) -echo "RUNNING ${id}" > {{ prefix }}.status - # Basic definitions case="{{ case }}" short="{{ short_name }}" @@ -46,12 +30,35 @@ fi mkdir -p ${model_root}/${case} cd ${model_root}/${case} +echo +echo ===== Copy nc files ===== +echo + # Create output directory # Create local links to input cmip time-series files lnd_ts_for_ilamb={{ output }}/post/lnd/{{ ts_land_grid }}/cmip_ts/monthly/ atm_ts_for_ilamb={{ output }}/post/atm/{{ ts_atm_grid }}/cmip_ts/monthly/ -cp -s ${lnd_ts_for_ilamb}/*_*_*_*_*_*_${Y1}??-${Y2}??.nc . -cp -s ${atm_ts_for_ilamb}/*_*_*_*_*_*_${Y1}??-${Y2}??.nc . +# Go through the time series files for between year1 and year2, +# using a step size equal to the number of years per time series file +for year in `seq ${Y1} {{ ts_num_years }} ${Y2}`; +do + start_year=`printf "%04d" ${year}` + end_year_int=$((${start_year} + {{ ts_num_years }} - 1)) + end_year=`printf "%04d" ${end_year_int}` + echo "Copying files for ${start_year} to ${end_year}" + cp -s ${lnd_ts_for_ilamb}/*_*_*_*_*_*_${start_year}??-${end_year}??.nc . + if [ $? != 0 ]; then + cd {{ scriptDir }} + echo 'ERROR (1)' > {{ prefix }}.status + exit 1 + fi + cp -s ${atm_ts_for_ilamb}/*_*_*_*_*_*_${start_year}??-${end_year}??.nc . + if [ $? != 0 ]; then + cd {{ scriptDir }} + echo 'ERROR (2)' > {{ prefix }}.status + exit 2 + fi +done cd ../.. echo @@ -74,8 +81,8 @@ srun -N 1 ilamb-run --config ilamb.cfg --model_root $model_root --regions globa if [ $? != 0 ]; then cd {{ scriptDir }} - echo 'ERROR (1)' > {{ prefix }}.status - exit 1 + echo 'ERROR (3)' > {{ prefix }}.status + exit 3 fi # Copy output to web server @@ -88,8 +95,8 @@ web_dir=${www}/${case}/ilamb/{{ sub }}_${Y1}-${Y2} mkdir -p ${web_dir} if [ $? != 0 ]; then cd {{ scriptDir }} - echo 'ERROR (2)' > {{ prefix }}.status - exit 2 + echo 'ERROR (4)' > {{ prefix }}.status + exit 4 fi {% if machine in ['pm-cpu', 'pm-gpu'] %} @@ -111,8 +118,8 @@ results_dir=_build/ rsync -a --delete ${results_dir} ${web_dir}/ if [ $? != 0 ]; then cd {{ scriptDir }} - echo 'ERROR (3)' > {{ prefix }}.status - exit 3 + echo 'ERROR (5)' > {{ prefix }}.status + exit 5 fi {% if machine in ['pm-cpu', 'pm-gpu'] %} diff --git a/zppy/templates/inclusions/boilerplate.bash b/zppy/templates/inclusions/boilerplate.bash new file mode 100644 index 00000000..ad8691d1 --- /dev/null +++ b/zppy/templates/inclusions/boilerplate.bash @@ -0,0 +1,15 @@ +# Turn on debug output if needed +debug={{ debug }} +if [[ "${debug,,}" == "true" ]]; then + set -x +fi + +# Script dir +cd {{ scriptDir }} + +# Get jobid +id=${SLURM_JOBID} + +# Update status file +STARTTIME=$(date +%s) +echo "RUNNING ${id}" > {{ prefix }}.status diff --git a/zppy/templates/inclusions/slurm_header.sh b/zppy/templates/inclusions/slurm_header.bash similarity index 100% rename from zppy/templates/inclusions/slurm_header.sh rename to zppy/templates/inclusions/slurm_header.bash diff --git a/zppy/templates/mpas_analysis.bash b/zppy/templates/mpas_analysis.bash index e0b5d29a..53e01583 100644 --- a/zppy/templates/mpas_analysis.bash +++ b/zppy/templates/mpas_analysis.bash @@ -1,27 +1,12 @@ #!/bin/bash -{% include 'inclusions/slurm_header.sh' %} +{% include 'inclusions/slurm_header.bash' %} +{% include 'inclusions/boilerplate.bash' %} {{ environment_commands }} # Additional settings for MPAS-Analysis export OMP_NUM_THREADS=1 export HDF5_USE_FILE_LOCKING=FALSE -# Turn on debug output if needed -debug={{ debug }} -if [[ "${debug,,}" == "true" ]]; then - set -x -fi - -# Script dir -cd {{ scriptDir }} - -# Get jobid -id=${SLURM_JOBID} - -# Update status file -STARTTIME=$(date +%s) -echo "RUNNING ${id}" > {{ scriptDir }}/{{ prefix }}.status - # Basic definitions case="{{ case }}" www="{{ www }}" diff --git a/zppy/templates/tc_analysis.bash b/zppy/templates/tc_analysis.bash index 53a47b34..61f4988c 100644 --- a/zppy/templates/tc_analysis.bash +++ b/zppy/templates/tc_analysis.bash @@ -1,26 +1,10 @@ #!/bin/bash -{% include 'inclusions/slurm_header.sh' %} - +{% include 'inclusions/slurm_header.bash' %} +{% include 'inclusions/boilerplate.bash' %} {{ environment_commands }} set -e # Stop running script on error -# Turn on debug output if needed -debug={{ debug }} -if [[ "${debug,,}" == "true" ]]; then - set -x -fi - -# Script dir -cd {{ scriptDir }} - -# Get jobid -id=${SLURM_JOBID} - -# Update status file -STARTTIME=$(date +%s) -echo "RUNNING ${id}" > {{ prefix }}.status - # A Bash script to post-process E3SM 6 hourly (h2) instantaneous output to generate a text file storing Tropical Cyclone tracks # tempestremap and tempestextremes are built in e3sm-unified from version 1.5.0. diff --git a/zppy/templates/ts.bash b/zppy/templates/ts.bash index 6979bb1b..de24489a 100644 --- a/zppy/templates/ts.bash +++ b/zppy/templates/ts.bash @@ -1,23 +1,8 @@ #!/bin/bash -{% include 'inclusions/slurm_header.sh' %} +{% include 'inclusions/slurm_header.bash' %} +{% include 'inclusions/boilerplate.bash' %} {{ environment_commands }} -# Turn on debug output if needed -debug={{ debug }} -if [[ "${debug,,}" == "true" ]]; then - set -x -fi - -# Script dir -cd {{ scriptDir }} - -# Get jobid -id=${SLURM_JOBID} - -# Update status file -STARTTIME=$(date +%s) -echo "RUNNING ${id}" > {{ prefix }}.status - # Create temporary workdir hash=`mktemp --dry-run -d XXXX` workdir=tmp.{{ prefix }}.${id}.${hash} @@ -35,9 +20,6 @@ do done done -ts_fmt={{ ts_fmt }} -echo $ts_fmt - {%- if frequency != 'monthly' %} # For non-monthly input files, need to add the last file of the previous year year={{ yr_start - 1 }} @@ -133,72 +115,6 @@ if [ $? != 0 ]; then exit 3 fi -{%- if ts_fmt != 'ts_only' %} -tmp_dir=tmp_{{ prefix }} - -# Generate CMIP ts -cat > default_metadata.json << EOF -{% include cmip_metadata %} -EOF -{ - export cmortables_dir={{ cmor_tables_prefix }}/cmip6-cmor-tables/Tables - #input_dir={{ output }}/post/{{ component }}/{{ grid }}/ts/{{ frequency }}/{{ '%dyr' % (ypf) }} - input_dir=${dest}/{{ '%04d' % (yr_start) }}_{{ '%04d' % (yr_end) }} - mkdir -p $input_dir - - cp -s $dest/*_{{ '%04d' % (yr_start) }}??_{{ '%04d' % (yr_end) }}??.nc $input_dir - dest_cmip={{ output }}/post/{{ component }}/{{ grid }}/cmip_ts/{{ frequency }} - mkdir -p ${dest_cmip} - {{ e3sm_to_cmip_environment_commands }} - srun -N 1 e3sm_to_cmip \ - --output-path \ - ${dest_cmip}/${tmp_dir} \ - {% if input_files.split(".")[0] == 'clm2' or input_files.split(".")[0] == 'elm' -%} - --var-list \ - 'mrsos, mrso, mrfso, mrros, mrro, prveg, evspsblveg, evspsblsoi, tran, tsl, lai, cLitter, cProduct, cSoilFast, cSoilMedium, cSoilSlow, fFire, fHarvest, cVeg, nbp, gpp, ra, rh' \ - --realm \ - lnd \ - {% endif -%} - {% if input_files.split(".")[0] == 'cam' or input_files.split(".")[0] == 'eam' -%} - --var-list \ - 'pr, tas, rsds, rlds, rsus' \ - --realm \ - atm \ - {% endif -%} - --input-path \ - ${input_dir} \ - --user-metadata \ - {{ scriptDir }}/${workdir}/default_metadata.json \ - --num-proc \ - 12 \ - --tables-path \ - ${cmortables_dir} - - if [ $? != 0 ]; then - cd {{ scriptDir }} - echo 'ERROR (4)' > {{ prefix }}.status - exit 4 - fi - {{ environment_commands }} - - # Move output ts files to final destination - mv ${dest_cmip}/${tmp_dir}/CMIP6/CMIP/*/*/*/*/*/*/*/*/*.nc ${dest_cmip} - if [ $? != 0 ]; then - cd {{ scriptDir }} - echo 'ERROR (5)' > {{ prefix }}.status - exit 5 - fi - - rm -r ${dest_cmip}/${tmp_dir} - -} -if [ $? != 0 ]; then - cd {{ scriptDir }} - echo 'ERROR (6)' > {{ prefix }}.status - exit 6 -fi -{%- endif %} - # Delete temporary workdir cd .. if [[ "${debug,,}" != "true" ]]; then diff --git a/zppy/ts.py b/zppy/ts.py index eab2a3d1..b9976057 100644 --- a/zppy/ts.py +++ b/zppy/ts.py @@ -35,7 +35,6 @@ def ts(config: ConfigObj, script_dir: str, existing_bundles, job_ids_file): set_mapping_file(c) set_grid(c) set_component_and_prc_typ(c) - c["cmor_tables_prefix"] = c["diagnostics_base_path"] year_sets: List[Tuple[int, int]] = get_years(c["years"]) # Loop over year sets for s in year_sets: @@ -78,8 +77,5 @@ def ts(config: ConfigObj, script_dir: str, existing_bundles, job_ids_file): print(f"...adding to bundle {c['bundle']}") print(f" environment_commands={c['environment_commands']}") - print( - f" e3sm_to_cmip_environment_commands={c['e3sm_to_cmip_environment_commands']}" - ) return existing_bundles diff --git a/zppy/utils.py b/zppy/utils.py index 83cb32be..8b5fb51b 100644 --- a/zppy/utils.py +++ b/zppy/utils.py @@ -201,7 +201,7 @@ def set_grid(c: Dict[str, Any]) -> None: # Output component (for directory structure) and procedure type for ncclimo def set_component_and_prc_typ(c: Dict[str, Any]) -> None: - if c["input_component"] != "": + if ("input_component" in c.keys()) and (c["input_component"] != ""): tmp = c["input_component"] else: tmp = c["input_files"].split(".")[0]