Add e3sm_to_cmip task (#650)

* Add e3sm_to_cmip task * Add dependency handling * Update defaults * Fix cmortables_dir * Currently testing on v3 * Image check tests passing * Clean up code * Address comments * Update parameters * Handle ts dependency and output * Unit tests passing * v3 image check test passing * Min cases with diff years * Simplified tests passing * New expected results for weekly tests * Reset utils.py
E3SM-Project · Dec 18, 2024 · 2891d44 · 2891d44
1 parent ea0a84d
commit 2891d44
Show file tree

Hide file tree

Showing 67 changed files with 691 additions and 398 deletions.
diff --git a/tests/integration/generated/test_min_case_add_dependencies_chrysalis.cfg b/tests/integration/generated/test_min_case_add_dependencies_chrysalis.cfg
@@ -25,7 +25,6 @@ years = "1985:1995:10",
 
 [ts]
 active = True
-e3sm_to_cmip_environment_commands = ""
 walltime = "00:50:00"
 # ts is in 5 year increments
 years = "1985:1995:5"
@@ -35,7 +34,6 @@ years = "1985:1995:5"
   frequency = "monthly"
   input_files = "eam.h0"
   input_subdir = "archive/atm/hist"
-  ts_fmt = "cmip"
   vars = "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,ICEFRAC,LANDFRAC,OCNFRAC,PS,CLDICE,CLDLIQ,T,AODDUST"
 
   [[ land_monthly ]]
@@ -45,7 +43,6 @@ years = "1985:1995:5"
   input_files = "elm.h0"
   input_subdir = "archive/lnd/hist"
   mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
-  ts_fmt = "cmip"
   vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
 
   [[ rof_monthly ]]
@@ -79,6 +76,21 @@ years = "1985:1995:5"
   mapping_file = "glb"
   vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
 
+[e3sm_to_cmip]
+active = True
+frequency = "monthly"
+walltime = "00:50:00"
+ts_num_years = 5
+years = "1985:1995:5"
+
+  [[ atm_monthly_180x360_aave ]]
+  # for ilamb
+  input_files = "eam.h0"
+
+  [[ land_monthly ]]
+  # for ilamb
+  input_files = "elm.h0"
+
 [e3sm_diags]
 active = True
 environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"

diff --git a/tests/integration/generated/test_min_case_carryover_dependencies_chrysalis.cfg b/tests/integration/generated/test_min_case_carryover_dependencies_chrysalis.cfg
@@ -60,14 +60,12 @@ walltime = "00:30:00"
 
 [ts]
 active = True
-e3sm_to_cmip_environment_commands = ""
 walltime = "00:30:00"
 
   [[ atm_monthly_180x360_aave ]]
   frequency = "monthly"
   input_files = "eam.h0"
   input_subdir = "archive/atm/hist"
-  ts_fmt = "cmip"
 
   [[ rof_monthly ]]
   extra_vars = 'areatotal2'
@@ -99,9 +97,20 @@ walltime = "00:30:00"
   input_files = "elm.h0"
   input_subdir = "archive/lnd/hist"
   mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
-  ts_fmt = "cmip"
   vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
 
+[e3sm_to_cmip]
+active = True
+frequency = "monthly"
+ts_num_years = 2
+walltime = "00:30:00"
+
+  [[ atm_monthly_180x360_aave ]]
+  input_files = "eam.h0"
+
+  [[ land_monthly ]]
+  input_files = "elm.h0"
+
 # TODO: Add "tc_analysis" back in after empty dat is resolved.
 # [tc_analysis]
 # # The second run should run in parallel with the first run.

diff --git a/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_chrysalis.cfg
@@ -14,14 +14,12 @@ years = "1985:1989:4",
 
 [ts]
 active = True
-e3sm_to_cmip_environment_commands = ""
 walltime = "00:30:00"
 
   [[ atm_monthly_180x360_aave ]]
   frequency = "monthly"
   input_files = "eam.h0"
   input_subdir = "archive/atm/hist"
-  #ts_fmt = "cmip"
 
 [e3sm_diags]
 active = True

diff --git a/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_mvm_1_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_mvm_1_chrysalis.cfg
@@ -14,11 +14,9 @@ years = "1985:1989:4",
 
 [ts]
 active = True
-e3sm_to_cmip_environment_commands = ""
 walltime = "00:30:00"
 
   [[ atm_monthly_180x360_aave ]]
   frequency = "monthly"
   input_files = "eam.h0"
   input_subdir = "archive/atm/hist"
-  #ts_fmt = "cmip"
diff --git a/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_mvm_2_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_mvm_2_chrysalis.cfg
@@ -14,14 +14,12 @@ years = "1995:1999:4",
 
 [ts]
 active = True
-e3sm_to_cmip_environment_commands = ""
 walltime = "00:30:00"
 
   [[ atm_monthly_180x360_aave ]]
   frequency = "monthly"
   input_files = "eam.h0"
   input_subdir = "archive/atm/hist"
-  #ts_fmt = "cmip"
 
 [e3sm_diags]
 active = True

diff --git a/tests/integration/generated/test_min_case_e3sm_diags_streamflow_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_streamflow_chrysalis.cfg
@@ -14,7 +14,6 @@ years = "1985:1989:4",
 
 [ts]
 active = True
-e3sm_to_cmip_environment_commands = ""
 walltime = "00:30:00"
 
   [[ rof_monthly ]]

diff --git a/tests/integration/generated/test_min_case_e3sm_diags_streamflow_mvm_1_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_streamflow_mvm_1_chrysalis.cfg
@@ -14,7 +14,6 @@ years = "1985:1989:4",
 
 [ts]
 active = True
-e3sm_to_cmip_environment_commands = ""
 walltime = "00:30:00"
 
   [[ rof_monthly ]]

diff --git a/tests/integration/generated/test_min_case_e3sm_diags_streamflow_mvm_2_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_streamflow_mvm_2_chrysalis.cfg
@@ -14,7 +14,6 @@ years = "1995:1999:4",
 
 [ts]
 active = True
-e3sm_to_cmip_environment_commands = ""
 walltime = "00:30:00"
 
   [[ rof_monthly ]]

diff --git a/tests/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_chrysalis.cfg
@@ -14,7 +14,6 @@ years = "1985:1989:4",
 
 [ts]
 active = True
-e3sm_to_cmip_environment_commands = ""
 walltime = "00:30:00"
 
   [[ atm_daily_180x360_aave ]]

diff --git a/...s/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_mvm_1_chrysalis.cfg b/...s/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_mvm_1_chrysalis.cfg
@@ -14,7 +14,6 @@ years = "1985:1989:4",
 
 [ts]
 active = True
-e3sm_to_cmip_environment_commands = ""
 walltime = "00:30:00"
 
   [[ atm_daily_180x360_aave ]]

diff --git a/...s/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_mvm_2_chrysalis.cfg b/...s/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_mvm_2_chrysalis.cfg
@@ -14,7 +14,6 @@ years = "1995:1999:4",
 
 [ts]
 active = True
-e3sm_to_cmip_environment_commands = ""
 walltime = "00:30:00"
 
   [[ atm_daily_180x360_aave ]]

diff --git a/...tion/generated/test_min_case_global_time_series_comprehensive_v3_setup_only_chrysalis.cfg b/...tion/generated/test_min_case_global_time_series_comprehensive_v3_setup_only_chrysalis.cfg
@@ -17,7 +17,6 @@ years = "1985:1989:2",
 
 [ts]
 active = True
-e3sm_to_cmip_environment_commands = ""
 walltime = "00:30:00"
 
   [[ atm_monthly_glb ]]
@@ -51,14 +50,3 @@ walltime = "00:30:00"
 
 # (This cfg is the setup portion only)
 # [global_time_series]
-# active = True
-# climo_years = "1985-1989", "1990-1995",
-# environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
-# experiment_name = "v3.LR.historical_0051"
-# figstr = "v3.LR.historical_0051"
-# moc_file=mocTimeSeries_1985-1995.nc
-# plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
-# ts_num_years = 5
-# ts_years = "1985-1989", "1985-1995",
-# walltime = "00:30:00"
-# years = "1985-1995",
diff --git a/tests/integration/generated/test_min_case_global_time_series_custom_chrysalis.cfg b/tests/integration/generated/test_min_case_global_time_series_custom_chrysalis.cfg
@@ -13,7 +13,6 @@ www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_
 
 [ts]
 active = True
-e3sm_to_cmip_environment_commands = ""
 walltime = "00:30:00"
 years = "1985:1995:5",
 

diff --git a/tests/integration/generated/test_min_case_global_time_series_original_8_chrysalis.cfg b/tests/integration/generated/test_min_case_global_time_series_original_8_chrysalis.cfg
@@ -13,7 +13,6 @@ www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_
 
 [ts]
 active = True
-e3sm_to_cmip_environment_commands = ""
 walltime = "00:30:00"
 
   [[ atm_monthly_glb ]]

diff --git a/tests/integration/generated/test_min_case_global_time_series_original_8_no_ocn_chrysalis.cfg b/tests/integration/generated/test_min_case_global_time_series_original_8_no_ocn_chrysalis.cfg
@@ -13,7 +13,6 @@ www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_
 
 [ts]
 active = True
-e3sm_to_cmip_environment_commands = ""
 walltime = "00:30:00"
 
   [[ atm_monthly_glb ]]

diff --git a/tests/integration/generated/test_min_case_ilamb_chrysalis.cfg b/tests/integration/generated/test_min_case_ilamb_chrysalis.cfg
@@ -9,13 +9,11 @@ mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
 output = "/lcrc/group/e3sm/ac.forsyth2/zppy_min_case_ilamb_output/unique_id/v3.LR.historical_0051"
 partition = "debug"
 qos = "regular"
+walltime = "00:30:00"
 www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_ilamb_www/unique_id"
 
 [ts]
 active = True
-e3sm_to_cmip_environment_commands = ""
-walltime = "00:50:00"
-ts_fmt = "cmip"
 years = "1985:2014:30"
 
   [[ atm_monthly_180x360_aave ]]
@@ -32,11 +30,21 @@ years = "1985:2014:30"
   mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
   vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
 
+[e3sm_to_cmip]
+active = True
+frequency = "monthly"
+ts_num_years = 30
+years = "1985:2014:30"
+
+  [[ atm_monthly_180x360_aave ]]
+  input_files = "eam.h0"
+
+  [[ land_monthly ]]
+  input_files = "elm.h0"
+
 [ilamb]
 active = True
 nodes = 8
-partition = "compute"
 short_name = "v3.LR.historical_0051"
 ts_num_years = 30
-walltime = "2:00:00"
 years = "1985:2014:30"
diff --git a/tests/integration/generated/test_min_case_ilamb_diff_years_chrysalis.cfg b/tests/integration/generated/test_min_case_ilamb_diff_years_chrysalis.cfg
@@ -0,0 +1,51 @@
+[default]
+case = "v3.LR.historical_0051"
+constraint = ""
+dry_run = "False"
+environment_commands = ""
+input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/v3.LR.historical_0051
+input_subdir = archive/atm/hist
+mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
+output = "/lcrc/group/e3sm/ac.forsyth2/zppy_min_case_ilamb_diff_years_output/unique_id/v3.LR.historical_0051"
+partition = "debug"
+qos = "regular"
+walltime = "00:30:00"
+www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_ilamb_diff_years_www/unique_id"
+
+[ts]
+active = True
+# 15 year increments for ts and e3sm_to_cmip, 30 for ilamb
+years = "1985:2014:15"
+
+  [[ atm_monthly_180x360_aave ]]
+  frequency = "monthly"
+  input_files = "eam.h0"
+  input_subdir = "archive/atm/hist"
+  vars = "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,ICEFRAC,LANDFRAC,OCNFRAC,PS,CLDICE,CLDLIQ,T,AODDUST"
+
+  [[ land_monthly ]]
+  extra_vars = "landfrac"
+  frequency = "monthly"
+  input_files = "elm.h0"
+  input_subdir = "archive/lnd/hist"
+  mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
+  vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
+
+[e3sm_to_cmip]
+active = True
+frequency = "monthly"
+ts_num_years = 15
+years = "1985:2014:15"
+
+  [[ atm_monthly_180x360_aave ]]
+  input_files = "eam.h0"
+
+  [[ land_monthly ]]
+  input_files = "elm.h0"
+
+[ilamb]
+active = True
+nodes = 8
+short_name = "v3.LR.historical_0051"
+ts_num_years = 15
+years = "1985:2014:30"
diff --git a/tests/integration/generated/test_min_case_ilamb_land_only_chrysalis.cfg b/tests/integration/generated/test_min_case_ilamb_land_only_chrysalis.cfg
@@ -13,9 +13,7 @@ www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_
 
 [ts]
 active = True
-e3sm_to_cmip_environment_commands = ""
 walltime = "00:50:00"
-ts_fmt = "cmip"
 years = "1985:2014:30"
 
   [[ land_monthly ]]
@@ -26,6 +24,16 @@ years = "1985:2014:30"
   mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
   vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
 
+[e3sm_to_cmip]
+active = True
+frequency = "monthly"
+ts_num_years = 30
+walltime = "00:50:00"
+years = "1985:2014:30"
+
+  [[ land_monthly ]]
+  input_files = "elm.h0"
+
 [ilamb]
 active = True
 land_only = True

diff --git a/tests/integration/generated/test_min_case_nco_chrysalis.cfg b/tests/integration/generated/test_min_case_nco_chrysalis.cfg
@@ -40,14 +40,12 @@ walltime = "00:30:00"
 
 [ts]
 active = True
-e3sm_to_cmip_environment_commands = ""
 walltime = "00:30:00"
 
   [[ atm_monthly_180x360_aave ]]
   frequency = "monthly"
   input_files = "eam.h0"
   input_subdir = "archive/atm/hist"
-  ts_fmt = "cmip"
 
   [[ atm_daily_180x360_aave ]]
   frequency = "daily"
@@ -85,5 +83,4 @@ walltime = "00:30:00"
   input_files = "elm.h0"
   input_subdir = "archive/lnd/hist"
   mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
-  ts_fmt = "cmip"
   vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"