change many wrappers to be consistent with finding input files. Error…

…s are now thrown when any input file is not found, checking other input types even when another input was not found. This increased errors reported in unit tests, so updated tests to reflect this.

internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py

@@ -39,12 +39,12 @@ def set_minimum_config_settings(config, set_fields=True, set_obs=True):
     config.set('config', 'INIT_INCREMENT', '12H')
     config.set('config', 'LEAD_SEQ', '12H')
     config.set('config', 'LOOP_ORDER', 'times')
-    config.set('config', 'ENSEMBLE_STAT_N_MEMBERS', 1)
+    config.set('config', 'ENSEMBLE_STAT_N_MEMBERS', 2)
     config.set('config', 'ENSEMBLE_STAT_CONFIG_FILE',
                '{PARM_BASE}/met_config/EnsembleStatConfig_wrapped')
     config.set('config', 'FCST_ENSEMBLE_STAT_INPUT_DIR', fcst_dir)
     config.set('config', 'FCST_ENSEMBLE_STAT_INPUT_TEMPLATE',
-               '{init?fmt=%Y%m%d%H}/fcst_file_F{lead?fmt=%3H}')
+               '{init?fmt=%Y%m%d%H}/fcst_file_F{lead?fmt=%3H},{init?fmt=%Y%m%d%H}/fcst_file_F{lead?fmt=%3H}')
     if set_obs:
         config.set('config', 'OBS_ENSEMBLE_STAT_GRID_INPUT_DIR', obs_dir)
         config.set('config', 'OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE',
@@ -67,16 +67,16 @@ def set_minimum_config_settings(config, set_fields=True, set_obs=True):
         (False, None, 3, 8, 0.7, 3),
         (True, 'obs_grid', 4, 8, 0.4, 0),
         (True, 'obs_grid', 4, 8, 0.7, 1),
-        (False, 'obs_grid', 4, 8, 0.7, 4),
+        (False, 'obs_grid', 4, 8, 0.7, 7),
         (True, 'point_grid', 4, 8, 0.4, 0),
         (True, 'point_grid', 4, 8, 0.7, 1),
-        (False, 'point_grid', 4, 8, 0.7, 4),
+        (False, 'point_grid', 4, 8, 0.7, 7),
         (True, 'ens_mean', 4, 8, 0.4, 0),
         (True, 'ens_mean', 4, 8, 0.7, 1),
-        (False, 'ens_mean', 4, 8, 0.7, 4),
+        (False, 'ens_mean', 4, 8, 0.7, 7),
         (True, 'ctrl', 4, 8, 0.4, 0),
         (True, 'ctrl', 4, 8, 0.7, 1),
-        (False, 'ctrl', 4, 8, 0.7, 4),
+        (False, 'ctrl', 4, 8, 0.7, 7),
         # still errors if more members than n_members found
         (True, 'low_n_member', 8, 8, 0.7, 6),
         (False, 'low_n_member', 8, 8, 0.7, 8),
@@ -135,11 +135,13 @@ def test_ensemble_stat_missing_inputs(metplus_config, get_test_data_dir, allow_m
           'FCST_VAR1_LEVELS': 'A06',
           'OBS_VAR1_NAME': 'obs_file',
           'OBS_VAR1_LEVELS': 'A06',
-          'FCST_ENSEMBLE_STAT_INPUT_TEMPLATE': '{fcst_name}_A{level?fmt=%3H}',
+          'FCST_ENSEMBLE_STAT_INPUT_TEMPLATE': '{fcst_name}_A{level?fmt=%3H},{fcst_name}_A{level?fmt=%3H}',
           },
          f'{fcst_dir}/fcst_file_A006'),
         # 1 - don't set forecast level
-        ({'FCST_ENSEMBLE_STAT_INPUT_TEMPLATE': 'fcst_file_A{level?fmt=%3H}'},
+        ({'FCST_VAR1_NAME': 'fcst_file',
+          'OBS_VAR1_NAME': 'obs_file',
+          'FCST_ENSEMBLE_STAT_INPUT_TEMPLATE': 'fcst_file_A{level?fmt=%3H},fcst_file_A{level?fmt=%3H}'},
          f'{fcst_dir}/fcst_file_A000'),
     ]
 )
@@ -159,16 +161,18 @@ def test_ensemble_stat_level_in_template(metplus_config, config_overrides,
     assert wrapper.isOK
 
     file_list_dir = wrapper.config.getdir('FILE_LISTS_DIR')
-    file_list_file = f"{file_list_dir}/20050807000000_12_ensemble_stat.txt"
+    file_list_file = f"{file_list_dir}/ensemble_stat_files_FCST_init_20050807000000_valid_20050807120000_lead_43200.txt"
     if os.path.exists(file_list_file):
         os.remove(file_list_file)
 
     wrapper.run_all_times()
+
     assert os.path.exists(file_list_file)
     with open(file_list_file, 'r') as file_handle:
         filenames = file_handle.read().splitlines()[1:]
-    assert len(filenames) == 1
+    assert len(filenames) == 2
     assert filenames[0] == expected_filename
+    assert filenames[1] == expected_filename
 
 
 @pytest.mark.parametrize(
@@ -860,20 +864,20 @@ def test_ensemble_stat_single_field(metplus_config, config_overrides,
     point_obs = ' '
     ens_mean = ' '
     if 'OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE' in config_overrides:
-        point_obs = f' -point_obs "{obs_dir}/{obs_point_template}" '
+        point_obs = f' -point_obs {obs_dir}/{obs_point_template} '
     if 'ENSEMBLE_STAT_ENS_MEAN_INPUT_TEMPLATE' in config_overrides:
         ens_mean = f' -ens_mean {ens_mean_dir}/{ens_mean_template} '
 
     expected_cmds = [(f"{app_path} {verbosity} "
-                      f"{file_list_dir}/20050807000000_12_ensemble_stat.txt "
-                      f"{config_file}{point_obs}"
-                      f'-grid_obs "{obs_dir}/2005080712/obs_file"{ens_mean}'
-                      f"-outdir {out_dir}/2005080712"),
+                      f"{file_list_dir}/ensemble_stat_files_FCST_init_20050807000000_valid_20050807120000_lead_43200.txt"
+                      f"{point_obs}"
+                      f'-grid_obs {obs_dir}/2005080712/obs_file{ens_mean}'
+                      f"{config_file} -outdir {out_dir}/2005080712"),
                      (f"{app_path} {verbosity} "
-                      f"{file_list_dir}/20050807120000_12_ensemble_stat.txt "
-                      f"{config_file}{point_obs}"
-                      f'-grid_obs "{obs_dir}/2005080800/obs_file"{ens_mean}'
-                      f"-outdir {out_dir}/2005080800"),
+                      f"{file_list_dir}/ensemble_stat_files_FCST_init_20050807120000_valid_20050808000000_lead_43200.txt"
+                      f"{point_obs}"
+                      f'-grid_obs {obs_dir}/2005080800/obs_file{ens_mean}'
+                      f"{config_file} -outdir {out_dir}/2005080800"),
                      ]
 
     all_cmds = wrapper.run_all_times()
@@ -905,7 +909,7 @@ def test_get_config_file(metplus_config):
 @pytest.mark.parametrize(
     'config_overrides, expected_num_files', [
         ({}, 4),
-        ({'ENSEMBLE_STAT_ENS_MEMBER_IDS': '1'}, 1),
+        ({'ENSEMBLE_STAT_ENS_MEMBER_IDS': '1'}, 2),
     ]
 )
 @pytest.mark.wrapper_c
@@ -926,13 +930,12 @@ def test_ensemble_stat_fill_missing(metplus_config, config_overrides,
     wrapper = EnsembleStatWrapper(config)
 
     file_list_file = os.path.join(wrapper.config.getdir('FILE_LISTS_DIR'),
-                                  '20050807000000_12_ensemble_stat.txt')
+                                  'ensemble_stat_files_FCST_init_20050807000000_valid_20050807120000_lead_43200.txt')
     if os.path.exists(file_list_file):
         os.remove(file_list_file)
 
     all_cmds = wrapper.run_all_times()
     assert len(all_cmds) == 1
-
     with open(file_list_file, 'r') as file_handle:
         actual_num_files = len(file_handle.read().splitlines()) - 1
 

internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py

@@ -3,6 +3,7 @@
 import pytest
 
 import os
+import re
 
 from datetime import datetime
 from dateutil.relativedelta import relativedelta
@@ -102,71 +103,60 @@ def test_grid_diag_missing_inputs(metplus_config, get_test_data_dir,
 
 
 @pytest.mark.parametrize(
-    'time_info, expected_subset', [
-        # all files
-        ({'init': '*', 'valid': '*', 'lead': '*'},
-         ['init_20141031213015_valid_20141031213015_lead_000.nc',
+    'runtime_freq,init_or_valid,expected_subset', [
+        # run once
+        ('RUN_ONCE', 'INIT',
+         [['init_20141031213015_valid_20141031213015_lead_000.nc',
           'init_20141031213015_valid_20141101213015_lead_024.nc',
           'init_20141101093015_valid_20141101093015_lead_000.nc',
           'init_20141101093015_valid_20141102093015_lead_024.nc',
+          ]]),
+        # once per init
+        ('RUN_ONCE_PER_INIT_OR_VALID', 'INIT',
+         [['init_20141031213015_valid_20141031213015_lead_000.nc',
+           'init_20141031213015_valid_20141101213015_lead_024.nc', ],
+          ['init_20141101093015_valid_20141101093015_lead_000.nc',
+           'init_20141101093015_valid_20141102093015_lead_024.nc',
+           ]]),
+
+        # once per valid
+        ('RUN_ONCE_PER_INIT_OR_VALID', 'VALID',
+         [['init_20141031213015_valid_20141031213015_lead_000.nc',
+           'init_20141030213015_valid_20141031213015_lead_024.nc'],
+          ['init_20141101093015_valid_20141101093015_lead_000.nc',
+           'init_20141031093015_valid_20141101093015_lead_024.nc'],
           ]),
-        # specific init
-        ({'init': datetime(2014, 10, 31, 21, 30, 15), 'valid': '*', 'lead': '*'},
-         ['init_20141031213015_valid_20141031213015_lead_000.nc',
-          'init_20141031213015_valid_20141101213015_lead_024.nc',
-          ]),
-        # specific valid
-        ({'init': '*', 'valid': datetime(2014, 11, 1, 9, 30, 15), 'lead': '*'},
-         ['init_20141101093015_valid_20141101093015_lead_000.nc',
-          ]),
-        # specific lead integer zero
-        ({'init': '*', 'valid': '*', 'lead': 0},
-         ['init_20141031213015_valid_20141031213015_lead_000.nc',
-          'init_20141101093015_valid_20141101093015_lead_000.nc',
-          ]),
-        # specific lead relativedelta non-zero
-        ({'init': '*', 'valid': '*', 'lead': relativedelta(hours=24)},
+        # once per lead
+        ('RUN_ONCE_PER_LEAD', 'INIT',
+         [['init_20141031213015_valid_20141031213015_lead_000.nc',
+           'init_20141101093015_valid_20141101093015_lead_000.nc'],
           ['init_20141031213015_valid_20141101213015_lead_024.nc',
-          'init_20141101093015_valid_20141102093015_lead_024.nc',
-          ]),
-        # specific lead integer non-zero
-        ({'init': '*', 'valid': '*', 'lead': 86400},
-          ['init_20141031213015_valid_20141101213015_lead_024.nc',
-          'init_20141101093015_valid_20141102093015_lead_024.nc',
-          ]),
-        # specific init/valid/lead integer zero
-        ({'init': datetime(2014, 10, 31, 21, 30, 15),
-          'valid': datetime(2014, 10, 31, 21, 30, 15),
-          'lead': 0},
-         ['init_20141031213015_valid_20141031213015_lead_000.nc',
-          ]),
-        # specific init/valid/lead relativedelta non-zero
-        ({'init': datetime(2014, 10, 31, 21, 30, 15),
-          'valid': datetime(2014, 11, 1, 21, 30, 15),
-          'lead': relativedelta(hours=24)},
-         ['init_20141031213015_valid_20141101213015_lead_024.nc',
-          ]),
-        # specific init/valid/lead integer non-zero
-        ({'init': datetime(2014, 10, 31, 21, 30, 15),
-          'valid': datetime(2014, 11, 1, 21, 30, 15),
-          'lead': 86400},
-         ['init_20141031213015_valid_20141101213015_lead_024.nc',
-          ]),
+           'init_20141101093015_valid_20141102093015_lead_024.nc',
+           ]]),
+        # once for each
+        ('RUN_ONCE_FOR_EACH', 'INIT',
+         [['init_20141031213015_valid_20141031213015_lead_000.nc'],
+          ['init_20141031213015_valid_20141101213015_lead_024.nc'],
+          ['init_20141101093015_valid_20141101093015_lead_000.nc'],
+          ['init_20141101093015_valid_20141102093015_lead_024.nc'],
+         ]),
     ]
 )
 @pytest.mark.wrapper
-def test_get_all_files_and_subset(metplus_config, time_info, expected_subset):
+def test_grid_diag_runtime_freq(metplus_config, runtime_freq, init_or_valid, expected_subset):
     """! Test to ensure that get_all_files only gets the files that are
     relevant to the runtime settings and not every file in the directory
     """
     config = metplus_config
-    config.set('config', 'LOOP_BY', 'INIT')
-    config.set('config', 'GRID_DIAG_RUNTIME_FREQ', 'RUN_ONCE')
-    config.set('config', 'INIT_TIME_FMT', '%Y%m%d%H%M%S')
-    config.set('config', 'INIT_BEG', '20141031213015')
-    config.set('config', 'INIT_END', '20141101093015')
-    config.set('config', 'INIT_INCREMENT', '12H')
+    config.set('config', 'LOOP_BY', init_or_valid)
+    config.set('config', 'GRID_DIAG_RUNTIME_FREQ', runtime_freq)
+    config.set('config', f'{init_or_valid}_TIME_FMT', '%Y%m%d%H%M%S')
+    config.set('config', f'{init_or_valid}_INCREMENT', '12H')
+    config.set('config', f'{init_or_valid}_BEG', '20141031213015')
+    config.set('config', f'{init_or_valid}_END', '20141101093015')
     config.set('config', 'LEAD_SEQ', '0H, 24H')
+    config.set('config', 'FCST_VAR1_NAME', 'FCST')
+    config.set('config', 'FCST_VAR1_LEVELS', 'L0')
 
     input_dir = os.path.join(config.getdir('METPLUS_BASE'),
                              'internal', 'tests',
@@ -177,38 +167,29 @@ def test_get_all_files_and_subset(metplus_config, time_info, expected_subset):
                ('init_{init?fmt=%Y%m%d%H%M%S}_valid_{valid?fmt=%Y%m%d%H%M%S}_'
                 'lead_{lead?fmt=%3H}.nc')
                )
-
-    expected_files = []
-    for init, valid, lead in [('20141031213015', '20141031213015', '000'),
-                              ('20141031213015', '20141101213015', '024'),
-                              ('20141101093015', '20141101093015', '000'),
-                              ('20141101093015', '20141102093015', '024')]:
-        filename = f'init_{init}_valid_{valid}_lead_{lead}.nc'
-        expected_files.append(os.path.join(input_dir, filename))
+    config.set('config', 'GRID_DIAG_OUTPUT_DIR', config.getdir('OUTPUT_BASE'))
 
     wrapper = GridDiagWrapper(config)
-    wrapper.c_dict['ALL_FILES'] = wrapper.get_all_files()
-
-    # convert list of lists into a single list to compare to expected results
-
-    actual_files = [item['input0'] for item in wrapper.c_dict['ALL_FILES']]
-    actual_files = [item for sub in actual_files for item in sub]
-    assert actual_files == expected_files
-
-    file_list_dict = wrapper.subset_input_files(time_info)
-    assert file_list_dict
-    if len(expected_subset) == 1:
-        file_list = [file_list_dict['input0']]
-    else:
-        with open(file_list_dict['input0'], 'r') as file_handle:
-            file_list = file_handle.readlines()
-
-        file_list = file_list[1:]
-        assert len(file_list) == len(expected_subset)
-
-    for actual_file, expected_file in zip(file_list, expected_subset):
-        actual_file = actual_file.strip()
-        assert os.path.basename(actual_file) == expected_file
+    wrapper.run_all_times()
+    assert len(wrapper.all_commands) == len(expected_subset)
+    file_list_files = []
+    print(wrapper.all_commands)
+    pattern = r'-data\s+([^\s]+)'
+    for cmd, _ in wrapper.all_commands:
+        match = re.search(pattern, cmd)
+        if match:
+            file_list_files.append(match.group(1))
+
+    assert len(file_list_files) == len(expected_subset)
+    for actual_file, expected_files in zip(file_list_files, expected_subset):
+        expected_files_full = [os.path.join(input_dir, item) for item in expected_files]
+        if len(expected_files) == 1:
+            assert actual_file == expected_files_full[0]
+        else:
+            with open(actual_file, 'r') as file_handle:
+                file_list = file_handle.read().splitlines()[1:]
+            print(f'ACTUAL: {file_list}')
+            assert sorted(file_list) == sorted(expected_files_full)
 
 
 @pytest.mark.parametrize(
@@ -367,16 +348,12 @@ def test_grid_diag(metplus_config, config_overrides, env_var_values,
     out_dir = wrapper.c_dict.get('OUTPUT_DIR')
 
     expected_cmds = [
-        (f"{app_path} -data {file_list_dir}/grid_diag_files_input0"
-         "_init_20160929000000_valid_ALL_lead_ALL.txt "
-         f"-data {file_list_dir}/grid_diag_files_input1"
-         "_init_20160929000000_valid_ALL_lead_ALL.txt "
-         f"-config {config_file} -out {out_dir}/grid_diag.all.nc {verbosity}"),
-        (f"{app_path} -data {file_list_dir}/grid_diag_files_input0"
-         "_init_20160929060000_valid_ALL_lead_ALL.txt "
-         f"-data {file_list_dir}/grid_diag_files_input1"
-         "_init_20160929060000_valid_ALL_lead_ALL.txt "
-         f"-config {config_file} -out {out_dir}/grid_diag.all.nc {verbosity}"),
+        (f"{app_path} -data {file_list_dir}/grid_diag_files_input0_init_20160929000000_valid_ALL_lead_ALL.txt"
+         f" -data {file_list_dir}/grid_diag_files_input1_init_20160929000000_valid_ALL_lead_ALL.txt"
+         f" -config {config_file} -out {out_dir}/grid_diag.all.nc {verbosity}"),
+        (f"{app_path} -data {file_list_dir}/grid_diag_files_input0_init_20160929060000_valid_ALL_lead_ALL.txt"
+         f" -data {file_list_dir}/grid_diag_files_input1_init_20160929060000_valid_ALL_lead_ALL.txt"
+         f" -config {config_file} -out {out_dir}/grid_diag.all.nc {verbosity}"),
     ]
 
     all_cmds = wrapper.run_all_times()

internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py

@@ -64,8 +64,8 @@ def set_minimum_config_settings(config):
         (False, 6, 12, 0.6, 1, True),
         (True, 12, 24, 0.5, 0, True),
         (True, 12, 24, 0.6, 1, True),
-        (False, 6, 12, 0.5, 6, False),
-        (True, 12, 24, 0.5, 12, False),
+        (False, 6, 12, 0.5, 10, False),
+        (True, 12, 24, 0.5, 20, False),
     ]
 )
 @pytest.mark.wrapper_b

internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py

@@ -37,12 +37,12 @@ def set_minimum_config_settings(config):
         (16, 24, 0.3, 16, False, 'RUN_ONCE_FOR_EACH'),
         (2, 4, 0.4, 0, True, 'RUN_ONCE_PER_INIT_OR_VALID'),
         (2, 4, 0.6, 1, True, 'RUN_ONCE_PER_INIT_OR_VALID'),
-        (2, 4, 0.6, 2, False, 'RUN_ONCE_PER_INIT_OR_VALID'),
+        (2, 4, 0.6, 16, False, 'RUN_ONCE_PER_INIT_OR_VALID'),
         (2, 5, 0.4, 0, True, 'RUN_ONCE_PER_LEAD'),
         (2, 5, 0.7, 1, True, 'RUN_ONCE_PER_LEAD'),
-        (2, 5, 0.4, 2, False, 'RUN_ONCE_PER_LEAD'),
+        (2, 5, 0.4, 17, False, 'RUN_ONCE_PER_LEAD'),
         (0, 1, 0.4, 0, True, 'RUN_ONCE'),
-        (0, 1, 0.4, 0, False, 'RUN_ONCE'),
+        (0, 1, 0.4, 16, False, 'RUN_ONCE'),
     ]
 )
 @pytest.mark.wrapper

internal/tests/pytests/wrappers/mode/test_mode_wrapper.py

@@ -62,7 +62,7 @@ def set_minimum_config_settings(config):
     'missing, run, thresh, errors, allow_missing', [
         (6, 12, 0.5, 0, True),
         (6, 12, 0.6, 1, True),
-        (6, 12, 0.5, 6, False),
+        (6, 12, 0.5, 10, False),
     ]
 )
 @pytest.mark.wrapper_a

internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py

@@ -80,13 +80,13 @@ def set_minimum_config_settings(config, set_inputs=True):
         (1, 3, 0.3, 0, True, 'CHOCOLATE'),
         (1, 3, 0.3, 0, True, 'BOTH'),
         (1, 3, 0.8, 1, True, 'BOTH'),
-        (1, 3, 0.8, 1, False, 'BOTH'),
+        (1, 3, 0.8, 22, False, 'BOTH'),
         (1, 3, 0.3, 0, True, 'FCST'),
         (1, 3, 0.8, 1, True, 'FCST'),
-        (1, 3, 0.8, 1, False, 'FCST'),
+        (1, 3, 0.8, 12, False, 'FCST'),
         (1, 3, 0.3, 0, True, 'OBS'),
         (1, 3, 0.8, 1, True, 'OBS'),
-        (1, 3, 0.8, 1, False, 'OBS'),
+        (1, 3, 0.8, 10, False, 'OBS'),
     ]
 )
 @pytest.mark.wrapper_a

internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py

@@ -33,12 +33,12 @@ def pb2nc_wrapper(metplus_config):
         (16, 24, 0.3, 16, False, 'RUN_ONCE_FOR_EACH'),
         (2, 4, 0.4, 0, True, 'RUN_ONCE_PER_INIT_OR_VALID'),
         (2, 4, 0.6, 1, True, 'RUN_ONCE_PER_INIT_OR_VALID'),
-        (2, 4, 0.6, 2, False, 'RUN_ONCE_PER_INIT_OR_VALID'),
+        (2, 4, 0.6, 16, False, 'RUN_ONCE_PER_INIT_OR_VALID'),
         (2, 5, 0.4, 0, True, 'RUN_ONCE_PER_LEAD'),
         (2, 5, 0.7, 1, True, 'RUN_ONCE_PER_LEAD'),
-        (2, 5, 0.4, 2, False, 'RUN_ONCE_PER_LEAD'),
+        (2, 5, 0.4, 17, False, 'RUN_ONCE_PER_LEAD'),
         (0, 1, 0.4, 0, True, 'RUN_ONCE'),
-        (0, 1, 0.4, 0, False, 'RUN_ONCE'),
+        (0, 1, 0.4, 16, False, 'RUN_ONCE'),
     ]
 )
 @pytest.mark.wrapper