Add IODA conversions to marine obs prep task ex-script

parm/soca/obsproc/obs_list_obsproc.yaml

@@ -0,0 +1,11 @@
+observers:
+- !INC ${OBS_YAML_DIR}/adt_all.yaml
+- !INC ${OBS_YAML_DIR}/adt_j3.yaml
+- !INC ${OBS_YAML_DIR}/adt_j2.yaml
+#- !INC ${OBS_YAML_DIR}/salt_profile_fnmoc.yaml
+- !INC ${OBS_YAML_DIR}/sss_smap.yaml
+- !INC ${OBS_YAML_DIR}/sss_smos.yaml
+- !INC ${OBS_YAML_DIR}/sst_noaa19_l3u.yaml
+- !INC ${OBS_YAML_DIR}/icec_emc.yaml
+- !INC ${OBS_YAML_DIR}/adt_3b_egm2008.yaml
+- !INC ${OBS_YAML_DIR}/icec_amsr2_north.yaml

parm/soca/obsproc/obsproc_config.yaml

@@ -0,0 +1,25 @@
+observations:
+- obs space:
+    name: sss_smap
+    obsproc subdir: SSS
+    obsproc regex: SMAP_L2B_SSS_NRT_?????_[AD]_????????T??????.h5
+    provider: SMAP
+    output file: sss_smap.ioda.nc
+- obs space:
+    name: sss_smos
+    provider: SMOS
+    obsproc subdir: SSS
+    output file: sss_smos.ioda.nc
+    obsproc regex: SM_OPER_MIR_OSUDP2_????????T??????_????????T??????_700_001_1.nc
+- obs space:
+    name: adt_3b_egm2008
+    obsproc subdir: ADT
+    obsproc regex: rads_adt_3b_???????.nc
+    provider: RADS
+    output file: adt_all.nc4
+- obs space:
+    name: icec_amsr2_north
+    provider: AMSR2
+    obsproc subdir: icec
+    output file: icec_amsr2_north_1.ioda.nc
+    obsproc regex: AMSR2-SEAICE-NH_v2r2_GW1_s???????????????_e???????????????_c???????????????.nc

parm/soca/obsproc/obsproc_config_test.yaml

@@ -0,0 +1,25 @@
+observations:
+- obs space:
+    name: sss_smap
+    obsproc subdir: '' 
+    obsproc regex: sss_smap_?.nc4
+    provider: SMAP
+    output file: sss_smap.ioda.nc
+- obs space:
+    name: sss_smos
+    provider: SMOS
+    obsproc subdir: ''
+    output file: sss_smos.ioda.nc
+    obsproc regex: sss_smos_?.nc4
+- obs space:
+    name: adt_3b_egm2008
+    obsproc subdir: ''
+    obsproc regex: rads_adt_??_???????.nc4
+    provider: RADS
+    output file: adt_all.nc4
+- obs space:
+    name: icec_amsr2_north
+    provider: AMSR2
+    obsproc subdir: ''
+    output file: icec_amsr2_north.ioda.nc
+    obsproc regex: icec_amsr2_north_?.nc4

scripts/exglobal_prep_ocean_obs.py

@@ -1,17 +1,53 @@
 #!/usr/bin/env python3
 # exglobal_prep_ocean_obs.py
 # Pepares observations for marine DA
+from datetime import datetime, timedelta
+import logging
 import os
-from wxflow import YAMLFile
 import prep_marine_obs
-import logging
+import subprocess
+from wxflow import YAMLFile, save_as_yaml
+
+# set up logger
+logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S')
+
+cyc = os.getenv('cyc')
+PDY = os.getenv('PDY')
+
+# set the window times
+cdateDatetime = datetime.strptime(PDY + cyc, '%Y%m%d%H')
+windowBeginDatetime = cdateDatetime - timedelta(hours=3)
+windowEndDatetime = cdateDatetime + timedelta(hours=3)
+windowBegin = windowBeginDatetime.strftime('%Y-%m-%dT%H:%M:%SZ')
+windowEnd = windowEndDatetime.strftime('%Y-%m-%dT%H:%M:%SZ')
+
+OCNOBS2IODAEXEC = os.getenv('OCNOBS2IODAEXEC')
 
 OBS_YAML = os.getenv('OBS_YAML')
+obsConfig = YAMLFile(OBS_YAML)
+
+OBSPROC_YAML = os.getenv('OBSPROC_YAML')
+obsprocConfig = YAMLFile(OBSPROC_YAML)
+
+# TODO (AFE): needs more error handling (missing sources, missing files)
+for observer in obsConfig['observers']:
+
+    obsSpaceName = observer['obs space']['name']
+    print(f"obsSpaceName: {obsSpaceName}")
+
+    for observation in obsprocConfig['observations']:
+
+        obsprocSpace = observation['obs space']
+        obsprocSpaceName = obsprocSpace['name']
+
+        if obsprocSpaceName == obsSpaceName:
+
+            matchingFiles = prep_marine_obs.obs_fetch(obsprocSpace)
+            obsprocSpace['input files'] = matchingFiles
+            obsprocSpace['window begin'] = windowBegin
+            obsprocSpace['window end'] = windowEnd
 
-data = YAMLFile(OBS_YAML)
-print(data)
+            iodaYamlFilename = obsprocSpaceName + '2ioda.yaml'
+            save_as_yaml(obsprocSpace, iodaYamlFilename)
 
-for observer in data['observers']:
-    obs_source_name = observer['obs space']['name']
-    logging.info(f"obs_source_name: {obs_source_name}")
-    prep_marine_obs.obs_fetch(obs_source_name)
+            subprocess.run([OCNOBS2IODAEXEC, iodaYamlFilename], check=True)

test/soca/gw/CMakeLists.txt

@@ -36,14 +36,22 @@ add_test(NAME test_gdasapp_soca_run_clean
 # Create scratch for testing apps
 file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/apps_scratch)
 
+# Create scratch for obs prep task
+file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/obsproc)
+
+# Set up obsproc, based on test_gdasapp_util_prepdata
+add_test(NAME test_gdasapp_soca_setup_obsproc
+         COMMAND ${PROJECT_SOURCE_DIR}/test/soca/gw/setup_obsproc.sh ${PROJECT_SOURCE_DIR}
+         WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/obsproc)
+
 # Test JGDAS_GLOBAL_OCEAN_ANALYSIS_*
-set(jjob_list "JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP"
+set(jjob_list "JGLOBAL_PREP_OCEAN_OBS"
+              "JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP"
               "JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT"
               "JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN"
               "JGDAS_GLOBAL_OCEAN_ANALYSIS_CHKPT"
               "JGDAS_GLOBAL_OCEAN_ANALYSIS_POST"
-              "JGDAS_GLOBAL_OCEAN_ANALYSIS_VRFY"
-              "JGLOBAL_PREP_OCEAN_OBS")
+              "JGDAS_GLOBAL_OCEAN_ANALYSIS_VRFY")
 
 set(setup "")
 foreach(jjob ${jjob_list})

test/soca/gw/run_jjobs.yaml.test

@@ -57,7 +57,9 @@ setup_expt config:
     NICAS_RESOL: 1
     NICAS_GRID_SIZE: 150
   prepoceanobs:
-    SOCA_OBS_LIST: @HOMEgfs@/sorc/gdas.cd/parm/soca/obs/obs_list_small.yaml
+    SOCA_OBS_LIST: @HOMEgfs@/sorc/gdas.cd/parm/soca/obsproc/obs_list_obsproc.yaml
+    OBSPROC_CONFIG: @HOMEgfs@/sorc/gdas.cd/parm/soca/obsproc/obsproc_config_test.yaml
+    DMPDIR: @HOMEgfs@/sorc/gdas.cd/build/test/soca/gw/obsproc
 
 job options:
   account: da-cpu

test/soca/gw/setup_obsproc.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -ex
+
+project_source_dir=$1
+
+# working directory should be ${PROJECT_BINARY_DIR}/test/soca/gw/obsproc, set in ctest command
+test_dmpdir="gdas.20180415/12"
+
+rm -rf ${test_dmpdir}
+mkdir -p ${test_dmpdir}
+
+cd ${test_dmpdir}
+
+${project_source_dir}/utils/test/prepdata.sh ${project_source_dir}/utils/test/
+
+
+

ush/soca/prep_marine_obs.py

@@ -63,32 +63,29 @@
 }
 
 
-def obs_fetch(obs_source_name):
+def obs_fetch(obsprocSpace):
 
-    try:
-        obs_source = obs_dict[obs_source_name]
-    except KeyError:
-        print(f'WARNING: no obs source {obs_source_name} defined, skipping')
-        return
-
-    subdir = obs_source[0]
-    filepattern = obs_source[1]
+    subdir = obsprocSpace['obsproc subdir']
+    filepattern = obsprocSpace['obsproc regex']
 
     datadir = os.path.join(cycdir, subdir)
     # TODO: check the existence of this
     print('datadir:', datadir)
-    matching_files = []
+    matchingFiles = []
 
     for root, _, files in os.walk(datadir):
         for filename in fnmatch.filter(files, filepattern):
-            matching_files.append((root, filename))
+            matchingFiles.append(filename)
 
     obs_cpy = []
-    for obs_src in matching_files:
-        obs_path = os.path.join(obs_src[0], obs_src[1])
-        obs_dst = os.path.join(COMIN_OBS, obs_src[1])
+    for obs_src in matchingFiles:
+        obs_path = os.path.join(datadir, obs_src)
+        obs_dst = os.path.join(COMIN_OBS, obs_src)
         obs_cpy.append([obs_path, obs_dst])
 
-    print(obs_cpy)
+    print(f"obs_cpy: {obs_cpy}")
+    print(f"matchingFiles: {matchingFiles}")
 
     FileHandler({'copy': obs_cpy}).sync()
+
+    return matchingFiles

ush/soca/run_jjobs.py

@@ -217,6 +217,12 @@ def fixconfigs(self):
                        'STMP': self.stmp,
                        'ROTDIR': self.rotdir,
                        'EXPDIRS': self.expdirs}
+
+        # needed for this ctest at least until R2D2 goes away
+        if 'JGLOBAL_PREP_OCEAN_OBS' in self.config['jjobs']:
+            dmpdir = self.config['setup_expt config']['prepoceanobs']['DMPDIR']
+            var2replace['DMPDIR'] = dmpdir
+
         with open(configbase, 'r') as f:
             newconfigbase = f.read()
         for key, value in var2replace.items():

utils/test/prepdata.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
+# called for test_gdasapp_util_prepdata, and by
+# test/soca/gw/setup_obsproc.sh for test_gdasapp_soca_setup_obsproc
+
+
 set -e
 
 cdl2nc4() {