Revise obs space localization for LETKF (#477)

* revise obs space localization for LETKF

* use all ob system

* this was missing

* works but ...

* Help with PR477 (#479)

* ...

* Add to localensembleda-tier1.yaml
    local_ensemble_save_posterior_ensemble: false
    local_ensemble_save_posterior_ensemble_increments: false
    local_ensemble_save_posterior_mean: false
    local_ensemble_save_posterior_mean_increment: true
output jedi_localensembleda_config.yaml shows right.

* In run_jedi_local_ensemble_da_executable.py, hard code if bypass_writing_hofxs:  del [obs space][obsdataout]

* update default for save_posterior_ensemble

* add back lienar_observer: true

---------

Co-authored-by: Yonggang Yu <[email protected]>
Co-authored-by: Yonggang Yu <[email protected]>

setup.py

@@ -45,6 +45,7 @@
                'configuration/*/*/*',
                'configuration/*/*/*/*',
                'configuration/*/*/*/*/*',
+               'configuration/*/*/*/*/*/*',
                'utilities/pinned_versions/*.yaml'
              ],
     },

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/aircraft.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/airs_aqua.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/amsr2_gcom-w1.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/amsua_aqua.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/amsua_metop-b.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/amsua_metop-c.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/amsua_n15.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/amsua_n18.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/amsua_n19.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/atms_n20.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/atms_npp.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/avhrr3_metop-b.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/avhrr3_n18.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/avhrr3_n19.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/cris-fsr_n20.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/cris-fsr_npp.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/gmi_gpm.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/gps.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/iasi_metop-b.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/iasi_metop-c.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/mhs_metop-b.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/mhs_metop-c.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/mhs_n19.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/mls55_aura.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/obsop_name_map.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/omi_aura.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/ompsnm_npp.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/pibal.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/satwind.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/scatwind.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/sfc.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/sfcship.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/sondes.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/localization/ssmis_f17.yaml

@@ -1,4 +1,4 @@
 obs localizations:
   localization method: Horizontal Gaspari-Cohn
-  lengthscale: 1000.e3
+  lengthscale: 500.e3
   max nobs: 100

src/swell/tasks/run_jedi_local_ensemble_da_executable.py

@@ -129,10 +129,10 @@ def execute(self) -> None:
         # Prevent both 'local_ensemble_save_posterior_mean' and
         # 'local_ensemble_save_posterior_ensemble' from being true
         # --------------------------------------------------------
-        if not self.config.local_ensemble_save_posterior_mean() ^ \
+        if self.config.local_ensemble_save_posterior_mean() or \
            self.config.local_ensemble_save_posterior_ensemble():
-            raise ValueError("Only one of 'local_ensemble_save_posterior_mean' and\
-            'local_ensemble_save_posterior_ensemble' may be true at once!")
+            raise ValueError("'local_ensemble_save_posterior_mean' and\
+            'local_ensemble_save_posterior_ensemble' cannot be both true!")
 
         # Jedi configuration file
         # -----------------------
@@ -186,6 +186,14 @@ def execute(self) -> None:
                 observer['obs space'].update(
                     {'distribution': {'name': 'Halo', 'halo size': 5000.e3}})
 
+        # bypass the writing of HofXs
+        # -------------------------------------------------------------------
+
+        bypass_HofXs = True
+        if bypass_HofXs:
+            for observer in jedi_config_dict['observations']['observers']:
+                del observer['obs space']['obsdataout']
+
         # Write the expanded dictionary to YAML file
         # ------------------------------------------
         with open(jedi_config_file, 'w') as jedi_config_file_open:

src/swell/tasks/task_questions.yaml

@@ -543,7 +543,7 @@ local_ensemble_inflation_rtps:
 
 local_ensemble_save_posterior_ensemble:
   ask_question: false
-  default_value: true
+  default_value: false
   models:
   - geos_atmosphere
   options:

src/swell/test/suite_tests/localensembleda-tier1.yaml

@@ -15,12 +15,16 @@ models:
     skip_ensemble_hofx: true
     local_ensemble_solver: GETKF
     local_ensemble_use_linear_observer: true
-    local_ensemble_save_posterior_mean: false
-    local_ensemble_save_posterior_ensemble_increments: true
     ensmean_only: false
+    local_ensemble_save_posterior_mean: false                 # default: false
+    local_ensemble_save_posterior_mean_increment: true        # default: true
+    local_ensemble_save_posterior_ensemble: false             # default: false
+    local_ensemble_save_posterior_ensemble_increments: false  # default: false
     path_to_ensemble: /discover/nobackup/projects/gmao/advda/SwellTestData/letk/ensemble/91/Y%Y/M%m/D%d/H%H/geos*%Y%m%d_%H%M%Sz.nc4
     observations:
+    - aircraft
     - sondes
+    - gps
     - amsua_aqua
     - amsua_n15
     - amsua_n18