Merge branch 'develop' into feature/NickE_gpsro_bufr

.gitignore

@@ -175,3 +175,4 @@ test-data-release/
 ufo/
 ufo-data/
 vader/
+/gdas-utils

parm/atm/obs/config/conv_ps.yaml

@@ -0,0 +1,295 @@
+obs space:
+  name: surface_ps 
+  obsdatain:
+    engine: 
+      type: H5File
+      obsfile: $(DATA)/obs/$(OPREFIX)sondes.{{ current_cycle | to_YMDH }}.nc4
+  obsdataout:
+    engine:
+      type: H5File
+      obsfile: $(DATA)/diags/diag_sondes_{{ current_cycle | to_YMDH }}.nc4
+  io pool:
+    max pool size: 1
+  simulated variables: [stationPressure]
+
+obs operator:
+  name: SfcPCorrected
+  variables:
+  - name: stationPressure
+  da_psfc_scheme: GSI
+  station_altitude: height
+  geovar_sfc_geomz: surface_altitude
+  geovar_geomz: geopotential_height
+
+obs prior filters:
+# Initial Error Assignments for SFC Observations 
+- filter: Perform Action
+  filter variables:
+  - name: stationPressure
+  where:
+  - variable: ObsType/stationPressure
+    is_in: [181]
+  action:
+    name: assign error
+    error function:
+      name: ObsFunction/ObsErrorModelStepwiseLinear
+      options:
+        round_to_the_nearest_integer: true
+        xvar:
+          name: ObsValue/stationPressure
+        xvals:  [80000, 75000, 70000, 65000, 60000, 55000  ]
+        errors: [110,    120,    120,   120,   120, 1.0e+11]
+
+- filter: Perform Action
+  filter variables:
+  - name: stationPressure
+  where:
+  - variable: ObsType/stationPressure
+    is_in: [187]
+  action:
+    name: assign error
+    error function:
+      name: ObsFunction/ObsErrorModelStepwiseLinear
+      options:
+        round_to_the_nearest_integer: true
+        xvar:
+          name: ObsValue/stationPressure
+        xvals:  [85000, 80000, 75000, 70000, 65000, 60000, 55000  ]
+        errors: [  120,   140,   140,   140,   140,   140, 1.0e+11]
+
+# Initial Error Assignments for SFCSHIP Observations 
+- filter: Perform Action
+  filter variables:
+  - name: stationPressure
+  where:
+  - variable: ObsType/stationPressure
+    is_in: [180]
+  action:
+    name: assign error
+    error function:
+      name: ObsFunction/ObsErrorModelStepwiseLinear
+      options:
+        round_to_the_nearest_integer: true
+        xvar:
+          name: ObsValue/stationPressure
+        xvals:  [60000, 55000  ]
+        errors: [  130, 1.0e+11]
+
+- filter: Perform Action
+  filter variables:
+  - name: stationPressure
+  where:
+  - variable: ObsType/stationPressure
+    is_in: [183]
+  action:
+    name: assign error
+    error parameter: 1.0e+11
+
+# Initial Error Assignments for Radiosonde
+- filter: Perform Action
+  filter variables:
+  - name: stationPressure
+  where:
+  - variable: ObsType/stationPressure
+    is_in: [120]
+  action:
+    name: assign error
+    error function:
+      name: ObsFunction/ObsErrorModelStepwiseLinear
+      options:
+        round_to_the_nearest_integer: true
+        xvar:
+          name: ObsValue/stationPressure
+        xvals:  [80000, 75000, 70000, 65000, 60000, 55000  ]
+        errors: [  110,   120,   120,   120,   120, 1.0e+11]
+
+obs post filters:
+# Observation range sanity check
+- filter: Bounds Check
+  filter variables:
+  - name: stationPressure
+  minvalue: 37499.0
+  maxvalue: 106999.0
+  action:
+    name: reject
+
+# Reject all ObsType 183
+- filter: RejectList
+  where:
+  - variable:
+      name: ObsType/stationPressure
+    is_in: 183
+
+# Reject surface pressure below 500 hPa 
+- filter: Bounds Check
+  filter variables:
+  - name: stationPressure
+  minvalue: 50000.00
+  action:
+    name: reject
+
+- filter: RejectList
+  where:
+  - variable:
+      name: PreQC/stationPressure
+    is_in: 4-15
+
+# Inflate obs error based on obs type
+- filter: Perform Action 
+  filter variables:
+  - name: stationPressure
+  where:
+  - variable: PreQC/stationPressure
+    is_in: 3, 7   
+  action:
+    name: inflate error
+    inflation factor: 1.2  
+
+# Calculate obs error inflation factors for duplicated observations at the same location
+- filter: Variable Assignment
+  assignments:
+  - name: ObsErrorFactorDuplicateCheck/stationPressure
+    type: float
+    function:
+      name: ObsFunction/ObsErrorFactorDuplicateCheck
+      options:
+        use_air_pressure: false 
+        variable: stationPressure 
+
+# Reduce effective observation error based on obs type and subtype
+# In this case: reduce effective obs error for buoy  
+- filter: Perform Action
+  filter variables:
+  - name: stationPressure
+  where:
+  - variable:
+      name: ObsType/stationPressure
+    is_in: 180
+  - variable:
+      name: ObsSubType/stationPressure
+    is_in: 0
+  action:
+    name: inflate error
+    inflation factor: 0.7
+
+# Reduce original observation error based on obs type and subtype
+# In this case: reduce original obs error for buoy 
+- filter: Variable Assignment
+  where:
+  - variable:
+      name: ObsType/stationPressure
+    is_in: 180
+  - variable:
+      name: ObsSubType/stationPressure
+    is_in: 0
+  assignments:
+  - name: ObsError/stationPressure
+    type: float
+    function:
+      name: ObsFunction/Arithmetic 
+      options: 
+        variables:
+        - name: ObsError/stationPressure
+        coefs: [0.7]
+
+# Calculate obs error inflation factors for large discrepancies between model and observations 
+- filter: Variable Assignment
+  assignments:
+  - name: ObsErrorFactorSfcPressure/stationPressure
+    type: float
+    function:
+      name: ObsFunction/ObsErrorFactorSfcPressure
+      options:
+        geovar_sfc_geomz: surface_altitude
+        geovar_geomz: geopotential_height
+        station_altitude: height 
+
+# Inflate surface pressure observation based on discrepancies between
+# model and observations due to terrian
+- filter: Perform Action
+  filter variables:
+  - name: stationPressure
+  action:
+    name: inflate error
+    inflation variable:
+      name: ObsErrorFactorSfcPressure/stationPressure
+
+- filter: Variable Assignment
+  assignments:
+  - name: DerivedMetaData/Innovation
+    type: float
+    function:
+      name: ObsFunction/Arithmetic
+      options:
+        variables:
+        - name: ObsValue/stationPressure
+        - name: HofX/stationPressure
+        coefs: [1, -1]
+
+- filter: Variable Assignment
+  assignments:
+  - name: DerivedMetaData/ObsErrorBoundSfcPressure1
+    type: float
+    function:
+      name: ObsFunction/ObsErrorBoundConventional
+      options:
+        obsvar: stationPressure
+        obserr_bound_min: 100
+        obserr_bound_max: 300
+        obserr_bound_factor: 5.0 
+
+- filter: Background Check 
+  filter variables:
+  - name: stationPressure
+  where:
+  - variable: PreQC/stationPressure
+    is_not_in: 3
+  function absolute threshold:
+  - name: DerivedMetaData/ObsErrorBoundSfcPressure1
+  action:
+    name: reject
+
+- filter: Variable Assignment
+  assignments:
+  - name: DerivedMetaData/ObsErrorBoundSfcPressure2
+    type: float
+    function:
+      name: ObsFunction/ObsErrorBoundConventional
+      options:
+        obsvar: stationPressure
+        obserr_bound_min: 100
+        obserr_bound_max: 300
+        obserr_bound_factor: 3.5 
+
+- filter: Background Check 
+  filter variables:
+  - name: stationPressure
+  where:
+  - variable: PreQC/stationPressure
+    is_in: 3
+  function absolute threshold:
+  - name: DerivedMetaData/ObsErrorBoundSfcPressure2
+  action:
+    name: reject
+
+# Inflate obs error based on duplicate check
+- filter: Perform Action
+  filter variables:
+  - name: stationPressure
+  action:
+    name: inflate error
+    inflation variable:
+      name: ObsErrorFactorDuplicateCheck/stationPressure
+
+# Reject data based on PreUseFlag (usage in GSI) 
+- filter: Perform Action
+  filter variables:
+  - name: stationPressure
+  where:
+  - variable: PreUseFlag/stationPressure
+    is_not_in: 0, 1
+  action:
+    name: reject 
+
+# End of Filters 
+

parm/atm/obs/config/omi_aura.yaml

@@ -11,8 +11,79 @@ obs space:
   io pool:
     max pool size: 1
   simulated variables: [ozoneTotal]
+
 obs operator:
   name: AtmVertInterpLay
-  geovals: [ozoneLayer]
+  geovals: [mole_fraction_of_ozone_in_air]
   coefficients: [0.007886131] # convert from ppmv to DU
   nlevels: [1]
+
+obs pre filters:
+- filter: Perform Action
+  filter variables:
+  - name: ozoneTotal
+  action:
+    name: assign error
+    error parameter: 6.0
+
+obs prior filters:
+# GSI read routine QC
+# range sanity check
+- filter: Bounds Check
+  filter variables:
+  - name: ozoneTotal
+  minvalue: 0
+  maxvalue: 10000
+  action:
+    name: reject
+
+# Do not use the data if row anomaly (bit 10)is 1
+- filter: RejectList
+  filter variables:
+  - name: ozoneTotal
+  where:
+  - variable:
+      name: MetaData/totalOzoneQualityFlag
+    any_bit_set_of: 9 
+
+# Scan position check: reject scan position >= 25
+- filter: RejectList
+  filter variables:
+  - name: ozoneTotal
+  where:
+  - variable:
+      name: MetaData/sensorScanPosition
+    minvalue: 25
+
+# Accept total_ozone_error_flag values of 0 and 1, but not any others.
+- filter: RejectList 
+  filter variables:
+  - name: ozoneTotal
+  where:
+  - variable:
+      name: MetaData/totalOzoneQualityCode
+    is_not_in: 0, 1
+
+# Use data with best ozone algorighm
+- filter: RejectList
+  filter variables:
+  - name: ozoneTotal
+  where:
+  - variable:
+      name: MetaData/bestOzoneAlgorithmFlag
+    is_in: 3, 13
+
+obs post filters:
+# GSI setup routine QC
+# Gross check
+- filter: Background Check
+  filter variables:
+  - name: ozoneTotal
+  threshold: 10.0
+  absolute threshold: 300.0
+  action:
+    name: reject
+
+# End of Filters
+
+