diff --git a/.gitignore b/.gitignore index 4eff16035..bd53d9c20 100644 --- a/.gitignore +++ b/.gitignore @@ -175,3 +175,4 @@ test-data-release/ ufo/ ufo-data/ vader/ +/gdas-utils diff --git a/parm/atm/obs/config/conv_ps.yaml b/parm/atm/obs/config/conv_ps.yaml new file mode 100644 index 000000000..8fa4ab185 --- /dev/null +++ b/parm/atm/obs/config/conv_ps.yaml @@ -0,0 +1,295 @@ +obs space: + name: surface_ps + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)sondes.{{ current_cycle | to_YMDH }}.nc4 + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_sondes_{{ current_cycle | to_YMDH }}.nc4 + io pool: + max pool size: 1 + simulated variables: [stationPressure] + +obs operator: + name: SfcPCorrected + variables: + - name: stationPressure + da_psfc_scheme: GSI + station_altitude: height + geovar_sfc_geomz: surface_altitude + geovar_geomz: geopotential_height + +obs prior filters: +# Initial Error Assignments for SFC Observations +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [181] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [80000, 75000, 70000, 65000, 60000, 55000 ] + errors: [110, 120, 120, 120, 120, 1.0e+11] + +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [187] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [85000, 80000, 75000, 70000, 65000, 60000, 55000 ] + errors: [ 120, 140, 140, 140, 140, 140, 1.0e+11] + +# Initial Error Assignments for SFCSHIP Observations +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [180] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [60000, 55000 ] + errors: [ 130, 1.0e+11] + +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [183] + action: + name: assign error + error parameter: 1.0e+11 + +# Initial Error Assignments for Radiosonde +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [120] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [80000, 75000, 70000, 65000, 60000, 55000 ] + errors: [ 110, 120, 120, 120, 120, 1.0e+11] + +obs post filters: +# Observation range sanity check +- filter: Bounds Check + filter variables: + - name: stationPressure + minvalue: 37499.0 + maxvalue: 106999.0 + action: + name: reject + +# Reject all ObsType 183 +- filter: RejectList + where: + - variable: + name: ObsType/stationPressure + is_in: 183 + +# Reject surface pressure below 500 hPa +- filter: Bounds Check + filter variables: + - name: stationPressure + minvalue: 50000.00 + action: + name: reject + +- filter: RejectList + where: + - variable: + name: PreQC/stationPressure + is_in: 4-15 + +# Inflate obs error based on obs type +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: PreQC/stationPressure + is_in: 3, 7 + action: + name: inflate error + inflation factor: 1.2 + +# Calculate obs error inflation factors for duplicated observations at the same location +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorDuplicateCheck/stationPressure + type: float + function: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: false + variable: stationPressure + +# Reduce effective observation error based on obs type and subtype +# In this case: reduce effective obs error for buoy +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: + name: ObsType/stationPressure + is_in: 180 + - variable: + name: ObsSubType/stationPressure + is_in: 0 + action: + name: inflate error + inflation factor: 0.7 + +# Reduce original observation error based on obs type and subtype +# In this case: reduce original obs error for buoy +- filter: Variable Assignment + where: + - variable: + name: ObsType/stationPressure + is_in: 180 + - variable: + name: ObsSubType/stationPressure + is_in: 0 + assignments: + - name: ObsError/stationPressure + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsError/stationPressure + coefs: [0.7] + +# Calculate obs error inflation factors for large discrepancies between model and observations +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorSfcPressure/stationPressure + type: float + function: + name: ObsFunction/ObsErrorFactorSfcPressure + options: + geovar_sfc_geomz: surface_altitude + geovar_geomz: geopotential_height + station_altitude: height + +# Inflate surface pressure observation based on discrepancies between +# model and observations due to terrian +- filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSfcPressure/stationPressure + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/Innovation + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsValue/stationPressure + - name: HofX/stationPressure + coefs: [1, -1] + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBoundSfcPressure1 + type: float + function: + name: ObsFunction/ObsErrorBoundConventional + options: + obsvar: stationPressure + obserr_bound_min: 100 + obserr_bound_max: 300 + obserr_bound_factor: 5.0 + +- filter: Background Check + filter variables: + - name: stationPressure + where: + - variable: PreQC/stationPressure + is_not_in: 3 + function absolute threshold: + - name: DerivedMetaData/ObsErrorBoundSfcPressure1 + action: + name: reject + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBoundSfcPressure2 + type: float + function: + name: ObsFunction/ObsErrorBoundConventional + options: + obsvar: stationPressure + obserr_bound_min: 100 + obserr_bound_max: 300 + obserr_bound_factor: 3.5 + +- filter: Background Check + filter variables: + - name: stationPressure + where: + - variable: PreQC/stationPressure + is_in: 3 + function absolute threshold: + - name: DerivedMetaData/ObsErrorBoundSfcPressure2 + action: + name: reject + +# Inflate obs error based on duplicate check +- filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsErrorFactorDuplicateCheck/stationPressure + +# Reject data based on PreUseFlag (usage in GSI) +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: PreUseFlag/stationPressure + is_not_in: 0, 1 + action: + name: reject + +# End of Filters + diff --git a/parm/atm/obs/config/omi_aura.yaml b/parm/atm/obs/config/omi_aura.yaml index 79973780c..20cabced5 100644 --- a/parm/atm/obs/config/omi_aura.yaml +++ b/parm/atm/obs/config/omi_aura.yaml @@ -11,8 +11,79 @@ obs space: io pool: max pool size: 1 simulated variables: [ozoneTotal] + obs operator: name: AtmVertInterpLay - geovals: [ozoneLayer] + geovals: [mole_fraction_of_ozone_in_air] coefficients: [0.007886131] # convert from ppmv to DU nlevels: [1] + +obs pre filters: +- filter: Perform Action + filter variables: + - name: ozoneTotal + action: + name: assign error + error parameter: 6.0 + +obs prior filters: +# GSI read routine QC +# range sanity check +- filter: Bounds Check + filter variables: + - name: ozoneTotal + minvalue: 0 + maxvalue: 10000 + action: + name: reject + +# Do not use the data if row anomaly (bit 10)is 1 +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/totalOzoneQualityFlag + any_bit_set_of: 9 + +# Scan position check: reject scan position >= 25 +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/sensorScanPosition + minvalue: 25 + +# Accept total_ozone_error_flag values of 0 and 1, but not any others. +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/totalOzoneQualityCode + is_not_in: 0, 1 + +# Use data with best ozone algorighm +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/bestOzoneAlgorithmFlag + is_in: 3, 13 + +obs post filters: +# GSI setup routine QC +# Gross check +- filter: Background Check + filter variables: + - name: ozoneTotal + threshold: 10.0 + absolute threshold: 300.0 + action: + name: reject + +# End of Filters + + diff --git a/parm/atm/obs/config/ompsnp_npp.yaml b/parm/atm/obs/config/ompsnp_npp.yaml index 5ba772fb3..c18b9abed 100644 --- a/parm/atm/obs/config/ompsnp_npp.yaml +++ b/parm/atm/obs/config/ompsnp_npp.yaml @@ -4,74 +4,284 @@ obs space: engine: type: H5File obsfile: $(DATA)/obs/$(OPREFIX)ompsnp_npp.${{ current_cycle | to_YMDH }}.nc4 + obsgrouping: + group variables: ["latitude"] + sort variable: "pressure" + sort order: "ascending" obsdataout: engine: type: H5File obsfile: $(DATA)/diags/diag_ompsnp_npp_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 - simulated variables: [ozoneTotal] + simulated variables: [ozoneLayer] + obs operator: name: AtmVertInterpLay - geovals: [ozoneLayer] + geovals: [mole_fraction_of_ozone_in_air] coefficients: [0.007886131] # convert from ppmv to DU nlevels: [22] -obs filters: -#- filter: BlackList -# filter variables: -# - name: ozoneTotal -# where: -# - variable: -# name: MetaData/total_ozone_error_flag -# minvalue: 0.1 -# maxvalue: 1.1 -# action: -# name: reject -#- filter: BlackList -# filter variables: -# - name: ozoneTotal -# where: -# - variable: -# name: MetaData/total_ozone_error_flag -# minvalue: 2.1 # toss toq>2, there are some 4 and 6 -# action: -# name: reject -#- filter: BlackList -# filter variables: -# - name: ozoneTotal -# where: -# - variable: -# name: MetaData/profile_ozone_error_flag -# minvalue: 1.1 -# action: -# name: reject -- filter: BlackList - filter variables: - - name: ozoneTotal + +obs pre filters: +# Observation error assignment +- filter: Perform Action + filter variables: + - name: ozoneLayer + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [0.001, 10.1325, 16.00935, 25.43258, 40.32735, 63.93607, 101.325, 160.0935, 254.3257, 403.2735, 639.3608, 1013.25, 1600.935, 2543.258, 4032.735, 6393.607, 10132.5, 16009.35, 25432.57, 40327.35, 63936.07, 101325] + errors: [7.7236, 0.02, 0.02, 0.025, 0.08, 0.15, 0.056, 0.125, 0.2, 0.299, 0.587, 0.864, 1.547, 2.718, 3.893, 4.353, 3.971, 4.407, 4.428, 3.312, 2.198, 2.285] + +obs prior filters: +# Do not assimilation where pressure is zero +# Zero pressure indicates the data is total column ozone +- filter: RejectList + filter variables: + - name: ozoneLayer where: - variable: name: MetaData/pressure - maxvalue: 64.00000 + maxvalue: 0.0001 + +# Sanity check on observaton values +- filter: Bounds Check + filter variables: + - name: ozoneLayer + minvalue: 0 + maxvalue: 1000 action: name: reject -- filter: Domain Check + +# Total Ozone Quality Check (keeps 0, 2) +# 0 indentifies good data +# 2 identifies good data with a solar zenith angle > 84 degrees +- filter: RejectList filter variables: - - name: ozoneTotal + - name: ozoneLayer where: - variable: - name: MetaData/latitude - minvalue: -90. - maxvalue: 90. + name: MetaData/totalOzoneQuality + is_not_in: 0, 2 + +# Profile Ozone Quality Check (keeps 0, 1, 7) +# 0 : good data +# 1 : good data with a solar zenith angle > 84 degrees +# 7 : profile for which stray light correction applied +- filter: RejectList + filter variables: + - name: ozoneLayer + where: - variable: - name: MetaData/longitude - minvalue: -180. - maxvalue: 360. -- filter: Bounds Check + name: MetaData/profileOzoneQuality + is_not_in: 0, 1, 7 + +obs post filters: +# Gross error check +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 120 + action: + name: reject + where: + - variable: + name: MetaData/pressure + maxvalue: 0.001 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 30 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 30000.0 + maxvalue: 110000.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 40 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 20000.0 + maxvalue: 30000.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 44.42 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 10100.0 + maxvalue: 20000.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 57.52 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 6400.0 + maxvalue: 10100.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 69.4 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 4000.0 + maxvalue: 6400.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 70 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 2600.0 + maxvalue: 4000.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 62.73 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 1600.0 + maxvalue: 2600.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 50.52 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 1100.0 + maxvalue: 1600.0 + +- filter: Background Check filter variables: - - name: ozoneTotal - minvalue: 0.000001 - maxvalue: 1000.0 + - name: ozoneLayer + absolute threshold: 35.9 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 700.0 + maxvalue: 1100.0 + - filter: Background Check filter variables: - - name: ozoneTotal - absolute threshold: 10.0 + - name: ozoneLayer + absolute threshold: 26.41 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 400.0 + maxvalue: 700.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 20.51 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 300.0 + maxvalue: 400.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 12.82 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 200.0 + maxvalue: 300.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 10 + action: + name: reject + where: + - variable: + name: MetaData/pressure + maxvalue: 70.0 + maxvalue: 200.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 5 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 40.0 + maxvalue: 70.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 2 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 30.0 + maxvalue: 40.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 1 + action: + name: reject + where: + - variable: + name: MetaData/pressure + maxvalue: 30.0 + +# End of Filters diff --git a/parm/atm/obs/config/ompstc8_npp.yaml b/parm/atm/obs/config/ompstc8_npp.yaml index e3b11ff64..545983d93 100644 --- a/parm/atm/obs/config/ompstc8_npp.yaml +++ b/parm/atm/obs/config/ompstc8_npp.yaml @@ -11,8 +11,79 @@ obs space: io pool: max pool size: 1 simulated variables: [ozoneTotal] + obs operator: name: AtmVertInterpLay - geovals: [ozoneLayer] + geovals: [mole_fraction_of_ozone_in_air] coefficients: [0.007886131] # convert from ppmv to DU nlevels: [1] + +obs pre filters: +- filter: Perform Action + filter variables: + - name: ozoneTotal + action: + name: assign error + error parameter: 6.0 + +obs prior filters: +# GSI read routine QC +# range sanity check +- filter: Bounds Check + filter variables: + - name: ozoneTotal + minvalue: 0 + maxvalue: 1000 + action: + name: reject + +# Accept total_ozone_error_flag values of 0 and 1, but not any others. +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/totalOzoneQualityCode + is_not_in: 0, 1 + +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/bestOzoneAlgorithmFlag + is_in: 3, 13 + +# GSI setup routine QC +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 1, 2, 3, 4, 35 + - variable: + name: MetaData/latitude + minvalue: 50.0 + +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 1, 2, 3, 4, 35 + - variable: + name: MetaData/latitude + maxvalue: -50.0 + +obs post filters: +- filter: Background Check + filter variables: + - name: ozoneTotal + threshold: 10.0 + absolute threshold: 300.0 + action: + name: reject + +# End of Filters diff --git a/parm/atm/obs/config/satwind_goes-16.yaml b/parm/atm/obs/config/satwind_goes-16.yaml index 6fd5699be..f5d7f3df6 100644 --- a/parm/atm/obs/config/satwind_goes-16.yaml +++ b/parm/atm/obs/config/satwind_goes-16.yaml @@ -11,540 +11,554 @@ obs space: io pool: max pool size: 1 simulated variables: [windEastward, windNorthward] + obs operator: name: VertInterp - -obs pre filters: + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + +obs prior filters: +# Apply variable changes needed for wind scaling +# For wind observations with pressure provided +- filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: False + +# Calculate error inflation factor for duplicate observations +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windEastward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windEastward + +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windNorthward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windNorthward + +obs post filters: # Assign the initial observation error, based on height/pressure # Hard-wiring to prepobs_errtable.global by Type # ObsError is currently not updating in diag file, but passes directly to EffectiveError when no inflation is specified in YAML - # Type 240 (GOES SWIR): Assigned all dummy values in prepobs_errtable.global - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 240 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,0.] #Pressure (Pa) - errors: [1000000000.,1000000000.] - - # Type 245 (GOES LWIR): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 245 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] - - # Type 246 (GOES cloud-top WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 246 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] - - # Type 247 (GOES clear-sky WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 247 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] - - # Type 251 (GOES VIS): Assigned all dummy values - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 251 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,0.] #Pressure (Pa) - errors: [1000000000.,1000000000.] - - obs prior filters: - # sanity-check criteria - # Observation Range Sanity Check - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - minvalue: -130. - maxvalue: 130. - action: - name: reject - - # Velocity Sanity Check - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - test variables: - - name: ObsFunction/Velocity - maxvalue: 130. - action: - name: reject - - # GSI read routine QC (part-1) - # Exclude Type 251 (VIS) --- obs tossed without passing to setup routine - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 251 - action: - name: reject - - # Exclude data with satellite zenith angle > 68 for all types --- obs tossed without passing to setup routine - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - test variables: - - name: MetaData/sensorZenithAngle - maxvalue: 68. - action: - name: reject - - # Reject obs with pressure < 12500 pa --- obs tosed without passing to setup routine - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - test variables: - - name: MetaData/pressure - minvalue: 12500. - action: - name: reject - - # Exclude data over non-water surface type where latitude > 20N for Type 240 (IRSW) and Type 245 (IRLW) --- obs tossed and not passed to setup routine - # Notes: This check was missing, so added (eliu) - # Replace land_type_index_NPOSS with water_area_fraction (eliu) - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 240, 245 - - variable: MetaData/latitude - minvalue: 20. - test variables: - - name: GeoVaLs/water_area_fraction - minvalue: 0.999 - action: - name: reject - - # GSI read routine QC (part-2) - # Reject obs with qualityInformationWithoutForecast < 90. OR > 100. - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - test variables: - - name: MetaData/qualityInformationWithoutForecast - minvalue: 90. - maxvalue: 100. - action: - name: reject - - # Reject obs with pressure < 15000 pa. - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - test variables: - - name: MetaData/pressure - minvalue: 15000. - action: - name: reject - - # Reject obs with pressure < 70000 pa. when Type=251 - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - - variable: ObsType/windEastward - is_in: 251 - test variables: - - name: MetaData/pressure - minvalue: 70000. - action: - name: reject - - # Reject obs with pressure > 30000. when Type=246 - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - - variable: ObsType/windEastward - is_in: 246 - test variables: - - name: MetaData/pressure - maxvalue: 30000. - action: - name: reject - - # Reject obs with pressure > 85000. when isli=1 (land surface) - # Notes: Replace land_type_index_NPOESS with land_area_fraction (eliu) - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - - variable: GeoVaLs/land_area_fraction - minvalue: 0.9999 - test variables: - - name: MetaData/pressure - maxvalue: 85000. - action: - name: reject - - # Reject obs with pct1 (Coeff. of Var.) outside of 0.04–0.5, Type [240,245,246,251] ONLY - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - - variable: ObsType/windEastward - is_in: 240, 245, 246, 251 - test variables: - - name: MetaData/coefficientOfVariation - minvalue: 0.04 - maxvalue: 0.5 - action: - name: reject - - # NESDIS obs are also subject to the experr_norm test defined as: - # - # if (10. - 0.1*(expectedError))/(ob_speed)>0.9, or ob_speed<0.1, reject, applies to NESDIS winds - # - # CLEARED: With caveat that float precision/handling differences can generate different acceptance criteria - # between UFO and GSI for observations with an experr_norm value right around the maxvalue. - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - test variables: - - name: ObsFunction/SatWindsErrnormCheck - maxvalue: 0.9 - action: - name: reject - - # Reject all Type=240 (GOES SWIR) AMVs: These are not currently assimilated in GSI and they have missing-values - # assigned to ob-errors - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 240 - action: - name: reject - - # GSI setupw routine QC - # Reject any ob Type [240–260] when pressure greater than 950 mb. - # CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 240-260 - test variables: - - name: MetaData/pressure - maxvalue: 95001. - action: - name: reject - - # GOES IR (245) reject when pressure between 399 and 801 mb. - # # CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/pressure - minvalue: 39901. - maxvalue: 80099. - - variable: ObsType/windEastward - is_in: 245 - action: - name: reject - - # GOES WV (246, 250, 254), reject when pressure greater than 399 mb. - # CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 246, 250, 254 - test variables: - - name: MetaData/pressure - maxvalue: 39900. - action: - name: reject - - # Multiple satellite platforms, reject when pressure is more than 50 mb above tropopause. - # CLEARED: minvalue is rejecting <, not <= as per a Perform Action, so threshold is unchanged - # Notes (eliu): This tropopause check reject too many obs; probably due to tropopause pressure estimation - # Turn this check off for now. - # Need to check if troposphere pressure was implemented correctly in fv3-jed - - filter: Difference Check - filter variables: - - name: windEastward - - name: windNorthward - reference: GeoVaLs/tropopause_pressure - value: MetaData/pressure - minvalue: -5000. # 50 hPa above tropopause level, negative p-diff - action: - name: reject - - # GOES (247) reject any observation with a /=0 surface type (non-water - # surface) within 110 hPa of the surface pressure (as part of the LNVD # check). - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - # Notes (eliu): Replace land_type_index_NPOESS with land_area_fraction. - - filter: Difference Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: - name: GeoVaLs/water_area_fraction - maxvalue: 0.001 - - variable: - name: ObsType/windEastward - is_in: 247 - reference: GeoVaLs/surface_pressure - value: MetaData/pressure - maxvalue: -11000. # within 110 hPa above surface pressure, negative p-diff - action: - name: reject - - obs post filters: - # Reject GOES (247) when difference of wind direction is more than 50 degrees. - # CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 247 - test variables: - - name: ObsFunction/WindDirAngleDiff - maxvalue: 50. - action: - name: reject - - # All satwinds must adjust errors based on ObsErrorFactorPressureCheck - # prior to the SPDB check (i.e. the gross-error check). The gross-error - # check uses the adjusted errors for error-bound tightening and rejection, - # so this check has to come first. This check will inflate errors for obs - # that are too close to either the model top or bottom. - # Notes (eliu): GMAO added a required parameter: adjusted_error_name. - - filter: Perform Action - filter variables: - - name: windEastward - where: - - variable: - name: ObsType/windEastward - is_in: 240-260 - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorPressureCheck - options: - surface_obs: false - variable: windEastward - inflation factor: 4.0 - - - filter: Perform Action - filter variables: - - name: windNorthward - where: - - variable: - name: ObsType/windNorthward - is_in: 240-260 - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorPressureCheck - options: - variable: windNorthward - inflation factor: 4.0 - - # All satwinds subject to a gross-error check that contains significant - # modifiers for satwinds with a negative speed-bias. ALL wind gross-error - # checks are currently being done by the SatWindsSPDBCheck. - # CLEARED - - filter: Background Check - filter variables: - - name: windEastward - function absolute threshold: - - name: ObsFunction/WindsSPDBCheck +# Type 240 (GOES SWIR): Assigned all dummy values in prepobs_errtable.global +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear options: - wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] - cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] - error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] - error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] - variable: windEastward - action: - name: reject - - - filter: Background Check - filter variables: - - name: windNorthward - function absolute threshold: - - name: ObsFunction/WindsSPDBCheck + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 245 (GOES LWIR): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 245 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear options: - wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] - cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] - error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] - error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] - variable: windNorthward - action: - name: reject - -# # The last error inflation check is for duplicate observations. This one needs -# # to come last, because we don't want to inflate errors for duplication if one -# # of the duplicates should be rejected. -# # Notes (eliu): ObsErrorFactorDuplicateCheck obsfunction requires PreUseFlag (usage parameter from read_satwnd.f90). -# # : Turn off duplicate check for now. -# - filter: RejectList -# filter variables: -# - name: windEastward -# action: -# name: inflate error -# inflation variable: -# name: ObsFunction/ObsErrorFactorDuplicateCheck -# options: -# use_air_pressure: true -# variable: windEastward + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] + +# Type 246 (GOES cloud-top WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 246 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] + +# Type 247 (GOES clear-sky WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 247 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] + +# Type 251 (GOES VIS): Assigned all dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 251 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# sanity-check criteria +# Observation Range Sanity Check +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + minvalue: -130. + maxvalue: 130. + action: + name: reject + +# Velocity Sanity Check +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Velocity + maxvalue: 130. + action: + name: reject + +# GSI read routine QC (part-1) +# Exclude Type 251 (VIS) --- obs tossed without passing to setup routine +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 251 + action: + name: reject + +# Exclude data with satellite zenith angle > 68 for all types --- obs tossed without passing to setup routine +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: MetaData/sensorZenithAngle + maxvalue: 68. + action: + name: reject + +# Reject obs with pressure < 12500 pa --- obs tosed without passing to setup routine +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + test variables: + - name: MetaData/pressure + minvalue: 12500. + action: + name: reject + +# Exclude data over non-water surface type where latitude > 20N for Type 240 (IRSW) and Type 245 (IRLW) --- obs tossed and not passed to setup routine +# Notes: This check was missing, so added (eliu) +# Replace land_type_index_NPOSS with water_area_fraction (eliu) +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240, 245 + - variable: MetaData/latitude + minvalue: 20. + test variables: + - name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: reject + +# GSI read routine QC (part-2) +# Reject obs with qualityInformationWithoutForecast < 90. OR > 100. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + test variables: + - name: MetaData/qualityInformationWithoutForecast + minvalue: 90. + maxvalue: 100. + action: + name: reject + +# Reject obs with pressure < 15000 pa. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + test variables: + - name: MetaData/pressure + minvalue: 15000. + action: + name: reject + +# Reject obs with pressure < 70000 pa. when Type=251 +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + - variable: ObsType/windEastward + is_in: 251 + test variables: + - name: MetaData/pressure + minvalue: 70000. + action: + name: reject + +# Reject obs with pressure > 30000. when Type=246 +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + - variable: ObsType/windEastward + is_in: 246 + test variables: + - name: MetaData/pressure + maxvalue: 30000. + action: + name: reject + +# Reject obs with pressure > 85000. when isli=1 (land surface) +# Notes: Replace land_type_index_NPOESS with land_area_fraction (eliu) +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + - variable: GeoVaLs/land_area_fraction + minvalue: 0.99 + test variables: + - name: MetaData/pressure + maxvalue: 85000. + action: + name: reject + +# Reject obs with pct1 (Coeff. of Var.) outside of 0.04–0.5, Type [240,245,246,251] ONLY +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + - variable: ObsType/windEastward + is_in: 240, 245, 246, 251 + test variables: + - name: MetaData/coefficientOfVariation + minvalue: 0.04 + maxvalue: 0.5 + action: + name: reject + +# NESDIS obs are also subject to the experr_norm test defined as: # -# - filter: RejectList -# filter variables: -# - name: windNorthward -# action: -# name: inflate error -# inflation variable: -# name: ObsFunction/ObsErrorFactorDuplicateCheck -# options: -# use_air_pressure: true -# variable: windNorthward +# if (10. - 0.1*(expectedError))/(ob_speed)>0.9, or ob_speed<0.1, reject, applies to NESDIS winds # - # We are extending this to an additional filter that inflates final ob-errors across-the-board by - # 1/0.8 = 1.25. This is caused by the GSI value of nvqc being set to .true. in the global operational - # configuration, see: https://github.com/NOAA-EMC/global-workflow/blob/d5ae3328fa4041b177357b1133f6b92e81c859d7/scripts/exglobal_atmos_analysis.sh#L750 - # This setting activates Line 1229 of setupw.f90 to scale ratio_errors by 0.8, which is applied in - # the denominator of the final ob-error, so 1/0.8 = 1.25 factor of ob-error inflation. - # - # If this nvqc functionality were to be switched off (i.e. if variational qc were to be turned off), - # you would want to remove this last inflation filter. - - filter: Perform Action - filter variables: - - name: windEastward - where: - - variable: ObsType/windEastward - is_in: 240-260 - action: - name: inflate error - inflation factor: 1.25 - - - filter: Perform Action - filter variables: - - name: windNorthward - where: - - variable: ObsType/windNorthward +# CLEARED: With caveat that float precision/handling differences can generate different acceptance criteria +# between UFO and GSI for observations with an experr_norm value right around the maxvalue. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + test variables: + - name: ObsFunction/SatWindsErrnormCheck + maxvalue: 0.9 + action: + name: reject + +# Reject all Type=240 (GOES SWIR) AMVs: These are not currently assimilated in GSI and they have missing-values +# assigned to ob-errors +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240 + action: + name: reject + +# GSI setupw routine QC +# Reject any ob Type [240–260] when pressure greater than 950 mb. +# CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240-260 + test variables: + - name: MetaData/pressure + maxvalue: 95001. + action: + name: reject + +# GOES IR (245) reject when pressure between 399 and 801 mb. +# # CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/pressure + minvalue: 39901. + maxvalue: 80099. + - variable: ObsType/windEastward + is_in: 245 + action: + name: reject + +# GOES WV (246, 250, 254), reject when pressure greater than 399 mb. +# CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 246, 250, 254 + test variables: + - name: MetaData/pressure + maxvalue: 39900. + action: + name: reject + +# Multiple satellite platforms, reject when pressure is more than 50 mb above tropopause. +# CLEARED: minvalue is rejecting <, not <= as per a Perform Action, so threshold is unchanged +# Notes (eliu): This tropopause check reject too many obs; probably due to tropopause pressure estimation +# Turn this check off for now. +# Need to check if troposphere pressure was implemented correctly in fv3-jed +- filter: Difference Check + filter variables: + - name: windEastward + - name: windNorthward + reference: GeoVaLs/tropopause_pressure + value: MetaData/pressure + minvalue: -5000. # 50 hPa above tropopause level, negative p-diff + action: + name: reject + +# GOES (247) reject any observation with a /=0 surface type (non-water +# surface) within 110 hPa of the surface pressure (as part of the LNVD # check). +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +# Notes (eliu): Replace land_type_index_NPOESS with land_area_fraction. +- filter: Difference Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + - variable: + name: ObsType/windEastward + is_in: 247 + reference: GeoVaLs/surface_pressure + value: MetaData/pressure + maxvalue: -11000. # within 110 hPa above surface pressure, negative p-diff + action: + name: reject + +obs post filters: +# Reject GOES (247) when difference of wind direction is more than 50 degrees. +# CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 247 + test variables: + - name: ObsFunction/WindDirAngleDiff + maxvalue: 50. + action: + name: reject + +# All satwinds must adjust errors based on ObsErrorFactorPressureCheck +# prior to the SPDB check (i.e. the gross-error check). The gross-error +# check uses the adjusted errors for error-bound tightening and rejection, +# so this check has to come first. This check will inflate errors for obs +# that are too close to either the model top or bottom. +# Notes (eliu): GMAO added a required parameter: adjusted_error_name. +- filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward is_in: 240-260 - action: - name: inflate error - inflation factor: 1.25 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + surface_obs: false + variable: windEastward + inflation factor: 4.0 - # END OF FILTERS +- filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + +# All satwinds subject to a gross-error check that contains significant +# modifiers for satwinds with a negative speed-bias. ALL wind gross-error +# checks are currently being done by the SatWindsSPDBCheck. +# CLEARED +- filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] + error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] + variable: windEastward + action: + name: reject + +- filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] + error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] + variable: windNorthward + action: + name: reject + +# The last error inflation check is for duplicate observations. This one needs +# to come last, because we don't want to inflate errors for duplication if one +# of the duplicates should be rejected. +# Notes (eliu): ObsErrorFactorDuplicateCheck obsfunction requires PreUseFlag (usage parameter from read_satwnd.f90). +# : Turn off duplicate check for now. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windEastward + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windNorthward + +# We are extending this to an additional filter that inflates final ob-errors across-the-board by +# 1/0.8 = 1.25. This is caused by the GSI value of nvqc being set to .true. in the global operational +# configuration, see: https://github.com/NOAA-EMC/global-workflow/blob/d5ae3328fa4041b177357b1133f6b92e81c859d7/scripts/exglobal_atmos_analysis.sh#L750 +# This setting activates Line 1229 of setupw.f90 to scale ratio_errors by 0.8, which is applied in +# the denominator of the final ob-error, so 1/0.8 = 1.25 factor of ob-error inflation. +# +# If this nvqc functionality were to be switched off (i.e. if variational qc were to be turned off), +# you would want to remove this last inflation filter. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# where: +# - variable: ObsType/windEastward +# is_in: 240-260 +# action: +# name: inflate error +# inflation factor: 1.25 + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# where: +# - variable: ObsType/windNorthward +# is_in: 240-260 +# action: +# name: inflate error +# inflation factor: 1.25 + +# End of Filters diff --git a/parm/atm/obs/config/satwind_goes-17.yaml b/parm/atm/obs/config/satwind_goes-17.yaml index ca3632fcb..240af22d3 100644 --- a/parm/atm/obs/config/satwind_goes-17.yaml +++ b/parm/atm/obs/config/satwind_goes-17.yaml @@ -11,540 +11,554 @@ obs space: io pool: max pool size: 1 simulated variables: [windEastward, windNorthward] + obs operator: name: VertInterp - -obs pre filters: + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + +obs prior filters: +# Apply variable changes needed for wind scaling +# For wind observations with pressure provided +- filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: False + +# Calculate error inflation factor for duplicate observations +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windEastward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windEastward + +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windNorthward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windNorthward + +obs post filters: # Assign the initial observation error, based on height/pressure # Hard-wiring to prepobs_errtable.global by Type # ObsError is currently not updating in diag file, but passes directly to EffectiveError when no inflation is specified in YAML - # Type 240 (GOES SWIR): Assigned all dummy values in prepobs_errtable.global - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 240 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,0.] #Pressure (Pa) - errors: [1000000000.,1000000000.] - - # Type 245 (GOES LWIR): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 245 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] - - # Type 246 (GOES cloud-top WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 246 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] - - # Type 247 (GOES clear-sky WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 247 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] - - # Type 251 (GOES VIS): Assigned all dummy values - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 251 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,0.] #Pressure (Pa) - errors: [1000000000.,1000000000.] - - obs prior filters: - # sanity-check criteria - # Observation Range Sanity Check - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - minvalue: -130. - maxvalue: 130. - action: - name: reject - - # Velocity Sanity Check - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - test variables: - - name: ObsFunction/Velocity - maxvalue: 130. - action: - name: reject - - # GSI read routine QC (part-1) - # Exclude Type 251 (VIS) --- obs tossed without passing to setup routine - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 251 - action: - name: reject - - # Exclude data with satellite zenith angle > 68 for all types --- obs tossed without passing to setup routine - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - test variables: - - name: MetaData/sensorZenithAngle - maxvalue: 68. - action: - name: reject - - # Reject obs with pressure < 12500 pa --- obs tosed without passing to setup routine - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - test variables: - - name: MetaData/pressure - minvalue: 12500. - action: - name: reject - - # Exclude data over non-water surface type where latitude > 20N for Type 240 (IRSW) and Type 245 (IRLW) --- obs tossed and not passed to setup routine - # Notes: This check was missing, so added (eliu) - # Replace land_type_index_NPOSS with water_area_fraction (eliu) - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 240, 245 - - variable: MetaData/latitude - minvalue: 20. - test variables: - - name: GeoVaLs/water_area_fraction - minvalue: 0.999 - action: - name: reject - - # GSI read routine QC (part-2) - # Reject obs with qualityInformationWithoutForecast < 90. OR > 100. - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - test variables: - - name: MetaData/qualityInformationWithoutForecast - minvalue: 90. - maxvalue: 100. - action: - name: reject - - # Reject obs with pressure < 15000 pa. - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - test variables: - - name: MetaData/pressure - minvalue: 15000. - action: - name: reject - - # Reject obs with pressure < 70000 pa. when Type=251 - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - - variable: ObsType/windEastward - is_in: 251 - test variables: - - name: MetaData/pressure - minvalue: 70000. - action: - name: reject - - # Reject obs with pressure > 30000. when Type=246 - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - - variable: ObsType/windEastward - is_in: 246 - test variables: - - name: MetaData/pressure - maxvalue: 30000. - action: - name: reject - - # Reject obs with pressure > 85000. when isli=1 (land surface) - # Notes: Replace land_type_index_NPOESS with land_area_fraction (eliu) - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - - variable: GeoVaLs/land_area_fraction - minvalue: 0.9999 - test variables: - - name: MetaData/pressure - maxvalue: 85000. - action: - name: reject - - # Reject obs with pct1 (Coeff. of Var.) outside of 0.04–0.5, Type [240,245,246,251] ONLY - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - - variable: ObsType/windEastward - is_in: 240, 245, 246, 251 - test variables: - - name: MetaData/coefficientOfVariation - minvalue: 0.04 - maxvalue: 0.5 - action: - name: reject - - # NESDIS obs are also subject to the experr_norm test defined as: - # - # if (10. - 0.1*(expectedError))/(ob_speed)>0.9, or ob_speed<0.1, reject, applies to NESDIS winds - # - # CLEARED: With caveat that float precision/handling differences can generate different acceptance criteria - # between UFO and GSI for observations with an experr_norm value right around the maxvalue. - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - test variables: - - name: ObsFunction/SatWindsErrnormCheck - maxvalue: 0.9 - action: - name: reject - - # Reject all Type=240 (GOES SWIR) AMVs: These are not currently assimilated in GSI and they have missing-values - # assigned to ob-errors - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 240 - action: - name: reject - - # GSI setupw routine QC - # Reject any ob Type [240–260] when pressure greater than 950 mb. - # CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 240-260 - test variables: - - name: MetaData/pressure - maxvalue: 95001. - action: - name: reject - - # GOES IR (245) reject when pressure between 399 and 801 mb. - # # CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/pressure - minvalue: 39901. - maxvalue: 80099. - - variable: ObsType/windEastward - is_in: 245 - action: - name: reject - - # GOES WV (246, 250, 254), reject when pressure greater than 399 mb. - # CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 246, 250, 254 - test variables: - - name: MetaData/pressure - maxvalue: 39900. - action: - name: reject - - # Multiple satellite platforms, reject when pressure is more than 50 mb above tropopause. - # CLEARED: minvalue is rejecting <, not <= as per a Perform Action, so threshold is unchanged - # Notes (eliu): This tropopause check reject too many obs; probably due to tropopause pressure estimation - # Turn this check off for now. - # Need to check if troposphere pressure was implemented correctly in fv3-jed - - filter: Difference Check - filter variables: - - name: windEastward - - name: windNorthward - reference: GeoVaLs/tropopause_pressure - value: MetaData/pressure - minvalue: -5000. # 50 hPa above tropopause level, negative p-diff - action: - name: reject - - # GOES (247) reject any observation with a /=0 surface type (non-water - # surface) within 110 hPa of the surface pressure (as part of the LNVD # check). - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - # Notes (eliu): Replace land_type_index_NPOESS with land_area_fraction. - - filter: Difference Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: - name: GeoVaLs/water_area_fraction - maxvalue: 0.001 - - variable: - name: ObsType/windEastward - is_in: 247 - reference: GeoVaLs/surface_pressure - value: MetaData/pressure - maxvalue: -11000. # within 110 hPa above surface pressure, negative p-diff - action: - name: reject - - obs post filters: - # Reject GOES (247) when difference of wind direction is more than 50 degrees. - # CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 247 - test variables: - - name: ObsFunction/WindDirAngleDiff - maxvalue: 50. - action: - name: reject - - # All satwinds must adjust errors based on ObsErrorFactorPressureCheck - # prior to the SPDB check (i.e. the gross-error check). The gross-error - # check uses the adjusted errors for error-bound tightening and rejection, - # so this check has to come first. This check will inflate errors for obs - # that are too close to either the model top or bottom. - # Notes (eliu): GMAO added a required parameter: adjusted_error_name. - - filter: Perform Action - filter variables: - - name: windEastward - where: - - variable: - name: ObsType/windEastward - is_in: 240-260 - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorPressureCheck - options: - surface_obs: false - variable: windEastward - inflation factor: 4.0 - - - filter: Perform Action - filter variables: - - name: windNorthward - where: - - variable: - name: ObsType/windNorthward - is_in: 240-260 - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorPressureCheck - options: - variable: windNorthward - inflation factor: 4.0 - - # All satwinds subject to a gross-error check that contains significant - # modifiers for satwinds with a negative speed-bias. ALL wind gross-error - # checks are currently being done by the SatWindsSPDBCheck. - # CLEARED - - filter: Background Check - filter variables: - - name: windEastward - function absolute threshold: - - name: ObsFunction/WindsSPDBCheck +# Type 240 (GOES SWIR): Assigned all dummy values in prepobs_errtable.global +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear options: - wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] - cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] - error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] - error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] - variable: windEastward - action: - name: reject - - - filter: Background Check - filter variables: - - name: windNorthward - function absolute threshold: - - name: ObsFunction/WindsSPDBCheck + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 245 (GOES LWIR): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 245 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear options: - wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] - cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] - error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] - error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] - variable: windNorthward - action: - name: reject - -# # The last error inflation check is for duplicate observations. This one needs -# # to come last, because we don't want to inflate errors for duplication if one -# # of the duplicates should be rejected. -# # Notes (eliu): ObsErrorFactorDuplicateCheck obsfunction requires PreUseFlag (usage parameter from read_satwnd.f90). -# # : Turn off duplicate check for now. -# - filter: RejectList -# filter variables: -# - name: windEastward -# action: -# name: inflate error -# inflation variable: -# name: ObsFunction/ObsErrorFactorDuplicateCheck -# options: -# use_air_pressure: true -# variable: windEastward + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] + +# Type 246 (GOES cloud-top WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 246 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] + +# Type 247 (GOES clear-sky WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 247 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] + +# Type 251 (GOES VIS): Assigned all dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 251 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# sanity-check criteria +# Observation Range Sanity Check +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + minvalue: -130. + maxvalue: 130. + action: + name: reject + +# Velocity Sanity Check +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Velocity + maxvalue: 130. + action: + name: reject + +# GSI read routine QC (part-1) +# Exclude Type 251 (VIS) --- obs tossed without passing to setup routine +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 251 + action: + name: reject + +# Exclude data with satellite zenith angle > 68 for all types --- obs tossed without passing to setup routine +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: MetaData/sensorZenithAngle + maxvalue: 68. + action: + name: reject + +# Reject obs with pressure < 12500 pa --- obs tosed without passing to setup routine +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + test variables: + - name: MetaData/pressure + minvalue: 12500. + action: + name: reject + +# Exclude data over non-water surface type where latitude > 20N for Type 240 (IRSW) and Type 245 (IRLW) --- obs tossed and not passed to setup routine +# Notes: This check was missing, so added (eliu) +# Replace land_type_index_NPOSS with water_area_fraction (eliu) +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240, 245 + - variable: MetaData/latitude + minvalue: 20. + test variables: + - name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: reject + +# GSI read routine QC (part-2) +# Reject obs with qualityInformationWithoutForecast < 90. OR > 100. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + test variables: + - name: MetaData/qualityInformationWithoutForecast + minvalue: 90. + maxvalue: 100. + action: + name: reject + +# Reject obs with pressure < 15000 pa. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + test variables: + - name: MetaData/pressure + minvalue: 15000. + action: + name: reject + +# Reject obs with pressure < 70000 pa. when Type=251 +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + - variable: ObsType/windEastward + is_in: 251 + test variables: + - name: MetaData/pressure + minvalue: 70000. + action: + name: reject + +# Reject obs with pressure > 30000. when Type=246 +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + - variable: ObsType/windEastward + is_in: 246 + test variables: + - name: MetaData/pressure + maxvalue: 30000. + action: + name: reject + +# Reject obs with pressure > 85000. when isli=1 (land surface) +# Notes: Replace land_type_index_NPOESS with land_area_fraction (eliu) +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + - variable: GeoVaLs/land_area_fraction + minvalue: 0.99 + test variables: + - name: MetaData/pressure + maxvalue: 85000. + action: + name: reject + +# Reject obs with pct1 (Coeff. of Var.) outside of 0.04–0.5, Type [240,245,246,251] ONLY +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + - variable: ObsType/windEastward + is_in: 240, 245, 246, 251 + test variables: + - name: MetaData/coefficientOfVariation + minvalue: 0.04 + maxvalue: 0.5 + action: + name: reject + +# NESDIS obs are also subject to the experr_norm test defined as: # -# - filter: RejectList -# filter variables: -# - name: windNorthward -# action: -# name: inflate error -# inflation variable: -# name: ObsFunction/ObsErrorFactorDuplicateCheck -# options: -# use_air_pressure: true -# variable: windNorthward +# if (10. - 0.1*(expectedError))/(ob_speed)>0.9, or ob_speed<0.1, reject, applies to NESDIS winds # - # We are extending this to an additional filter that inflates final ob-errors across-the-board by - # 1/0.8 = 1.25. This is caused by the GSI value of nvqc being set to .true. in the global operational - # configuration, see: https://github.com/NOAA-EMC/global-workflow/blob/d5ae3328fa4041b177357b1133f6b92e81c859d7/scripts/exglobal_atmos_analysis.sh#L750 - # This setting activates Line 1229 of setupw.f90 to scale ratio_errors by 0.8, which is applied in - # the denominator of the final ob-error, so 1/0.8 = 1.25 factor of ob-error inflation. - # - # If this nvqc functionality were to be switched off (i.e. if variational qc were to be turned off), - # you would want to remove this last inflation filter. - - filter: Perform Action - filter variables: - - name: windEastward - where: - - variable: ObsType/windEastward - is_in: 240-260 - action: - name: inflate error - inflation factor: 1.25 - - - filter: Perform Action - filter variables: - - name: windNorthward - where: - - variable: ObsType/windNorthward +# CLEARED: With caveat that float precision/handling differences can generate different acceptance criteria +# between UFO and GSI for observations with an experr_norm value right around the maxvalue. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + test variables: + - name: ObsFunction/SatWindsErrnormCheck + maxvalue: 0.9 + action: + name: reject + +# Reject all Type=240 (GOES SWIR) AMVs: These are not currently assimilated in GSI and they have missing-values +# assigned to ob-errors +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240 + action: + name: reject + +# GSI setupw routine QC +# Reject any ob Type [240–260] when pressure greater than 950 mb. +# CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240-260 + test variables: + - name: MetaData/pressure + maxvalue: 95001. + action: + name: reject + +# GOES IR (245) reject when pressure between 399 and 801 mb. +# # CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/pressure + minvalue: 39901. + maxvalue: 80099. + - variable: ObsType/windEastward + is_in: 245 + action: + name: reject + +# GOES WV (246, 250, 254), reject when pressure greater than 399 mb. +# CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 246, 250, 254 + test variables: + - name: MetaData/pressure + maxvalue: 39900. + action: + name: reject + +# Multiple satellite platforms, reject when pressure is more than 50 mb above tropopause. +# CLEARED: minvalue is rejecting <, not <= as per a Perform Action, so threshold is unchanged +# Notes (eliu): This tropopause check reject too many obs; probably due to tropopause pressure estimation +# Turn this check off for now. +# Need to check if troposphere pressure was implemented correctly in fv3-jed +- filter: Difference Check + filter variables: + - name: windEastward + - name: windNorthward + reference: GeoVaLs/tropopause_pressure + value: MetaData/pressure + minvalue: -5000. # 50 hPa above tropopause level, negative p-diff + action: + name: reject + +# GOES (247) reject any observation with a /=0 surface type (non-water +# surface) within 110 hPa of the surface pressure (as part of the LNVD # check). +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +# Notes (eliu): Replace land_type_index_NPOESS with land_area_fraction. +- filter: Difference Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + - variable: + name: ObsType/windEastward + is_in: 247 + reference: GeoVaLs/surface_pressure + value: MetaData/pressure + maxvalue: -11000. # within 110 hPa above surface pressure, negative p-diff + action: + name: reject + +obs post filters: +# Reject GOES (247) when difference of wind direction is more than 50 degrees. +# CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 247 + test variables: + - name: ObsFunction/WindDirAngleDiff + maxvalue: 50. + action: + name: reject + +# All satwinds must adjust errors based on ObsErrorFactorPressureCheck +# prior to the SPDB check (i.e. the gross-error check). The gross-error +# check uses the adjusted errors for error-bound tightening and rejection, +# so this check has to come first. This check will inflate errors for obs +# that are too close to either the model top or bottom. +# Notes (eliu): GMAO added a required parameter: adjusted_error_name. +- filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward is_in: 240-260 - action: - name: inflate error - inflation factor: 1.25 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + surface_obs: false + variable: windEastward + inflation factor: 4.0 - # END OF FILTERS +- filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + +# All satwinds subject to a gross-error check that contains significant +# modifiers for satwinds with a negative speed-bias. ALL wind gross-error +# checks are currently being done by the SatWindsSPDBCheck. +# CLEARED +- filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] + error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] + variable: windEastward + action: + name: reject + +- filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] + error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] + variable: windNorthward + action: + name: reject + +# The last error inflation check is for duplicate observations. This one needs +# to come last, because we don't want to inflate errors for duplication if one +# of the duplicates should be rejected. +# Notes (eliu): ObsErrorFactorDuplicateCheck obsfunction requires PreUseFlag (usage parameter from read_satwnd.f90). +# : Turn off duplicate check for now. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windEastward + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windNorthward + +# We are extending this to an additional filter that inflates final ob-errors across-the-board by +# 1/0.8 = 1.25. This is caused by the GSI value of nvqc being set to .true. in the global operational +# configuration, see: https://github.com/NOAA-EMC/global-workflow/blob/d5ae3328fa4041b177357b1133f6b92e81c859d7/scripts/exglobal_atmos_analysis.sh#L750 +# This setting activates Line 1229 of setupw.f90 to scale ratio_errors by 0.8, which is applied in +# the denominator of the final ob-error, so 1/0.8 = 1.25 factor of ob-error inflation. +# +# If this nvqc functionality were to be switched off (i.e. if variational qc were to be turned off), +# you would want to remove this last inflation filter. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# where: +# - variable: ObsType/windEastward +# is_in: 240-260 +# action: +# name: inflate error +# inflation factor: 1.25 + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# where: +# - variable: ObsType/windNorthward +# is_in: 240-260 +# action: +# name: inflate error +# inflation factor: 1.25 + +# End of Filters diff --git a/parm/atm/obs/config/scatwind_metop-a.yaml b/parm/atm/obs/config/scatwind_metop-a.yaml new file mode 100644 index 000000000..4735c31fe --- /dev/null +++ b/parm/atm/obs/config/scatwind_metop-a.yaml @@ -0,0 +1,280 @@ +obs space: + name: ascatw_ascat_metop-a + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)ascatw.ascat_metop-a.{{ current_cycle | to_YMDH }}.nc4 + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_ascatw_ascat_metop-a_{{ current_cycle | to_YMDH }}.nc4 + io pool: + max pool size: 1 + simulated variables: [windEastward, windNorthward] + +obs operator: + name: VertInterp + # Use height vertical coordinate first +# vertical coordinate: geometric_height + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + +obs prior filters: +# Apply variable changes needed for rescaled height coordinate +- filter: Variable Transforms + Transform: AdjustedHeightCoordinate + SkipWhenNoObs: False + +# Apply variable changes needed for wind scaling +- filter: Variable Transforms + Transform: SurfaceWindScalingHeight + SkipWhenNoObs: False + +# Assign the initial observation error (constant value, 1.5 m/s right now). +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 1.5 + +# Calculate error inflation factor for duplicate observations +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windEastward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windEastward + +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windNorthward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windNorthward + +# Reject all obs with PreQC mark already set above 3 +# NOTE: All scatwinds have an automatic PreQC mark of 2 (hard-wired default from GSI) +# - filter: PreQC +# maxvalue: 3 +# action: +# name: reject + +obs post filters: +# Reject all ASCAT (Type 290) winds with tsavg <= 273.0 (surface temperature) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 290 +# - variable: GeoVaLs/surface_temperature + - variable: GeoVaLs/surface_temperature_where_land + maxvalue: 273. + action: + name: reject + +# Reject all ASCAT (Type 290) winds with isflg /= 0 (non-water surface) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 290 + - variable: GeoVaLs/water_area_fraction + maxvalue: 0.99 + action: + name: reject + +# Reject ASCAT (Type 290) when observed component deviates from background by more than 5.0 m/s +# NOTE: This check can reject a u- or v-component of the same observation independently, which +# is fundamentally different from how GSI rejects obs (both components are rejected if +# either component fails a check). +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Arithmetic + options: + variables: + - name: ObsValue/windEastward + - name: HofX/windEastward + coefs: [1.0, -1.0] + minvalue: -5.0 + maxvalue: 5.0 + +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Arithmetic + options: + variables: + - name: ObsValue/windNorthward + - name: HofX/windNorthward + coefs: [1.0, -1.0] + minvalue: -5.0 + maxvalue: 5.0 + +# Reject OSCAT (Type 291) when observed component deviates from background by more than 6.0 m/s +# NOTE: This check can reject a u- or v-component of the same observation independently, which +# is fundamentally different from how GSI rejects obs (both components are rejected if +# either component fails a check). +- filter: Background Check + filter variables: + - name: windEastward + - name: windNorthward + threshold: 6. + absolute threshold: 6. + where: + - variable: ObsType/windEastward + is_in: 291 + action: + name: reject + +# Reject ASCAT (Type 290) when ambiguity check fails (returned value is negative) +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 290 + test variables: + - name: ObsFunction/ScatWindsAmbiguityCheck + options: + minimum_uv: 0.0001 # hard-coding a minimum-uv for transparancy, want this to basically be zero + maxvalue: 0. + action: + name: reject + +# All scatwinds must adjust errors based on ObsErrorFactorPressureCheck. +# This check will inflate errors for obs that are too close to either +# the model top or bottom. +- filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 290-291 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4.0 + +- filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 290-291 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + +# All scatwinds subject to a gross error check. This is contained within +# the WindsSPDBCheck, although it is not exclusive to satwinds. +- filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 290, 291] + cgross: [ 5.0, 5.0] + error_min: [1.4, 1.4] + error_max: [6.1, 6.1] + variable: windEastward + action: + name: reject + +- filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 290, 291] + cgross: [ 5.0, 5.0] + error_min: [1.4, 1.4] + error_max: [6.1, 6.1] + variable: windNorthward + action: + name: reject + +# The last error inflation check is for duplicate observations. This one needs +# to come last, because we don't want to inflate errors for duplication if one +# of the duplicates should be rejected. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windEastward + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windNorthward + +# There is no across-the-board inflation for nvqc=.true. for scatwinds, presumably because for +# this inflation to take place both nvqc must be .true. AND ibeta must be >0, see: +# https://github.com/NOAA-EMC/GSI/blob/14ae595af1b03471287d322596d35c0665336e95/src/gsi/setupw.f90#L1229 +# GSI settings must have ibeta>0 for satwinds, but not for scatwinds. +# +# If the ibeta settings for scatwinds were to change while nvqc remained .true., we would extend YAML to +# an additional filter that inflates final ob-errors across-the-board by 1/0.8 = 1.25. NOTE: the nvqc setting +# is defaulted to .false. in GSI code, but is overridden in global operational configuration. See: +# configuration, see: https://github.com/NOAA-EMC/global-workflow/blob/d5ae3328fa4041b177357b1133f6b92e81c859d7/scripts/exglobal_atmos_analysis.sh#L750 +# This setting activates Line 1229 of setupw.f90 to scale ratio_errors by 0.8, which is applied in +# the denominator of the final ob-error, so 1/0.8 = 1.25 factor of ob-error inflation. +# +# If this functionality were to be activated for scatwinds, you would want to include this last inflation filter. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# where: +# - variable: ObsType/windEastward +# is_in: 290-291 +# action: +# name: inflate error +# inflation factor: 1.25 +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# where: +# - variable: ObsType/windNorthward +# is_in: 290-291 +# action: +# name: inflate error +# inflation factor: 1.25 + +# END OF FILTERS# diff --git a/parm/atm/obs/config/scatwind_metop-b.yaml b/parm/atm/obs/config/scatwind_metop-b.yaml new file mode 100644 index 000000000..5ed7c38bc --- /dev/null +++ b/parm/atm/obs/config/scatwind_metop-b.yaml @@ -0,0 +1,280 @@ +obs space: + name: ascatw_ascat_metop-b + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)ascatw.ascat_metop-b.{{ current_cycle | to_YMDH }}.nc4 + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_ascatw_ascat_metop-b_{{ current_cycle | to_YMDH }}.nc4 + io pool: + max pool size: 1 + simulated variables: [windEastward, windNorthward] + +obs operator: + name: VertInterp + # Use height vertical coordinate first +# vertical coordinate: geometric_height + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + +obs prior filters: +# Apply variable changes needed for rescaled height coordinate +- filter: Variable Transforms + Transform: AdjustedHeightCoordinate + SkipWhenNoObs: False + +# Apply variable changes needed for wind scaling +- filter: Variable Transforms + Transform: SurfaceWindScalingHeight + SkipWhenNoObs: False + +# Assign the initial observation error (constant value, 1.5 m/s right now). +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 1.5 + +# Calculate error inflation factor for duplicate observations +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windEastward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windEastward + +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windNorthward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windNorthward + +# Reject all obs with PreQC mark already set above 3 +# NOTE: All scatwinds have an automatic PreQC mark of 2 (hard-wired default from GSI) +# - filter: PreQC +# maxvalue: 3 +# action: +# name: reject + +obs post filters: +# Reject all ASCAT (Type 290) winds with tsavg <= 273.0 (surface temperature) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 290 +# - variable: GeoVaLs/surface_temperature + - variable: GeoVaLs/surface_temperature_where_land + maxvalue: 273. + action: + name: reject + +# Reject all ASCAT (Type 290) winds with isflg /= 0 (non-water surface) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 290 + - variable: GeoVaLs/water_area_fraction + maxvalue: 0.99 + action: + name: reject + +# Reject ASCAT (Type 290) when observed component deviates from background by more than 5.0 m/s +# NOTE: This check can reject a u- or v-component of the same observation independently, which +# is fundamentally different from how GSI rejects obs (both components are rejected if +# either component fails a check). +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Arithmetic + options: + variables: + - name: ObsValue/windEastward + - name: HofX/windEastward + coefs: [1.0, -1.0] + minvalue: -5.0 + maxvalue: 5.0 + +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Arithmetic + options: + variables: + - name: ObsValue/windNorthward + - name: HofX/windNorthward + coefs: [1.0, -1.0] + minvalue: -5.0 + maxvalue: 5.0 + +# Reject OSCAT (Type 291) when observed component deviates from background by more than 6.0 m/s +# NOTE: This check can reject a u- or v-component of the same observation independently, which +# is fundamentally different from how GSI rejects obs (both components are rejected if +# either component fails a check). +- filter: Background Check + filter variables: + - name: windEastward + - name: windNorthward + threshold: 6. + absolute threshold: 6. + where: + - variable: ObsType/windEastward + is_in: 291 + action: + name: reject + +# Reject ASCAT (Type 290) when ambiguity check fails (returned value is negative) +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 290 + test variables: + - name: ObsFunction/ScatWindsAmbiguityCheck + options: + minimum_uv: 0.0001 # hard-coding a minimum-uv for transparancy, want this to basically be zero + maxvalue: 0. + action: + name: reject + +# All scatwinds must adjust errors based on ObsErrorFactorPressureCheck. +# This check will inflate errors for obs that are too close to either +# the model top or bottom. +- filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 290-291 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4.0 + +- filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 290-291 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + +# All scatwinds subject to a gross error check. This is contained within +# the WindsSPDBCheck, although it is not exclusive to satwinds. +- filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 290, 291] + cgross: [ 5.0, 5.0] + error_min: [1.4, 1.4] + error_max: [6.1, 6.1] + variable: windEastward + action: + name: reject + +- filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 290, 291] + cgross: [ 5.0, 5.0] + error_min: [1.4, 1.4] + error_max: [6.1, 6.1] + variable: windNorthward + action: + name: reject + +# The last error inflation check is for duplicate observations. This one needs +# to come last, because we don't want to inflate errors for duplication if one +# of the duplicates should be rejected. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windEastward + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windNorthward + +# There is no across-the-board inflation for nvqc=.true. for scatwinds, presumably because for +# this inflation to take place both nvqc must be .true. AND ibeta must be >0, see: +# https://github.com/NOAA-EMC/GSI/blob/14ae595af1b03471287d322596d35c0665336e95/src/gsi/setupw.f90#L1229 +# GSI settings must have ibeta>0 for satwinds, but not for scatwinds. +# +# If the ibeta settings for scatwinds were to change while nvqc remained .true., we would extend YAML to +# an additional filter that inflates final ob-errors across-the-board by 1/0.8 = 1.25. NOTE: the nvqc setting +# is defaulted to .false. in GSI code, but is overridden in global operational configuration. See: +# configuration, see: https://github.com/NOAA-EMC/global-workflow/blob/d5ae3328fa4041b177357b1133f6b92e81c859d7/scripts/exglobal_atmos_analysis.sh#L750 +# This setting activates Line 1229 of setupw.f90 to scale ratio_errors by 0.8, which is applied in +# the denominator of the final ob-error, so 1/0.8 = 1.25 factor of ob-error inflation. +# +# If this functionality were to be activated for scatwinds, you would want to include this last inflation filter. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# where: +# - variable: ObsType/windEastward +# is_in: 290-291 +# action: +# name: inflate error +# inflation factor: 1.25 +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# where: +# - variable: ObsType/windNorthward +# is_in: 290-291 +# action: +# name: inflate error +# inflation factor: 1.25 + +# END OF FILTERS# diff --git a/parm/atm/obs/testing/conv_ps.yaml b/parm/atm/obs/testing/conv_ps.yaml index f43bc9f41..20a449a8a 100644 --- a/parm/atm/obs/testing/conv_ps.yaml +++ b/parm/atm/obs/testing/conv_ps.yaml @@ -8,8 +8,8 @@ obs space: engine: type: H5File obsfile: !ENV conv_ps_diag_${CDATE}.nc4 - overwrite: true simulated variables: [stationPressure] + geovals: filename: !ENV conv_ps_geoval_${CDATE}.nc4 @@ -22,191 +22,276 @@ obs operator: geovar_sfc_geomz: surface_altitude geovar_geomz: geopotential_height +obs prior filters: +# Initial Error Assignments for SFC Observations +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [181] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [80000, 75000, 70000, 65000, 60000, 55000 ] + errors: [110, 120, 120, 120, 120, 1.0e+11] + +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [187] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [85000, 80000, 75000, 70000, 65000, 60000, 55000 ] + errors: [ 120, 140, 140, 140, 140, 140, 1.0e+11] + +# Initial Error Assignments for SFCSHIP Observations +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [180] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [60000, 55000 ] + errors: [ 130, 1.0e+11] + +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [183] + action: + name: assign error + error parameter: 1.0e+11 + +# Initial Error Assignments for Radiosonde +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [120] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [80000, 75000, 70000, 65000, 60000, 55000 ] + errors: [ 110, 120, 120, 120, 120, 1.0e+11] + obs post filters: - # Observation range sanity check - - filter: Bounds Check - filter variables: - - name: stationPressure - minvalue: 37499.0 - maxvalue: 106999.0 - action: - name: reject - - # Reject all ObsType 183 - - filter: RejectList - where: - - variable: - name: ObsType/stationPressure - is_in: 183 - - # Reject surface pressure below 500 hPa - - filter: Bounds Check - filter variables: - - name: stationPressure - minvalue: 50000.00 - action: - name: reject - - - filter: RejectList - where: - - variable: - name: PreQC/stationPressure - is_in: 4-15 - - # Inflate obs error based on obs type - - filter: Perform Action - filter variables: - - name: stationPressure - where: - - variable: PreQC/stationPressure - is_in: 3, 7 - action: - name: inflate error - inflation factor: 1.2 - - # Calculate obs error inflation factors for duplicated observations at the same location - - filter: Variable Assignment - assignments: - - name: ObsErrorFactorDuplicateCheck/stationPressure - type: float - function: - name: ObsFunction/ObsErrorFactorDuplicateCheck - options: - use_air_pressure: false - variable: stationPressure - - # Reduce effective observation error based on obs type and subtype - # In this case: reduce effective obs error for buoy - - filter: Perform Action - filter variables: - - name: stationPressure - where: - - variable: - name: ObsType/stationPressure - is_in: 180 - - variable: - name: ObsSubType/stationPressure - is_in: 0 - action: - name: inflate error - inflation factor: 0.7 - - # Reduce original observation error based on obs type and subtype - # In this case: reduce original obs error for buoy - - filter: Variable Assignment - where: - - variable: - name: ObsType/stationPressure - is_in: 180 - - variable: - name: ObsSubType/stationPressure - is_in: 0 - assignments: - - name: ObsError/stationPressure - type: float - function: - name: ObsFunction/Arithmetic - options: - variables: - - name: ObsError/stationPressure - coefs: [0.7] - - # Calculate obs error inflation factors for large discrepancies between model and observations - - filter: Variable Assignment - assignments: - - name: ObsErrorFactorSfcPressure/stationPressure - type: float - function: - name: ObsFunction/ObsErrorFactorSfcPressure - options: - geovar_sfc_geomz: surface_altitude - - # Inflate surface pressure observation based on discrepancies between - # model and observations due to terrian - - filter: Perform Action - filter variables: - - name: stationPressure - action: - name: inflate error - inflation variable: - name: ObsErrorFactorSfcPressure/stationPressure - - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/Innovation - type: float - function: - name: ObsFunction/Arithmetic - options: - variables: - - name: ObsValue/stationPressure - - name: HofX/stationPressure - coefs: [1, -1] - - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ObsErrorBoundSfcPressure1 - type: float - function: - name: ObsFunction/ObsErrorBoundConventional - options: - obsvar: stationPressure - obserr_bound_min: 100 - obserr_bound_max: 300 - obserr_bound_factor: 5.0 - - - filter: Background Check - filter variables: - - name: stationPressure - where: - - variable: PreQC/stationPressure - is_not_in: 3 - function absolute threshold: - - name: DerivedMetaData/ObsErrorBoundSfcPressure1 - action: - name: reject - - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ObsErrorBoundSfcPressure2 - type: float - function: - name: ObsFunction/ObsErrorBoundConventional - options: - obsvar: stationPressure - obserr_bound_min: 100 - obserr_bound_max: 300 - obserr_bound_factor: 3.5 - - - filter: Background Check - filter variables: - - name: stationPressure - where: - - variable: PreQC/stationPressure - is_in: 3 - function absolute threshold: - - name: DerivedMetaData/ObsErrorBoundSfcPressure2 - action: - name: reject - - # Inflate obs error based on duplicate check - - filter: Perform Action - filter variables: - - name: stationPressure - action: - name: inflate error - inflation variable: - name: ObsErrorFactorDuplicateCheck/stationPressure - - # Reject data based on PreUseFlag (usage in GSI) - - filter: Perform Action - filter variables: - - name: stationPressure - where: - - variable: PreUseFlag/stationPressure - is_not_in: 0, 1 - action: - name: reject +# Observation range sanity check +- filter: Bounds Check + filter variables: + - name: stationPressure + minvalue: 37499.0 + maxvalue: 106999.0 + action: + name: reject + +# Reject all ObsType 183 +- filter: RejectList + where: + - variable: + name: ObsType/stationPressure + is_in: 183 + +# Reject surface pressure below 500 hPa +- filter: Bounds Check + filter variables: + - name: stationPressure + minvalue: 50000.00 + action: + name: reject + +- filter: RejectList + where: + - variable: + name: PreQC/stationPressure + is_in: 4-15 + +# Inflate obs error based on obs type +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: PreQC/stationPressure + is_in: 3, 7 + action: + name: inflate error + inflation factor: 1.2 + +# Calculate obs error inflation factors for duplicated observations at the same location +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorDuplicateCheck/stationPressure + type: float + function: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: false + variable: stationPressure + +# Reduce effective observation error based on obs type and subtype +# In this case: reduce effective obs error for buoy +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: + name: ObsType/stationPressure + is_in: 180 + - variable: + name: ObsSubType/stationPressure + is_in: 0 + action: + name: inflate error + inflation factor: 0.7 + +# Reduce original observation error based on obs type and subtype +# In this case: reduce original obs error for buoy +- filter: Variable Assignment + where: + - variable: + name: ObsType/stationPressure + is_in: 180 + - variable: + name: ObsSubType/stationPressure + is_in: 0 + assignments: + - name: ObsError/stationPressure + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsError/stationPressure + coefs: [0.7] + +# Calculate obs error inflation factors for large discrepancies between model and observations +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorSfcPressure/stationPressure + type: float + function: + name: ObsFunction/ObsErrorFactorSfcPressure + options: + geovar_sfc_geomz: surface_altitude + geovar_geomz: geopotential_height + station_altitude: height + +# Inflate surface pressure observation based on discrepancies between +# model and observations due to terrian +- filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSfcPressure/stationPressure + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/Innovation + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsValue/stationPressure + - name: HofX/stationPressure + coefs: [1, -1] + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBoundSfcPressure1 + type: float + function: + name: ObsFunction/ObsErrorBoundConventional + options: + obsvar: stationPressure + obserr_bound_min: 100 + obserr_bound_max: 300 + obserr_bound_factor: 5.0 + +- filter: Background Check + filter variables: + - name: stationPressure + where: + - variable: PreQC/stationPressure + is_not_in: 3 + function absolute threshold: + - name: DerivedMetaData/ObsErrorBoundSfcPressure1 + action: + name: reject + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBoundSfcPressure2 + type: float + function: + name: ObsFunction/ObsErrorBoundConventional + options: + obsvar: stationPressure + obserr_bound_min: 100 + obserr_bound_max: 300 + obserr_bound_factor: 3.5 + +- filter: Background Check + filter variables: + - name: stationPressure + where: + - variable: PreQC/stationPressure + is_in: 3 + function absolute threshold: + - name: DerivedMetaData/ObsErrorBoundSfcPressure2 + action: + name: reject + +# Inflate obs error based on duplicate check +- filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsErrorFactorDuplicateCheck/stationPressure + +# Reject data based on PreUseFlag (usage in GSI) +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: PreUseFlag/stationPressure + is_not_in: 0, 1 + action: + name: reject +# End of Filters passedBenchmark: 85378 diff --git a/parm/atm/obs/testing/conv_ps_noqc.yaml b/parm/atm/obs/testing/conv_ps_noqc.yaml index 44a6ef4c8..8d469729f 100644 --- a/parm/atm/obs/testing/conv_ps_noqc.yaml +++ b/parm/atm/obs/testing/conv_ps_noqc.yaml @@ -8,8 +8,8 @@ obs space: engine: type: H5File obsfile: !ENV conv_ps_diag_${CDATE}.nc4 - overwrite: true simulated variables: [stationPressure] + geovals: filename: !ENV conv_ps_geoval_${CDATE}.nc4 @@ -22,10 +22,89 @@ obs operator: geovar_sfc_geomz: surface_altitude geovar_geomz: geopotential_height -vector ref: GsiHofXBc -tolerance: 1.e-4 -#linear obs operator test: -# coef TL: 0.1 -# tolerance TL: 1.0e-13 -# tolerance AD: 1.0e-11 +obs prior filters: +# Initial Error Assignments for SFC Observations +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [181] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [80000, 75000, 70000, 65000, 60000, 55000 ] + errors: [110, 120, 120, 120, 120, 1.0e+11] + +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [187] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [85000, 80000, 75000, 70000, 65000, 60000, 55000 ] + errors: [ 120, 140, 140, 140, 140, 140, 1.0e+11] + +# Initial Error Assignments for SFCSHIP Observations +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [180] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [60000, 55000 ] + errors: [ 130, 1.0e+11] + +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [183] + action: + name: assign error + error parameter: 1.0e+11 + + # Initial Error Assignments for Radiosonde +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [120] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [80000, 75000, 70000, 65000, 60000, 55000 ] + errors: [ 110, 120, 120, 120, 120, 1.0e+11] + +passedBenchmark: 92824 # total: 92842; missing: 18 +#vector ref: GsiHofXBc +#tolerance: 1.e-4 diff --git a/parm/atm/obs/testing/omi_aura.yaml b/parm/atm/obs/testing/omi_aura.yaml index a090f5945..8c5bb52da 100644 --- a/parm/atm/obs/testing/omi_aura.yaml +++ b/parm/atm/obs/testing/omi_aura.yaml @@ -11,8 +11,10 @@ obs space: io pool: max pool size: 1 simulated variables: [ozoneTotal] + geovals: filename: !ENV omi_aura_geoval_${CDATE}.nc4 + obs operator: name: AtmVertInterpLay geovals: [mole_fraction_of_ozone_in_air] @@ -85,4 +87,5 @@ obs post filters: action: name: reject -passedBenchmark: 1182 +# End of Filters +passedBenchmark: 1170 diff --git a/parm/atm/obs/testing/omi_aura_noqc.yaml b/parm/atm/obs/testing/omi_aura_noqc.yaml index 8929c38b6..d32954a8f 100644 --- a/parm/atm/obs/testing/omi_aura_noqc.yaml +++ b/parm/atm/obs/testing/omi_aura_noqc.yaml @@ -19,9 +19,14 @@ obs operator: coefficients: [0.007886131] # convert from ppmv to DU nlevels: [1] -vector ref: GsiHofXBc -tolerance: 1.e-5 -#linear obs operator test: -## coef TL: 0.1 -## tolerance TL: 1.0e-13 -## tolerance AD: 1.0e-11 +obs pre filters: +- filter: Perform Action + filter variables: + - name: ozoneTotal + action: + name: assign error + error parameter: 6.0 + +passedBenchmark: 4927 # total:6082; missing:1155 +#vector ref: GsiHofXBc +#tolerance: 1.e-5 diff --git a/parm/atm/obs/testing/ompsnp_npp.yaml b/parm/atm/obs/testing/ompsnp_npp.yaml index f7ee9d4ec..b7974f5cd 100644 --- a/parm/atm/obs/testing/ompsnp_npp.yaml +++ b/parm/atm/obs/testing/ompsnp_npp.yaml @@ -15,8 +15,10 @@ obs space: io pool: max pool size: 1 simulated variables: [ozoneLayer] + geovals: filename: !ENV ompsnp_npp_geoval_${CDATE}.nc4 + obs operator: name: AtmVertInterpLay geovals: [mole_fraction_of_ozone_in_air] @@ -285,4 +287,5 @@ obs post filters: name: MetaData/pressure maxvalue: 30.0 +# End of Filters passedBenchmark: 4914 diff --git a/parm/atm/obs/testing/ompsnp_npp_noqc.yaml b/parm/atm/obs/testing/ompsnp_npp_noqc.yaml index 219f9e942..639a8dda5 100644 --- a/parm/atm/obs/testing/ompsnp_npp_noqc.yaml +++ b/parm/atm/obs/testing/ompsnp_npp_noqc.yaml @@ -22,11 +22,23 @@ obs operator: geovals: [mole_fraction_of_ozone_in_air] coefficients: [0.007886131] # convert from ppmv to DU nlevels: [22] - -vector ref: GsiHofXBc -tolerance: 1.e-5 -#linear obs operator test: -# coef TL: 0.1 -# tolerance TL: 1.0e-13 -# tolerance AD: 1.0e-11 + +# Observation Error Assignment +obs pre filters: +- filter: Perform Action + filter variables: + - name: ozoneLayer + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [0.001, 10.1325, 16.00935, 25.43258, 40.32735, 63.93607, 101.325, 160.0935, 254.3257, 403.2735, 639.3608, 1013.25, 1600.935, 2543.258, 4032.735, 6393.607, 10132.5, 16009.35, 25432.57, 40327.35, 63936.07, 101325] + errors: [7.7236, 0.02, 0.02, 0.025, 0.08, 0.15, 0.056, 0.125, 0.2, 0.299, 0.587, 0.864, 1.547, 2.718, 3.893, 4.353, 3.971, 4.407, 4.428, 3.312, 2.198, 2.285] + +passedBenchmark: 6314 # total:6314; missing:0 +#vector ref: GsiHofXBc +#tolerance: 1.e-5 diff --git a/parm/atm/obs/testing/ompstc8_npp.yaml b/parm/atm/obs/testing/ompstc8_npp.yaml index aea18a5dc..8fb3d6165 100644 --- a/parm/atm/obs/testing/ompstc8_npp.yaml +++ b/parm/atm/obs/testing/ompstc8_npp.yaml @@ -15,8 +15,10 @@ obs space: io pool: max pool size: 1 simulated variables: [ozoneTotal] + geovals: filename: !ENV ompstc8_npp_geoval_${CDATE}.nc4 + obs operator: name: AtmVertInterpLay geovals: [mole_fraction_of_ozone_in_air] @@ -91,4 +93,5 @@ obs post filters: action: name: reject +# End of Filters passedBenchmark: 6130 diff --git a/parm/atm/obs/testing/ompstc8_npp_noqc.yaml b/parm/atm/obs/testing/ompstc8_npp_noqc.yaml index 31027007a..de9ef7839 100644 --- a/parm/atm/obs/testing/ompstc8_npp_noqc.yaml +++ b/parm/atm/obs/testing/ompstc8_npp_noqc.yaml @@ -22,11 +22,15 @@ obs operator: geovals: [mole_fraction_of_ozone_in_air] coefficients: [0.007886131] # convert from ppmv to DU nlevels: [1] +obs pre filters: +- filter: Perform Action + filter variables: + - name: ozoneTotal + action: + name: assign error + error parameter: 6.0 -vector ref: GsiHofXBc -tolerance: 1.e-5 -#linear obs operator test: -# coef TL: 0.1 -# tolerance TL: 1.0e-13 -# tolerance AD: 1.0e-11 +passedBenchmark: 6693 # total:6870; missing:177 +#vector ref: GsiHofXBc +#tolerance: 1.e-5 diff --git a/parm/atm/obs/testing/satwind.yaml b/parm/atm/obs/testing/satwind.yaml index bdf10f646..a43d107a4 100644 --- a/parm/atm/obs/testing/satwind.yaml +++ b/parm/atm/obs/testing/satwind.yaml @@ -9,988 +9,1024 @@ obs space: type: H5File obsfile: !ENV satwind_diag_${CDATE}.nc4 simulated variables: [windEastward, windNorthward] + geovals: filename: !ENV satwind_geoval_${CDATE}.nc4 -vector ref: GsiHofXBc -tolerance: 0.01 + obs operator: name: VertInterp -obs pre filters: + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + +obs prior filters: +# Apply variable changes needed for wind scaling +# For wind observations with pressure provided +- filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: False + +# Calculate obs error inflation factors for duplicated observations at the same location +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windEastward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windEastward + +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windNorthward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windNorthward + +obs post filters: # Assign the initial observation error, based on height/pressure # Hard-wiring to prepobs_errtable.global by Type # ObsError is currently not updating in diag file, but passes directly to EffectiveError when no inflation is specified in YAML - # Type 240 (GOES SWIR): Assigned all dummy values in prepobs_errtable.global - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 240 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,0.] #Pressure (Pa) - errors: [1000000000.,1000000000.] - # Type 241 (Multi Spec. Imager LWIR): Assigned all dummy values - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 241 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,0.] #Pressure (Pa) - errors: [1000000000.,1000000000.] - # Type 242 (Himawari VIS) - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 242 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] - # Type 243 (MVIRI/SEVIRI VIS) - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 243 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] - # Type 244 (AVHRR LWIR) - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 244 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] - # Type 245 (GOES LWIR): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 245 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] - # Type 246 (GOES cloud-top WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 246 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] - # Type 247 (GOES clear-sky WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 247 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] - # Type 248 (GOES Sounder cloud-top WV): Assigned all dummy values - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 248 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,0.] #Pressure (Pa) - errors: [1000000000.,1000000000.] - # Type 249 (GOES Sounder clear-sky WV): Assigned all dummy values - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 249 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,0.] #Pressure (Pa) - errors: [1000000000.,1000000000.] - # Type 250 (Himawari AHI WV, cloud-top or clear-sky) - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 250 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,7.,7.3,7.6,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.] - # Type 251 (GOES VIS): Assigned all dummy values - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 251 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,0.] #Pressure (Pa) - errors: [1000000000.,1000000000.] - # Type 252 (Himawari AHI LWIR) - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 252 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] - # Type 253 (MVIRI/SEVERI LWIR) - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 253 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] - # Type 254 (MVIRI/SEVIRI WV, both cloud-top and clear-sky) - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 254 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.5,6.1,6.,6.5,7.3,7.6,7.,7.5,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] - # Type 255 (LEOGEO): - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 255 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] - # Type 256 (Multi Spec. Imager WV, both clear-sky and cloud-top): Assigned all dummy values - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 256 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,0.] #Pressure (Pa) - errors: [1000000000.,1000000000.] - # Type 257 (MODIS LWIR) - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 257 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] - # Type 258 (MODIS cloud-top WV): Some levels assigned dummy values - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 258 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [1000000000.,1000000000.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] - # Type 259 (MODIS clear-sky WV): Some levels assigned dummy values - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 259 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [1000000000.,1000000000.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] - # Type 260 (VIIRS LWIR): All levels assigned dummy values in prepobs_errtable.global, HOWEVER the GSI values appear - # to be a standard profile (borrowed from e.g., Type=244). Using the standard profile here. - # It's possibly that my prepobs_errtable.global file is out-of-date. - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 260 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] -obs prior filters: - # - # sanity-check criteria - # - # Observation Range Sanity Check - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - minvalue: -130. - maxvalue: 130. - action: - name: reject - # Velocity Sanity Check - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - test variables: - - name: ObsFunction/Velocity - maxvalue: 130. - action: - name: reject -# +# Type 240 (GOES SWIR): Assigned all dummy values in prepobs_errtable.global +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 241 (Multi Spec. Imager LWIR): Assigned all dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 241 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 242 (Himawari VIS) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 242 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 243 (MVIRI/SEVIRI VIS) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 243 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 244 (AVHRR LWIR) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 244 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 245 (GOES LWIR): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 245 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] + +# Type 246 (GOES cloud-top WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 246 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] + +# Type 247 (GOES clear-sky WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 247 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] + +# Type 248 (GOES Sounder cloud-top WV): Assigned all dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 248 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 249 (GOES Sounder clear-sky WV): Assigned all dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 249 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 250 (Himawari AHI WV, cloud-top or clear-sky) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 250 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,7.,7.3,7.6,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.] + +# Type 251 (GOES VIS): Assigned all dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 251 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 252 (Himawari AHI LWIR) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 252 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 253 (MVIRI/SEVERI LWIR) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 253 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 254 (MVIRI/SEVIRI WV, both cloud-top and clear-sky) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 254 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.5,6.1,6.,6.5,7.3,7.6,7.,7.5,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 255 (LEOGEO): +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 255 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 256 (Multi Spec. Imager WV, both clear-sky and cloud-top): Assigned all dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 256 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 257 (MODIS LWIR) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 257 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 258 (MODIS cloud-top WV): Some levels assigned dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 258 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 259 (MODIS clear-sky WV): Some levels assigned dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 259 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 260 (VIIRS LWIR): All levels assigned dummy values in prepobs_errtable.global, HOWEVER the GSI values appear +# to be a standard profile (borrowed from e.g., Type=244). Using the standard profile here. +# It's possibly that my prepobs_errtable.global file is out-of-date. +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 260 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# sanity-check criteria +# Observation Range Sanity Check +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + minvalue: -130. + maxvalue: 130. + action: + name: reject + +# Velocity Sanity Check +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Velocity + maxvalue: 130. + action: + name: reject + # preQC (read_satwnd) criteria -# # EUMETSAT winds: satelliteIdentifer [50–79] (>49, <80) - # Reject obs with satelliteZenithAngle > 68 deg - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 50-79 - test variables: - - name: MetaData/satelliteZenithAngle - maxvalue: 68. - action: - name: reject - # Reject obs with windComputationMethod = 5 (clear-sky WV AMV) - # CLEARED - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 50-79 - - variable: MetaData/windComputationMethod - is_in: 5 - action: - name: reject - # Reject obs with qualityInformationWithoutForecast < 85 - # CLEARED - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 50-79 - test variables: - - name: MetaData/qualityInformationWithoutForecast - minvalue: 85. - maxvalue: 100. - action: - name: reject +# Reject obs with satelliteZenithAngle > 68 deg +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 50-79 + test variables: + - name: MetaData/satelliteZenithAngle + maxvalue: 68. + action: + name: reject + +# Reject obs with windComputationMethod = 5 (clear-sky WV AMV) +# CLEARED +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 50-79 + - variable: MetaData/windComputationMethod + is_in: 5 + action: + name: reject + +# Reject obs with qualityInformationWithoutForecast < 85 +# CLEARED +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 50-79 + test variables: + - name: MetaData/qualityInformationWithoutForecast + minvalue: 85. + maxvalue: 100. + action: + name: reject + # JMA: satelliteIdentifier [100–199] (>99, <200) - # Reject obs with satelliteZenithAngle > 68 deg - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 100-199 - test variables: - - name: MetaData/satelliteZenithAngle - maxvalue: 68. - action: - name: reject - # Reject obs with windComputationMethod = 5 (clear-sky WV AMV) - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 100-199 - - variable: MetaData/windComputationMethod - is_in: 5 - action: - name: reject - # Reject obs with qualityInformationWithoutForecast < 85. - # CLEARED - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 100-199 - test variables: - - name: MetaData/qualityInformationWithoutForecast - minvalue: 85. - maxvalue: 100. - action: - name: reject +# Reject obs with satelliteZenithAngle > 68 deg +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 100-199 + test variables: + - name: MetaData/satelliteZenithAngle + maxvalue: 68. + action: + name: reject + +# Reject obs with windComputationMethod = 5 (clear-sky WV AMV) +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 100-199 + - variable: MetaData/windComputationMethod + is_in: 5 + action: + name: reject + +# Reject obs with qualityInformationWithoutForecast < 85. +# CLEARED +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 100-199 + test variables: + - name: MetaData/qualityInformationWithoutForecast + minvalue: 85. + maxvalue: 100. + action: + name: reject + # NESDIS: satelliteIdentifier [250–299] (>249, <300) - # Reject obs with satelliteZenithAngle > 68 deg - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - test variables: - - name: MetaData/satelliteZenithAngle - maxvalue: 68. - action: - name: reject - # Reject obs with qualityInformationWithoutForecast < 90. OR > 100. - # CLEARED - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - test variables: - - name: MetaData/qualityInformationWithoutForecast - minvalue: 90. - maxvalue: 100. - action: - name: reject - # Reject obs with pressure < 15000. - # CLEARED - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - test variables: - - name: MetaData/pressure - minvalue: 15000. - action: - name: reject - # Reject obs with pressure < 70000. when Type=251 - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - - variable: ObsType/windEastward - is_in: 251 - test variables: - - name: MetaData/pressure - minvalue: 70000. - action: - name: reject - # Reject obs with pressure > 30000. when Type=246 - # CLEARED - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - - variable: ObsType/windEastward - is_in: 246 - test variables: - - name: MetaData/pressure - maxvalue: 30000. - action: - name: reject - # Reject obs with pressure > 85000. when isli=1 (land surface) - # CLEARED - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - - variable: GeoVaLs/land_area_fraction - minvalue: 0.999 - test variables: - - name: MetaData/pressure - maxvalue: 85000. - action: - name: reject - # Reject obs with pct1 (Coeff. of Var.) outside of 0.04–0.5, Type [240,245,246,251] ONLY - # CLEARED - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - - variable: ObsType/windEastward - is_in: 240,245,246,251 - test variables: - - name: MetaData/coefficientOfVariation - minvalue: 0.04 - maxvalue: 0.5 - action: - name: reject +# Reject obs with satelliteZenithAngle > 68 deg +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + test variables: + - name: MetaData/satelliteZenithAngle + maxvalue: 68. + action: + name: reject + +# Reject obs with qualityInformationWithoutForecast < 90. OR > 100. +# CLEARED +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + test variables: + - name: MetaData/qualityInformationWithoutForecast + minvalue: 90. + maxvalue: 100. + action: + name: reject + +# Reject obs with pressure < 15000. +# CLEARED +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + test variables: + - name: MetaData/pressure + minvalue: 15000. + action: + name: reject + +# Reject obs with pressure < 70000. when Type=251 +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + - variable: ObsType/windEastward + is_in: 251 + test variables: + - name: MetaData/pressure + minvalue: 70000. + action: + name: reject + +# Reject obs with pressure > 30000. when Type=246 +# CLEARED +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + - variable: ObsType/windEastward + is_in: 246 + test variables: + - name: MetaData/pressure + maxvalue: 30000. + action: + name: reject + +# Reject obs with pressure > 85000. when isli=1 (land surface) +# CLEARED +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + - variable: GeoVaLs/land_area_fraction + minvalue: 0.99 +# - variable: GeoVaLs/dominant_surface_type +# maxvalue: 1.0001 +# minvalue: 0.9999 + test variables: + - name: MetaData/pressure + maxvalue: 85000. + action: + name: reject + +# Reject obs with pct1 (Coeff. of Var.) outside of 0.04–0.5, Type [240,245,246,251] ONLY +# CLEARED +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + - variable: ObsType/windEastward + is_in: 240,245,246,251 + test variables: + - name: MetaData/coefficientOfVariation + minvalue: 0.04 + maxvalue: 0.5 + action: + name: reject + # NESDIS obs are also subject to the experr_norm test defined as: # # if (10. - 0.1*(expectedError))/(ob_speed)>0.9, or ob_speed<0.1, reject, applies to NESDIS winds # # CLEARED: With caveat that float precision/handling differences can generate different acceptance criteria # between UFO and GSO for observations with an experr_norm value right around the maxvalue. - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - test variables: - - name: ObsFunction/SatWindsErrnormCheck - maxvalue: 0.9 - action: - name: reject -# - # Reject all Type=240 (GOES SWIR) AMVs: These are not currently assimilated in GSI and they have missing-values - # assigned to ob-errors - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 240 - action: - name: reject +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + test variables: + - name: ObsFunction/SatWindsErrnormCheck + maxvalue: 0.9 + action: + name: reject + +# Reject all Type=240 (GOES SWIR) AMVs: These are not currently assimilated in GSI and they have missing-values +# assigned to ob-errors +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240 + action: + name: reject # # setupw criteria # - # Reject any ob Type [240–260] when pressure greater than 950 mb. - # CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 240-260 - test variables: - - name: MetaData/pressure - maxvalue: 95001. - action: - name: reject - # GOES IR (245) reject when pressure between 399 and 801 mb. - # CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/pressure - minvalue: 39901. - maxvalue: 80099. - - variable: ObsType/windEastward - is_in: 245 - action: - name: reject - # JMA IR (252) reject when pressure between 499 and 801 mb. - # CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/pressure - minvalue: 49901. - maxvalue: 80099. - - variable: ObsType/windEastward - is_in: 252 - action: - name: reject - # EUMETSAT IR (253) reject when pressure between 401 and 801 mb. - # CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/pressure - minvalue: 40101. - maxvalue: 80099. - - variable: ObsType/windEastward - is_in: 253 - action: - name: reject - # GOES WV (246, 250, 254), reject when pressure greater than 399 mb. - # CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 246, 250, 254 - test variables: - - name: MetaData/pressure - maxvalue: 39900. - action: - name: reject - # EUMET (242) and JMA (243) vis, reject when pressure less than 700 mb. - # CLEARED: minvalue is rejecting <, not <= as per a Perform Action, so threshold is unchanged - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 242, 243 - test variables: - - name: MetaData/pressure - minvalue: 70000. - action: - name: reject - # MODIS-Aqua/Terra (257) and (259), reject when pressure less than 249 mb. - # CLEARED: minvalue is rejecting <, not <= as per a Perform Action, so threshold is unchanged - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 257,259 - test variables: - - name: MetaData/pressure - minvalue: 24900. - action: - name: reject - # MODIS-Aqua/Terra (258) and (259), reject when pressure greater than 600 mb. - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - # maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 258, 259 - test variables: - - name: MetaData/pressure - maxvalue: 60000. - action: - name: reject - # Multiple satellite platforms, reject when pressure is more than 50 mb above tropopause. - # CLEARED: minvalue is rejecting <, not <= as per a Perform Action, so threshold is unchanged - - filter: Difference Check - filter variables: - - name: windEastward - - name: windNorthward - reference: GeoVaLs/tropopause_pressure - value: MetaData/pressure - minvalue: -5000. # 50 hPa above tropopause level, negative p-diff - action: - name: reject - # GOES (247) reject any observation with a /=0 surface type (non-water - # surface) within 110 hPa of the surface pressure (as part of the LNVD - # check). - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - - filter: Difference Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: - name: GeoVaLs/water_area_fraction - maxvalue: 0.001 - - variable: - name: ObsType/windEastward - is_in: 247 - reference: GeoVaLs/surface_pressure - value: MetaData/pressure - maxvalue: -11000. # within 110 hPa above surface pressure, negative p-diff - action: - name: reject - # AVHRR (244), MODIS (257,258,259), and VIIRS (260) reject any - # observation with a /=0 surface type (non-water surface) within - # 200 hPa of the surface pressure (as part of the LNVD check). - # CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged - - filter: Difference Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: - name: GeoVaLs/water_area_fraction - maxvalue: 0.001 +# Reject any ob Type [240–260] when pressure greater than 950 mb. +# CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240-260 + test variables: + - name: MetaData/pressure + maxvalue: 95001. + action: + name: reject + +# GOES IR (245) reject when pressure between 399 and 801 mb. +# CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/pressure + minvalue: 39901. + maxvalue: 80099. + - variable: ObsType/windEastward + is_in: 245 + action: + name: reject + +# JMA IR (252) reject when pressure between 499 and 801 mb. +# CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/pressure + minvalue: 49901. + maxvalue: 80099. + - variable: ObsType/windEastward + is_in: 252 + action: + name: reject + +# EUMETSAT IR (253) reject when pressure between 401 and 801 mb. +# CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/pressure + minvalue: 40101. + maxvalue: 80099. + - variable: ObsType/windEastward + is_in: 253 + action: + name: reject + +# GOES WV (246, 250, 254), reject when pressure greater than 399 mb. +# CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 246, 250, 254 + test variables: + - name: MetaData/pressure + maxvalue: 39900. + action: + name: reject + +# EUMET (242) and JMA (243) vis, reject when pressure less than 700 mb. +# CLEARED: minvalue is rejecting <, not <= as per a Perform Action, so threshold is unchanged +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 242, 243 + test variables: + - name: MetaData/pressure + minvalue: 70000. + action: + name: reject + +# MODIS-Aqua/Terra (257) and (259), reject when pressure less than 249 mb. +# CLEARED: minvalue is rejecting <, not <= as per a Perform Action, so threshold is unchanged +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 257,259 + test variables: + - name: MetaData/pressure + minvalue: 24900. + action: + name: reject + +# MODIS-Aqua/Terra (258) and (259), reject when pressure greater than 600 mb. +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +# maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 258, 259 + test variables: + - name: MetaData/pressure + maxvalue: 60000. + action: + name: reject + +# Multiple satellite platforms, reject when pressure is more than 50 mb above tropopause. +# CLEARED: minvalue is rejecting <, not <= as per a Perform Action, so threshold is unchanged +- filter: Difference Check + filter variables: + - name: windEastward + - name: windNorthward + reference: GeoVaLs/tropopause_pressure + value: MetaData/pressure + minvalue: -5000. # 50 hPa above tropopause level, negative p-diff + action: + name: reject + +# GOES (247) reject any observation with a /=0 surface type (non-water +# surface) within 110 hPa of the surface pressure (as part of the LNVD +# check). +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Difference Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 +# name: GeoVaLs/dominant_surface_type +# minvalue: 0.001 + - variable: + name: ObsType/windEastward + is_in: 247 + reference: GeoVaLs/surface_pressure + value: MetaData/pressure + maxvalue: -11000. # within 110 hPa above surface pressure, negative p-diff + action: + name: reject + +# AVHRR (244), MODIS (257,258,259), and VIIRS (260) reject any +# observation with a /=0 surface type (non-water surface) within +# 200 hPa of the surface pressure (as part of the LNVD check). +# CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged +- filter: Difference Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 +# name: GeoVaLs/dominant_surface_type +# minvalue: 0.001 + - variable: + name: ObsType/windEastward + is_in: 244, 257-260 + reference: GeoVaLs/surface_pressure + value: MetaData/pressure + maxvalue: -20000. # within 200 hPa above surface pressure, negative p-diff + action: + name: reject + +# Reject GOES (247) when difference of wind direction is more than 50 degrees. +# CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 247 + test variables: + - name: ObsFunction/WindDirAngleDiff + maxvalue: 50. + action: + name: reject + +# AVHRR (244), MODIS (257,258,259), VIIRS (260), GOES (247) use a LNVD check. +# CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 244, 247, 257-260 + test variables: + - name: ObsFunction/SatWindsLNVDCheck + maxvalue: 3. + action: + name: reject + +# All satwinds must adjust errors based on ObsErrorFactorPressureCheck +# prior to the SPDB check (i.e. the gross-error check). The gross-error +# check uses the adjusted errors for error-bound tightening and rejection, +# so this check has to come first. This check will inflate errors for obs +# that are too close to either the model top or bottom. +- filter: Perform Action + filter variables: + - name: windEastward + where: - variable: name: ObsType/windEastward - is_in: 244, 257-260 - reference: GeoVaLs/surface_pressure - value: MetaData/pressure - maxvalue: -20000. # within 200 hPa above surface pressure, negative p-diff - action: - name: reject -obs post filters: - # Reject GOES (247) when difference of wind direction is more than 50 degrees. - # CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 247 - test variables: - - name: ObsFunction/WindDirAngleDiff - maxvalue: 50. - action: - name: reject - # AVHRR (244), MODIS (257,258,259), VIIRS (260), GOES (247) use a LNVD check. - # CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 244, 247, 257-260 - test variables: - - name: ObsFunction/SatWindsLNVDCheck - maxvalue: 3. - action: - name: reject - # All satwinds must adjust errors based on ObsErrorFactorPressureCheck - # prior to the SPDB check (i.e. the gross-error check). The gross-error - # check uses the adjusted errors for error-bound tightening and rejection, - # so this check has to come first. This check will inflate errors for obs - # that are too close to either the model top or bottom. - - filter: Perform Action - filter variables: - - name: windEastward - where: - - variable: - name: ObsType/windEastward - is_in: 240-260 - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorPressureCheck - options: - variable: windEastward - inflation factor: 4.0 - - - filter: Perform Action - filter variables: - - name: windNorthward - where: - - variable: - name: ObsType/windNorthward - is_in: 240-260 - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorPressureCheck - options: - variable: windNorthward - inflation factor: 4.0 - # - # All satwinds subject to a gross-error check that contains significant - # modifiers for satwinds with a negative speed-bias. ALL wind gross-error - # checks are currently being done by the SatWindsSPDBCheck. - # CLEARED - - filter: Background Check - filter variables: - - name: windEastward - function absolute threshold: - - name: ObsFunction/WindsSPDBCheck + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck options: - wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] - cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] - error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] - error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] variable: windEastward - action: - name: reject - - - filter: Background Check - filter variables: - - name: windNorthward - function absolute threshold: - - name: ObsFunction/WindsSPDBCheck + inflation factor: 4.0 + +- filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck options: - wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] - cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] - error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] - error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] variable: windNorthward - action: - name: reject - # The last error inflation check is for duplicate observations. This one needs - # to come last, because we don't want to inflate errors for duplication if one - # of the duplicates should be rejected. - - filter: RejectList - filter variables: - - name: windEastward - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorDuplicateCheck - options: - use_air_pressure: true - variable: windEastward - - filter: RejectList - filter variables: - - name: windNorthward - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorDuplicateCheck - options: - use_air_pressure: true - variable: windNorthward - # We are extending this to an additional filter that inflates final ob-errors across-the-board by - # 1/0.8 = 1.25. This is caused by the GSI value of nvqc being set to .true. in the global operational - # configuration, see: https://github.com/NOAA-EMC/global-workflow/blob/d5ae3328fa4041b177357b1133f6b92e81c859d7/scripts/exglobal_atmos_analysis.sh#L750 - # This setting activates Line 1229 of setupw.f90 to scale ratio_errors by 0.8, which is applied in - # the denominator of the final ob-error, so 1/0.8 = 1.25 factor of ob-error inflation. - # - # If this nvqc functionality were to be switched off (i.e. if variational qc were to be turned off), - # you would want to remove this last inflation filter. - - filter: Perform Action - filter variables: - - name: windEastward - where: - - variable: ObsType/windEastward - is_in: 240-260 - action: - name: inflate error - inflation factor: 1.25 + inflation factor: 4.0 + +# All satwinds subject to a gross-error check that contains significant +# modifiers for satwinds with a negative speed-bias. ALL wind gross-error +# checks are currently being done by the SatWindsSPDBCheck. +# CLEARED +- filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] + error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] + variable: windEastward + action: + name: reject + +- filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] + error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] + variable: windNorthward + action: + name: reject + +# The last error inflation check is for duplicate observations. This one needs +# to come last, because we don't want to inflate errors for duplication if one +# of the duplicates should be rejected. +# Inflate obs error for duplicated observations at the same location +#- filter: Perform Action +# filter variables: +# - name: windEastward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windEastward + +# Inflate obs error for duplicated observations at the same location +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windNorthward + +# We are extending this to an additional filter that inflates final ob-errors across-the-board by +# 1/0.8 = 1.25. This is caused by the GSI value of nvqc being set to .true. in the global operational +# configuration, see: https://github.com/NOAA-EMC/global-workflow/blob/d5ae3328fa4041b177357b1133f6b92e81c859d7/scripts/exglobal_atmos_analysis.sh#L750 +# This setting activates Line 1229 of setupw.f90 to scale ratio_errors by 0.8, which is applied in +# the denominator of the final ob-error, so 1/0.8 = 1.25 factor of ob-error inflation. # - - filter: Perform Action - filter variables: - - name: windNorthward - where: - - variable: ObsType/windNorthward - is_in: 240-260 - action: - name: inflate error - inflation factor: 1.25 -# END OF FILTERS -linear obs operator: - name: Identity - -#passedBenchmark: 1025814# 2 variables (u,v), both passing 512907 obs, including 512907 GSI/UFO agreements, and: -passedBenchmark: 1150870 # 2 variables (u,v), both passing 575435 obs, including 512907 GSI/UFO agreements, and: +# If this nvqc functionality were to be switched off (i.e. if variational qc were to be turned off), +# you would want to remove this last inflation filter. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# where: +# - variable: ObsType/windEastward +# is_in: 240-260 +# action: +# name: inflate error +# inflation factor: 1.25 + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# where: +# - variable: ObsType/windNorthward +# is_in: 240-260 +# action: +# name: inflate error +# inflation factor: 1.25 + +# End of Filters +#passedBenchmark: 1150870 # 2 variables (u,v), both passing 575435 obs, including 575435 GSI/UFO agreements, and: +passedBenchmark: 1025814 # 2 variables (u,v), both passing 512907 obs, including 512907 GSI/UFO agreements, and: # 16 GOES AMVs (6 u-obs, 6 v-obs) that are rejected by the SatWindsErrnormCheck in UFO # but are retained in GSI's equivalent experr_norm check. All 6 of these disagreements # have error norm values at almost exactly 0.9, indicating a float precision/handling diff --git a/parm/atm/obs/testing/satwind_noqc.yaml b/parm/atm/obs/testing/satwind_noqc.yaml index cef238455..b2535470d 100644 --- a/parm/atm/obs/testing/satwind_noqc.yaml +++ b/parm/atm/obs/testing/satwind_noqc.yaml @@ -9,10 +9,404 @@ obs space: type: H5File obsfile: !ENV satwind_diag_${CDATE}.nc4 simulated variables: [windEastward, windNorthward] + geovals: filename: !ENV satwind_geoval_${CDATE}.nc4 + obs operator: name: VertInterp + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + +obs prior filters: +# Apply variable changes needed wind scaling +- filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: False + +obs post filters: +# Assign the initial observation error, based on height/pressure +# Hard-wiring to prepobs_errtable.global by Type +# ObsError is currently not updating in diag file, but passes directly to EffectiveError when no inflation is specified in YAML +# Type 240 (GOES SWIR): Assigned all dummy values in prepobs_errtable.global +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 241 (Multi Spec. Imager LWIR): Assigned all dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 241 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 242 (Himawari VIS) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 242 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 243 (MVIRI/SEVIRI VIS) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 243 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 244 (AVHRR LWIR) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 244 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 245 (GOES LWIR): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 245 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] + +# Type 246 (GOES cloud-top WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 246 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] + +# Type 247 (GOES clear-sky WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 247 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] + +# Type 248 (GOES Sounder cloud-top WV): Assigned all dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 248 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 249 (GOES Sounder clear-sky WV): Assigned all dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 249 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 250 (Himawari AHI WV, cloud-top or clear-sky) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 250 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,7.,7.3,7.6,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.] + +# Type 251 (GOES VIS): Assigned all dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 251 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 252 (Himawari AHI LWIR) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 252 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 253 (MVIRI/SEVERI LWIR) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 253 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 254 (MVIRI/SEVIRI WV, both cloud-top and clear-sky) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 254 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.5,6.1,6.,6.5,7.3,7.6,7.,7.5,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 255 (LEOGEO): +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 255 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 256 (Multi Spec. Imager WV, both clear-sky and cloud-top): Assigned all dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 256 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 257 (MODIS LWIR) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 257 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 258 (MODIS cloud-top WV): Some levels assigned dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 258 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 259 (MODIS clear-sky WV): Some levels assigned dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 259 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 260 (VIIRS LWIR): All levels assigned dummy values in prepobs_errtable.global, HOWEVER the GSI values appear +# to be a standard profile (borrowed from e.g., Type=244). Using the standard profile here. +# It's possibly that my prepobs_errtable.global file is out-of-date. +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 260 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] -vector ref: GsiHofXBc -tolerance: 1.e-5 +passedBenchmark: 2459862 # with enforced time window +#passedBenchmark: 2752378 # without enforcing time window diff --git a/parm/atm/obs/testing/scatwind.yaml b/parm/atm/obs/testing/scatwind.yaml index 30a1af6f7..9befdda81 100644 --- a/parm/atm/obs/testing/scatwind.yaml +++ b/parm/atm/obs/testing/scatwind.yaml @@ -9,228 +9,283 @@ obs space: type: H5File obsfile: !ENV scatwind_diag_${CDATE}.nc4 simulated variables: [windEastward, windNorthward] + geovals: filename: !ENV scatwind_geoval_${CDATE}.nc4 -vector ref: GsiHofXBc -tolerance: 0.01 + obs operator: name: VertInterp - apply near surface wind scaling: true -# -obs pre filters: - # Assign the initial observation error (constant value, 1.5 m/s right now). - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - action: - name: assign error - error parameter: 1.5 -#obs prior filters: - # - # Reject all obs with PreQC mark already set above 3 - # NOTE: All scatwinds have an automatic PreQC mark of 2 (hard-wired default from GSI) - # - filter: PreQC - # maxvalue: 3 - # action: - # name: reject - # + # Use height vertical coordinate first +# vertical coordinate: geometric_height + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + +obs prior filters: +# Apply variable changes needed for rescaled height coordinate +- filter: Variable Transforms + Transform: AdjustedHeightCoordinate + SkipWhenNoObs: False + +# Apply variable changes needed for wind scaling +- filter: Variable Transforms + Transform: SurfaceWindScalingHeight + SkipWhenNoObs: False + +# Assign the initial observation error (constant value, 1.5 m/s right now). +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 1.5 + +# Calculate error inflation factor for duplicate observations +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windEastward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windEastward + +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windNorthward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windNorthward + +# Reject all obs with PreQC mark already set above 3 +# NOTE: All scatwinds have an automatic PreQC mark of 2 (hard-wired default from GSI) +# - filter: PreQC +# maxvalue: 3 +# action: +# name: reject + obs post filters: - # - # Reject all ASCAT (Type 290) winds with tsavg <= 273.0 (surface temperature) - # - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 290 - - variable: GeoVaLs/surface_temperature - maxvalue: 273. - action: - name: reject - # - # Reject all ASCAT (Type 290) winds with isflg /= 0 (non-water surface) - # - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 290 - - variable: GeoVaLs/water_area_fraction - maxvalue: 0.001 - action: - name: reject - # Reject ASCAT (Type 290) when observed component deviates from background by more than 5.0 m/s - # NOTE: This check can reject a u- or v-component of the same observation independently, which - # is fundamentally different from how GSI rejects obs (both components are rejected if - # either component fails a check). - - filter: Background Check - filter variables: - - name: windEastward - - name: windNorthward - threshold: 5. - absolute threshold: 5. - where: - - variable: ObsType/windEastward - is_in: 290 - action: - name: reject - # Reject OSCAT (Type 291) when observed component deviates from background by more than 6.0 m/s - # NOTE: This check can reject a u- or v-component of the same observation independently, which - # is fundamentally different from how GSI rejects obs (both components are rejected if - # either component fails a check). - - filter: Background Check - filter variables: - - name: windEastward - - name: windNorthward - threshold: 6. - absolute threshold: 6. - where: - - variable: ObsType/windEastward - is_in: 291 - action: - name: reject - # Reject ASCAT (Type 290) when ambiguity check fails (returned value is negative) - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 290 - test variables: - - name: ObsFunction/ScatWindsAmbiguityCheck - options: - test_hofx: GsiHofX - minimum_uv: 0.0001 # hard-coding a minimum-uv for transparancy, want this to basically be zero - maxvalue: 0. - action: - name: reject -# - # All scatwinds must adjust errors based on ObsErrorFactorPressureCheck. - # This check will inflate errors for obs that are too close to either - # the model top or bottom. - - filter: Perform Action - filter variables: - - name: windEastward - where: - - variable: - name: ObsType/windEastward - is_in: 290-291 - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorPressureCheck - options: - variable: windEastward - inflation factor: 4.0 -# - - filter: Perform Action - filter variables: - - name: windNorthward - where: - - variable: - name: ObsType/windNorthward - is_in: 290-291 - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorPressureCheck - options: - variable: windNorthward - inflation factor: 4.0 - # All scatwinds subject to a gross error check. This is contained within - # the WindsSPDBCheck, although it is not exclusive to satwinds. - - filter: Background Check - filter variables: - - name: windEastward - function absolute threshold: - - name: ObsFunction/WindsSPDBCheck +# Reject all ASCAT (Type 290) winds with tsavg <= 273.0 (surface temperature) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 290 +# - variable: GeoVaLs/surface_temperature + - variable: GeoVaLs/surface_temperature_where_land + maxvalue: 273. + action: + name: reject + +# Reject all ASCAT (Type 290) winds with isflg /= 0 (non-water surface) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 290 + - variable: GeoVaLs/water_area_fraction + maxvalue: 0.99 +# - variable: GeoVaLs/surface_type +# minvalue: 0.001 + action: + name: reject +#passedBenchmark: 53558 # 2 variables (u,v), u=26779 passing, v=26779 passing + +# Reject ASCAT (Type 290) when observed component deviates from background by more than 5.0 m/s +# NOTE: This check can reject a u- or v-component of the same observation independently, which +# is fundamentally different from how GSI rejects obs (both components are rejected if +# either component fails a check). +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Arithmetic + options: + variables: + - name: ObsValue/windEastward + - name: HofX/windEastward + coefs: [1.0, -1.0] + minvalue: -5.0 + maxvalue: 5.0 + +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Arithmetic + options: + variables: + - name: ObsValue/windNorthward + - name: HofX/windNorthward + coefs: [1.0, -1.0] + minvalue: -5.0 + maxvalue: 5.0 +#passedBenchmark: 52362 # 2 variables (u,v), u=26181 passing, v=26181 passing + +# Reject OSCAT (Type 291) when observed component deviates from background by more than 6.0 m/s +# NOTE: This check can reject a u- or v-component of the same observation independently, which +# is fundamentally different from how GSI rejects obs (both components are rejected if +# either component fails a check). +- filter: Background Check + filter variables: + - name: windEastward + - name: windNorthward + threshold: 6. + absolute threshold: 6. + where: + - variable: ObsType/windEastward + is_in: 291 + action: + name: reject + +# Reject ASCAT (Type 290) when ambiguity check fails (returned value is negative) +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 290 + test variables: + - name: ObsFunction/ScatWindsAmbiguityCheck + options: +# test_hofx: GsiHofX + minimum_uv: 0.0001 # hard-coding a minimum-uv for transparancy, want this to basically be zero + maxvalue: 0. + action: + name: reject +#passedBenchmark: 51776 # 2 variables (u,v), u=25888 passing, v=25888 passing + +# All scatwinds must adjust errors based on ObsErrorFactorPressureCheck. +# This check will inflate errors for obs that are too close to either +# the model top or bottom. +- filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 290-291 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck options: - wndtype: [ 290, 291] - cgross: [ 5.0, 5.0] - error_min: [1.4, 1.4] - error_max: [6.1, 6.1] variable: windEastward - action: - name: reject - - - filter: Background Check - filter variables: - - name: windNorthward - function absolute threshold: - - name: ObsFunction/WindsSPDBCheck + inflation factor: 4.0 + +- filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 290-291 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck options: - wndtype: [ 290, 291] - cgross: [ 5.0, 5.0] - error_min: [1.4, 1.4] - error_max: [6.1, 6.1] variable: windNorthward - action: - name: reject - # The last error inflation check is for duplicate observations. This one needs - # to come last, because we don't want to inflate errors for duplication if one - # of the duplicates should be rejected. - - filter: RejectList - filter variables: - - name: windEastward - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorDuplicateCheck - options: - use_air_pressure: true - variable: windEastward - - filter: RejectList - filter variables: - - name: windNorthward - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorDuplicateCheck - options: - use_air_pressure: true - variable: windNorthward - # There is no across-the-board inflation for nvqc=.true. for scatwinds, presumably because for - # this inflation to take place both nvqc must be .true. AND ibeta must be >0, see: - # https://github.com/NOAA-EMC/GSI/blob/14ae595af1b03471287d322596d35c0665336e95/src/gsi/setupw.f90#L1229 - # GSI settings must have ibeta>0 for satwinds, but not for scatwinds. - # - # - # If the ibeta settings for scatwinds were to change while nvqc remained .true., we would extend YAML to - # an additional filter that inflates final ob-errors across-the-board by 1/0.8 = 1.25. NOTE: the nvqc setting - # is defaulted to .false. in GSI code, but is overridden in global operational configuration. See: - # configuration, see: https://github.com/NOAA-EMC/global-workflow/blob/d5ae3328fa4041b177357b1133f6b92e81c859d7/scripts/exglobal_atmos_analysis.sh#L750 - # This setting activates Line 1229 of setupw.f90 to scale ratio_errors by 0.8, which is applied in - # the denominator of the final ob-error, so 1/0.8 = 1.25 factor of ob-error inflation. - # - # If this functionality were to be activated for scatwinds, you would want to include this last inflation filter. -# - filter: Perform Action -# filter variables: -# - name: windEastward -# where: -# - variable: ObsType/windEastward -# is_in: 290-291 -# action: -# name: inflate error -# inflation factor: 1.25 -# - filter: Perform Action -# filter variables: -# - name: windNorthward -# where: -# - variable: ObsType/windNorthward -# is_in: 290-291 -# action: -# name: inflate error -# inflation factor: 1.25 -linear obs operator: - name: Identity -#passedBenchmark: 52097 # 2 variables (u,v), u=26087 passing, v=26010 passing -passedBenchmark: 52109 # 2 variables (u,v), u=26093 passing, v=26016 passing + inflation factor: 4.0 +#passedBenchmark: 51776 # 2 variables (u,v), u=25888 passing, v=25888 passing + +# All scatwinds subject to a gross error check. This is contained within +# the WindsSPDBCheck, although it is not exclusive to satwinds. +- filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 290, 291] + cgross: [ 5.0, 5.0] + error_min: [1.4, 1.4] + error_max: [6.1, 6.1] + variable: windEastward + action: + name: reject + +- filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 290, 291] + cgross: [ 5.0, 5.0] + error_min: [1.4, 1.4] + error_max: [6.1, 6.1] + variable: windNorthward + action: + name: reject +#passedBenchmark: 51776 # 2 variables (u,v), u=25888 passing, v=25888 passing + +# The last error inflation check is for duplicate observations. This one needs +# to come last, because we don't want to inflate errors for duplication if one +# of the duplicates should be rejected. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windEastward + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windNorthward + +# There is no across-the-board inflation for nvqc=.true. for scatwinds, presumably because for +# this inflation to take place both nvqc must be .true. AND ibeta must be >0, see: +# https://github.com/NOAA-EMC/GSI/blob/14ae595af1b03471287d322596d35c0665336e95/src/gsi/setupw.f90#L1229 +# GSI settings must have ibeta>0 for satwinds, but not for scatwinds. +# +# If the ibeta settings for scatwinds were to change while nvqc remained .true., we would extend YAML to +# an additional filter that inflates final ob-errors across-the-board by 1/0.8 = 1.25. NOTE: the nvqc setting +# is defaulted to .false. in GSI code, but is overridden in global operational configuration. See: +# configuration, see: https://github.com/NOAA-EMC/global-workflow/blob/d5ae3328fa4041b177357b1133f6b92e81c859d7/scripts/exglobal_atmos_analysis.sh#L750 +# This setting activates Line 1229 of setupw.f90 to scale ratio_errors by 0.8, which is applied in +# the denominator of the final ob-error, so 1/0.8 = 1.25 factor of ob-error inflation. +# +# If this functionality were to be activated for scatwinds, you would want to include this last inflation filter. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# where: +# - variable: ObsType/windEastward +# is_in: 290-291 +# action: +# name: inflate error +# inflation factor: 1.25 +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# where: +# - variable: ObsType/windNorthward +# is_in: 290-291 +# action: +# name: inflate error +# inflation factor: 1.25 +passedBenchmark: 51764 # 2 variables (u,v), u=25882 passing, v=25882 passing # GSI rejects both u- and v-component in first-guess check, UFO does not, but when UFO rej is reconciled btwn u and v these match GSI rej # u: 207 obs pass with corresponding v being rejected, 25880 obs in UFO/GSI agreement # v: 130 obs pass with corresponding u being rejected, 25880 obs in UFO/GSI agreement diff --git a/parm/atm/obs/testing/scatwind_noqc.yaml b/parm/atm/obs/testing/scatwind_noqc.yaml index f8f36efeb..ec4aef452 100644 --- a/parm/atm/obs/testing/scatwind_noqc.yaml +++ b/parm/atm/obs/testing/scatwind_noqc.yaml @@ -9,13 +9,42 @@ obs space: type: H5File obsfile: !ENV scatwind_diag_${CDATE}.nc4 simulated variables: [windEastward, windNorthward] + geovals: filename: !ENV scatwind_geoval_${CDATE}.nc4 -vector ref: GsiHofXBc -tolerance: 0.01 + obs operator: name: VertInterp - apply near surface wind scaling: true + # Use height vertical coordinate first +# vertical coordinate: geometric_height + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + +obs prior filters: +# Apply variable changes needed for rescaled height coordinate +- filter: Variable Transforms + Transform: AdjustedHeightCoordinate + SkipWhenNoObs: False + +# Apply variable changes needed for wind scaling +- filter: Variable Transforms + Transform: SurfaceWindScalingHeight + SkipWhenNoObs: False + +# Assign the initial observation error (constant value, 1.5 m/s) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 1.5 -vector ref: GsiHofXBc -tolerance: 1.e-5 +passedBenchmark: 53546 +#passedBenchmark: 53558 +#vector ref: GsiHofXBc +#tolerance: 1.e-5 diff --git a/parm/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.json b/parm/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.json new file mode 100644 index 000000000..bdde19135 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.json @@ -0,0 +1,12 @@ +{ + "data_format" : "prepbufr", + "subsets" : [ "ADPSFC" ], + "source" : "prepBUFR", + "data_type" : "ADPSFC", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "data_description" : "ADPSFC_prepbufr", + "data_provider" : "U.S. NOAA" +} diff --git a/parm/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.json b/parm/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.json new file mode 100644 index 000000000..4bdc8f781 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.json @@ -0,0 +1,12 @@ +{ + "data_format" : "prepbufr", + "subsets" : [ "SFCSHP" ], + "source" : "prepBUFR", + "data_type" : "SFCSHP", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "data_description" : "SFCSHP_prepbufr", + "data_provider" : "U.S. NOAA" +} diff --git a/parm/soca/berror/soca_ensb.yaml b/parm/soca/berror/soca_ensb.yaml index 814dfc7ac..447d5b5fd 100644 --- a/parm/soca/berror/soca_ensb.yaml +++ b/parm/soca/berror/soca_ensb.yaml @@ -30,6 +30,12 @@ steric height: linear variable changes: - linear variable change name: BalanceSOCA # Only the steric balance is applied +ensemble mean output: + datadir: ./static_ens + date: '{{ATM_WINDOW_BEGIN}}' + exp: ens_mean + type: incr + ssh output: unbalanced: datadir: ./static_ens diff --git a/parm/soca/berror/soca_ensweights.yaml b/parm/soca/berror/soca_ensweights.yaml index c7ea40a98..094377f95 100644 --- a/parm/soca/berror/soca_ensweights.yaml +++ b/parm/soca/berror/soca_ensweights.yaml @@ -19,6 +19,16 @@ weights: # Need to provide weights^2 when reading from file ice: 0.0025 # 5% of original variance ocean: 0.0625 # 25% " " + # Apply localized weights to the ocean ens. B + ocean local weights: + - lon: -172.0 + lat: 11.0 + amplitude: -1.0 + length scale: 700.0 + - lon: -160.0 + lat: 12.0 + amplitude: -1.0 + length scale: 700.0 output: datadir: ./ diff --git a/parm/soca/obs/config/salt_profile_argo.yaml b/parm/soca/obs/config/salt_profile_argo.yaml new file mode 100644 index 000000000..9f3e2521a --- /dev/null +++ b/parm/soca/obs/config/salt_profile_argo.yaml @@ -0,0 +1,31 @@ +obs space: + name: salt_profile_argo + obsdatain: + engine: + type: H5File + obsfile: !ENV ${DATA}/obs/${OPREFIX}salt_profile_argo.${PDY}${cyc}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ${DATA}/diags/salt_profile_argo.${PDY}${cyc}.nc4 + simulated variables: [salinity] + io pool: + max pool size: 1 +obs operator: + name: VertInterp + observation alias file: ./obsop_name_map.yaml + vertical coordinate: sea_water_depth + observation vertical coordinate: depth + interpolation method: linear +obs error: + covariance model: diagonal + +obs filter: + # Passivate obs where ocean fraction is > 90% + - filter: Domain Check + action: + name: passivate + where: + - variable: {name: GeoVaLs/sea_area_fraction} + maxvalue: 0.9 + diff --git a/parm/soca/obs/config/salt_profile_glider.yaml b/parm/soca/obs/config/salt_profile_glider.yaml new file mode 100644 index 000000000..a1504886b --- /dev/null +++ b/parm/soca/obs/config/salt_profile_glider.yaml @@ -0,0 +1,31 @@ +obs space: + name: salt_profile_glider + obsdatain: + engine: + type: H5File + obsfile: !ENV ${DATA}/obs/${OPREFIX}salt_profile_glider.${PDY}${cyc}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ${DATA}/diags/salt_profile_glider.${PDY}${cyc}.nc4 + simulated variables: [salinity] + io pool: + max pool size: 1 +obs operator: + name: VertInterp + observation alias file: ./obsop_name_map.yaml + vertical coordinate: sea_water_depth + observation vertical coordinate: depth + interpolation method: linear +obs error: + covariance model: diagonal + +obs filter: + # Passivate obs where ocean fraction is > 90% + - filter: Domain Check + action: + name: passivate + where: + - variable: {name: GeoVaLs/sea_area_fraction} + maxvalue: 0.9 + diff --git a/parm/soca/obs/config/salt_profile_tesac.yaml b/parm/soca/obs/config/salt_profile_tesac.yaml new file mode 100644 index 000000000..53cc4496e --- /dev/null +++ b/parm/soca/obs/config/salt_profile_tesac.yaml @@ -0,0 +1,31 @@ +obs space: + name: salt_profile_tesac + obsdatain: + engine: + type: H5File + obsfile: !ENV ${DATA}/obs/${OPREFIX}salt_profile_tesac.${PDY}${cyc}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ${DATA}/diags/salt_profile_tesac.${PDY}${cyc}.nc4 + simulated variables: [salinity] + io pool: + max pool size: 1 +obs operator: + name: VertInterp + observation alias file: ./obsop_name_map.yaml + vertical coordinate: sea_water_depth + observation vertical coordinate: depth + interpolation method: linear +obs error: + covariance model: diagonal + +obs filter: + # Passivate obs where ocean fraction is > 90% + - filter: Domain Check + action: + name: passivate + where: + - variable: {name: GeoVaLs/sea_area_fraction} + maxvalue: 0.9 + diff --git a/parm/soca/obs/config/temp_profile_argo.yaml b/parm/soca/obs/config/temp_profile_argo.yaml new file mode 100644 index 000000000..ff1b8ce9d --- /dev/null +++ b/parm/soca/obs/config/temp_profile_argo.yaml @@ -0,0 +1,28 @@ +obs space: + name: temp_profile_argo + obsdatain: + engine: + type: H5File + obsfile: !ENV ${DATA}/obs/${OPREFIX}temp_profile_argo.${PDY}${cyc}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ${DATA}/diags/temp_profile_argo.${PDY}${cyc}.nc4 + simulated variables: [waterTemperature] + io pool: + max pool size: 1 +obs operator: + name: InsituTemperature + +obs error: + covariance model: diagonal + +obs filters: + # Passivate obs where ocean fraction is > 90% + - filter: Domain Check + action: + name: passivate + where: + - variable: {name: GeoVaLs/sea_area_fraction} + maxvalue: 0.9 + diff --git a/parm/soca/obs/config/temp_profile_bathy.yaml b/parm/soca/obs/config/temp_profile_bathy.yaml new file mode 100644 index 000000000..90c0569ea --- /dev/null +++ b/parm/soca/obs/config/temp_profile_bathy.yaml @@ -0,0 +1,28 @@ +obs space: + name: temp_profile_bathy + obsdatain: + engine: + type: H5File + obsfile: !ENV ${DATA}/obs/${OPREFIX}temp_profile_bathy.${PDY}${cyc}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ${DATA}/diags/temp_profile_bathy.${PDY}${cyc}.nc4 + simulated variables: [waterTemperature] + io pool: + max pool size: 1 +obs operator: + name: InsituTemperature + +obs error: + covariance model: diagonal + +obs filters: + # Passivate obs where ocean fraction is > 90% + - filter: Domain Check + action: + name: passivate + where: + - variable: {name: GeoVaLs/sea_area_fraction} + maxvalue: 0.9 + diff --git a/parm/soca/obs/config/temp_profile_glider.yaml b/parm/soca/obs/config/temp_profile_glider.yaml new file mode 100644 index 000000000..721fccafb --- /dev/null +++ b/parm/soca/obs/config/temp_profile_glider.yaml @@ -0,0 +1,28 @@ +obs space: + name: temp_profile_glider + obsdatain: + engine: + type: H5File + obsfile: !ENV ${DATA}/obs/${OPREFIX}temp_profile_glider.${PDY}${cyc}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ${DATA}/diags/temp_profile_glider.${PDY}${cyc}.nc4 + simulated variables: [waterTemperature] + io pool: + max pool size: 1 +obs operator: + name: InsituTemperature + +obs error: + covariance model: diagonal + +obs filters: + # Passivate obs where ocean fraction is > 90% + - filter: Domain Check + action: + name: passivate + where: + - variable: {name: GeoVaLs/sea_area_fraction} + maxvalue: 0.9 + diff --git a/parm/soca/obs/config/temp_profile_tesac.yaml b/parm/soca/obs/config/temp_profile_tesac.yaml new file mode 100644 index 000000000..8cbe84bd6 --- /dev/null +++ b/parm/soca/obs/config/temp_profile_tesac.yaml @@ -0,0 +1,28 @@ +obs space: + name: temp_profile_tesac + obsdatain: + engine: + type: H5File + obsfile: !ENV ${DATA}/obs/${OPREFIX}temp_profile_tesac.${PDY}${cyc}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ${DATA}/diags/temp_profile_tesac.${PDY}${cyc}.nc4 + simulated variables: [waterTemperature] + io pool: + max pool size: 1 +obs operator: + name: InsituTemperature + +obs error: + covariance model: diagonal + +obs filters: + # Passivate obs where ocean fraction is > 90% + - filter: Domain Check + action: + name: passivate + where: + - variable: {name: GeoVaLs/sea_area_fraction} + maxvalue: 0.9 + diff --git a/parm/soca/obs/obs_list.yaml b/parm/soca/obs/obs_list.yaml index d710f3461..89b2659a1 100644 --- a/parm/soca/obs/obs_list.yaml +++ b/parm/soca/obs/obs_list.yaml @@ -13,3 +13,10 @@ observers: #- !INC ${OBS_YAML_DIR}/icec_ssmis_f17_south.yaml #- !INC ${OBS_YAML_DIR}/icec_ssmis_f18_north.yaml - !INC ${OBS_YAML_DIR}/icec_ssmis_f18_south.yaml +#- !INC ${OBS_YAML_DIR}/temp_profile_argo.yaml +#- !INC ${OBS_YAML_DIR}/temp_profile_bathy.yaml +#- !INC ${OBS_YAML_DIR}/temp_profile_glider.yaml +#- !INC ${OBS_YAML_DIR}/temp_profile_tesac.yaml +#- !INC ${OBS_YAML_DIR}/salt_profile_argo.yaml +#- !INC ${OBS_YAML_DIR}/salt_profile_glider.yaml +#- !INC ${OBS_YAML_DIR}/salt_profile_tesac.yaml \ No newline at end of file diff --git a/scripts/exgdas_global_marine_analysis_vrfy.py b/scripts/exgdas_global_marine_analysis_vrfy.py index 52d657563..6179c00e7 100755 --- a/scripts/exgdas_global_marine_analysis_vrfy.py +++ b/scripts/exgdas_global_marine_analysis_vrfy.py @@ -81,10 +81,15 @@ config = plotConfig(grid_file=grid_file, data_file=data_file, lats=np.arange(-60, 60, 10), + lons=np.arange(-280, 80, 30), variables_zonal={'Temp': [0, 2], 'Salt': [0, 0.2], 'u': [0, 0.2], 'v': [0, 0.2]}, + variables_meridional={'Temp': [0, 2], + 'Salt': [0, 0.2], + 'u': [0, 0.2], + 'v': [0, 0.2]}, variables_horiz={'Temp': [0, 2], 'Salt': [0, 0.2], 'u': [0, 0.2], @@ -103,7 +108,7 @@ config = plotConfig(grid_file=grid_file, data_file=data_file, lats=np.arange(-60, 60, 10), - lons=np.arange(-280, 80, 10), + lons=np.arange(-280, 80, 30), variables_zonal={'Temp': [-0.5, 0.5], 'Salt': [-0.1, 0.1]}, variables_horiz={'Temp': [-0.5, 0.5], diff --git a/test/land/letkfoi_land.yaml b/test/land/letkfoi_land.yaml index c9069f0e8..98cd3d171 100644 --- a/test/land/letkfoi_land.yaml +++ b/test/land/letkfoi_land.yaml @@ -64,7 +64,7 @@ observations: where: - minvalue: '-999.0' variable: - name: MetaData/height + name: MetaData/stationElevation - filter: Domain Check where: - maxvalue: '1.5' diff --git a/test/soca/gw/run_jjobs.yaml.test b/test/soca/gw/run_jjobs.yaml.test index f773a8c79..f908c841f 100644 --- a/test/soca/gw/run_jjobs.yaml.test +++ b/test/soca/gw/run_jjobs.yaml.test @@ -33,6 +33,9 @@ gw environement: OOPS_DEBUG: 1 OMP_NUM_THREADS: 1 + run scripts: + GDASPREPOCNOBSPY: @HOMEgfs@/sorc/gdas.cd/scripts/exglobal_prep_ocean_obs.py + setup_expt config: base: DO_JEDIATMVAR: "NO" @@ -51,6 +54,8 @@ setup_expt config: SABER_BLOCKS_YAML: @HOMEgfs@/sorc/gdas.cd/parm/soca/berror/saber_blocks.yaml NICAS_RESOL: 1 NICAS_GRID_SIZE: 150 + prepoceanobs: + SOCA_OBS_LIST: @HOMEgfs@/sorc/gdas.cd/parm/soca/obs/obs_list_small.yaml job options: account: da-cpu diff --git a/test/soca/testinput/socahybridweights.yaml b/test/soca/testinput/socahybridweights.yaml index cdad1ab27..855dd6d60 100644 --- a/test/soca/testinput/socahybridweights.yaml +++ b/test/soca/testinput/socahybridweights.yaml @@ -16,8 +16,19 @@ background: read_from_file: 1 weights: - ice: 0.1 - ocean: 0.5 + # Need to provide weights^2 when reading from file + ice: 0.0025 # 5% of original variance + ocean: 0.0625 # 25% " " + # Apply localized weights to the ocean ens. B + ocean local weights: + - lon: -172.0 + lat: 11.0 + amplitude: -1.0 + length scale: 700.0 + - lon: -160.0 + lat: 12.0 + amplitude: -1.0 + length scale: 700.0 output: datadir: ./ diff --git a/ush/eva/jedi_gsi_compare_conv.yaml b/ush/eva/jedi_gsi_compare_conv.yaml index 02293a7b0..c82a0dbd9 100644 --- a/ush/eva/jedi_gsi_compare_conv.yaml +++ b/ush/eva/jedi_gsi_compare_conv.yaml @@ -107,6 +107,24 @@ transforms: for: variable: *variables + # Obs Error that passed QC for JEDI + - transform: accept where + new name: experiment::EffectiveErrorQc::${variable} + starting field: experiment::EffectiveError::${variable} + where: + - experiment::EffectiveQC::${variable} == 0 + for: + variable: *variables + + # ObsError that passed QC for GSI + - transform: accept where + new name: experiment::GsiFinalObsErrorQc::${variable} + starting field: experiment::GsiFinalObsError::${variable} + where: + - experiment::GsiEffectiveQC::${variable} == 0 + for: + variable: *variables + graphics: # ---------- Scatter Plots ---------- @@ -276,6 +294,7 @@ graphics: markersize: 5 color: 'red' label: 'hofxdiff vs pressure' + do_linear_regression: False statistics: fields: - field_name: experiment::HofXDiff::${variable} @@ -354,6 +373,42 @@ graphics: markersize: 5 color: 'red' label: 'errordiff vs pressure' + do_linear_regression: False + # Error comparison as a function of pressure + - batch figure: + variables: *variables + @CHANNELSKEY@ + figure: + layout: [1,1] + title: 'JEDI-GSI Comparison (Passed QC) | @NAME@ @CYCLE@ | ${variable_title}' + output name: observation_scatter_plots/errorcomp_vs_pressure_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png + plots: + - add_xlabel: 'Observation Error' + add_ylabel: 'Observation Pressure' + add_grid: + add_legend: + loc: 'lower left' + layers: + - type: Scatter + x: + variable: experiment::GsiFinalObsErrorQc::${variable} + y: + variable: experiment::MetaData::pressure + @CHANNELKEY@ + markersize: 3 + color: 'red' + label: 'GSI Error' + do_linear_regression: False + - type: Scatter + x: + variable: experiment::EffectiveErrorQc::${variable} + y: + variable: experiment::MetaData::pressure + @CHANNELKEY@ + markersize: 5 + color: 'green' + label: 'JEDI Error' + do_linear_regression: False # ---------- Histograms ---------- # Histogram of h(x) difference diff --git a/ush/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.py b/ush/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.py new file mode 100755 index 000000000..a5700574a --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.py @@ -0,0 +1,347 @@ +#!/usr/bin/env python3 +# (C) Copyright 2023 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import sys +import numpy as np +import os +import argparse +import math +import calendar +import time +from datetime import datetime +import json +from pyiodaconv import bufr +from collections import namedtuple +from pyioda import ioda_obs_space as ioda_ospace +from wxflow import Logger + + +def Compute_dateTime(cycleTimeSinceEpoch, dhr): + + dhr = np.int64(dhr*3600) + dateTime = dhr + cycleTimeSinceEpoch + + return dateTime + + +def bufr_to_ioda(config, logger): + + subsets = config["subsets"] + logger.debug(f"Checking subsets = {subsets}") + + # Get parameters from configuration + data_format = config["data_format"] + source = config["source"] + data_type = config["data_type"] + data_description = config["data_description"] + data_provider = config["data_provider"] + cycle_type = config["cycle_type"] + cycle_datetime = config["cycle_datetime"] + dump_dir = config["dump_directory"] + ioda_dir = config["ioda_directory"] + cycle = config["cycle_datetime"] + + # Get derived parameters + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + + reference_time = datetime.strptime(cycle, "%Y%m%d%H") + reference_time = reference_time.strftime("%Y-%m-%dT%H:%M:%SZ") + reference_time_full = f"{yyyymmdd}{hh}00" + + logger.debug(f"reference_time = {reference_time}") + + # General informaton + converter = 'BUFR to IODA Converter' + platform_description = 'SFCSHP data from prepBUFR format' + + bufrfile = f"{cycle_type}.t{hh}z.{data_format}" + DATA_PATH = os.path.join(dump_dir, f"{cycle_type}.{yyyymmdd}", + str(hh), bufrfile) + + logger.debug(f"The DATA_PATH is: {DATA_PATH}") + + # ============================================ + # Make the QuerySet for all the data we want + # ============================================ + start_time = time.time() + + logger.debug('Making QuerySet ...') + q = bufr.QuerySet(subsets) + + # ObsType + q.add('observationType', '*/TYP') + + # MetaData + q.add('stationIdentification', '*/SID') + q.add('latitude', '*/YOB') + q.add('longitude', '*/XOB') + q.add('obsTimeMinusCycleTime', '*/DHR') + q.add('heightOfStation', '*/Z___INFO/Z__EVENT{1}/ZOB') + q.add('pressure', '*/P___INFO/P__EVENT{1}/POB') + +# # Quality Infomation (Quality Indicator) + q.add('qualityMarkerStationPressure', '*/P___INFO/P__EVENT{1}/PQM') + q.add('qualityMarkerStationElevation', '*/Z___INFO/Z__EVENT{1}/ZQM') + + # ObsValue + q.add('stationPressure', '*/P___INFO/P__EVENT{1}/POB') + q.add('stationElevation', '*/ELV') + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for making QuerySet: {running_time} seconds") + + # ============================================================== + # Open the BUFR file and execute the QuerySet to get ResultSet + # Use the ResultSet returned to get numpy arrays of the data + # ============================================================== + start_time = time.time() + + logger.debug(f"Executing QuerySet to get ResultSet ...") + with bufr.File(DATA_PATH) as f: + r = f.execute(q) + + logger.debug(" ... Executing QuerySet: get ObsType ...") + # ObsType + typ = r.get('observationType') + + logger.debug(" ... Executing QuerySet: get MetaData ...") + # MetaData + sid = r.get('stationIdentification') + lat = r.get('latitude') + lon = r.get('longitude') + lon[lon > 180] -= 360 + zob = r.get('heightOfStation', type='float') + pressure = r.get('pressure') + pressure *= 100 + + logger.debug(f" ... Executing QuerySet: get QualityMarker information ...") + # Quality Information + pobqm = r.get('qualityMarkerStationPressure') + zobqm = r.get('qualityMarkerStationElevation') + + logger.debug(f" ... Executing QuerySet: get ObsValue ...") + # ObsValue + elv = r.get('stationElevation', type='float') + pob = r.get('stationPressure') + pob *= 100 + + logger.debug(f" ... Executing QuerySet: get dateTime ...") + # DateTime: seconds since Epoch time + # IODA has no support for numpy datetime arrays dtype=datetime64[s] + dhr = r.get('obsTimeMinusCycleTime', type='int64') + + logger.debug(f" ... Executing QuerySet: Done!") + + logger.debug(f" ... Executing QuerySet: Check BUFR variable generic \ + dimension and type ...") + # Check BUFR variable generic dimension and type + logger.debug(f" typ shape = {typ.shape}") + logger.debug(f" sid shape = {sid.shape}") + logger.debug(f" dhr shape = {dhr.shape}") + logger.debug(f" lat shape = {lat.shape}") + logger.debug(f" lon shape = {lon.shape}") + logger.debug(f" zob shape = {zob.shape}") + logger.debug(f" pressure shape = {pressure.shape}") + + logger.debug(f" pobqm shape = {pobqm.shape}") + logger.debug(f" zobqm shape = {zobqm.shape}") + + logger.debug(f" elv shape = {elv.shape}") + logger.debug(f" pob shape = {pob.shape}") + + logger.debug(f" dhr type = {dhr.shape}") + + logger.debug(f" sid type = {sid.dtype}") + logger.debug(f" dhr type = {dhr.dtype}") + logger.debug(f" lat type = {lat.dtype}") + logger.debug(f" lon type = {lon.dtype}") + logger.debug(f" zob type = {zob.dtype}") + logger.debug(f" typ type = {typ.dtype}") + logger.debug(f" pressure type = {pressure.dtype}") + + logger.debug(f" pobqm type = {pobqm.dtype}") + logger.debug(f" zobqm type = {zobqm.dtype}") + + logger.debug(f" elv type = {elv.dtype}") + logger.debug(f" pob type = {pob.dtype}") + + logger.debug(f" dhr type = {dhr.dtype}") + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for executing QuerySet to get ResultSet: \ + {running_time} seconds") + + # ========================= + # Create derived variables + # ========================= + start_time = time.time() + + logger.debug(f"Creating derived variables - dateTime ...") + + cycleTimeSinceEpoch = np.int64(calendar.timegm(time.strptime( + reference_time_full, '%Y%m%d%H%M'))) + dateTime = Compute_dateTime(cycleTimeSinceEpoch, dhr) + + logger.debug(f" Check derived variables type ... ") + logger.debug(f" dateTime shape = {dateTime.shape}") + logger.debug(f" dateTime type = {dateTime.dtype}") + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for creating derived variables: \ + {running_time} seconds") + + # ===================================== + # Create IODA ObsSpace + # Write IODA output + # ===================================== + + # Create the dimensions + dims = {'Location': np.arange(0, lat.shape[0])} + + iodafile = f"{cycle_type}.t{hh}z.{data_type}.{data_format}.nc" + OUTPUT_PATH = os.path.join(ioda_dir, iodafile) + logger.debug(f" ... ... Create OUTPUT file: {OUTPUT_PATH}") + + path, fname = os.path.split(OUTPUT_PATH) + if path and not os.path.exists(path): + os.makedirs(path) + + obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) + + # Create Global attributes + logger.debug(f" ... ... Create global attributes") + + obsspace.write_attr('Converter', converter) + obsspace.write_attr('source', source) + obsspace.write_attr('sourceFiles', bufrfile) + obsspace.write_attr('dataProviderOrigin', data_provider) + obsspace.write_attr('description', data_description) + obsspace.write_attr('datetimeReference', reference_time) + obsspace.write_attr('datetimeRange', + [str(min(dateTime)), str(max(dateTime))]) + obsspace.write_attr('platformLongDescription', platform_description) + + # Create IODA variables + logger.debug(f" ... ... Create variables: name, type, units, & attributes") + + # Observation Type - Station Elevation + obsspace.create_var('ObsType/stationElevation', dtype=typ.dtype, + fillval=typ.fill_value) \ + .write_attr('long_name', 'Station Elevation Observation Type') \ + .write_data(typ) + + # Observation Type - Station Pressure + obsspace.create_var('ObsType/stationPressure', dtype=typ.dtype, + fillval=typ.fill_value) \ + .write_attr('long_name', 'Station Pressure Observation Type') \ + .write_data(typ) + + # Longitude + obsspace.create_var('MetaData/longitude', dtype=lon.dtype, + fillval=lon.fill_value) \ + .write_attr('units', 'degrees_east') \ + .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ + .write_attr('long_name', 'Longitude') \ + .write_data(lon) + + # Latitude + obsspace.create_var('MetaData/latitude', dtype=lat.dtype, + fillval=lat.fill_value) \ + .write_attr('units', 'degrees_north') \ + .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Latitude') \ + .write_data(lat) + + # Datetime + obsspace.create_var('MetaData/dateTime', dtype=dateTime.dtype, + fillval=dateTime.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'Datetime') \ + .write_data(dateTime) + + # Station Identification + obsspace.create_var('MetaData/stationIdentification', dtype=sid.dtype, + fillval=sid.fill_value) \ + .write_attr('long_name', 'Station Identification') \ + .write_data(sid) + + # Height Of Station + obsspace.create_var('MetaData/heightOfStation', dtype=zob.dtype, + fillval=zob.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Height Of Station') \ + .write_data(zob) + + # Pressure + obsspace.create_var('MetaData/pressure', dtype=pressure.dtype, + fillval=pressure.fill_value) \ + .write_attr('units', 'Pa') \ + .write_attr('long_name', 'Pressure') \ + .write_data(pressure) + + # QualityMarker - Station Elevation + obsspace.create_var('QualityMarker/stationElevation', dtype=zobqm.dtype, + fillval=zobqm.fill_value) \ + .write_attr('long_name', 'Station Elevation Quality Marker') \ + .write_data(zobqm) + + # QualityMarker - Station Pressure + obsspace.create_var('QualityMarker/stationPressure', dtype=pobqm.dtype, + fillval=pobqm.fill_value) \ + .write_attr('long_name', 'Station Pressure Quality Marker') \ + .write_data(pobqm) + + # Station Elevation + obsspace.create_var('ObsValue/stationElevation', dtype=elv.dtype, + fillval=elv.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Station Elevation') \ + .write_data(elv) + + # Station Pressure + obsspace.create_var('ObsValue/stationPressure', dtype=pob.dtype, + fillval=pob.fill_value) \ + .write_attr('units', 'Pa') \ + .write_attr('long_name', 'Station Pressure') \ + .write_data(pob) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for splitting and output IODA: \ + {running_time} seconds") + + logger.debug("All Done!") + + +if __name__ == '__main__': + + start_time = time.time() + + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, + help='Input JSON configuration', + required=True) + parser.add_argument('-v', '--verbose', + help='print debug logging information', + action='store_true') + args = parser.parse_args() + + log_level = 'DEBUG' if args.verbose else 'INFO' + logger = Logger('bufr2ioda_adpsfc_prepbufr.py', level=log_level, + colored_log=True) + + with open(args.config, "r") as json_file: + config = json.load(json_file) + + bufr_to_ioda(config, logger) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Total running time: {running_time} seconds") diff --git a/ush/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.py b/ush/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.py new file mode 100755 index 000000000..47207a3e9 --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.py @@ -0,0 +1,515 @@ +#!/usr/bin/env python3 +# (C) Copyright 2023 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import sys +import numpy as np +import os +import argparse +import math +import calendar +import time +from datetime import datetime +import json +from pyiodaconv import bufr +from collections import namedtuple +from pyioda import ioda_obs_space as ioda_ospace +from wxflow import Logger + + +def Compute_dateTime(cycleTimeSinceEpoch, dhr): + + dhr = np.int64(dhr*3600) + dateTime = dhr + cycleTimeSinceEpoch + + return dateTime + + +def bufr_to_ioda(config, logger): + + subsets = config["subsets"] + logger.debug(f"Checking subsets = {subsets}") + + # Get parameters from configuration + data_format = config["data_format"] + source = config["source"] + data_type = config["data_type"] + data_description = config["data_description"] + data_provider = config["data_provider"] + cycle_type = config["cycle_type"] + cycle_datetime = config["cycle_datetime"] + dump_dir = config["dump_directory"] + ioda_dir = config["ioda_directory"] + cycle = config["cycle_datetime"] + + # Get derived parameters + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + + reference_time = datetime.strptime(cycle, "%Y%m%d%H") + reference_time = reference_time.strftime("%Y-%m-%dT%H:%M:%SZ") + reference_time_full = f"{yyyymmdd}{hh}00" + + logger.debug(f"Reference time = {reference_time}") + + # General informaton + converter = 'BUFR to IODA Converter' + platform_description = 'SFCSHP data from prepBUFR format' + + bufrfile = f"{cycle_type}.t{hh}z.{data_format}" + DATA_PATH = os.path.join(dump_dir, f"{cycle_type}.{yyyymmdd}", + str(hh), bufrfile) + + logger.debug(f"The DATA_PATH is: {DATA_PATH}") + + # ============================================ + # Make the QuerySet for all the data we want + # ============================================ + start_time = time.time() + + logger.debug("Making QuerySet ...") + q = bufr.QuerySet(subsets) + + # ObsType + q.add('observationType', '*/TYP') + + # MetaData + q.add('stationIdentification', '*/SID') + q.add('latitude', '*/YOB') + q.add('longitude', '*/XOB') + q.add('obsTimeMinusCycleTime', '*/DHR') + q.add('heightOfStation', '*/Z___INFO/Z__EVENT{1}/ZOB') + q.add('pressure', '*/P___INFO/P__EVENT{1}/POB') + q.add('temperatureEventCode', '*/T___INFO/T__EVENT{1}/TPC') + +# # Quality Infomation (Quality Indicator) + q.add('qualityMarkerStationElevation', '*/Z___INFO/Z__EVENT{1}/ZQM') + q.add('qualityMarkerStationPressure', '*/P___INFO/P__EVENT{1}/PQM') + q.add('qualityMarkerAirTemperature', '*/T___INFO/T__EVENT{1}/TQM') + q.add('qualityMarkerSpecificHumidity', '*/Q___INFO/Q__EVENT{1}/QQM') + q.add('qualityMarkerWindNorthward', '*/W___INFO/W__EVENT{1}/WQM') + q.add('qualityMarkerSeaSurfaceTemperature', '*/SST_INFO/SSTEVENT{1}/SSTQM') + + # ObsValue + q.add('stationElevation', '*/ELV') + q.add('stationPressure', '*/P___INFO/P__EVENT{1}/POB') + q.add('airTemperature', '*/T___INFO/T__EVENT{1}/TOB') + q.add('specificHumidity', '*/Q___INFO/Q__EVENT{1}/QOB') + q.add('windNorthward', '*/W___INFO/W__EVENT{1}/VOB') + q.add('windEastward', '*/W___INFO/W__EVENT{1}/UOB') + q.add('seaSurfaceTemperature', '*/SST_INFO/SSTEVENT{1}/SST1') + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for making QuerySet: {running_time} seconds") + + # ============================================================== + # Open the BUFR file and execute the QuerySet to get ResultSet + # Use the ResultSet returned to get numpy arrays of the data + # ============================================================== + start_time = time.time() + + logger.debug(f"Executing QuerySet to get ResultSet ...") + with bufr.File(DATA_PATH) as f: + r = f.execute(q) + + logger.debug(f" ... Executing QuerySet: get metadata: basic ...") + # ObsType + logger.debug(" ... Executing QuerySet: get ObsType ...") + typ = r.get('observationType') + + # MetaData + logger.debug(" ... Executing QuerySet: MetaData ...") + sid = r.get('stationIdentification') + lat = r.get('latitude') + lon = r.get('longitude') + lon[lon > 180] -= 360 + zob = r.get('heightOfStation', type='float') + pressure = r.get('pressure') + pressure *= 100 + tpc = r.get('temperatureEventCode') + + # Quality Information + logger.debug(f" ... Executing QuerySet: QualityMarker ...") + zobqm = r.get('qualityMarkerStationElevation') + pobqm = r.get('qualityMarkerStationPressure') + tobqm = r.get('qualityMarkerAirTemperature') + tsenqm = np.full(tobqm.shape[0], tobqm.fill_value) + tsenqm = np.where(((tpc >= 1) & (tpc < 8)), tobqm, tsenqm) + tvoqm = np.full(tobqm.shape[0], tobqm.fill_value) + tvoqm = np.where((tpc == 8), tobqm, tvoqm) + qobqm = r.get('qualityMarkerSpecificHumidity') + wobqm = r.get('qualityMarkerWindNorthward') + sstqm = r.get('qualityMarkerSeaSurfaceTemperature') + + logger.debug(f" ... Executing QuerySet: ObsValue ...") + # ObsValue + elv = r.get('stationElevation', type='float') + pob = r.get('stationPressure') + pob *= 100 + tob = r.get('airTemperature') + tob += 273.15 + tsen = np.full(tob.shape[0], tob.fill_value) + tsen = np.where(((tpc >= 1) & (tpc < 8)), tob, tsen) + tvo = np.full(tob.shape[0], tob.fill_value) + tvo = np.where((tpc == 8), tob, tvo) + qob = r.get('specificHumidity', type='float') + qob *= 0.000001 + uob = r.get('windEastward') + vob = r.get('windNorthward') + sst1 = r.get('seaSurfaceTemperature') + + logger.debug(f" ... Executing QuerySet: get datatime: observation time ...") + # DateTime: seconds since Epoch time + # IODA has no support for numpy datetime arrays dtype=datetime64[s] + dhr = r.get('obsTimeMinusCycleTime', type='int64') + + logger.debug(f" ... Executing QuerySet: Done!") + + logger.debug(f" ... Executing QuerySet: Check BUFR variable generic \ + dimension and type ...") + # Check BUFR variable generic dimension and type + logger.debug(f" typ shape = {typ.shape}") + logger.debug(f" sid shape = {sid.shape}") + logger.debug(f" dhr shape = {dhr.shape}") + logger.debug(f" lat shape = {lat.shape}") + logger.debug(f" lon shape = {lon.shape}") + logger.debug(f" zob shape = {zob.shape}") + logger.debug(f" pressure shape = {pressure.shape}") + logger.debug(f" tpc shape = {tpc.shape}") + + logger.debug(f" zobqm shape = {zobqm.shape}") + logger.debug(f" pobqm shape = {pobqm.shape}") + logger.debug(f" tobqm shape = {tobqm.shape}") + logger.debug(f" tsenqm shape = {tsenqm.shape}") + logger.debug(f" tvoqm shape = {tvoqm.shape}") + logger.debug(f" qobqm shape = {qobqm.shape}") + logger.debug(f" wobqm shape = {wobqm.shape}") + logger.debug(f" sstqm shape = {sstqm.shape}") + + logger.debug(f" elv shape = {elv.shape}") + logger.debug(f" pob shape = {pob.shape}") + logger.debug(f" tob shape = {tob.shape}") + logger.debug(f" tsen shape = {tsen.shape}") + logger.debug(f" tvo shape = {tvo.shape}") + logger.debug(f" qob shape = {qob.shape}") + logger.debug(f" uob shape = {uob.shape}") + logger.debug(f" vob shape = {vob.shape}") + logger.debug(f" sst1 shape = {sst1.shape}") + + logger.debug(f" typ type = {typ.dtype}") + logger.debug(f" sid type = {sid.dtype}") + logger.debug(f" dhr type = {dhr.dtype}") + logger.debug(f" lat type = {lat.dtype}") + logger.debug(f" lon type = {lon.dtype}") + logger.debug(f" zob type = {zob.dtype}") + logger.debug(f" pressure type = {pressure.dtype}") + logger.debug(f" tpc type = {tpc.dtype}") + + logger.debug(f" pobqm type = {pobqm.dtype}") + logger.debug(f" tobqm type = {tobqm.dtype}") + logger.debug(f" tsenqm type = {tsenqm.dtype}") + logger.debug(f" tvoqm type = {tvoqm.dtype}") + logger.debug(f" qobqm type = {qobqm.dtype}") + logger.debug(f" wobqm type = {wobqm.dtype}") + logger.debug(f" sstqm type = {sstqm.dtype}") + + logger.debug(f" elv type = {elv.dtype}") + logger.debug(f" pob type = {pob.dtype}") + logger.debug(f" tob type = {tob.dtype}") + logger.debug(f" tsen type = {tsen.dtype}") + logger.debug(f" tvo type = {tvo.dtype}") + logger.debug(f" qob type = {qob.dtype}") + logger.debug(f" uob type = {uob.dtype}") + logger.debug(f" vob type = {vob.dtype}") + logger.debug(f" sst1 type = {sst1.dtype}") + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for executing QuerySet to get ResultSet: \ + {running_time} seconds") + + # ========================= + # Create derived variables + # ========================= + start_time = time.time() + + logger.debug(f"Creating derived variables - dateTime ...") + + cycleTimeSinceEpoch = np.int64(calendar.timegm(time.strptime( + reference_time_full, '%Y%m%d%H%M'))) + dateTime = Compute_dateTime(cycleTimeSinceEpoch, dhr) + + logger.debug(f" Check derived variables type ... ") + logger.debug(f" dateTime shape = {dateTime.shape}") + logger.debug(f" dateTime type = {dateTime.dtype}") + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for creating derived variables: {running_time} \ + seconds") + + # ===================================== + # Create IODA ObsSpace + # Write IODA output + # ===================================== + + # Create the dimensions + dims = {'Location': np.arange(0, lat.shape[0])} + + iodafile = f"{cycle_type}.t{hh}z.{data_type}.{data_format}.nc" + OUTPUT_PATH = os.path.join(ioda_dir, iodafile) + logger.debug(f" ... ... Create OUTPUT file: {OUTPUT_PATH}") + + path, fname = os.path.split(OUTPUT_PATH) + if path and not os.path.exists(path): + os.makedirs(path) + + obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) + + # Create Global attributes + logger.debug(f" ... ... Create global attributes") + + obsspace.write_attr('Converter', converter) + obsspace.write_attr('source', source) + obsspace.write_attr('sourceFiles', bufrfile) + obsspace.write_attr('dataProviderOrigin', data_provider) + obsspace.write_attr('description', data_description) + obsspace.write_attr('datetimeReference', reference_time) + obsspace.write_attr('datetimeRange', + [str(min(dateTime)), str(max(dateTime))]) + obsspace.write_attr('platformLongDescription', platform_description) + + # Create IODA variables + logger.debug(f" ... ... Create variables: name, type, units, & attributes") + + # ObsType: stationElevation + obsspace.create_var('ObsType/stationElevation', dtype=typ.dtype, + fillval=typ.fill_value) \ + .write_attr('long_name', 'Station Elevation Observation Type') \ + .write_data(typ) + + # ObsType: stationPressure + obsspace.create_var('ObsType/stationPressure', dtype=typ.dtype, + fillval=typ.fill_value) \ + .write_attr('long_name', 'Station Pressure Observation Type') \ + .write_data(typ) + + # ObsType: air Temperature + obsspace.create_var('ObsType/airTemperature', dtype=typ.dtype, + fillval=typ.fill_value) \ + .write_attr('long_name', 'Air Temperature Observation Type') \ + .write_data(typ) + + # ObsType: virtual Temperature + obsspace.create_var('ObsType/virtualTemperature', dtype=typ.dtype, + fillval=typ.fill_value) \ + .write_attr('long_name', 'Virtual Temperature Observation Type') \ + .write_data(typ) + + # ObsType: Specific Humidity + obsspace.create_var('ObsType/specificHumidity', dtype=typ.dtype, + fillval=typ.fill_value) \ + .write_attr('long_name', 'Specific Humidity Observation Type') \ + .write_data(typ) + + # ObsType: wind Eastward + obsspace.create_var('ObsType/windEastward', dtype=typ.dtype, + fillval=typ.fill_value) \ + .write_attr('long_name', 'Wind Eastward Observation Type') \ + .write_data(typ) + + # ObsType: wind Northward + obsspace.create_var('ObsType/windNorthward', dtype=typ.dtype, + fillval=typ.fill_value) \ + .write_attr('long_name', 'Wind Northward Observation Type') \ + .write_data(typ) + + # ObsType: Sea Surface Temperature + obsspace.create_var('ObsType/seaSurfaceTemperature', dtype=typ.dtype, + fillval=typ.fill_value) \ + .write_attr('long_name', 'Sea Surface Temperature Observation Type') \ + .write_data(typ) + + # Longitude + obsspace.create_var('MetaData/longitude', dtype=lon.dtype, + fillval=lon.fill_value) \ + .write_attr('units', 'degrees_east') \ + .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ + .write_attr('long_name', 'Longitude') \ + .write_data(lon) + + # Latitude + obsspace.create_var('MetaData/latitude', dtype=lat.dtype, + fillval=lat.fill_value) \ + .write_attr('units', 'degrees_north') \ + .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Latitude') \ + .write_data(lat) + + # Datetime + obsspace.create_var('MetaData/dateTime', dtype=dateTime.dtype, + fillval=dateTime.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'Datetime') \ + .write_data(dateTime) + + # Station Identification + obsspace.create_var('MetaData/stationIdentification', dtype=sid.dtype, + fillval=sid.fill_value) \ + .write_attr('long_name', 'Station Identification') \ + .write_data(sid) + + # Station Elevation + obsspace.create_var('MetaData/heightOfStation', dtype=zob.dtype, + fillval=zob.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Height Of Station') \ + .write_data(zob) + + # Pressure + obsspace.create_var('MetaData/pressure', dtype=pressure.dtype, + fillval=pressure.fill_value) \ + .write_attr('units', 'Pa') \ + .write_attr('long_name', 'Pressure') \ + .write_data(pressure) + + # Temperature Event Code + obsspace.create_var('MetaData/temperatureEventCode', dtype=tpc.dtype, + fillval=tpc.fill_value) \ + .write_attr('long_name', 'Temperature Event Code') \ + .write_data(tpc) + + # Quality Marker: Station Pressure + obsspace.create_var('QualityMarker/stationPressure', dtype=pobqm.dtype, + fillval=pobqm.fill_value) \ + .write_attr('long_name', 'Station Pressure Quality Marker') \ + .write_data(pobqm) + + # Quality Marker: Air Temperature + obsspace.create_var('QualityMarker/airTemperature', dtype=tobqm.dtype, + fillval=tobqm.fill_value) \ + .write_attr('long_name', 'Air Temperature Quality Marker') \ + .write_data(tsenqm) + + # Quality Marker: Virtual Temperature + obsspace.create_var('QualityMarker/virtualTemperature', dtype=tobqm.dtype, + fillval=tobqm.fill_value) \ + .write_attr('long_name', 'Virtual Temperature Quality Marker') \ + .write_data(tvoqm) + + # Quality Marker: Specific Humidity + obsspace.create_var('QualityMarker/specificHumidity', dtype=qobqm.dtype, + fillval=qobqm.fill_value) \ + .write_attr('long_name', 'Specific Humidity Quality Marker') \ + .write_data(qobqm) + + # Quality Marker: Eastward Wind + obsspace.create_var('QualityMarker/windEastward', dtype=wobqm.dtype, + fillval=wobqm.fill_value) \ + .write_attr('long_name', 'Eastward Wind Quality Marker') \ + .write_data(wobqm) + + # Quality Marker: Northward Wind + obsspace.create_var('QualityMarker/windNorthward', dtype=wobqm.dtype, + fillval=wobqm.fill_value) \ + .write_attr('long_name', 'Northward Wind Quality Marker') \ + .write_data(wobqm) + + # Quality Marker: Sea Surface Temperature + obsspace.create_var('QualityMarker/seaSurfaceTemperature', + dtype=sstqm.dtype, fillval=sstqm.fill_value) \ + .write_attr('long_name', 'Sea Surface Temperature Quality Marker') \ + .write_data(sstqm) + + # Station Elevation + obsspace.create_var('ObsValue/stationElevation', dtype=elv.dtype, + fillval=elv.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Station Elevation') \ + .write_data(elv) + + # Station Pressure + obsspace.create_var('ObsValue/stationPressure', dtype=pob.dtype, + fillval=pob.fill_value) \ + .write_attr('units', 'Pa') \ + .write_attr('long_name', 'Station Pressure') \ + .write_data(pob) + + # Air Temperature + obsspace.create_var('ObsValue/airTemperature', dtype=tob.dtype, + fillval=tob.fill_value) \ + .write_attr('units', 'K') \ + .write_attr('long_name', 'Air Temperature') \ + .write_data(tsen) + + # Virtual Temperature + obsspace.create_var('ObsValue/virtualTemperature', dtype=tob.dtype, + fillval=tob.fill_value) \ + .write_attr('units', 'K') \ + .write_attr('long_name', 'Virtual Temperature') \ + .write_data(tvo) + + # Specific Humidity + obsspace.create_var('ObsValue/specificHumidity', dtype=qob.dtype, + fillval=qob.fill_value) \ + .write_attr('units', 'kg kg-1') \ + .write_attr('long_name', 'Specific Humidity') \ + .write_data(qob) + + # Eastward Wind + obsspace.create_var('ObsValue/windEastward', dtype=uob.dtype, + fillval=uob.fill_value) \ + .write_attr('units', 'm s-1') \ + .write_attr('long_name', 'Eastward Wind') \ + .write_data(uob) + + # Northward Wind + obsspace.create_var('ObsValue/windNorthward', dtype=vob.dtype, + fillval=vob.fill_value) \ + .write_attr('units', 'm s-1') \ + .write_attr('long_name', 'Northward Wind') \ + .write_data(vob) + + # Sea Surface Temperature + obsspace.create_var('ObsValue/seaSurfaceTemperature', dtype=sst1.dtype, + fillval=sst1.fill_value) \ + .write_attr('units', 'K') \ + .write_attr('long_name', 'Sea Surface Temperature') \ + .write_data(sst1) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for splitting and output IODA: \ + {running_time} seconds") + + logger.debug("All Done!") + + +if __name__ == '__main__': + + start_time = time.time() + + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, + help='Input JSON configuration', + required=True) + parser.add_argument('-v', '--verbose', + help='print debug logging information', + action='store_true') + args = parser.parse_args() + + log_level = 'DEBUG' if args.verbose else 'INFO' + logger = Logger('bufr2ioda_sfcshp_prepbufr.py', level=log_level, + colored_log=True) + + with open(args.config, "r") as json_file: + config = json.load(json_file) + + bufr_to_ioda(config, logger) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Total running time: {running_time} seconds") diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index d7a7bb130..da01dd131 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -30,7 +30,8 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): } # Specify observation types to be processed by a script - BUFR_py = ["satwind_amv_goes", "satwind_scat", "adpupa_prepbufr"] + + BUFR_py = ["satwind_amv_goes", "satwind_scat", "adpupa_prepbufr", "adpsfc_prepbufr", "sfcshp_prepbufr"] for obtype in BUFR_py: logger.info(f"Convert {obtype}...") diff --git a/ush/soca/soca_vrfy.py b/ush/soca/soca_vrfy.py index 0f6c5b654..0741e9d28 100755 --- a/ush/soca/soca_vrfy.py +++ b/ush/soca/soca_vrfy.py @@ -33,7 +33,7 @@ def plotConfig(grid_file=[], lat=np.nan, lats=np.arange(-60, 60, 10), lon=np.nan, - lons=np.arange(-280, 80, 10), + lons=np.arange(-280, 80, 30), proj='set me', projs=['Global']): @@ -52,8 +52,8 @@ def plotConfig(grid_file=[], config['bounds'] = bounds config['lats'] = lats # all the lats to plot config['lat'] = lat # the lat being currently plotted - config['lons'] = lons - config['lon'] = lon + config['lons'] = lons # all the lons to plot + config['lon'] = lon # the lon being currently plotted config['max depths'] = max_depths # all the max depths to plot config['max depth'] = max_depth # the max depth currently plotted config['horiz variables'] = variables_horiz # all the vars for horiz plots @@ -159,13 +159,13 @@ def plotMeridionalSlice(config): lon = float(config['lon']) grid = xr.open_dataset(config['grid file']) data = xr.open_dataset(config['fields file']) - lon_index = np.argmin(np.array(np.abs(np.squeeze(grid.lon)[:, 0]-lon))) - slice_data = np.squeeze(np.array(data[config['variable']]))[:, lon_index, :] - depth = np.squeeze(np.array(grid['h']))[:, lon_index, :] + lon_index = np.argmin(np.array(np.abs(np.squeeze(grid.lon)[0, :]-lon))) + slice_data = np.squeeze(np.array(data[config['variable']]))[:, :, lon_index] + depth = np.squeeze(np.array(grid['h']))[:, :, lon_index] depth[np.where(np.abs(depth) > 10000.0)] = 0.0 depth = np.cumsum(depth, axis=0) bounds = config['bounds'] - y = np.tile(np.squeeze(grid.lon[:, lon_index]), (np.shape(depth)[0], 1)) + y = np.tile(np.squeeze(grid.lat)[:, lon_index], (np.shape(depth)[0], 1)) fig, ax = plt.subplots(figsize=(8, 5)) plt.pcolormesh(y, -depth, slice_data, vmin=bounds[0], vmax=bounds[1], diff --git a/ush/ufoeval/run_ufo_hofx_test.sh b/ush/ufoeval/run_ufo_hofx_test.sh index 85fed17cf..230966284 100755 --- a/ush/ufoeval/run_ufo_hofx_test.sh +++ b/ush/ufoeval/run_ufo_hofx_test.sh @@ -33,6 +33,7 @@ run_filtering=YES run_eva=YES eva_stats_only=NO keep_output=NO +window_shift=False while getopts "c:hsxq" opt; do case $opt in @@ -99,13 +100,7 @@ if [ $run_filtering == NO ]; then else yamlpath=$GDASApp/parm/atm/obs/testing/${obtype}.yaml fi - -#exename=test_ObsFilters.x -if [ $run_filtering == NO ]; then - exename=test_ObsOperator.x -else - exename=test_ObsFilters.x -fi +exename=test_ObsFilters.x #-------------- Do not modify below this line ---------------- # paths that should only be changed by an expert user @@ -183,11 +178,13 @@ export OPREFIX=gdas.t${cyc}z export APREFIX=gdas.t${cyc}z export GPREFIX=gdas.t${gcyc}z +if [ $obtype == "conv_ps" ]; then window_shift=True; fi cat > $workdir/temp.yaml << EOF window begin: '{{ WINDOW_BEGIN | to_isotime }}' window end: '{{ WINDOW_END | to_isotime }}' observations: - !INC $yamlpath +window shift: ${window_shift} EOF $GDASApp/ush/genYAML --input $workdir/temp.yaml --output $workdir/${obtype}_${cycle}.yaml diff --git a/ush/ufoeval/test_yamls.sh b/ush/ufoeval/test_yamls.sh index 7e98d80e2..d46056bd8 100755 --- a/ush/ufoeval/test_yamls.sh +++ b/ush/ufoeval/test_yamls.sh @@ -13,10 +13,23 @@ if [ $? -ne 0 ]; then exit 1 fi -for file in `ls ../../parm/atm/obs/testing/*yaml`; do +# Process tests wiht QC +for file in `find ../../parm/atm/obs/testing/*.yaml -type f -not -name "*noqc*"`; do basefile=${file##*/} - inst="${basefile%.*}" - ./run_ufo_hofx_test.sh -x $inst > $WORKDIR/$inst.log 2> $WORKDIR/$inst.err + obtype="${basefile%.*}" + ./run_ufo_hofx_test.sh -x $obtype > $WORKDIR/$obtype.log 2> $WORKDIR/$obtype.err + if [ $? == 0 ]; then + echo $basefile Passes \(yay!\) + else + echo $basefile Fails \(boo!\) + fi +done + +# Process tests without QC (HofX + Observation error assignment) +for file in `ls ../../parm/atm/obs/testing/*_noqc.yaml`; do + basefile=${file##*/} + obtype="${basefile%_noqc.*}" + ./run_ufo_hofx_test.sh -x -q $obtype > $WORKDIR/${obtype}_noqc.log 2> $WORKDIR/${obtype}_noqc.err if [ $? == 0 ]; then echo $basefile Passes \(yay!\) else diff --git a/utils/obsproc/NetCDFToIodaConverter.h b/utils/obsproc/NetCDFToIodaConverter.h index 4686ec804..6942c1c0c 100644 --- a/utils/obsproc/NetCDFToIodaConverter.h +++ b/utils/obsproc/NetCDFToIodaConverter.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -43,6 +44,9 @@ namespace gdasapp { Eigen::ArrayXXf intMetadata_; // Optional array of integer metadata std::vector intMetadataName_; // String descriptor of the integer metadata + // Optional global attributes + std::map strGlobalAttr_; + // Constructor explicit IodaVars(const int nobs = 0, const std::vector fmnames = {}, @@ -233,6 +237,14 @@ namespace gdasapp { ogrp.vars.createWithScales("PreQC/"+variable_, {ogrp.vars["Location"]}, int_params); + // add input filenames to IODA file global attributes + ogrp.atts.add("obs_source_files", inputFilenames_); + + // add global attributes collected from the specific converter + for (const auto& globalAttr : iodaVars.strGlobalAttr_) { + ogrp.atts.add(globalAttr.first , globalAttr.second); + } + // Create the optional IODA integer metadata ioda::Variable tmpIntMeta; int count = 0; @@ -254,7 +266,7 @@ namespace gdasapp { } // Write obs info to group - oops::Log::info() << "Writting ioda file" << std::endl; + oops::Log::info() << "Writing ioda file" << std::endl; iodaLon.writeWithEigenRegular(iodaVarsAll.longitude_); iodaLat.writeWithEigenRegular(iodaVarsAll.latitude_); iodaDatetime.writeWithEigenRegular(iodaVarsAll.datetime_); diff --git a/utils/obsproc/Rads2Ioda.h b/utils/obsproc/Rads2Ioda.h index a31953d2b..6a2368a2d 100644 --- a/utils/obsproc/Rads2Ioda.h +++ b/utils/obsproc/Rads2Ioda.h @@ -57,10 +57,31 @@ namespace gdasapp { ncFile.getVar("time_mjd").getVar(datetime); iodaVars.referenceDate_ = "seconds since 1858-11-17T00:00:00Z"; + std::map altimeterMap; + // TODO(All): This is incomplete, add missions to the list below + altimeterMap["SNTNL-3A"] = 1; + altimeterMap["SNTNL-3B"] = 2; + altimeterMap["JASON-1"] = 3; + altimeterMap["JASON-2"] = 4; + altimeterMap["JASON-3"] = 5; + altimeterMap["CRYOSAT2"] = 6; + altimeterMap["SARAL"] = 7; + // Read optional integer metadata "mission" std::string mission_name; ncFile.getAtt("mission_name").getValues(mission_name); - int mission_index = altimeterMissions(mission_name); // mission name mapped to integer + int mission_index = altimeterMap[mission_name]; // mission name mapped to integer + + // convert mission map to string to add to global attributes + std::stringstream mapStr; + for (const auto& mapEntry : altimeterMap) { + mapStr << mapEntry.first << " = " << mapEntry.second << " "; + } + iodaVars.strGlobalAttr_["mission_index"] = mapStr.str(); + + std::string references; + ncFile.getAtt("references").getValues(references); + iodaVars.strGlobalAttr_["references"] = references; // Read optional integer metadata "pass" and "cycle" int pass[iodaVars.location_]; // NOLINT @@ -96,18 +117,5 @@ namespace gdasapp { } return iodaVars; }; - int altimeterMissions(std::string missionName) { - std::map altimeterMap; - // TODO(All): This is incomplete, add missions to the list below - // and add to global attribute - altimeterMap["SNTNL-3A"] = 1; - altimeterMap["SNTNL-3B"] = 2; - altimeterMap["JASON-1"] = 3; - altimeterMap["JASON-2"] = 4; - altimeterMap["JASON-3"] = 5; - altimeterMap["CRYOSAT2"] = 6; - altimeterMap["SARAL"] = 7; - return altimeterMap[missionName]; - } }; // class Rads2Ioda } // namespace gdasapp diff --git a/utils/soca/gdas_ens_handler.h b/utils/soca/gdas_ens_handler.h index 44e994a0b..5cf862651 100644 --- a/utils/soca/gdas_ens_handler.h +++ b/utils/soca/gdas_ens_handler.h @@ -104,6 +104,10 @@ namespace gdasapp { gdasapp_ens_utils::ensMoments(ensMembers, ensMean, ensStd, ensVariance); oops::Log::info() << "mean: " << ensMean << std::endl; oops::Log::info() << "std: " << ensStd << std::endl; + if ( fullConfig.has("ensemble mean output") ) { + const eckit::LocalConfiguration ensMeanOutputConfig(fullConfig, "ensemble mean output"); + ensMean.write(ensMeanOutputConfig); + } // Remove mean from ensemble members for (size_t i = 0; i < postProcIncr.ensSize_; ++i) { @@ -115,6 +119,7 @@ namespace gdasapp { eckit::LocalConfiguration stericVarChangeConfig; fullConfig.get("steric height", stericVarChangeConfig); oops::Log::info() << "steric config 0000: " << stericVarChangeConfig << std::endl; + // Initialize trajectories const eckit::LocalConfiguration trajConfig(fullConfig, "trajectory"); soca::State cycleTraj(geom, trajConfig); // trajectory of the cycle @@ -148,7 +153,7 @@ namespace gdasapp { soca::Increment incr = postProcIncr.appendLayer(ensMembers[i]); // Save total ssh - oops::Log::info() << "ssh ensemble memnber " << i << std::endl; + oops::Log::info() << "ssh ensemble member " << i << std::endl; soca::Increment ssh_tmp(geom, socaSshVar, postProcIncr.dt_); ssh_tmp = ensMembers[i]; sshTotal.push_back(ssh_tmp); @@ -186,14 +191,14 @@ namespace gdasapp { // Add the unbalanced ssh to the recentered perturbation // this assumes ssh_u is independent of the trajectory - oops::Log::info() << "&&&&& before adding ssh_u " << incr << std::endl; + oops::Log::debug() << "&&&&& before adding ssh_u " << incr << std::endl; atlas::FieldSet incrFs; incr.toFieldSet(incrFs); atlas::FieldSet sshNonStericFs; sshNonSteric[i].toFieldSet(sshNonStericFs); util::addFieldSets(incrFs, sshNonStericFs); incr.fromFieldSet(incrFs); - oops::Log::info() << "&&&&& after adding ssh_u " << incr << std::endl; + oops::Log::debug() << "&&&&& after adding ssh_u " << incr << std::endl; // Save final perturbation, used in the offline EnVAR result = postProcIncr.save(incr, i+1); diff --git a/utils/soca/gdas_socahybridweights.h b/utils/soca/gdas_socahybridweights.h index adf20188a..219450a8e 100644 --- a/utils/soca/gdas_socahybridweights.h +++ b/utils/soca/gdas_socahybridweights.h @@ -3,12 +3,17 @@ #include #include #include +#include #include "eckit/config/LocalConfiguration.h" #include "atlas/field.h" +#include "atlas/util/Earth.h" +#include "atlas/util/Geometry.h" +#include "atlas/util/Point.h" #include "oops/base/PostProcessor.h" +#include "oops/generic/gc99.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" #include "oops/util/DateTime.h" @@ -20,6 +25,38 @@ #include "soca/State/State.h" namespace gdasapp { + // Create a simple mask based on a Gaussian function + void gaussianMask(const soca::Geometry & geom, soca::Increment & gaussIncr, + eckit::LocalConfiguration conf) { + // Get the 2D grid + std::vector lats; + std::vector lons; + bool halo = true; + geom.latlon(lats, lons, halo); + + // Prepare fieldset from increment + atlas::FieldSet gaussIncrFs; + gaussIncr.toFieldSet(gaussIncrFs); + + // Get the GC99 parameters from config + double amp = conf.getDouble("amplitude"); + double scale = conf.getDouble("length scale"); + const atlas::PointLonLat p0(conf.getDouble("lon"), conf.getDouble("lat")); + + // Recompute weights + for (auto & field : gaussIncrFs) { + oops::Log::info() << "---------- Field name: " << field.name() << std::endl; + auto view = atlas::array::make_view(field); + for (int jnode = 0; jnode < field.shape(0); ++jnode) { + atlas::PointLonLat p1(lons[jnode], lats[jnode]); + double d = atlas::util::Earth::distance(p0, p1)/1000.0; + for (int jlevel = 0; jlevel < field.shape(1); ++jlevel) { + view(jnode, jlevel) += amp * oops::gc99(d/scale); + } + } + } + gaussIncr.fromFieldSet(gaussIncrFs); + } class SocaHybridWeights : public oops::Application { public: @@ -45,8 +82,7 @@ namespace gdasapp { socaVars += socaOcnVars; /// Read the background - // TODO(guillaume): Use the ice extent to set the weights ... no clue if this is - // possible at this level + // TODO(guillaume): Use the ice extent to set the weights soca::State socaBkg(geom, socaVars, dt); const eckit::LocalConfiguration socaBkgConfig(fullConfig, "background"); socaBkg.read(socaBkgConfig); @@ -70,6 +106,17 @@ namespace gdasapp { /// Create fields of weights for the ocean soca::Increment socaOcnHW(geom, socaOcnVars, dt); socaOcnHW.ones(); + + /// Apply localized gaussians to the weights + eckit::LocalConfiguration localWeightsConfigs(fullConfig, "weights.ocean local weights"); + std::vector localWeightsList = + localWeightsConfigs.getSubConfigurations(); + for (auto & conf : localWeightsList) { + gaussianMask(geom, socaOcnHW, conf); + oops::Log::info() << "Local weights for socaOcnHW: " << std::endl << conf << std::endl; + oops::Log::info() << socaOcnHW << std::endl; + } + socaOcnHW *= wOcean; oops::Log::info() << "socaOcnHW: " << std::endl << socaOcnHW << std::endl; socaOcnHW.write(socaHWOutConfig);