From 2b0674787f06e220c142cd5840d6ebf7df6b1fb3 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Thu, 26 Oct 2023 13:12:53 -0400 Subject: [PATCH 01/14] add gdas-utils symlink to gitignore (#694) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 4eff16035..bd53d9c20 100644 --- a/.gitignore +++ b/.gitignore @@ -175,3 +175,4 @@ test-data-release/ ufo/ ufo-data/ vader/ +/gdas-utils From 08e0c4a1c0d74574a5683279ef82304346acff18 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA <58948505+AndrewEichmann-NOAA@users.noreply.github.com> Date: Thu, 26 Oct 2023 13:32:07 -0400 Subject: [PATCH 02/14] Add global attributes to IODA conversion output files (#686) * initial additions for global attributes * added string global attributes * cleanup, extra attribute --------- Co-authored-by: Guillaume Vernieres Co-authored-by: Cory Martin --- utils/obsproc/NetCDFToIodaConverter.h | 14 ++++++++++- utils/obsproc/Rads2Ioda.h | 36 ++++++++++++++++----------- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/utils/obsproc/NetCDFToIodaConverter.h b/utils/obsproc/NetCDFToIodaConverter.h index 4686ec804..6942c1c0c 100644 --- a/utils/obsproc/NetCDFToIodaConverter.h +++ b/utils/obsproc/NetCDFToIodaConverter.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -43,6 +44,9 @@ namespace gdasapp { Eigen::ArrayXXf intMetadata_; // Optional array of integer metadata std::vector intMetadataName_; // String descriptor of the integer metadata + // Optional global attributes + std::map strGlobalAttr_; + // Constructor explicit IodaVars(const int nobs = 0, const std::vector fmnames = {}, @@ -233,6 +237,14 @@ namespace gdasapp { ogrp.vars.createWithScales("PreQC/"+variable_, {ogrp.vars["Location"]}, int_params); + // add input filenames to IODA file global attributes + ogrp.atts.add("obs_source_files", inputFilenames_); + + // add global attributes collected from the specific converter + for (const auto& globalAttr : iodaVars.strGlobalAttr_) { + ogrp.atts.add(globalAttr.first , globalAttr.second); + } + // Create the optional IODA integer metadata ioda::Variable tmpIntMeta; int count = 0; @@ -254,7 +266,7 @@ namespace gdasapp { } // Write obs info to group - oops::Log::info() << "Writting ioda file" << std::endl; + oops::Log::info() << "Writing ioda file" << std::endl; iodaLon.writeWithEigenRegular(iodaVarsAll.longitude_); iodaLat.writeWithEigenRegular(iodaVarsAll.latitude_); iodaDatetime.writeWithEigenRegular(iodaVarsAll.datetime_); diff --git a/utils/obsproc/Rads2Ioda.h b/utils/obsproc/Rads2Ioda.h index a31953d2b..6a2368a2d 100644 --- a/utils/obsproc/Rads2Ioda.h +++ b/utils/obsproc/Rads2Ioda.h @@ -57,10 +57,31 @@ namespace gdasapp { ncFile.getVar("time_mjd").getVar(datetime); iodaVars.referenceDate_ = "seconds since 1858-11-17T00:00:00Z"; + std::map altimeterMap; + // TODO(All): This is incomplete, add missions to the list below + altimeterMap["SNTNL-3A"] = 1; + altimeterMap["SNTNL-3B"] = 2; + altimeterMap["JASON-1"] = 3; + altimeterMap["JASON-2"] = 4; + altimeterMap["JASON-3"] = 5; + altimeterMap["CRYOSAT2"] = 6; + altimeterMap["SARAL"] = 7; + // Read optional integer metadata "mission" std::string mission_name; ncFile.getAtt("mission_name").getValues(mission_name); - int mission_index = altimeterMissions(mission_name); // mission name mapped to integer + int mission_index = altimeterMap[mission_name]; // mission name mapped to integer + + // convert mission map to string to add to global attributes + std::stringstream mapStr; + for (const auto& mapEntry : altimeterMap) { + mapStr << mapEntry.first << " = " << mapEntry.second << " "; + } + iodaVars.strGlobalAttr_["mission_index"] = mapStr.str(); + + std::string references; + ncFile.getAtt("references").getValues(references); + iodaVars.strGlobalAttr_["references"] = references; // Read optional integer metadata "pass" and "cycle" int pass[iodaVars.location_]; // NOLINT @@ -96,18 +117,5 @@ namespace gdasapp { } return iodaVars; }; - int altimeterMissions(std::string missionName) { - std::map altimeterMap; - // TODO(All): This is incomplete, add missions to the list below - // and add to global attribute - altimeterMap["SNTNL-3A"] = 1; - altimeterMap["SNTNL-3B"] = 2; - altimeterMap["JASON-1"] = 3; - altimeterMap["JASON-2"] = 4; - altimeterMap["JASON-3"] = 5; - altimeterMap["CRYOSAT2"] = 6; - altimeterMap["SARAL"] = 7; - return altimeterMap[missionName]; - } }; // class Rads2Ioda } // namespace gdasapp From f7bb53b8364588611720253d1db31a4353778589 Mon Sep 17 00:00:00 2001 From: Mindo Choi <141867620+apchoiCMD@users.noreply.github.com> Date: Mon, 30 Oct 2023 10:50:34 -0400 Subject: [PATCH 03/14] Fix a bug in vrfy section for plotting meridional slice (#695) --- scripts/exgdas_global_marine_analysis_vrfy.py | 7 ++++++- ush/soca/soca_vrfy.py | 14 +++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/scripts/exgdas_global_marine_analysis_vrfy.py b/scripts/exgdas_global_marine_analysis_vrfy.py index 52d657563..6179c00e7 100755 --- a/scripts/exgdas_global_marine_analysis_vrfy.py +++ b/scripts/exgdas_global_marine_analysis_vrfy.py @@ -81,10 +81,15 @@ config = plotConfig(grid_file=grid_file, data_file=data_file, lats=np.arange(-60, 60, 10), + lons=np.arange(-280, 80, 30), variables_zonal={'Temp': [0, 2], 'Salt': [0, 0.2], 'u': [0, 0.2], 'v': [0, 0.2]}, + variables_meridional={'Temp': [0, 2], + 'Salt': [0, 0.2], + 'u': [0, 0.2], + 'v': [0, 0.2]}, variables_horiz={'Temp': [0, 2], 'Salt': [0, 0.2], 'u': [0, 0.2], @@ -103,7 +108,7 @@ config = plotConfig(grid_file=grid_file, data_file=data_file, lats=np.arange(-60, 60, 10), - lons=np.arange(-280, 80, 10), + lons=np.arange(-280, 80, 30), variables_zonal={'Temp': [-0.5, 0.5], 'Salt': [-0.1, 0.1]}, variables_horiz={'Temp': [-0.5, 0.5], diff --git a/ush/soca/soca_vrfy.py b/ush/soca/soca_vrfy.py index 0f6c5b654..0741e9d28 100755 --- a/ush/soca/soca_vrfy.py +++ b/ush/soca/soca_vrfy.py @@ -33,7 +33,7 @@ def plotConfig(grid_file=[], lat=np.nan, lats=np.arange(-60, 60, 10), lon=np.nan, - lons=np.arange(-280, 80, 10), + lons=np.arange(-280, 80, 30), proj='set me', projs=['Global']): @@ -52,8 +52,8 @@ def plotConfig(grid_file=[], config['bounds'] = bounds config['lats'] = lats # all the lats to plot config['lat'] = lat # the lat being currently plotted - config['lons'] = lons - config['lon'] = lon + config['lons'] = lons # all the lons to plot + config['lon'] = lon # the lon being currently plotted config['max depths'] = max_depths # all the max depths to plot config['max depth'] = max_depth # the max depth currently plotted config['horiz variables'] = variables_horiz # all the vars for horiz plots @@ -159,13 +159,13 @@ def plotMeridionalSlice(config): lon = float(config['lon']) grid = xr.open_dataset(config['grid file']) data = xr.open_dataset(config['fields file']) - lon_index = np.argmin(np.array(np.abs(np.squeeze(grid.lon)[:, 0]-lon))) - slice_data = np.squeeze(np.array(data[config['variable']]))[:, lon_index, :] - depth = np.squeeze(np.array(grid['h']))[:, lon_index, :] + lon_index = np.argmin(np.array(np.abs(np.squeeze(grid.lon)[0, :]-lon))) + slice_data = np.squeeze(np.array(data[config['variable']]))[:, :, lon_index] + depth = np.squeeze(np.array(grid['h']))[:, :, lon_index] depth[np.where(np.abs(depth) > 10000.0)] = 0.0 depth = np.cumsum(depth, axis=0) bounds = config['bounds'] - y = np.tile(np.squeeze(grid.lon[:, lon_index]), (np.shape(depth)[0], 1)) + y = np.tile(np.squeeze(grid.lat)[:, lon_index], (np.shape(depth)[0], 1)) fig, ax = plt.subplots(figsize=(8, 5)) plt.pcolormesh(y, -depth, slice_data, vmin=bounds[0], vmax=bounds[1], From 073d811b0467d855ffac9134d2759287a332b36b Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Tue, 31 Oct 2023 15:56:15 -0400 Subject: [PATCH 04/14] Update satwind yamls for testing and end-to-end (#696) * Update satwind yaml due to update in UFO for the following: (1) Update test data set with error inflation for duplicate obs and varqc turned off. (2) remove minvalue/maxvalue thresholds in error assignments. The use of the threshold is not correct. The updated UFO detects the mistake. The removal does not affect results (3) update observation operator options and parameters * Update test branchmark (restuls wtih enforcing fixed time window) and some minor changes for run hofx script * Move filters two spacings left to line up with filter header.t p * Modify comments * Fix comments --- parm/atm/obs/config/satwind_goes-16.yaml | 1066 ++++++------ parm/atm/obs/config/satwind_goes-17.yaml | 1066 ++++++------ parm/atm/obs/testing/satwind.yaml | 1946 +++++++++++----------- parm/atm/obs/testing/satwind_noqc.yaml | 398 ++++- ush/ufoeval/run_ufo_hofx_test.sh | 7 +- 5 files changed, 2468 insertions(+), 2015 deletions(-) diff --git a/parm/atm/obs/config/satwind_goes-16.yaml b/parm/atm/obs/config/satwind_goes-16.yaml index 6fd5699be..f5d7f3df6 100644 --- a/parm/atm/obs/config/satwind_goes-16.yaml +++ b/parm/atm/obs/config/satwind_goes-16.yaml @@ -11,540 +11,554 @@ obs space: io pool: max pool size: 1 simulated variables: [windEastward, windNorthward] + obs operator: name: VertInterp - -obs pre filters: + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + +obs prior filters: +# Apply variable changes needed for wind scaling +# For wind observations with pressure provided +- filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: False + +# Calculate error inflation factor for duplicate observations +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windEastward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windEastward + +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windNorthward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windNorthward + +obs post filters: # Assign the initial observation error, based on height/pressure # Hard-wiring to prepobs_errtable.global by Type # ObsError is currently not updating in diag file, but passes directly to EffectiveError when no inflation is specified in YAML - # Type 240 (GOES SWIR): Assigned all dummy values in prepobs_errtable.global - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 240 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,0.] #Pressure (Pa) - errors: [1000000000.,1000000000.] - - # Type 245 (GOES LWIR): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 245 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] - - # Type 246 (GOES cloud-top WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 246 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] - - # Type 247 (GOES clear-sky WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 247 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] - - # Type 251 (GOES VIS): Assigned all dummy values - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 251 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,0.] #Pressure (Pa) - errors: [1000000000.,1000000000.] - - obs prior filters: - # sanity-check criteria - # Observation Range Sanity Check - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - minvalue: -130. - maxvalue: 130. - action: - name: reject - - # Velocity Sanity Check - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - test variables: - - name: ObsFunction/Velocity - maxvalue: 130. - action: - name: reject - - # GSI read routine QC (part-1) - # Exclude Type 251 (VIS) --- obs tossed without passing to setup routine - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 251 - action: - name: reject - - # Exclude data with satellite zenith angle > 68 for all types --- obs tossed without passing to setup routine - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - test variables: - - name: MetaData/sensorZenithAngle - maxvalue: 68. - action: - name: reject - - # Reject obs with pressure < 12500 pa --- obs tosed without passing to setup routine - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - test variables: - - name: MetaData/pressure - minvalue: 12500. - action: - name: reject - - # Exclude data over non-water surface type where latitude > 20N for Type 240 (IRSW) and Type 245 (IRLW) --- obs tossed and not passed to setup routine - # Notes: This check was missing, so added (eliu) - # Replace land_type_index_NPOSS with water_area_fraction (eliu) - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 240, 245 - - variable: MetaData/latitude - minvalue: 20. - test variables: - - name: GeoVaLs/water_area_fraction - minvalue: 0.999 - action: - name: reject - - # GSI read routine QC (part-2) - # Reject obs with qualityInformationWithoutForecast < 90. OR > 100. - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - test variables: - - name: MetaData/qualityInformationWithoutForecast - minvalue: 90. - maxvalue: 100. - action: - name: reject - - # Reject obs with pressure < 15000 pa. - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - test variables: - - name: MetaData/pressure - minvalue: 15000. - action: - name: reject - - # Reject obs with pressure < 70000 pa. when Type=251 - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - - variable: ObsType/windEastward - is_in: 251 - test variables: - - name: MetaData/pressure - minvalue: 70000. - action: - name: reject - - # Reject obs with pressure > 30000. when Type=246 - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - - variable: ObsType/windEastward - is_in: 246 - test variables: - - name: MetaData/pressure - maxvalue: 30000. - action: - name: reject - - # Reject obs with pressure > 85000. when isli=1 (land surface) - # Notes: Replace land_type_index_NPOESS with land_area_fraction (eliu) - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - - variable: GeoVaLs/land_area_fraction - minvalue: 0.9999 - test variables: - - name: MetaData/pressure - maxvalue: 85000. - action: - name: reject - - # Reject obs with pct1 (Coeff. of Var.) outside of 0.04–0.5, Type [240,245,246,251] ONLY - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - - variable: ObsType/windEastward - is_in: 240, 245, 246, 251 - test variables: - - name: MetaData/coefficientOfVariation - minvalue: 0.04 - maxvalue: 0.5 - action: - name: reject - - # NESDIS obs are also subject to the experr_norm test defined as: - # - # if (10. - 0.1*(expectedError))/(ob_speed)>0.9, or ob_speed<0.1, reject, applies to NESDIS winds - # - # CLEARED: With caveat that float precision/handling differences can generate different acceptance criteria - # between UFO and GSI for observations with an experr_norm value right around the maxvalue. - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - test variables: - - name: ObsFunction/SatWindsErrnormCheck - maxvalue: 0.9 - action: - name: reject - - # Reject all Type=240 (GOES SWIR) AMVs: These are not currently assimilated in GSI and they have missing-values - # assigned to ob-errors - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 240 - action: - name: reject - - # GSI setupw routine QC - # Reject any ob Type [240–260] when pressure greater than 950 mb. - # CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 240-260 - test variables: - - name: MetaData/pressure - maxvalue: 95001. - action: - name: reject - - # GOES IR (245) reject when pressure between 399 and 801 mb. - # # CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/pressure - minvalue: 39901. - maxvalue: 80099. - - variable: ObsType/windEastward - is_in: 245 - action: - name: reject - - # GOES WV (246, 250, 254), reject when pressure greater than 399 mb. - # CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 246, 250, 254 - test variables: - - name: MetaData/pressure - maxvalue: 39900. - action: - name: reject - - # Multiple satellite platforms, reject when pressure is more than 50 mb above tropopause. - # CLEARED: minvalue is rejecting <, not <= as per a Perform Action, so threshold is unchanged - # Notes (eliu): This tropopause check reject too many obs; probably due to tropopause pressure estimation - # Turn this check off for now. - # Need to check if troposphere pressure was implemented correctly in fv3-jed - - filter: Difference Check - filter variables: - - name: windEastward - - name: windNorthward - reference: GeoVaLs/tropopause_pressure - value: MetaData/pressure - minvalue: -5000. # 50 hPa above tropopause level, negative p-diff - action: - name: reject - - # GOES (247) reject any observation with a /=0 surface type (non-water - # surface) within 110 hPa of the surface pressure (as part of the LNVD # check). - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - # Notes (eliu): Replace land_type_index_NPOESS with land_area_fraction. - - filter: Difference Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: - name: GeoVaLs/water_area_fraction - maxvalue: 0.001 - - variable: - name: ObsType/windEastward - is_in: 247 - reference: GeoVaLs/surface_pressure - value: MetaData/pressure - maxvalue: -11000. # within 110 hPa above surface pressure, negative p-diff - action: - name: reject - - obs post filters: - # Reject GOES (247) when difference of wind direction is more than 50 degrees. - # CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 247 - test variables: - - name: ObsFunction/WindDirAngleDiff - maxvalue: 50. - action: - name: reject - - # All satwinds must adjust errors based on ObsErrorFactorPressureCheck - # prior to the SPDB check (i.e. the gross-error check). The gross-error - # check uses the adjusted errors for error-bound tightening and rejection, - # so this check has to come first. This check will inflate errors for obs - # that are too close to either the model top or bottom. - # Notes (eliu): GMAO added a required parameter: adjusted_error_name. - - filter: Perform Action - filter variables: - - name: windEastward - where: - - variable: - name: ObsType/windEastward - is_in: 240-260 - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorPressureCheck - options: - surface_obs: false - variable: windEastward - inflation factor: 4.0 - - - filter: Perform Action - filter variables: - - name: windNorthward - where: - - variable: - name: ObsType/windNorthward - is_in: 240-260 - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorPressureCheck - options: - variable: windNorthward - inflation factor: 4.0 - - # All satwinds subject to a gross-error check that contains significant - # modifiers for satwinds with a negative speed-bias. ALL wind gross-error - # checks are currently being done by the SatWindsSPDBCheck. - # CLEARED - - filter: Background Check - filter variables: - - name: windEastward - function absolute threshold: - - name: ObsFunction/WindsSPDBCheck +# Type 240 (GOES SWIR): Assigned all dummy values in prepobs_errtable.global +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear options: - wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] - cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] - error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] - error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] - variable: windEastward - action: - name: reject - - - filter: Background Check - filter variables: - - name: windNorthward - function absolute threshold: - - name: ObsFunction/WindsSPDBCheck + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 245 (GOES LWIR): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 245 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear options: - wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] - cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] - error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] - error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] - variable: windNorthward - action: - name: reject - -# # The last error inflation check is for duplicate observations. This one needs -# # to come last, because we don't want to inflate errors for duplication if one -# # of the duplicates should be rejected. -# # Notes (eliu): ObsErrorFactorDuplicateCheck obsfunction requires PreUseFlag (usage parameter from read_satwnd.f90). -# # : Turn off duplicate check for now. -# - filter: RejectList -# filter variables: -# - name: windEastward -# action: -# name: inflate error -# inflation variable: -# name: ObsFunction/ObsErrorFactorDuplicateCheck -# options: -# use_air_pressure: true -# variable: windEastward + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] + +# Type 246 (GOES cloud-top WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 246 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] + +# Type 247 (GOES clear-sky WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 247 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] + +# Type 251 (GOES VIS): Assigned all dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 251 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# sanity-check criteria +# Observation Range Sanity Check +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + minvalue: -130. + maxvalue: 130. + action: + name: reject + +# Velocity Sanity Check +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Velocity + maxvalue: 130. + action: + name: reject + +# GSI read routine QC (part-1) +# Exclude Type 251 (VIS) --- obs tossed without passing to setup routine +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 251 + action: + name: reject + +# Exclude data with satellite zenith angle > 68 for all types --- obs tossed without passing to setup routine +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: MetaData/sensorZenithAngle + maxvalue: 68. + action: + name: reject + +# Reject obs with pressure < 12500 pa --- obs tosed without passing to setup routine +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + test variables: + - name: MetaData/pressure + minvalue: 12500. + action: + name: reject + +# Exclude data over non-water surface type where latitude > 20N for Type 240 (IRSW) and Type 245 (IRLW) --- obs tossed and not passed to setup routine +# Notes: This check was missing, so added (eliu) +# Replace land_type_index_NPOSS with water_area_fraction (eliu) +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240, 245 + - variable: MetaData/latitude + minvalue: 20. + test variables: + - name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: reject + +# GSI read routine QC (part-2) +# Reject obs with qualityInformationWithoutForecast < 90. OR > 100. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + test variables: + - name: MetaData/qualityInformationWithoutForecast + minvalue: 90. + maxvalue: 100. + action: + name: reject + +# Reject obs with pressure < 15000 pa. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + test variables: + - name: MetaData/pressure + minvalue: 15000. + action: + name: reject + +# Reject obs with pressure < 70000 pa. when Type=251 +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + - variable: ObsType/windEastward + is_in: 251 + test variables: + - name: MetaData/pressure + minvalue: 70000. + action: + name: reject + +# Reject obs with pressure > 30000. when Type=246 +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + - variable: ObsType/windEastward + is_in: 246 + test variables: + - name: MetaData/pressure + maxvalue: 30000. + action: + name: reject + +# Reject obs with pressure > 85000. when isli=1 (land surface) +# Notes: Replace land_type_index_NPOESS with land_area_fraction (eliu) +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + - variable: GeoVaLs/land_area_fraction + minvalue: 0.99 + test variables: + - name: MetaData/pressure + maxvalue: 85000. + action: + name: reject + +# Reject obs with pct1 (Coeff. of Var.) outside of 0.04–0.5, Type [240,245,246,251] ONLY +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + - variable: ObsType/windEastward + is_in: 240, 245, 246, 251 + test variables: + - name: MetaData/coefficientOfVariation + minvalue: 0.04 + maxvalue: 0.5 + action: + name: reject + +# NESDIS obs are also subject to the experr_norm test defined as: # -# - filter: RejectList -# filter variables: -# - name: windNorthward -# action: -# name: inflate error -# inflation variable: -# name: ObsFunction/ObsErrorFactorDuplicateCheck -# options: -# use_air_pressure: true -# variable: windNorthward +# if (10. - 0.1*(expectedError))/(ob_speed)>0.9, or ob_speed<0.1, reject, applies to NESDIS winds # - # We are extending this to an additional filter that inflates final ob-errors across-the-board by - # 1/0.8 = 1.25. This is caused by the GSI value of nvqc being set to .true. in the global operational - # configuration, see: https://github.com/NOAA-EMC/global-workflow/blob/d5ae3328fa4041b177357b1133f6b92e81c859d7/scripts/exglobal_atmos_analysis.sh#L750 - # This setting activates Line 1229 of setupw.f90 to scale ratio_errors by 0.8, which is applied in - # the denominator of the final ob-error, so 1/0.8 = 1.25 factor of ob-error inflation. - # - # If this nvqc functionality were to be switched off (i.e. if variational qc were to be turned off), - # you would want to remove this last inflation filter. - - filter: Perform Action - filter variables: - - name: windEastward - where: - - variable: ObsType/windEastward - is_in: 240-260 - action: - name: inflate error - inflation factor: 1.25 - - - filter: Perform Action - filter variables: - - name: windNorthward - where: - - variable: ObsType/windNorthward +# CLEARED: With caveat that float precision/handling differences can generate different acceptance criteria +# between UFO and GSI for observations with an experr_norm value right around the maxvalue. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + test variables: + - name: ObsFunction/SatWindsErrnormCheck + maxvalue: 0.9 + action: + name: reject + +# Reject all Type=240 (GOES SWIR) AMVs: These are not currently assimilated in GSI and they have missing-values +# assigned to ob-errors +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240 + action: + name: reject + +# GSI setupw routine QC +# Reject any ob Type [240–260] when pressure greater than 950 mb. +# CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240-260 + test variables: + - name: MetaData/pressure + maxvalue: 95001. + action: + name: reject + +# GOES IR (245) reject when pressure between 399 and 801 mb. +# # CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/pressure + minvalue: 39901. + maxvalue: 80099. + - variable: ObsType/windEastward + is_in: 245 + action: + name: reject + +# GOES WV (246, 250, 254), reject when pressure greater than 399 mb. +# CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 246, 250, 254 + test variables: + - name: MetaData/pressure + maxvalue: 39900. + action: + name: reject + +# Multiple satellite platforms, reject when pressure is more than 50 mb above tropopause. +# CLEARED: minvalue is rejecting <, not <= as per a Perform Action, so threshold is unchanged +# Notes (eliu): This tropopause check reject too many obs; probably due to tropopause pressure estimation +# Turn this check off for now. +# Need to check if troposphere pressure was implemented correctly in fv3-jed +- filter: Difference Check + filter variables: + - name: windEastward + - name: windNorthward + reference: GeoVaLs/tropopause_pressure + value: MetaData/pressure + minvalue: -5000. # 50 hPa above tropopause level, negative p-diff + action: + name: reject + +# GOES (247) reject any observation with a /=0 surface type (non-water +# surface) within 110 hPa of the surface pressure (as part of the LNVD # check). +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +# Notes (eliu): Replace land_type_index_NPOESS with land_area_fraction. +- filter: Difference Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + - variable: + name: ObsType/windEastward + is_in: 247 + reference: GeoVaLs/surface_pressure + value: MetaData/pressure + maxvalue: -11000. # within 110 hPa above surface pressure, negative p-diff + action: + name: reject + +obs post filters: +# Reject GOES (247) when difference of wind direction is more than 50 degrees. +# CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 247 + test variables: + - name: ObsFunction/WindDirAngleDiff + maxvalue: 50. + action: + name: reject + +# All satwinds must adjust errors based on ObsErrorFactorPressureCheck +# prior to the SPDB check (i.e. the gross-error check). The gross-error +# check uses the adjusted errors for error-bound tightening and rejection, +# so this check has to come first. This check will inflate errors for obs +# that are too close to either the model top or bottom. +# Notes (eliu): GMAO added a required parameter: adjusted_error_name. +- filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward is_in: 240-260 - action: - name: inflate error - inflation factor: 1.25 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + surface_obs: false + variable: windEastward + inflation factor: 4.0 - # END OF FILTERS +- filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + +# All satwinds subject to a gross-error check that contains significant +# modifiers for satwinds with a negative speed-bias. ALL wind gross-error +# checks are currently being done by the SatWindsSPDBCheck. +# CLEARED +- filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] + error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] + variable: windEastward + action: + name: reject + +- filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] + error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] + variable: windNorthward + action: + name: reject + +# The last error inflation check is for duplicate observations. This one needs +# to come last, because we don't want to inflate errors for duplication if one +# of the duplicates should be rejected. +# Notes (eliu): ObsErrorFactorDuplicateCheck obsfunction requires PreUseFlag (usage parameter from read_satwnd.f90). +# : Turn off duplicate check for now. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windEastward + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windNorthward + +# We are extending this to an additional filter that inflates final ob-errors across-the-board by +# 1/0.8 = 1.25. This is caused by the GSI value of nvqc being set to .true. in the global operational +# configuration, see: https://github.com/NOAA-EMC/global-workflow/blob/d5ae3328fa4041b177357b1133f6b92e81c859d7/scripts/exglobal_atmos_analysis.sh#L750 +# This setting activates Line 1229 of setupw.f90 to scale ratio_errors by 0.8, which is applied in +# the denominator of the final ob-error, so 1/0.8 = 1.25 factor of ob-error inflation. +# +# If this nvqc functionality were to be switched off (i.e. if variational qc were to be turned off), +# you would want to remove this last inflation filter. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# where: +# - variable: ObsType/windEastward +# is_in: 240-260 +# action: +# name: inflate error +# inflation factor: 1.25 + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# where: +# - variable: ObsType/windNorthward +# is_in: 240-260 +# action: +# name: inflate error +# inflation factor: 1.25 + +# End of Filters diff --git a/parm/atm/obs/config/satwind_goes-17.yaml b/parm/atm/obs/config/satwind_goes-17.yaml index ca3632fcb..240af22d3 100644 --- a/parm/atm/obs/config/satwind_goes-17.yaml +++ b/parm/atm/obs/config/satwind_goes-17.yaml @@ -11,540 +11,554 @@ obs space: io pool: max pool size: 1 simulated variables: [windEastward, windNorthward] + obs operator: name: VertInterp - -obs pre filters: + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + +obs prior filters: +# Apply variable changes needed for wind scaling +# For wind observations with pressure provided +- filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: False + +# Calculate error inflation factor for duplicate observations +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windEastward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windEastward + +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windNorthward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windNorthward + +obs post filters: # Assign the initial observation error, based on height/pressure # Hard-wiring to prepobs_errtable.global by Type # ObsError is currently not updating in diag file, but passes directly to EffectiveError when no inflation is specified in YAML - # Type 240 (GOES SWIR): Assigned all dummy values in prepobs_errtable.global - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 240 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,0.] #Pressure (Pa) - errors: [1000000000.,1000000000.] - - # Type 245 (GOES LWIR): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 245 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] - - # Type 246 (GOES cloud-top WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 246 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] - - # Type 247 (GOES clear-sky WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 247 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] - - # Type 251 (GOES VIS): Assigned all dummy values - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 251 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,0.] #Pressure (Pa) - errors: [1000000000.,1000000000.] - - obs prior filters: - # sanity-check criteria - # Observation Range Sanity Check - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - minvalue: -130. - maxvalue: 130. - action: - name: reject - - # Velocity Sanity Check - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - test variables: - - name: ObsFunction/Velocity - maxvalue: 130. - action: - name: reject - - # GSI read routine QC (part-1) - # Exclude Type 251 (VIS) --- obs tossed without passing to setup routine - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 251 - action: - name: reject - - # Exclude data with satellite zenith angle > 68 for all types --- obs tossed without passing to setup routine - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - test variables: - - name: MetaData/sensorZenithAngle - maxvalue: 68. - action: - name: reject - - # Reject obs with pressure < 12500 pa --- obs tosed without passing to setup routine - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - test variables: - - name: MetaData/pressure - minvalue: 12500. - action: - name: reject - - # Exclude data over non-water surface type where latitude > 20N for Type 240 (IRSW) and Type 245 (IRLW) --- obs tossed and not passed to setup routine - # Notes: This check was missing, so added (eliu) - # Replace land_type_index_NPOSS with water_area_fraction (eliu) - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 240, 245 - - variable: MetaData/latitude - minvalue: 20. - test variables: - - name: GeoVaLs/water_area_fraction - minvalue: 0.999 - action: - name: reject - - # GSI read routine QC (part-2) - # Reject obs with qualityInformationWithoutForecast < 90. OR > 100. - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - test variables: - - name: MetaData/qualityInformationWithoutForecast - minvalue: 90. - maxvalue: 100. - action: - name: reject - - # Reject obs with pressure < 15000 pa. - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - test variables: - - name: MetaData/pressure - minvalue: 15000. - action: - name: reject - - # Reject obs with pressure < 70000 pa. when Type=251 - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - - variable: ObsType/windEastward - is_in: 251 - test variables: - - name: MetaData/pressure - minvalue: 70000. - action: - name: reject - - # Reject obs with pressure > 30000. when Type=246 - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - - variable: ObsType/windEastward - is_in: 246 - test variables: - - name: MetaData/pressure - maxvalue: 30000. - action: - name: reject - - # Reject obs with pressure > 85000. when isli=1 (land surface) - # Notes: Replace land_type_index_NPOESS with land_area_fraction (eliu) - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - - variable: GeoVaLs/land_area_fraction - minvalue: 0.9999 - test variables: - - name: MetaData/pressure - maxvalue: 85000. - action: - name: reject - - # Reject obs with pct1 (Coeff. of Var.) outside of 0.04–0.5, Type [240,245,246,251] ONLY - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - - variable: ObsType/windEastward - is_in: 240, 245, 246, 251 - test variables: - - name: MetaData/coefficientOfVariation - minvalue: 0.04 - maxvalue: 0.5 - action: - name: reject - - # NESDIS obs are also subject to the experr_norm test defined as: - # - # if (10. - 0.1*(expectedError))/(ob_speed)>0.9, or ob_speed<0.1, reject, applies to NESDIS winds - # - # CLEARED: With caveat that float precision/handling differences can generate different acceptance criteria - # between UFO and GSI for observations with an experr_norm value right around the maxvalue. - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - test variables: - - name: ObsFunction/SatWindsErrnormCheck - maxvalue: 0.9 - action: - name: reject - - # Reject all Type=240 (GOES SWIR) AMVs: These are not currently assimilated in GSI and they have missing-values - # assigned to ob-errors - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 240 - action: - name: reject - - # GSI setupw routine QC - # Reject any ob Type [240–260] when pressure greater than 950 mb. - # CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 240-260 - test variables: - - name: MetaData/pressure - maxvalue: 95001. - action: - name: reject - - # GOES IR (245) reject when pressure between 399 and 801 mb. - # # CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/pressure - minvalue: 39901. - maxvalue: 80099. - - variable: ObsType/windEastward - is_in: 245 - action: - name: reject - - # GOES WV (246, 250, 254), reject when pressure greater than 399 mb. - # CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 246, 250, 254 - test variables: - - name: MetaData/pressure - maxvalue: 39900. - action: - name: reject - - # Multiple satellite platforms, reject when pressure is more than 50 mb above tropopause. - # CLEARED: minvalue is rejecting <, not <= as per a Perform Action, so threshold is unchanged - # Notes (eliu): This tropopause check reject too many obs; probably due to tropopause pressure estimation - # Turn this check off for now. - # Need to check if troposphere pressure was implemented correctly in fv3-jed - - filter: Difference Check - filter variables: - - name: windEastward - - name: windNorthward - reference: GeoVaLs/tropopause_pressure - value: MetaData/pressure - minvalue: -5000. # 50 hPa above tropopause level, negative p-diff - action: - name: reject - - # GOES (247) reject any observation with a /=0 surface type (non-water - # surface) within 110 hPa of the surface pressure (as part of the LNVD # check). - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - # Notes (eliu): Replace land_type_index_NPOESS with land_area_fraction. - - filter: Difference Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: - name: GeoVaLs/water_area_fraction - maxvalue: 0.001 - - variable: - name: ObsType/windEastward - is_in: 247 - reference: GeoVaLs/surface_pressure - value: MetaData/pressure - maxvalue: -11000. # within 110 hPa above surface pressure, negative p-diff - action: - name: reject - - obs post filters: - # Reject GOES (247) when difference of wind direction is more than 50 degrees. - # CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 247 - test variables: - - name: ObsFunction/WindDirAngleDiff - maxvalue: 50. - action: - name: reject - - # All satwinds must adjust errors based on ObsErrorFactorPressureCheck - # prior to the SPDB check (i.e. the gross-error check). The gross-error - # check uses the adjusted errors for error-bound tightening and rejection, - # so this check has to come first. This check will inflate errors for obs - # that are too close to either the model top or bottom. - # Notes (eliu): GMAO added a required parameter: adjusted_error_name. - - filter: Perform Action - filter variables: - - name: windEastward - where: - - variable: - name: ObsType/windEastward - is_in: 240-260 - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorPressureCheck - options: - surface_obs: false - variable: windEastward - inflation factor: 4.0 - - - filter: Perform Action - filter variables: - - name: windNorthward - where: - - variable: - name: ObsType/windNorthward - is_in: 240-260 - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorPressureCheck - options: - variable: windNorthward - inflation factor: 4.0 - - # All satwinds subject to a gross-error check that contains significant - # modifiers for satwinds with a negative speed-bias. ALL wind gross-error - # checks are currently being done by the SatWindsSPDBCheck. - # CLEARED - - filter: Background Check - filter variables: - - name: windEastward - function absolute threshold: - - name: ObsFunction/WindsSPDBCheck +# Type 240 (GOES SWIR): Assigned all dummy values in prepobs_errtable.global +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear options: - wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] - cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] - error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] - error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] - variable: windEastward - action: - name: reject - - - filter: Background Check - filter variables: - - name: windNorthward - function absolute threshold: - - name: ObsFunction/WindsSPDBCheck + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 245 (GOES LWIR): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 245 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear options: - wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] - cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] - error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] - error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] - variable: windNorthward - action: - name: reject - -# # The last error inflation check is for duplicate observations. This one needs -# # to come last, because we don't want to inflate errors for duplication if one -# # of the duplicates should be rejected. -# # Notes (eliu): ObsErrorFactorDuplicateCheck obsfunction requires PreUseFlag (usage parameter from read_satwnd.f90). -# # : Turn off duplicate check for now. -# - filter: RejectList -# filter variables: -# - name: windEastward -# action: -# name: inflate error -# inflation variable: -# name: ObsFunction/ObsErrorFactorDuplicateCheck -# options: -# use_air_pressure: true -# variable: windEastward + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] + +# Type 246 (GOES cloud-top WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 246 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] + +# Type 247 (GOES clear-sky WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 247 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] + +# Type 251 (GOES VIS): Assigned all dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 251 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# sanity-check criteria +# Observation Range Sanity Check +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + minvalue: -130. + maxvalue: 130. + action: + name: reject + +# Velocity Sanity Check +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Velocity + maxvalue: 130. + action: + name: reject + +# GSI read routine QC (part-1) +# Exclude Type 251 (VIS) --- obs tossed without passing to setup routine +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 251 + action: + name: reject + +# Exclude data with satellite zenith angle > 68 for all types --- obs tossed without passing to setup routine +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: MetaData/sensorZenithAngle + maxvalue: 68. + action: + name: reject + +# Reject obs with pressure < 12500 pa --- obs tosed without passing to setup routine +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + test variables: + - name: MetaData/pressure + minvalue: 12500. + action: + name: reject + +# Exclude data over non-water surface type where latitude > 20N for Type 240 (IRSW) and Type 245 (IRLW) --- obs tossed and not passed to setup routine +# Notes: This check was missing, so added (eliu) +# Replace land_type_index_NPOSS with water_area_fraction (eliu) +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240, 245 + - variable: MetaData/latitude + minvalue: 20. + test variables: + - name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: reject + +# GSI read routine QC (part-2) +# Reject obs with qualityInformationWithoutForecast < 90. OR > 100. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + test variables: + - name: MetaData/qualityInformationWithoutForecast + minvalue: 90. + maxvalue: 100. + action: + name: reject + +# Reject obs with pressure < 15000 pa. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + test variables: + - name: MetaData/pressure + minvalue: 15000. + action: + name: reject + +# Reject obs with pressure < 70000 pa. when Type=251 +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + - variable: ObsType/windEastward + is_in: 251 + test variables: + - name: MetaData/pressure + minvalue: 70000. + action: + name: reject + +# Reject obs with pressure > 30000. when Type=246 +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + - variable: ObsType/windEastward + is_in: 246 + test variables: + - name: MetaData/pressure + maxvalue: 30000. + action: + name: reject + +# Reject obs with pressure > 85000. when isli=1 (land surface) +# Notes: Replace land_type_index_NPOESS with land_area_fraction (eliu) +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + - variable: GeoVaLs/land_area_fraction + minvalue: 0.99 + test variables: + - name: MetaData/pressure + maxvalue: 85000. + action: + name: reject + +# Reject obs with pct1 (Coeff. of Var.) outside of 0.04–0.5, Type [240,245,246,251] ONLY +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + - variable: ObsType/windEastward + is_in: 240, 245, 246, 251 + test variables: + - name: MetaData/coefficientOfVariation + minvalue: 0.04 + maxvalue: 0.5 + action: + name: reject + +# NESDIS obs are also subject to the experr_norm test defined as: # -# - filter: RejectList -# filter variables: -# - name: windNorthward -# action: -# name: inflate error -# inflation variable: -# name: ObsFunction/ObsErrorFactorDuplicateCheck -# options: -# use_air_pressure: true -# variable: windNorthward +# if (10. - 0.1*(expectedError))/(ob_speed)>0.9, or ob_speed<0.1, reject, applies to NESDIS winds # - # We are extending this to an additional filter that inflates final ob-errors across-the-board by - # 1/0.8 = 1.25. This is caused by the GSI value of nvqc being set to .true. in the global operational - # configuration, see: https://github.com/NOAA-EMC/global-workflow/blob/d5ae3328fa4041b177357b1133f6b92e81c859d7/scripts/exglobal_atmos_analysis.sh#L750 - # This setting activates Line 1229 of setupw.f90 to scale ratio_errors by 0.8, which is applied in - # the denominator of the final ob-error, so 1/0.8 = 1.25 factor of ob-error inflation. - # - # If this nvqc functionality were to be switched off (i.e. if variational qc were to be turned off), - # you would want to remove this last inflation filter. - - filter: Perform Action - filter variables: - - name: windEastward - where: - - variable: ObsType/windEastward - is_in: 240-260 - action: - name: inflate error - inflation factor: 1.25 - - - filter: Perform Action - filter variables: - - name: windNorthward - where: - - variable: ObsType/windNorthward +# CLEARED: With caveat that float precision/handling differences can generate different acceptance criteria +# between UFO and GSI for observations with an experr_norm value right around the maxvalue. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + test variables: + - name: ObsFunction/SatWindsErrnormCheck + maxvalue: 0.9 + action: + name: reject + +# Reject all Type=240 (GOES SWIR) AMVs: These are not currently assimilated in GSI and they have missing-values +# assigned to ob-errors +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240 + action: + name: reject + +# GSI setupw routine QC +# Reject any ob Type [240–260] when pressure greater than 950 mb. +# CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240-260 + test variables: + - name: MetaData/pressure + maxvalue: 95001. + action: + name: reject + +# GOES IR (245) reject when pressure between 399 and 801 mb. +# # CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/pressure + minvalue: 39901. + maxvalue: 80099. + - variable: ObsType/windEastward + is_in: 245 + action: + name: reject + +# GOES WV (246, 250, 254), reject when pressure greater than 399 mb. +# CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 246, 250, 254 + test variables: + - name: MetaData/pressure + maxvalue: 39900. + action: + name: reject + +# Multiple satellite platforms, reject when pressure is more than 50 mb above tropopause. +# CLEARED: minvalue is rejecting <, not <= as per a Perform Action, so threshold is unchanged +# Notes (eliu): This tropopause check reject too many obs; probably due to tropopause pressure estimation +# Turn this check off for now. +# Need to check if troposphere pressure was implemented correctly in fv3-jed +- filter: Difference Check + filter variables: + - name: windEastward + - name: windNorthward + reference: GeoVaLs/tropopause_pressure + value: MetaData/pressure + minvalue: -5000. # 50 hPa above tropopause level, negative p-diff + action: + name: reject + +# GOES (247) reject any observation with a /=0 surface type (non-water +# surface) within 110 hPa of the surface pressure (as part of the LNVD # check). +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +# Notes (eliu): Replace land_type_index_NPOESS with land_area_fraction. +- filter: Difference Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + - variable: + name: ObsType/windEastward + is_in: 247 + reference: GeoVaLs/surface_pressure + value: MetaData/pressure + maxvalue: -11000. # within 110 hPa above surface pressure, negative p-diff + action: + name: reject + +obs post filters: +# Reject GOES (247) when difference of wind direction is more than 50 degrees. +# CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 247 + test variables: + - name: ObsFunction/WindDirAngleDiff + maxvalue: 50. + action: + name: reject + +# All satwinds must adjust errors based on ObsErrorFactorPressureCheck +# prior to the SPDB check (i.e. the gross-error check). The gross-error +# check uses the adjusted errors for error-bound tightening and rejection, +# so this check has to come first. This check will inflate errors for obs +# that are too close to either the model top or bottom. +# Notes (eliu): GMAO added a required parameter: adjusted_error_name. +- filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward is_in: 240-260 - action: - name: inflate error - inflation factor: 1.25 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + surface_obs: false + variable: windEastward + inflation factor: 4.0 - # END OF FILTERS +- filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + +# All satwinds subject to a gross-error check that contains significant +# modifiers for satwinds with a negative speed-bias. ALL wind gross-error +# checks are currently being done by the SatWindsSPDBCheck. +# CLEARED +- filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] + error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] + variable: windEastward + action: + name: reject + +- filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] + error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] + variable: windNorthward + action: + name: reject + +# The last error inflation check is for duplicate observations. This one needs +# to come last, because we don't want to inflate errors for duplication if one +# of the duplicates should be rejected. +# Notes (eliu): ObsErrorFactorDuplicateCheck obsfunction requires PreUseFlag (usage parameter from read_satwnd.f90). +# : Turn off duplicate check for now. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windEastward + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windNorthward + +# We are extending this to an additional filter that inflates final ob-errors across-the-board by +# 1/0.8 = 1.25. This is caused by the GSI value of nvqc being set to .true. in the global operational +# configuration, see: https://github.com/NOAA-EMC/global-workflow/blob/d5ae3328fa4041b177357b1133f6b92e81c859d7/scripts/exglobal_atmos_analysis.sh#L750 +# This setting activates Line 1229 of setupw.f90 to scale ratio_errors by 0.8, which is applied in +# the denominator of the final ob-error, so 1/0.8 = 1.25 factor of ob-error inflation. +# +# If this nvqc functionality were to be switched off (i.e. if variational qc were to be turned off), +# you would want to remove this last inflation filter. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# where: +# - variable: ObsType/windEastward +# is_in: 240-260 +# action: +# name: inflate error +# inflation factor: 1.25 + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# where: +# - variable: ObsType/windNorthward +# is_in: 240-260 +# action: +# name: inflate error +# inflation factor: 1.25 + +# End of Filters diff --git a/parm/atm/obs/testing/satwind.yaml b/parm/atm/obs/testing/satwind.yaml index bdf10f646..a43d107a4 100644 --- a/parm/atm/obs/testing/satwind.yaml +++ b/parm/atm/obs/testing/satwind.yaml @@ -9,988 +9,1024 @@ obs space: type: H5File obsfile: !ENV satwind_diag_${CDATE}.nc4 simulated variables: [windEastward, windNorthward] + geovals: filename: !ENV satwind_geoval_${CDATE}.nc4 -vector ref: GsiHofXBc -tolerance: 0.01 + obs operator: name: VertInterp -obs pre filters: + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + +obs prior filters: +# Apply variable changes needed for wind scaling +# For wind observations with pressure provided +- filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: False + +# Calculate obs error inflation factors for duplicated observations at the same location +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windEastward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windEastward + +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windNorthward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windNorthward + +obs post filters: # Assign the initial observation error, based on height/pressure # Hard-wiring to prepobs_errtable.global by Type # ObsError is currently not updating in diag file, but passes directly to EffectiveError when no inflation is specified in YAML - # Type 240 (GOES SWIR): Assigned all dummy values in prepobs_errtable.global - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 240 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,0.] #Pressure (Pa) - errors: [1000000000.,1000000000.] - # Type 241 (Multi Spec. Imager LWIR): Assigned all dummy values - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 241 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,0.] #Pressure (Pa) - errors: [1000000000.,1000000000.] - # Type 242 (Himawari VIS) - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 242 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] - # Type 243 (MVIRI/SEVIRI VIS) - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 243 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] - # Type 244 (AVHRR LWIR) - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 244 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] - # Type 245 (GOES LWIR): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 245 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] - # Type 246 (GOES cloud-top WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 246 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] - # Type 247 (GOES clear-sky WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 247 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] - # Type 248 (GOES Sounder cloud-top WV): Assigned all dummy values - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 248 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,0.] #Pressure (Pa) - errors: [1000000000.,1000000000.] - # Type 249 (GOES Sounder clear-sky WV): Assigned all dummy values - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 249 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,0.] #Pressure (Pa) - errors: [1000000000.,1000000000.] - # Type 250 (Himawari AHI WV, cloud-top or clear-sky) - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 250 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,7.,7.3,7.6,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.] - # Type 251 (GOES VIS): Assigned all dummy values - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 251 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,0.] #Pressure (Pa) - errors: [1000000000.,1000000000.] - # Type 252 (Himawari AHI LWIR) - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 252 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] - # Type 253 (MVIRI/SEVERI LWIR) - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 253 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] - # Type 254 (MVIRI/SEVIRI WV, both cloud-top and clear-sky) - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 254 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.5,6.1,6.,6.5,7.3,7.6,7.,7.5,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] - # Type 255 (LEOGEO): - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 255 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] - # Type 256 (Multi Spec. Imager WV, both clear-sky and cloud-top): Assigned all dummy values - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 256 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,0.] #Pressure (Pa) - errors: [1000000000.,1000000000.] - # Type 257 (MODIS LWIR) - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 257 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] - # Type 258 (MODIS cloud-top WV): Some levels assigned dummy values - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 258 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [1000000000.,1000000000.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] - # Type 259 (MODIS clear-sky WV): Some levels assigned dummy values - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 259 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [1000000000.,1000000000.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] - # Type 260 (VIIRS LWIR): All levels assigned dummy values in prepobs_errtable.global, HOWEVER the GSI values appear - # to be a standard profile (borrowed from e.g., Type=244). Using the standard profile here. - # It's possibly that my prepobs_errtable.global file is out-of-date. - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 260 - minvalue: -135. - maxvalue: 135. - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) - errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] -obs prior filters: - # - # sanity-check criteria - # - # Observation Range Sanity Check - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - minvalue: -130. - maxvalue: 130. - action: - name: reject - # Velocity Sanity Check - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - test variables: - - name: ObsFunction/Velocity - maxvalue: 130. - action: - name: reject -# +# Type 240 (GOES SWIR): Assigned all dummy values in prepobs_errtable.global +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 241 (Multi Spec. Imager LWIR): Assigned all dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 241 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 242 (Himawari VIS) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 242 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 243 (MVIRI/SEVIRI VIS) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 243 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 244 (AVHRR LWIR) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 244 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 245 (GOES LWIR): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 245 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] + +# Type 246 (GOES cloud-top WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 246 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] + +# Type 247 (GOES clear-sky WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 247 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] + +# Type 248 (GOES Sounder cloud-top WV): Assigned all dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 248 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 249 (GOES Sounder clear-sky WV): Assigned all dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 249 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 250 (Himawari AHI WV, cloud-top or clear-sky) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 250 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,7.,7.3,7.6,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.] + +# Type 251 (GOES VIS): Assigned all dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 251 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 252 (Himawari AHI LWIR) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 252 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 253 (MVIRI/SEVERI LWIR) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 253 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 254 (MVIRI/SEVIRI WV, both cloud-top and clear-sky) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 254 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.5,6.1,6.,6.5,7.3,7.6,7.,7.5,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 255 (LEOGEO): +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 255 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 256 (Multi Spec. Imager WV, both clear-sky and cloud-top): Assigned all dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 256 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 257 (MODIS LWIR) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 257 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 258 (MODIS cloud-top WV): Some levels assigned dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 258 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 259 (MODIS clear-sky WV): Some levels assigned dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 259 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 260 (VIIRS LWIR): All levels assigned dummy values in prepobs_errtable.global, HOWEVER the GSI values appear +# to be a standard profile (borrowed from e.g., Type=244). Using the standard profile here. +# It's possibly that my prepobs_errtable.global file is out-of-date. +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 260 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# sanity-check criteria +# Observation Range Sanity Check +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + minvalue: -130. + maxvalue: 130. + action: + name: reject + +# Velocity Sanity Check +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Velocity + maxvalue: 130. + action: + name: reject + # preQC (read_satwnd) criteria -# # EUMETSAT winds: satelliteIdentifer [50–79] (>49, <80) - # Reject obs with satelliteZenithAngle > 68 deg - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 50-79 - test variables: - - name: MetaData/satelliteZenithAngle - maxvalue: 68. - action: - name: reject - # Reject obs with windComputationMethod = 5 (clear-sky WV AMV) - # CLEARED - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 50-79 - - variable: MetaData/windComputationMethod - is_in: 5 - action: - name: reject - # Reject obs with qualityInformationWithoutForecast < 85 - # CLEARED - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 50-79 - test variables: - - name: MetaData/qualityInformationWithoutForecast - minvalue: 85. - maxvalue: 100. - action: - name: reject +# Reject obs with satelliteZenithAngle > 68 deg +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 50-79 + test variables: + - name: MetaData/satelliteZenithAngle + maxvalue: 68. + action: + name: reject + +# Reject obs with windComputationMethod = 5 (clear-sky WV AMV) +# CLEARED +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 50-79 + - variable: MetaData/windComputationMethod + is_in: 5 + action: + name: reject + +# Reject obs with qualityInformationWithoutForecast < 85 +# CLEARED +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 50-79 + test variables: + - name: MetaData/qualityInformationWithoutForecast + minvalue: 85. + maxvalue: 100. + action: + name: reject + # JMA: satelliteIdentifier [100–199] (>99, <200) - # Reject obs with satelliteZenithAngle > 68 deg - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 100-199 - test variables: - - name: MetaData/satelliteZenithAngle - maxvalue: 68. - action: - name: reject - # Reject obs with windComputationMethod = 5 (clear-sky WV AMV) - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 100-199 - - variable: MetaData/windComputationMethod - is_in: 5 - action: - name: reject - # Reject obs with qualityInformationWithoutForecast < 85. - # CLEARED - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 100-199 - test variables: - - name: MetaData/qualityInformationWithoutForecast - minvalue: 85. - maxvalue: 100. - action: - name: reject +# Reject obs with satelliteZenithAngle > 68 deg +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 100-199 + test variables: + - name: MetaData/satelliteZenithAngle + maxvalue: 68. + action: + name: reject + +# Reject obs with windComputationMethod = 5 (clear-sky WV AMV) +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 100-199 + - variable: MetaData/windComputationMethod + is_in: 5 + action: + name: reject + +# Reject obs with qualityInformationWithoutForecast < 85. +# CLEARED +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 100-199 + test variables: + - name: MetaData/qualityInformationWithoutForecast + minvalue: 85. + maxvalue: 100. + action: + name: reject + # NESDIS: satelliteIdentifier [250–299] (>249, <300) - # Reject obs with satelliteZenithAngle > 68 deg - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - test variables: - - name: MetaData/satelliteZenithAngle - maxvalue: 68. - action: - name: reject - # Reject obs with qualityInformationWithoutForecast < 90. OR > 100. - # CLEARED - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - test variables: - - name: MetaData/qualityInformationWithoutForecast - minvalue: 90. - maxvalue: 100. - action: - name: reject - # Reject obs with pressure < 15000. - # CLEARED - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - test variables: - - name: MetaData/pressure - minvalue: 15000. - action: - name: reject - # Reject obs with pressure < 70000. when Type=251 - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - - variable: ObsType/windEastward - is_in: 251 - test variables: - - name: MetaData/pressure - minvalue: 70000. - action: - name: reject - # Reject obs with pressure > 30000. when Type=246 - # CLEARED - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - - variable: ObsType/windEastward - is_in: 246 - test variables: - - name: MetaData/pressure - maxvalue: 30000. - action: - name: reject - # Reject obs with pressure > 85000. when isli=1 (land surface) - # CLEARED - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - - variable: GeoVaLs/land_area_fraction - minvalue: 0.999 - test variables: - - name: MetaData/pressure - maxvalue: 85000. - action: - name: reject - # Reject obs with pct1 (Coeff. of Var.) outside of 0.04–0.5, Type [240,245,246,251] ONLY - # CLEARED - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - - variable: ObsType/windEastward - is_in: 240,245,246,251 - test variables: - - name: MetaData/coefficientOfVariation - minvalue: 0.04 - maxvalue: 0.5 - action: - name: reject +# Reject obs with satelliteZenithAngle > 68 deg +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + test variables: + - name: MetaData/satelliteZenithAngle + maxvalue: 68. + action: + name: reject + +# Reject obs with qualityInformationWithoutForecast < 90. OR > 100. +# CLEARED +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + test variables: + - name: MetaData/qualityInformationWithoutForecast + minvalue: 90. + maxvalue: 100. + action: + name: reject + +# Reject obs with pressure < 15000. +# CLEARED +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + test variables: + - name: MetaData/pressure + minvalue: 15000. + action: + name: reject + +# Reject obs with pressure < 70000. when Type=251 +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + - variable: ObsType/windEastward + is_in: 251 + test variables: + - name: MetaData/pressure + minvalue: 70000. + action: + name: reject + +# Reject obs with pressure > 30000. when Type=246 +# CLEARED +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + - variable: ObsType/windEastward + is_in: 246 + test variables: + - name: MetaData/pressure + maxvalue: 30000. + action: + name: reject + +# Reject obs with pressure > 85000. when isli=1 (land surface) +# CLEARED +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + - variable: GeoVaLs/land_area_fraction + minvalue: 0.99 +# - variable: GeoVaLs/dominant_surface_type +# maxvalue: 1.0001 +# minvalue: 0.9999 + test variables: + - name: MetaData/pressure + maxvalue: 85000. + action: + name: reject + +# Reject obs with pct1 (Coeff. of Var.) outside of 0.04–0.5, Type [240,245,246,251] ONLY +# CLEARED +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + - variable: ObsType/windEastward + is_in: 240,245,246,251 + test variables: + - name: MetaData/coefficientOfVariation + minvalue: 0.04 + maxvalue: 0.5 + action: + name: reject + # NESDIS obs are also subject to the experr_norm test defined as: # # if (10. - 0.1*(expectedError))/(ob_speed)>0.9, or ob_speed<0.1, reject, applies to NESDIS winds # # CLEARED: With caveat that float precision/handling differences can generate different acceptance criteria # between UFO and GSO for observations with an experr_norm value right around the maxvalue. - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/satelliteIdentifier - is_in: 250-299 - test variables: - - name: ObsFunction/SatWindsErrnormCheck - maxvalue: 0.9 - action: - name: reject -# - # Reject all Type=240 (GOES SWIR) AMVs: These are not currently assimilated in GSI and they have missing-values - # assigned to ob-errors - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 240 - action: - name: reject +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/satelliteIdentifier + is_in: 250-299 + test variables: + - name: ObsFunction/SatWindsErrnormCheck + maxvalue: 0.9 + action: + name: reject + +# Reject all Type=240 (GOES SWIR) AMVs: These are not currently assimilated in GSI and they have missing-values +# assigned to ob-errors +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240 + action: + name: reject # # setupw criteria # - # Reject any ob Type [240–260] when pressure greater than 950 mb. - # CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 240-260 - test variables: - - name: MetaData/pressure - maxvalue: 95001. - action: - name: reject - # GOES IR (245) reject when pressure between 399 and 801 mb. - # CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/pressure - minvalue: 39901. - maxvalue: 80099. - - variable: ObsType/windEastward - is_in: 245 - action: - name: reject - # JMA IR (252) reject when pressure between 499 and 801 mb. - # CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/pressure - minvalue: 49901. - maxvalue: 80099. - - variable: ObsType/windEastward - is_in: 252 - action: - name: reject - # EUMETSAT IR (253) reject when pressure between 401 and 801 mb. - # CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: MetaData/pressure - minvalue: 40101. - maxvalue: 80099. - - variable: ObsType/windEastward - is_in: 253 - action: - name: reject - # GOES WV (246, 250, 254), reject when pressure greater than 399 mb. - # CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 246, 250, 254 - test variables: - - name: MetaData/pressure - maxvalue: 39900. - action: - name: reject - # EUMET (242) and JMA (243) vis, reject when pressure less than 700 mb. - # CLEARED: minvalue is rejecting <, not <= as per a Perform Action, so threshold is unchanged - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 242, 243 - test variables: - - name: MetaData/pressure - minvalue: 70000. - action: - name: reject - # MODIS-Aqua/Terra (257) and (259), reject when pressure less than 249 mb. - # CLEARED: minvalue is rejecting <, not <= as per a Perform Action, so threshold is unchanged - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 257,259 - test variables: - - name: MetaData/pressure - minvalue: 24900. - action: - name: reject - # MODIS-Aqua/Terra (258) and (259), reject when pressure greater than 600 mb. - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - # maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 258, 259 - test variables: - - name: MetaData/pressure - maxvalue: 60000. - action: - name: reject - # Multiple satellite platforms, reject when pressure is more than 50 mb above tropopause. - # CLEARED: minvalue is rejecting <, not <= as per a Perform Action, so threshold is unchanged - - filter: Difference Check - filter variables: - - name: windEastward - - name: windNorthward - reference: GeoVaLs/tropopause_pressure - value: MetaData/pressure - minvalue: -5000. # 50 hPa above tropopause level, negative p-diff - action: - name: reject - # GOES (247) reject any observation with a /=0 surface type (non-water - # surface) within 110 hPa of the surface pressure (as part of the LNVD - # check). - # NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested - - filter: Difference Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: - name: GeoVaLs/water_area_fraction - maxvalue: 0.001 - - variable: - name: ObsType/windEastward - is_in: 247 - reference: GeoVaLs/surface_pressure - value: MetaData/pressure - maxvalue: -11000. # within 110 hPa above surface pressure, negative p-diff - action: - name: reject - # AVHRR (244), MODIS (257,258,259), and VIIRS (260) reject any - # observation with a /=0 surface type (non-water surface) within - # 200 hPa of the surface pressure (as part of the LNVD check). - # CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged - - filter: Difference Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: - name: GeoVaLs/water_area_fraction - maxvalue: 0.001 +# Reject any ob Type [240–260] when pressure greater than 950 mb. +# CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240-260 + test variables: + - name: MetaData/pressure + maxvalue: 95001. + action: + name: reject + +# GOES IR (245) reject when pressure between 399 and 801 mb. +# CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/pressure + minvalue: 39901. + maxvalue: 80099. + - variable: ObsType/windEastward + is_in: 245 + action: + name: reject + +# JMA IR (252) reject when pressure between 499 and 801 mb. +# CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/pressure + minvalue: 49901. + maxvalue: 80099. + - variable: ObsType/windEastward + is_in: 252 + action: + name: reject + +# EUMETSAT IR (253) reject when pressure between 401 and 801 mb. +# CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/pressure + minvalue: 40101. + maxvalue: 80099. + - variable: ObsType/windEastward + is_in: 253 + action: + name: reject + +# GOES WV (246, 250, 254), reject when pressure greater than 399 mb. +# CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 246, 250, 254 + test variables: + - name: MetaData/pressure + maxvalue: 39900. + action: + name: reject + +# EUMET (242) and JMA (243) vis, reject when pressure less than 700 mb. +# CLEARED: minvalue is rejecting <, not <= as per a Perform Action, so threshold is unchanged +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 242, 243 + test variables: + - name: MetaData/pressure + minvalue: 70000. + action: + name: reject + +# MODIS-Aqua/Terra (257) and (259), reject when pressure less than 249 mb. +# CLEARED: minvalue is rejecting <, not <= as per a Perform Action, so threshold is unchanged +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 257,259 + test variables: + - name: MetaData/pressure + minvalue: 24900. + action: + name: reject + +# MODIS-Aqua/Terra (258) and (259), reject when pressure greater than 600 mb. +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +# maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 258, 259 + test variables: + - name: MetaData/pressure + maxvalue: 60000. + action: + name: reject + +# Multiple satellite platforms, reject when pressure is more than 50 mb above tropopause. +# CLEARED: minvalue is rejecting <, not <= as per a Perform Action, so threshold is unchanged +- filter: Difference Check + filter variables: + - name: windEastward + - name: windNorthward + reference: GeoVaLs/tropopause_pressure + value: MetaData/pressure + minvalue: -5000. # 50 hPa above tropopause level, negative p-diff + action: + name: reject + +# GOES (247) reject any observation with a /=0 surface type (non-water +# surface) within 110 hPa of the surface pressure (as part of the LNVD +# check). +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Difference Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 +# name: GeoVaLs/dominant_surface_type +# minvalue: 0.001 + - variable: + name: ObsType/windEastward + is_in: 247 + reference: GeoVaLs/surface_pressure + value: MetaData/pressure + maxvalue: -11000. # within 110 hPa above surface pressure, negative p-diff + action: + name: reject + +# AVHRR (244), MODIS (257,258,259), and VIIRS (260) reject any +# observation with a /=0 surface type (non-water surface) within +# 200 hPa of the surface pressure (as part of the LNVD check). +# CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged +- filter: Difference Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 +# name: GeoVaLs/dominant_surface_type +# minvalue: 0.001 + - variable: + name: ObsType/windEastward + is_in: 244, 257-260 + reference: GeoVaLs/surface_pressure + value: MetaData/pressure + maxvalue: -20000. # within 200 hPa above surface pressure, negative p-diff + action: + name: reject + +# Reject GOES (247) when difference of wind direction is more than 50 degrees. +# CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 247 + test variables: + - name: ObsFunction/WindDirAngleDiff + maxvalue: 50. + action: + name: reject + +# AVHRR (244), MODIS (257,258,259), VIIRS (260), GOES (247) use a LNVD check. +# CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 244, 247, 257-260 + test variables: + - name: ObsFunction/SatWindsLNVDCheck + maxvalue: 3. + action: + name: reject + +# All satwinds must adjust errors based on ObsErrorFactorPressureCheck +# prior to the SPDB check (i.e. the gross-error check). The gross-error +# check uses the adjusted errors for error-bound tightening and rejection, +# so this check has to come first. This check will inflate errors for obs +# that are too close to either the model top or bottom. +- filter: Perform Action + filter variables: + - name: windEastward + where: - variable: name: ObsType/windEastward - is_in: 244, 257-260 - reference: GeoVaLs/surface_pressure - value: MetaData/pressure - maxvalue: -20000. # within 200 hPa above surface pressure, negative p-diff - action: - name: reject -obs post filters: - # Reject GOES (247) when difference of wind direction is more than 50 degrees. - # CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 247 - test variables: - - name: ObsFunction/WindDirAngleDiff - maxvalue: 50. - action: - name: reject - # AVHRR (244), MODIS (257,258,259), VIIRS (260), GOES (247) use a LNVD check. - # CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 244, 247, 257-260 - test variables: - - name: ObsFunction/SatWindsLNVDCheck - maxvalue: 3. - action: - name: reject - # All satwinds must adjust errors based on ObsErrorFactorPressureCheck - # prior to the SPDB check (i.e. the gross-error check). The gross-error - # check uses the adjusted errors for error-bound tightening and rejection, - # so this check has to come first. This check will inflate errors for obs - # that are too close to either the model top or bottom. - - filter: Perform Action - filter variables: - - name: windEastward - where: - - variable: - name: ObsType/windEastward - is_in: 240-260 - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorPressureCheck - options: - variable: windEastward - inflation factor: 4.0 - - - filter: Perform Action - filter variables: - - name: windNorthward - where: - - variable: - name: ObsType/windNorthward - is_in: 240-260 - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorPressureCheck - options: - variable: windNorthward - inflation factor: 4.0 - # - # All satwinds subject to a gross-error check that contains significant - # modifiers for satwinds with a negative speed-bias. ALL wind gross-error - # checks are currently being done by the SatWindsSPDBCheck. - # CLEARED - - filter: Background Check - filter variables: - - name: windEastward - function absolute threshold: - - name: ObsFunction/WindsSPDBCheck + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck options: - wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] - cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] - error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] - error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] variable: windEastward - action: - name: reject - - - filter: Background Check - filter variables: - - name: windNorthward - function absolute threshold: - - name: ObsFunction/WindsSPDBCheck + inflation factor: 4.0 + +- filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck options: - wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] - cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] - error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] - error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] variable: windNorthward - action: - name: reject - # The last error inflation check is for duplicate observations. This one needs - # to come last, because we don't want to inflate errors for duplication if one - # of the duplicates should be rejected. - - filter: RejectList - filter variables: - - name: windEastward - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorDuplicateCheck - options: - use_air_pressure: true - variable: windEastward - - filter: RejectList - filter variables: - - name: windNorthward - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorDuplicateCheck - options: - use_air_pressure: true - variable: windNorthward - # We are extending this to an additional filter that inflates final ob-errors across-the-board by - # 1/0.8 = 1.25. This is caused by the GSI value of nvqc being set to .true. in the global operational - # configuration, see: https://github.com/NOAA-EMC/global-workflow/blob/d5ae3328fa4041b177357b1133f6b92e81c859d7/scripts/exglobal_atmos_analysis.sh#L750 - # This setting activates Line 1229 of setupw.f90 to scale ratio_errors by 0.8, which is applied in - # the denominator of the final ob-error, so 1/0.8 = 1.25 factor of ob-error inflation. - # - # If this nvqc functionality were to be switched off (i.e. if variational qc were to be turned off), - # you would want to remove this last inflation filter. - - filter: Perform Action - filter variables: - - name: windEastward - where: - - variable: ObsType/windEastward - is_in: 240-260 - action: - name: inflate error - inflation factor: 1.25 + inflation factor: 4.0 + +# All satwinds subject to a gross-error check that contains significant +# modifiers for satwinds with a negative speed-bias. ALL wind gross-error +# checks are currently being done by the SatWindsSPDBCheck. +# CLEARED +- filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] + error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] + variable: windEastward + action: + name: reject + +- filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] + error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] + variable: windNorthward + action: + name: reject + +# The last error inflation check is for duplicate observations. This one needs +# to come last, because we don't want to inflate errors for duplication if one +# of the duplicates should be rejected. +# Inflate obs error for duplicated observations at the same location +#- filter: Perform Action +# filter variables: +# - name: windEastward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windEastward + +# Inflate obs error for duplicated observations at the same location +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windNorthward + +# We are extending this to an additional filter that inflates final ob-errors across-the-board by +# 1/0.8 = 1.25. This is caused by the GSI value of nvqc being set to .true. in the global operational +# configuration, see: https://github.com/NOAA-EMC/global-workflow/blob/d5ae3328fa4041b177357b1133f6b92e81c859d7/scripts/exglobal_atmos_analysis.sh#L750 +# This setting activates Line 1229 of setupw.f90 to scale ratio_errors by 0.8, which is applied in +# the denominator of the final ob-error, so 1/0.8 = 1.25 factor of ob-error inflation. # - - filter: Perform Action - filter variables: - - name: windNorthward - where: - - variable: ObsType/windNorthward - is_in: 240-260 - action: - name: inflate error - inflation factor: 1.25 -# END OF FILTERS -linear obs operator: - name: Identity - -#passedBenchmark: 1025814# 2 variables (u,v), both passing 512907 obs, including 512907 GSI/UFO agreements, and: -passedBenchmark: 1150870 # 2 variables (u,v), both passing 575435 obs, including 512907 GSI/UFO agreements, and: +# If this nvqc functionality were to be switched off (i.e. if variational qc were to be turned off), +# you would want to remove this last inflation filter. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# where: +# - variable: ObsType/windEastward +# is_in: 240-260 +# action: +# name: inflate error +# inflation factor: 1.25 + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# where: +# - variable: ObsType/windNorthward +# is_in: 240-260 +# action: +# name: inflate error +# inflation factor: 1.25 + +# End of Filters +#passedBenchmark: 1150870 # 2 variables (u,v), both passing 575435 obs, including 575435 GSI/UFO agreements, and: +passedBenchmark: 1025814 # 2 variables (u,v), both passing 512907 obs, including 512907 GSI/UFO agreements, and: # 16 GOES AMVs (6 u-obs, 6 v-obs) that are rejected by the SatWindsErrnormCheck in UFO # but are retained in GSI's equivalent experr_norm check. All 6 of these disagreements # have error norm values at almost exactly 0.9, indicating a float precision/handling diff --git a/parm/atm/obs/testing/satwind_noqc.yaml b/parm/atm/obs/testing/satwind_noqc.yaml index cef238455..b2535470d 100644 --- a/parm/atm/obs/testing/satwind_noqc.yaml +++ b/parm/atm/obs/testing/satwind_noqc.yaml @@ -9,10 +9,404 @@ obs space: type: H5File obsfile: !ENV satwind_diag_${CDATE}.nc4 simulated variables: [windEastward, windNorthward] + geovals: filename: !ENV satwind_geoval_${CDATE}.nc4 + obs operator: name: VertInterp + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + +obs prior filters: +# Apply variable changes needed wind scaling +- filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: False + +obs post filters: +# Assign the initial observation error, based on height/pressure +# Hard-wiring to prepobs_errtable.global by Type +# ObsError is currently not updating in diag file, but passes directly to EffectiveError when no inflation is specified in YAML +# Type 240 (GOES SWIR): Assigned all dummy values in prepobs_errtable.global +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 241 (Multi Spec. Imager LWIR): Assigned all dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 241 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 242 (Himawari VIS) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 242 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 243 (MVIRI/SEVIRI VIS) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 243 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 244 (AVHRR LWIR) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 244 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 245 (GOES LWIR): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 245 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] + +# Type 246 (GOES cloud-top WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 246 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] + +# Type 247 (GOES clear-sky WV): I am assuming these are halved relative to prepobs_errtable.global, based on read_satwnd.f90: L1410–1416 +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 247 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.6,7.8,7.8,8.,8.,8.2,10.,12.,12.6,13.2,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.,14.] + +# Type 248 (GOES Sounder cloud-top WV): Assigned all dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 248 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 249 (GOES Sounder clear-sky WV): Assigned all dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 249 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 250 (Himawari AHI WV, cloud-top or clear-sky) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 250 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,7.,7.3,7.6,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.] + +# Type 251 (GOES VIS): Assigned all dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 251 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 252 (Himawari AHI LWIR) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 252 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 253 (MVIRI/SEVERI LWIR) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 253 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 254 (MVIRI/SEVIRI WV, both cloud-top and clear-sky) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 254 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.5,6.1,6.,6.5,7.3,7.6,7.,7.5,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 255 (LEOGEO): +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 255 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 256 (Multi Spec. Imager WV, both clear-sky and cloud-top): Assigned all dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 256 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.] + +# Type 257 (MODIS LWIR) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 257 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 258 (MODIS cloud-top WV): Some levels assigned dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 258 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 259 (MODIS clear-sky WV): Some levels assigned dummy values +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 259 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [1000000000.,1000000000.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 260 (VIIRS LWIR): All levels assigned dummy values in prepobs_errtable.global, HOWEVER the GSI values appear +# to be a standard profile (borrowed from e.g., Type=244). Using the standard profile here. +# It's possibly that my prepobs_errtable.global file is out-of-date. +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 260 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] -vector ref: GsiHofXBc -tolerance: 1.e-5 +passedBenchmark: 2459862 # with enforced time window +#passedBenchmark: 2752378 # without enforcing time window diff --git a/ush/ufoeval/run_ufo_hofx_test.sh b/ush/ufoeval/run_ufo_hofx_test.sh index 85fed17cf..ccc043bd4 100755 --- a/ush/ufoeval/run_ufo_hofx_test.sh +++ b/ush/ufoeval/run_ufo_hofx_test.sh @@ -100,12 +100,7 @@ else yamlpath=$GDASApp/parm/atm/obs/testing/${obtype}.yaml fi -#exename=test_ObsFilters.x -if [ $run_filtering == NO ]; then - exename=test_ObsOperator.x -else - exename=test_ObsFilters.x -fi +exename=test_ObsFilters.x #-------------- Do not modify below this line ---------------- # paths that should only be changed by an expert user From 7eebf431b0b6c17a89e1e510256f69c31bad4acf Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Tue, 31 Oct 2023 16:07:11 -0400 Subject: [PATCH 05/14] Update Ozone for testing and end-to-end (#699) * Adding YAML files for Ozone in config. Update test benchmark for omi_aura. Tide up the YAMLs. * Fix comments --- parm/atm/obs/config/omi_aura.yaml | 73 ++++- parm/atm/obs/config/ompsnp_npp.yaml | 308 +++++++++++++++++---- parm/atm/obs/config/ompstc8_npp.yaml | 73 ++++- parm/atm/obs/testing/omi_aura.yaml | 5 +- parm/atm/obs/testing/omi_aura_noqc.yaml | 17 +- parm/atm/obs/testing/ompsnp_npp.yaml | 3 + parm/atm/obs/testing/ompsnp_npp_noqc.yaml | 26 +- parm/atm/obs/testing/ompstc8_npp.yaml | 3 + parm/atm/obs/testing/ompstc8_npp_noqc.yaml | 16 +- 9 files changed, 453 insertions(+), 71 deletions(-) diff --git a/parm/atm/obs/config/omi_aura.yaml b/parm/atm/obs/config/omi_aura.yaml index 79973780c..20cabced5 100644 --- a/parm/atm/obs/config/omi_aura.yaml +++ b/parm/atm/obs/config/omi_aura.yaml @@ -11,8 +11,79 @@ obs space: io pool: max pool size: 1 simulated variables: [ozoneTotal] + obs operator: name: AtmVertInterpLay - geovals: [ozoneLayer] + geovals: [mole_fraction_of_ozone_in_air] coefficients: [0.007886131] # convert from ppmv to DU nlevels: [1] + +obs pre filters: +- filter: Perform Action + filter variables: + - name: ozoneTotal + action: + name: assign error + error parameter: 6.0 + +obs prior filters: +# GSI read routine QC +# range sanity check +- filter: Bounds Check + filter variables: + - name: ozoneTotal + minvalue: 0 + maxvalue: 10000 + action: + name: reject + +# Do not use the data if row anomaly (bit 10)is 1 +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/totalOzoneQualityFlag + any_bit_set_of: 9 + +# Scan position check: reject scan position >= 25 +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/sensorScanPosition + minvalue: 25 + +# Accept total_ozone_error_flag values of 0 and 1, but not any others. +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/totalOzoneQualityCode + is_not_in: 0, 1 + +# Use data with best ozone algorighm +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/bestOzoneAlgorithmFlag + is_in: 3, 13 + +obs post filters: +# GSI setup routine QC +# Gross check +- filter: Background Check + filter variables: + - name: ozoneTotal + threshold: 10.0 + absolute threshold: 300.0 + action: + name: reject + +# End of Filters + + diff --git a/parm/atm/obs/config/ompsnp_npp.yaml b/parm/atm/obs/config/ompsnp_npp.yaml index 5ba772fb3..c18b9abed 100644 --- a/parm/atm/obs/config/ompsnp_npp.yaml +++ b/parm/atm/obs/config/ompsnp_npp.yaml @@ -4,74 +4,284 @@ obs space: engine: type: H5File obsfile: $(DATA)/obs/$(OPREFIX)ompsnp_npp.${{ current_cycle | to_YMDH }}.nc4 + obsgrouping: + group variables: ["latitude"] + sort variable: "pressure" + sort order: "ascending" obsdataout: engine: type: H5File obsfile: $(DATA)/diags/diag_ompsnp_npp_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 - simulated variables: [ozoneTotal] + simulated variables: [ozoneLayer] + obs operator: name: AtmVertInterpLay - geovals: [ozoneLayer] + geovals: [mole_fraction_of_ozone_in_air] coefficients: [0.007886131] # convert from ppmv to DU nlevels: [22] -obs filters: -#- filter: BlackList -# filter variables: -# - name: ozoneTotal -# where: -# - variable: -# name: MetaData/total_ozone_error_flag -# minvalue: 0.1 -# maxvalue: 1.1 -# action: -# name: reject -#- filter: BlackList -# filter variables: -# - name: ozoneTotal -# where: -# - variable: -# name: MetaData/total_ozone_error_flag -# minvalue: 2.1 # toss toq>2, there are some 4 and 6 -# action: -# name: reject -#- filter: BlackList -# filter variables: -# - name: ozoneTotal -# where: -# - variable: -# name: MetaData/profile_ozone_error_flag -# minvalue: 1.1 -# action: -# name: reject -- filter: BlackList - filter variables: - - name: ozoneTotal + +obs pre filters: +# Observation error assignment +- filter: Perform Action + filter variables: + - name: ozoneLayer + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [0.001, 10.1325, 16.00935, 25.43258, 40.32735, 63.93607, 101.325, 160.0935, 254.3257, 403.2735, 639.3608, 1013.25, 1600.935, 2543.258, 4032.735, 6393.607, 10132.5, 16009.35, 25432.57, 40327.35, 63936.07, 101325] + errors: [7.7236, 0.02, 0.02, 0.025, 0.08, 0.15, 0.056, 0.125, 0.2, 0.299, 0.587, 0.864, 1.547, 2.718, 3.893, 4.353, 3.971, 4.407, 4.428, 3.312, 2.198, 2.285] + +obs prior filters: +# Do not assimilation where pressure is zero +# Zero pressure indicates the data is total column ozone +- filter: RejectList + filter variables: + - name: ozoneLayer where: - variable: name: MetaData/pressure - maxvalue: 64.00000 + maxvalue: 0.0001 + +# Sanity check on observaton values +- filter: Bounds Check + filter variables: + - name: ozoneLayer + minvalue: 0 + maxvalue: 1000 action: name: reject -- filter: Domain Check + +# Total Ozone Quality Check (keeps 0, 2) +# 0 indentifies good data +# 2 identifies good data with a solar zenith angle > 84 degrees +- filter: RejectList filter variables: - - name: ozoneTotal + - name: ozoneLayer where: - variable: - name: MetaData/latitude - minvalue: -90. - maxvalue: 90. + name: MetaData/totalOzoneQuality + is_not_in: 0, 2 + +# Profile Ozone Quality Check (keeps 0, 1, 7) +# 0 : good data +# 1 : good data with a solar zenith angle > 84 degrees +# 7 : profile for which stray light correction applied +- filter: RejectList + filter variables: + - name: ozoneLayer + where: - variable: - name: MetaData/longitude - minvalue: -180. - maxvalue: 360. -- filter: Bounds Check + name: MetaData/profileOzoneQuality + is_not_in: 0, 1, 7 + +obs post filters: +# Gross error check +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 120 + action: + name: reject + where: + - variable: + name: MetaData/pressure + maxvalue: 0.001 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 30 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 30000.0 + maxvalue: 110000.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 40 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 20000.0 + maxvalue: 30000.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 44.42 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 10100.0 + maxvalue: 20000.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 57.52 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 6400.0 + maxvalue: 10100.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 69.4 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 4000.0 + maxvalue: 6400.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 70 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 2600.0 + maxvalue: 4000.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 62.73 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 1600.0 + maxvalue: 2600.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 50.52 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 1100.0 + maxvalue: 1600.0 + +- filter: Background Check filter variables: - - name: ozoneTotal - minvalue: 0.000001 - maxvalue: 1000.0 + - name: ozoneLayer + absolute threshold: 35.9 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 700.0 + maxvalue: 1100.0 + - filter: Background Check filter variables: - - name: ozoneTotal - absolute threshold: 10.0 + - name: ozoneLayer + absolute threshold: 26.41 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 400.0 + maxvalue: 700.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 20.51 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 300.0 + maxvalue: 400.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 12.82 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 200.0 + maxvalue: 300.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 10 + action: + name: reject + where: + - variable: + name: MetaData/pressure + maxvalue: 70.0 + maxvalue: 200.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 5 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 40.0 + maxvalue: 70.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 2 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 30.0 + maxvalue: 40.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 1 + action: + name: reject + where: + - variable: + name: MetaData/pressure + maxvalue: 30.0 + +# End of Filters diff --git a/parm/atm/obs/config/ompstc8_npp.yaml b/parm/atm/obs/config/ompstc8_npp.yaml index e3b11ff64..545983d93 100644 --- a/parm/atm/obs/config/ompstc8_npp.yaml +++ b/parm/atm/obs/config/ompstc8_npp.yaml @@ -11,8 +11,79 @@ obs space: io pool: max pool size: 1 simulated variables: [ozoneTotal] + obs operator: name: AtmVertInterpLay - geovals: [ozoneLayer] + geovals: [mole_fraction_of_ozone_in_air] coefficients: [0.007886131] # convert from ppmv to DU nlevels: [1] + +obs pre filters: +- filter: Perform Action + filter variables: + - name: ozoneTotal + action: + name: assign error + error parameter: 6.0 + +obs prior filters: +# GSI read routine QC +# range sanity check +- filter: Bounds Check + filter variables: + - name: ozoneTotal + minvalue: 0 + maxvalue: 1000 + action: + name: reject + +# Accept total_ozone_error_flag values of 0 and 1, but not any others. +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/totalOzoneQualityCode + is_not_in: 0, 1 + +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/bestOzoneAlgorithmFlag + is_in: 3, 13 + +# GSI setup routine QC +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 1, 2, 3, 4, 35 + - variable: + name: MetaData/latitude + minvalue: 50.0 + +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 1, 2, 3, 4, 35 + - variable: + name: MetaData/latitude + maxvalue: -50.0 + +obs post filters: +- filter: Background Check + filter variables: + - name: ozoneTotal + threshold: 10.0 + absolute threshold: 300.0 + action: + name: reject + +# End of Filters diff --git a/parm/atm/obs/testing/omi_aura.yaml b/parm/atm/obs/testing/omi_aura.yaml index a090f5945..8c5bb52da 100644 --- a/parm/atm/obs/testing/omi_aura.yaml +++ b/parm/atm/obs/testing/omi_aura.yaml @@ -11,8 +11,10 @@ obs space: io pool: max pool size: 1 simulated variables: [ozoneTotal] + geovals: filename: !ENV omi_aura_geoval_${CDATE}.nc4 + obs operator: name: AtmVertInterpLay geovals: [mole_fraction_of_ozone_in_air] @@ -85,4 +87,5 @@ obs post filters: action: name: reject -passedBenchmark: 1182 +# End of Filters +passedBenchmark: 1170 diff --git a/parm/atm/obs/testing/omi_aura_noqc.yaml b/parm/atm/obs/testing/omi_aura_noqc.yaml index 8929c38b6..d32954a8f 100644 --- a/parm/atm/obs/testing/omi_aura_noqc.yaml +++ b/parm/atm/obs/testing/omi_aura_noqc.yaml @@ -19,9 +19,14 @@ obs operator: coefficients: [0.007886131] # convert from ppmv to DU nlevels: [1] -vector ref: GsiHofXBc -tolerance: 1.e-5 -#linear obs operator test: -## coef TL: 0.1 -## tolerance TL: 1.0e-13 -## tolerance AD: 1.0e-11 +obs pre filters: +- filter: Perform Action + filter variables: + - name: ozoneTotal + action: + name: assign error + error parameter: 6.0 + +passedBenchmark: 4927 # total:6082; missing:1155 +#vector ref: GsiHofXBc +#tolerance: 1.e-5 diff --git a/parm/atm/obs/testing/ompsnp_npp.yaml b/parm/atm/obs/testing/ompsnp_npp.yaml index f7ee9d4ec..b7974f5cd 100644 --- a/parm/atm/obs/testing/ompsnp_npp.yaml +++ b/parm/atm/obs/testing/ompsnp_npp.yaml @@ -15,8 +15,10 @@ obs space: io pool: max pool size: 1 simulated variables: [ozoneLayer] + geovals: filename: !ENV ompsnp_npp_geoval_${CDATE}.nc4 + obs operator: name: AtmVertInterpLay geovals: [mole_fraction_of_ozone_in_air] @@ -285,4 +287,5 @@ obs post filters: name: MetaData/pressure maxvalue: 30.0 +# End of Filters passedBenchmark: 4914 diff --git a/parm/atm/obs/testing/ompsnp_npp_noqc.yaml b/parm/atm/obs/testing/ompsnp_npp_noqc.yaml index 219f9e942..639a8dda5 100644 --- a/parm/atm/obs/testing/ompsnp_npp_noqc.yaml +++ b/parm/atm/obs/testing/ompsnp_npp_noqc.yaml @@ -22,11 +22,23 @@ obs operator: geovals: [mole_fraction_of_ozone_in_air] coefficients: [0.007886131] # convert from ppmv to DU nlevels: [22] - -vector ref: GsiHofXBc -tolerance: 1.e-5 -#linear obs operator test: -# coef TL: 0.1 -# tolerance TL: 1.0e-13 -# tolerance AD: 1.0e-11 + +# Observation Error Assignment +obs pre filters: +- filter: Perform Action + filter variables: + - name: ozoneLayer + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [0.001, 10.1325, 16.00935, 25.43258, 40.32735, 63.93607, 101.325, 160.0935, 254.3257, 403.2735, 639.3608, 1013.25, 1600.935, 2543.258, 4032.735, 6393.607, 10132.5, 16009.35, 25432.57, 40327.35, 63936.07, 101325] + errors: [7.7236, 0.02, 0.02, 0.025, 0.08, 0.15, 0.056, 0.125, 0.2, 0.299, 0.587, 0.864, 1.547, 2.718, 3.893, 4.353, 3.971, 4.407, 4.428, 3.312, 2.198, 2.285] + +passedBenchmark: 6314 # total:6314; missing:0 +#vector ref: GsiHofXBc +#tolerance: 1.e-5 diff --git a/parm/atm/obs/testing/ompstc8_npp.yaml b/parm/atm/obs/testing/ompstc8_npp.yaml index aea18a5dc..8fb3d6165 100644 --- a/parm/atm/obs/testing/ompstc8_npp.yaml +++ b/parm/atm/obs/testing/ompstc8_npp.yaml @@ -15,8 +15,10 @@ obs space: io pool: max pool size: 1 simulated variables: [ozoneTotal] + geovals: filename: !ENV ompstc8_npp_geoval_${CDATE}.nc4 + obs operator: name: AtmVertInterpLay geovals: [mole_fraction_of_ozone_in_air] @@ -91,4 +93,5 @@ obs post filters: action: name: reject +# End of Filters passedBenchmark: 6130 diff --git a/parm/atm/obs/testing/ompstc8_npp_noqc.yaml b/parm/atm/obs/testing/ompstc8_npp_noqc.yaml index 31027007a..de9ef7839 100644 --- a/parm/atm/obs/testing/ompstc8_npp_noqc.yaml +++ b/parm/atm/obs/testing/ompstc8_npp_noqc.yaml @@ -22,11 +22,15 @@ obs operator: geovals: [mole_fraction_of_ozone_in_air] coefficients: [0.007886131] # convert from ppmv to DU nlevels: [1] +obs pre filters: +- filter: Perform Action + filter variables: + - name: ozoneTotal + action: + name: assign error + error parameter: 6.0 -vector ref: GsiHofXBc -tolerance: 1.e-5 -#linear obs operator test: -# coef TL: 0.1 -# tolerance TL: 1.0e-13 -# tolerance AD: 1.0e-11 +passedBenchmark: 6693 # total:6870; missing:177 +#vector ref: GsiHofXBc +#tolerance: 1.e-5 From 9a3c226237c79b6a74312c8b1c3b99cf33ac4b28 Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Tue, 31 Oct 2023 16:12:13 -0400 Subject: [PATCH 06/14] Update scatwind yamls for testing and end-to-end (#697) * Update scatwind YAML files based on UFO update accordingly. * Remove extra comments * Add YAML files for scatwind under config * Move filters two spacings left to line up with the filter header. --- parm/atm/obs/config/scatwind_metop-a.yaml | 280 +++++++++++++ parm/atm/obs/config/scatwind_metop-b.yaml | 280 +++++++++++++ parm/atm/obs/testing/scatwind.yaml | 481 ++++++++++++---------- parm/atm/obs/testing/scatwind_noqc.yaml | 39 +- 4 files changed, 862 insertions(+), 218 deletions(-) create mode 100644 parm/atm/obs/config/scatwind_metop-a.yaml create mode 100644 parm/atm/obs/config/scatwind_metop-b.yaml diff --git a/parm/atm/obs/config/scatwind_metop-a.yaml b/parm/atm/obs/config/scatwind_metop-a.yaml new file mode 100644 index 000000000..4735c31fe --- /dev/null +++ b/parm/atm/obs/config/scatwind_metop-a.yaml @@ -0,0 +1,280 @@ +obs space: + name: ascatw_ascat_metop-a + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)ascatw.ascat_metop-a.{{ current_cycle | to_YMDH }}.nc4 + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_ascatw_ascat_metop-a_{{ current_cycle | to_YMDH }}.nc4 + io pool: + max pool size: 1 + simulated variables: [windEastward, windNorthward] + +obs operator: + name: VertInterp + # Use height vertical coordinate first +# vertical coordinate: geometric_height + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + +obs prior filters: +# Apply variable changes needed for rescaled height coordinate +- filter: Variable Transforms + Transform: AdjustedHeightCoordinate + SkipWhenNoObs: False + +# Apply variable changes needed for wind scaling +- filter: Variable Transforms + Transform: SurfaceWindScalingHeight + SkipWhenNoObs: False + +# Assign the initial observation error (constant value, 1.5 m/s right now). +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 1.5 + +# Calculate error inflation factor for duplicate observations +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windEastward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windEastward + +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windNorthward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windNorthward + +# Reject all obs with PreQC mark already set above 3 +# NOTE: All scatwinds have an automatic PreQC mark of 2 (hard-wired default from GSI) +# - filter: PreQC +# maxvalue: 3 +# action: +# name: reject + +obs post filters: +# Reject all ASCAT (Type 290) winds with tsavg <= 273.0 (surface temperature) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 290 +# - variable: GeoVaLs/surface_temperature + - variable: GeoVaLs/surface_temperature_where_land + maxvalue: 273. + action: + name: reject + +# Reject all ASCAT (Type 290) winds with isflg /= 0 (non-water surface) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 290 + - variable: GeoVaLs/water_area_fraction + maxvalue: 0.99 + action: + name: reject + +# Reject ASCAT (Type 290) when observed component deviates from background by more than 5.0 m/s +# NOTE: This check can reject a u- or v-component of the same observation independently, which +# is fundamentally different from how GSI rejects obs (both components are rejected if +# either component fails a check). +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Arithmetic + options: + variables: + - name: ObsValue/windEastward + - name: HofX/windEastward + coefs: [1.0, -1.0] + minvalue: -5.0 + maxvalue: 5.0 + +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Arithmetic + options: + variables: + - name: ObsValue/windNorthward + - name: HofX/windNorthward + coefs: [1.0, -1.0] + minvalue: -5.0 + maxvalue: 5.0 + +# Reject OSCAT (Type 291) when observed component deviates from background by more than 6.0 m/s +# NOTE: This check can reject a u- or v-component of the same observation independently, which +# is fundamentally different from how GSI rejects obs (both components are rejected if +# either component fails a check). +- filter: Background Check + filter variables: + - name: windEastward + - name: windNorthward + threshold: 6. + absolute threshold: 6. + where: + - variable: ObsType/windEastward + is_in: 291 + action: + name: reject + +# Reject ASCAT (Type 290) when ambiguity check fails (returned value is negative) +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 290 + test variables: + - name: ObsFunction/ScatWindsAmbiguityCheck + options: + minimum_uv: 0.0001 # hard-coding a minimum-uv for transparancy, want this to basically be zero + maxvalue: 0. + action: + name: reject + +# All scatwinds must adjust errors based on ObsErrorFactorPressureCheck. +# This check will inflate errors for obs that are too close to either +# the model top or bottom. +- filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 290-291 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4.0 + +- filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 290-291 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + +# All scatwinds subject to a gross error check. This is contained within +# the WindsSPDBCheck, although it is not exclusive to satwinds. +- filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 290, 291] + cgross: [ 5.0, 5.0] + error_min: [1.4, 1.4] + error_max: [6.1, 6.1] + variable: windEastward + action: + name: reject + +- filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 290, 291] + cgross: [ 5.0, 5.0] + error_min: [1.4, 1.4] + error_max: [6.1, 6.1] + variable: windNorthward + action: + name: reject + +# The last error inflation check is for duplicate observations. This one needs +# to come last, because we don't want to inflate errors for duplication if one +# of the duplicates should be rejected. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windEastward + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windNorthward + +# There is no across-the-board inflation for nvqc=.true. for scatwinds, presumably because for +# this inflation to take place both nvqc must be .true. AND ibeta must be >0, see: +# https://github.com/NOAA-EMC/GSI/blob/14ae595af1b03471287d322596d35c0665336e95/src/gsi/setupw.f90#L1229 +# GSI settings must have ibeta>0 for satwinds, but not for scatwinds. +# +# If the ibeta settings for scatwinds were to change while nvqc remained .true., we would extend YAML to +# an additional filter that inflates final ob-errors across-the-board by 1/0.8 = 1.25. NOTE: the nvqc setting +# is defaulted to .false. in GSI code, but is overridden in global operational configuration. See: +# configuration, see: https://github.com/NOAA-EMC/global-workflow/blob/d5ae3328fa4041b177357b1133f6b92e81c859d7/scripts/exglobal_atmos_analysis.sh#L750 +# This setting activates Line 1229 of setupw.f90 to scale ratio_errors by 0.8, which is applied in +# the denominator of the final ob-error, so 1/0.8 = 1.25 factor of ob-error inflation. +# +# If this functionality were to be activated for scatwinds, you would want to include this last inflation filter. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# where: +# - variable: ObsType/windEastward +# is_in: 290-291 +# action: +# name: inflate error +# inflation factor: 1.25 +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# where: +# - variable: ObsType/windNorthward +# is_in: 290-291 +# action: +# name: inflate error +# inflation factor: 1.25 + +# END OF FILTERS# diff --git a/parm/atm/obs/config/scatwind_metop-b.yaml b/parm/atm/obs/config/scatwind_metop-b.yaml new file mode 100644 index 000000000..5ed7c38bc --- /dev/null +++ b/parm/atm/obs/config/scatwind_metop-b.yaml @@ -0,0 +1,280 @@ +obs space: + name: ascatw_ascat_metop-b + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)ascatw.ascat_metop-b.{{ current_cycle | to_YMDH }}.nc4 + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_ascatw_ascat_metop-b_{{ current_cycle | to_YMDH }}.nc4 + io pool: + max pool size: 1 + simulated variables: [windEastward, windNorthward] + +obs operator: + name: VertInterp + # Use height vertical coordinate first +# vertical coordinate: geometric_height + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + +obs prior filters: +# Apply variable changes needed for rescaled height coordinate +- filter: Variable Transforms + Transform: AdjustedHeightCoordinate + SkipWhenNoObs: False + +# Apply variable changes needed for wind scaling +- filter: Variable Transforms + Transform: SurfaceWindScalingHeight + SkipWhenNoObs: False + +# Assign the initial observation error (constant value, 1.5 m/s right now). +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 1.5 + +# Calculate error inflation factor for duplicate observations +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windEastward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windEastward + +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windNorthward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windNorthward + +# Reject all obs with PreQC mark already set above 3 +# NOTE: All scatwinds have an automatic PreQC mark of 2 (hard-wired default from GSI) +# - filter: PreQC +# maxvalue: 3 +# action: +# name: reject + +obs post filters: +# Reject all ASCAT (Type 290) winds with tsavg <= 273.0 (surface temperature) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 290 +# - variable: GeoVaLs/surface_temperature + - variable: GeoVaLs/surface_temperature_where_land + maxvalue: 273. + action: + name: reject + +# Reject all ASCAT (Type 290) winds with isflg /= 0 (non-water surface) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 290 + - variable: GeoVaLs/water_area_fraction + maxvalue: 0.99 + action: + name: reject + +# Reject ASCAT (Type 290) when observed component deviates from background by more than 5.0 m/s +# NOTE: This check can reject a u- or v-component of the same observation independently, which +# is fundamentally different from how GSI rejects obs (both components are rejected if +# either component fails a check). +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Arithmetic + options: + variables: + - name: ObsValue/windEastward + - name: HofX/windEastward + coefs: [1.0, -1.0] + minvalue: -5.0 + maxvalue: 5.0 + +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Arithmetic + options: + variables: + - name: ObsValue/windNorthward + - name: HofX/windNorthward + coefs: [1.0, -1.0] + minvalue: -5.0 + maxvalue: 5.0 + +# Reject OSCAT (Type 291) when observed component deviates from background by more than 6.0 m/s +# NOTE: This check can reject a u- or v-component of the same observation independently, which +# is fundamentally different from how GSI rejects obs (both components are rejected if +# either component fails a check). +- filter: Background Check + filter variables: + - name: windEastward + - name: windNorthward + threshold: 6. + absolute threshold: 6. + where: + - variable: ObsType/windEastward + is_in: 291 + action: + name: reject + +# Reject ASCAT (Type 290) when ambiguity check fails (returned value is negative) +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 290 + test variables: + - name: ObsFunction/ScatWindsAmbiguityCheck + options: + minimum_uv: 0.0001 # hard-coding a minimum-uv for transparancy, want this to basically be zero + maxvalue: 0. + action: + name: reject + +# All scatwinds must adjust errors based on ObsErrorFactorPressureCheck. +# This check will inflate errors for obs that are too close to either +# the model top or bottom. +- filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 290-291 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4.0 + +- filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 290-291 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + +# All scatwinds subject to a gross error check. This is contained within +# the WindsSPDBCheck, although it is not exclusive to satwinds. +- filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 290, 291] + cgross: [ 5.0, 5.0] + error_min: [1.4, 1.4] + error_max: [6.1, 6.1] + variable: windEastward + action: + name: reject + +- filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 290, 291] + cgross: [ 5.0, 5.0] + error_min: [1.4, 1.4] + error_max: [6.1, 6.1] + variable: windNorthward + action: + name: reject + +# The last error inflation check is for duplicate observations. This one needs +# to come last, because we don't want to inflate errors for duplication if one +# of the duplicates should be rejected. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windEastward + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windNorthward + +# There is no across-the-board inflation for nvqc=.true. for scatwinds, presumably because for +# this inflation to take place both nvqc must be .true. AND ibeta must be >0, see: +# https://github.com/NOAA-EMC/GSI/blob/14ae595af1b03471287d322596d35c0665336e95/src/gsi/setupw.f90#L1229 +# GSI settings must have ibeta>0 for satwinds, but not for scatwinds. +# +# If the ibeta settings for scatwinds were to change while nvqc remained .true., we would extend YAML to +# an additional filter that inflates final ob-errors across-the-board by 1/0.8 = 1.25. NOTE: the nvqc setting +# is defaulted to .false. in GSI code, but is overridden in global operational configuration. See: +# configuration, see: https://github.com/NOAA-EMC/global-workflow/blob/d5ae3328fa4041b177357b1133f6b92e81c859d7/scripts/exglobal_atmos_analysis.sh#L750 +# This setting activates Line 1229 of setupw.f90 to scale ratio_errors by 0.8, which is applied in +# the denominator of the final ob-error, so 1/0.8 = 1.25 factor of ob-error inflation. +# +# If this functionality were to be activated for scatwinds, you would want to include this last inflation filter. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# where: +# - variable: ObsType/windEastward +# is_in: 290-291 +# action: +# name: inflate error +# inflation factor: 1.25 +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# where: +# - variable: ObsType/windNorthward +# is_in: 290-291 +# action: +# name: inflate error +# inflation factor: 1.25 + +# END OF FILTERS# diff --git a/parm/atm/obs/testing/scatwind.yaml b/parm/atm/obs/testing/scatwind.yaml index 30a1af6f7..9befdda81 100644 --- a/parm/atm/obs/testing/scatwind.yaml +++ b/parm/atm/obs/testing/scatwind.yaml @@ -9,228 +9,283 @@ obs space: type: H5File obsfile: !ENV scatwind_diag_${CDATE}.nc4 simulated variables: [windEastward, windNorthward] + geovals: filename: !ENV scatwind_geoval_${CDATE}.nc4 -vector ref: GsiHofXBc -tolerance: 0.01 + obs operator: name: VertInterp - apply near surface wind scaling: true -# -obs pre filters: - # Assign the initial observation error (constant value, 1.5 m/s right now). - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - action: - name: assign error - error parameter: 1.5 -#obs prior filters: - # - # Reject all obs with PreQC mark already set above 3 - # NOTE: All scatwinds have an automatic PreQC mark of 2 (hard-wired default from GSI) - # - filter: PreQC - # maxvalue: 3 - # action: - # name: reject - # + # Use height vertical coordinate first +# vertical coordinate: geometric_height + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + +obs prior filters: +# Apply variable changes needed for rescaled height coordinate +- filter: Variable Transforms + Transform: AdjustedHeightCoordinate + SkipWhenNoObs: False + +# Apply variable changes needed for wind scaling +- filter: Variable Transforms + Transform: SurfaceWindScalingHeight + SkipWhenNoObs: False + +# Assign the initial observation error (constant value, 1.5 m/s right now). +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 1.5 + +# Calculate error inflation factor for duplicate observations +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windEastward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windEastward + +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windNorthward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windNorthward + +# Reject all obs with PreQC mark already set above 3 +# NOTE: All scatwinds have an automatic PreQC mark of 2 (hard-wired default from GSI) +# - filter: PreQC +# maxvalue: 3 +# action: +# name: reject + obs post filters: - # - # Reject all ASCAT (Type 290) winds with tsavg <= 273.0 (surface temperature) - # - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 290 - - variable: GeoVaLs/surface_temperature - maxvalue: 273. - action: - name: reject - # - # Reject all ASCAT (Type 290) winds with isflg /= 0 (non-water surface) - # - - filter: Perform Action - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 290 - - variable: GeoVaLs/water_area_fraction - maxvalue: 0.001 - action: - name: reject - # Reject ASCAT (Type 290) when observed component deviates from background by more than 5.0 m/s - # NOTE: This check can reject a u- or v-component of the same observation independently, which - # is fundamentally different from how GSI rejects obs (both components are rejected if - # either component fails a check). - - filter: Background Check - filter variables: - - name: windEastward - - name: windNorthward - threshold: 5. - absolute threshold: 5. - where: - - variable: ObsType/windEastward - is_in: 290 - action: - name: reject - # Reject OSCAT (Type 291) when observed component deviates from background by more than 6.0 m/s - # NOTE: This check can reject a u- or v-component of the same observation independently, which - # is fundamentally different from how GSI rejects obs (both components are rejected if - # either component fails a check). - - filter: Background Check - filter variables: - - name: windEastward - - name: windNorthward - threshold: 6. - absolute threshold: 6. - where: - - variable: ObsType/windEastward - is_in: 291 - action: - name: reject - # Reject ASCAT (Type 290) when ambiguity check fails (returned value is negative) - - filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - where: - - variable: ObsType/windEastward - is_in: 290 - test variables: - - name: ObsFunction/ScatWindsAmbiguityCheck - options: - test_hofx: GsiHofX - minimum_uv: 0.0001 # hard-coding a minimum-uv for transparancy, want this to basically be zero - maxvalue: 0. - action: - name: reject -# - # All scatwinds must adjust errors based on ObsErrorFactorPressureCheck. - # This check will inflate errors for obs that are too close to either - # the model top or bottom. - - filter: Perform Action - filter variables: - - name: windEastward - where: - - variable: - name: ObsType/windEastward - is_in: 290-291 - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorPressureCheck - options: - variable: windEastward - inflation factor: 4.0 -# - - filter: Perform Action - filter variables: - - name: windNorthward - where: - - variable: - name: ObsType/windNorthward - is_in: 290-291 - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorPressureCheck - options: - variable: windNorthward - inflation factor: 4.0 - # All scatwinds subject to a gross error check. This is contained within - # the WindsSPDBCheck, although it is not exclusive to satwinds. - - filter: Background Check - filter variables: - - name: windEastward - function absolute threshold: - - name: ObsFunction/WindsSPDBCheck +# Reject all ASCAT (Type 290) winds with tsavg <= 273.0 (surface temperature) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 290 +# - variable: GeoVaLs/surface_temperature + - variable: GeoVaLs/surface_temperature_where_land + maxvalue: 273. + action: + name: reject + +# Reject all ASCAT (Type 290) winds with isflg /= 0 (non-water surface) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 290 + - variable: GeoVaLs/water_area_fraction + maxvalue: 0.99 +# - variable: GeoVaLs/surface_type +# minvalue: 0.001 + action: + name: reject +#passedBenchmark: 53558 # 2 variables (u,v), u=26779 passing, v=26779 passing + +# Reject ASCAT (Type 290) when observed component deviates from background by more than 5.0 m/s +# NOTE: This check can reject a u- or v-component of the same observation independently, which +# is fundamentally different from how GSI rejects obs (both components are rejected if +# either component fails a check). +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Arithmetic + options: + variables: + - name: ObsValue/windEastward + - name: HofX/windEastward + coefs: [1.0, -1.0] + minvalue: -5.0 + maxvalue: 5.0 + +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Arithmetic + options: + variables: + - name: ObsValue/windNorthward + - name: HofX/windNorthward + coefs: [1.0, -1.0] + minvalue: -5.0 + maxvalue: 5.0 +#passedBenchmark: 52362 # 2 variables (u,v), u=26181 passing, v=26181 passing + +# Reject OSCAT (Type 291) when observed component deviates from background by more than 6.0 m/s +# NOTE: This check can reject a u- or v-component of the same observation independently, which +# is fundamentally different from how GSI rejects obs (both components are rejected if +# either component fails a check). +- filter: Background Check + filter variables: + - name: windEastward + - name: windNorthward + threshold: 6. + absolute threshold: 6. + where: + - variable: ObsType/windEastward + is_in: 291 + action: + name: reject + +# Reject ASCAT (Type 290) when ambiguity check fails (returned value is negative) +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 290 + test variables: + - name: ObsFunction/ScatWindsAmbiguityCheck + options: +# test_hofx: GsiHofX + minimum_uv: 0.0001 # hard-coding a minimum-uv for transparancy, want this to basically be zero + maxvalue: 0. + action: + name: reject +#passedBenchmark: 51776 # 2 variables (u,v), u=25888 passing, v=25888 passing + +# All scatwinds must adjust errors based on ObsErrorFactorPressureCheck. +# This check will inflate errors for obs that are too close to either +# the model top or bottom. +- filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 290-291 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck options: - wndtype: [ 290, 291] - cgross: [ 5.0, 5.0] - error_min: [1.4, 1.4] - error_max: [6.1, 6.1] variable: windEastward - action: - name: reject - - - filter: Background Check - filter variables: - - name: windNorthward - function absolute threshold: - - name: ObsFunction/WindsSPDBCheck + inflation factor: 4.0 + +- filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 290-291 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck options: - wndtype: [ 290, 291] - cgross: [ 5.0, 5.0] - error_min: [1.4, 1.4] - error_max: [6.1, 6.1] variable: windNorthward - action: - name: reject - # The last error inflation check is for duplicate observations. This one needs - # to come last, because we don't want to inflate errors for duplication if one - # of the duplicates should be rejected. - - filter: RejectList - filter variables: - - name: windEastward - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorDuplicateCheck - options: - use_air_pressure: true - variable: windEastward - - filter: RejectList - filter variables: - - name: windNorthward - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorDuplicateCheck - options: - use_air_pressure: true - variable: windNorthward - # There is no across-the-board inflation for nvqc=.true. for scatwinds, presumably because for - # this inflation to take place both nvqc must be .true. AND ibeta must be >0, see: - # https://github.com/NOAA-EMC/GSI/blob/14ae595af1b03471287d322596d35c0665336e95/src/gsi/setupw.f90#L1229 - # GSI settings must have ibeta>0 for satwinds, but not for scatwinds. - # - # - # If the ibeta settings for scatwinds were to change while nvqc remained .true., we would extend YAML to - # an additional filter that inflates final ob-errors across-the-board by 1/0.8 = 1.25. NOTE: the nvqc setting - # is defaulted to .false. in GSI code, but is overridden in global operational configuration. See: - # configuration, see: https://github.com/NOAA-EMC/global-workflow/blob/d5ae3328fa4041b177357b1133f6b92e81c859d7/scripts/exglobal_atmos_analysis.sh#L750 - # This setting activates Line 1229 of setupw.f90 to scale ratio_errors by 0.8, which is applied in - # the denominator of the final ob-error, so 1/0.8 = 1.25 factor of ob-error inflation. - # - # If this functionality were to be activated for scatwinds, you would want to include this last inflation filter. -# - filter: Perform Action -# filter variables: -# - name: windEastward -# where: -# - variable: ObsType/windEastward -# is_in: 290-291 -# action: -# name: inflate error -# inflation factor: 1.25 -# - filter: Perform Action -# filter variables: -# - name: windNorthward -# where: -# - variable: ObsType/windNorthward -# is_in: 290-291 -# action: -# name: inflate error -# inflation factor: 1.25 -linear obs operator: - name: Identity -#passedBenchmark: 52097 # 2 variables (u,v), u=26087 passing, v=26010 passing -passedBenchmark: 52109 # 2 variables (u,v), u=26093 passing, v=26016 passing + inflation factor: 4.0 +#passedBenchmark: 51776 # 2 variables (u,v), u=25888 passing, v=25888 passing + +# All scatwinds subject to a gross error check. This is contained within +# the WindsSPDBCheck, although it is not exclusive to satwinds. +- filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 290, 291] + cgross: [ 5.0, 5.0] + error_min: [1.4, 1.4] + error_max: [6.1, 6.1] + variable: windEastward + action: + name: reject + +- filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 290, 291] + cgross: [ 5.0, 5.0] + error_min: [1.4, 1.4] + error_max: [6.1, 6.1] + variable: windNorthward + action: + name: reject +#passedBenchmark: 51776 # 2 variables (u,v), u=25888 passing, v=25888 passing + +# The last error inflation check is for duplicate observations. This one needs +# to come last, because we don't want to inflate errors for duplication if one +# of the duplicates should be rejected. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windEastward + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windNorthward + +# There is no across-the-board inflation for nvqc=.true. for scatwinds, presumably because for +# this inflation to take place both nvqc must be .true. AND ibeta must be >0, see: +# https://github.com/NOAA-EMC/GSI/blob/14ae595af1b03471287d322596d35c0665336e95/src/gsi/setupw.f90#L1229 +# GSI settings must have ibeta>0 for satwinds, but not for scatwinds. +# +# If the ibeta settings for scatwinds were to change while nvqc remained .true., we would extend YAML to +# an additional filter that inflates final ob-errors across-the-board by 1/0.8 = 1.25. NOTE: the nvqc setting +# is defaulted to .false. in GSI code, but is overridden in global operational configuration. See: +# configuration, see: https://github.com/NOAA-EMC/global-workflow/blob/d5ae3328fa4041b177357b1133f6b92e81c859d7/scripts/exglobal_atmos_analysis.sh#L750 +# This setting activates Line 1229 of setupw.f90 to scale ratio_errors by 0.8, which is applied in +# the denominator of the final ob-error, so 1/0.8 = 1.25 factor of ob-error inflation. +# +# If this functionality were to be activated for scatwinds, you would want to include this last inflation filter. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# where: +# - variable: ObsType/windEastward +# is_in: 290-291 +# action: +# name: inflate error +# inflation factor: 1.25 +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# where: +# - variable: ObsType/windNorthward +# is_in: 290-291 +# action: +# name: inflate error +# inflation factor: 1.25 +passedBenchmark: 51764 # 2 variables (u,v), u=25882 passing, v=25882 passing # GSI rejects both u- and v-component in first-guess check, UFO does not, but when UFO rej is reconciled btwn u and v these match GSI rej # u: 207 obs pass with corresponding v being rejected, 25880 obs in UFO/GSI agreement # v: 130 obs pass with corresponding u being rejected, 25880 obs in UFO/GSI agreement diff --git a/parm/atm/obs/testing/scatwind_noqc.yaml b/parm/atm/obs/testing/scatwind_noqc.yaml index f8f36efeb..ec4aef452 100644 --- a/parm/atm/obs/testing/scatwind_noqc.yaml +++ b/parm/atm/obs/testing/scatwind_noqc.yaml @@ -9,13 +9,42 @@ obs space: type: H5File obsfile: !ENV scatwind_diag_${CDATE}.nc4 simulated variables: [windEastward, windNorthward] + geovals: filename: !ENV scatwind_geoval_${CDATE}.nc4 -vector ref: GsiHofXBc -tolerance: 0.01 + obs operator: name: VertInterp - apply near surface wind scaling: true + # Use height vertical coordinate first +# vertical coordinate: geometric_height + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + +obs prior filters: +# Apply variable changes needed for rescaled height coordinate +- filter: Variable Transforms + Transform: AdjustedHeightCoordinate + SkipWhenNoObs: False + +# Apply variable changes needed for wind scaling +- filter: Variable Transforms + Transform: SurfaceWindScalingHeight + SkipWhenNoObs: False + +# Assign the initial observation error (constant value, 1.5 m/s) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 1.5 -vector ref: GsiHofXBc -tolerance: 1.e-5 +passedBenchmark: 53546 +#passedBenchmark: 53558 +#vector ref: GsiHofXBc +#tolerance: 1.e-5 From e609c1102db6450c6db03d5de0bc97c79d6530aa Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Wed, 1 Nov 2023 08:02:46 -0400 Subject: [PATCH 07/14] Add functionality to locally change the weights used for the hybrid B (#700) * patch ens. B * saving ens. mean * adjusted weights locations * tidy * uups --- parm/soca/berror/soca_ensb.yaml | 6 +++ parm/soca/berror/soca_ensweights.yaml | 10 +++++ test/soca/testinput/socahybridweights.yaml | 15 ++++++- utils/soca/gdas_ens_handler.h | 11 +++-- utils/soca/gdas_socahybridweights.h | 51 +++++++++++++++++++++- 5 files changed, 86 insertions(+), 7 deletions(-) diff --git a/parm/soca/berror/soca_ensb.yaml b/parm/soca/berror/soca_ensb.yaml index 814dfc7ac..447d5b5fd 100644 --- a/parm/soca/berror/soca_ensb.yaml +++ b/parm/soca/berror/soca_ensb.yaml @@ -30,6 +30,12 @@ steric height: linear variable changes: - linear variable change name: BalanceSOCA # Only the steric balance is applied +ensemble mean output: + datadir: ./static_ens + date: '{{ATM_WINDOW_BEGIN}}' + exp: ens_mean + type: incr + ssh output: unbalanced: datadir: ./static_ens diff --git a/parm/soca/berror/soca_ensweights.yaml b/parm/soca/berror/soca_ensweights.yaml index c7ea40a98..094377f95 100644 --- a/parm/soca/berror/soca_ensweights.yaml +++ b/parm/soca/berror/soca_ensweights.yaml @@ -19,6 +19,16 @@ weights: # Need to provide weights^2 when reading from file ice: 0.0025 # 5% of original variance ocean: 0.0625 # 25% " " + # Apply localized weights to the ocean ens. B + ocean local weights: + - lon: -172.0 + lat: 11.0 + amplitude: -1.0 + length scale: 700.0 + - lon: -160.0 + lat: 12.0 + amplitude: -1.0 + length scale: 700.0 output: datadir: ./ diff --git a/test/soca/testinput/socahybridweights.yaml b/test/soca/testinput/socahybridweights.yaml index cdad1ab27..855dd6d60 100644 --- a/test/soca/testinput/socahybridweights.yaml +++ b/test/soca/testinput/socahybridweights.yaml @@ -16,8 +16,19 @@ background: read_from_file: 1 weights: - ice: 0.1 - ocean: 0.5 + # Need to provide weights^2 when reading from file + ice: 0.0025 # 5% of original variance + ocean: 0.0625 # 25% " " + # Apply localized weights to the ocean ens. B + ocean local weights: + - lon: -172.0 + lat: 11.0 + amplitude: -1.0 + length scale: 700.0 + - lon: -160.0 + lat: 12.0 + amplitude: -1.0 + length scale: 700.0 output: datadir: ./ diff --git a/utils/soca/gdas_ens_handler.h b/utils/soca/gdas_ens_handler.h index 44e994a0b..5cf862651 100644 --- a/utils/soca/gdas_ens_handler.h +++ b/utils/soca/gdas_ens_handler.h @@ -104,6 +104,10 @@ namespace gdasapp { gdasapp_ens_utils::ensMoments(ensMembers, ensMean, ensStd, ensVariance); oops::Log::info() << "mean: " << ensMean << std::endl; oops::Log::info() << "std: " << ensStd << std::endl; + if ( fullConfig.has("ensemble mean output") ) { + const eckit::LocalConfiguration ensMeanOutputConfig(fullConfig, "ensemble mean output"); + ensMean.write(ensMeanOutputConfig); + } // Remove mean from ensemble members for (size_t i = 0; i < postProcIncr.ensSize_; ++i) { @@ -115,6 +119,7 @@ namespace gdasapp { eckit::LocalConfiguration stericVarChangeConfig; fullConfig.get("steric height", stericVarChangeConfig); oops::Log::info() << "steric config 0000: " << stericVarChangeConfig << std::endl; + // Initialize trajectories const eckit::LocalConfiguration trajConfig(fullConfig, "trajectory"); soca::State cycleTraj(geom, trajConfig); // trajectory of the cycle @@ -148,7 +153,7 @@ namespace gdasapp { soca::Increment incr = postProcIncr.appendLayer(ensMembers[i]); // Save total ssh - oops::Log::info() << "ssh ensemble memnber " << i << std::endl; + oops::Log::info() << "ssh ensemble member " << i << std::endl; soca::Increment ssh_tmp(geom, socaSshVar, postProcIncr.dt_); ssh_tmp = ensMembers[i]; sshTotal.push_back(ssh_tmp); @@ -186,14 +191,14 @@ namespace gdasapp { // Add the unbalanced ssh to the recentered perturbation // this assumes ssh_u is independent of the trajectory - oops::Log::info() << "&&&&& before adding ssh_u " << incr << std::endl; + oops::Log::debug() << "&&&&& before adding ssh_u " << incr << std::endl; atlas::FieldSet incrFs; incr.toFieldSet(incrFs); atlas::FieldSet sshNonStericFs; sshNonSteric[i].toFieldSet(sshNonStericFs); util::addFieldSets(incrFs, sshNonStericFs); incr.fromFieldSet(incrFs); - oops::Log::info() << "&&&&& after adding ssh_u " << incr << std::endl; + oops::Log::debug() << "&&&&& after adding ssh_u " << incr << std::endl; // Save final perturbation, used in the offline EnVAR result = postProcIncr.save(incr, i+1); diff --git a/utils/soca/gdas_socahybridweights.h b/utils/soca/gdas_socahybridweights.h index adf20188a..219450a8e 100644 --- a/utils/soca/gdas_socahybridweights.h +++ b/utils/soca/gdas_socahybridweights.h @@ -3,12 +3,17 @@ #include #include #include +#include #include "eckit/config/LocalConfiguration.h" #include "atlas/field.h" +#include "atlas/util/Earth.h" +#include "atlas/util/Geometry.h" +#include "atlas/util/Point.h" #include "oops/base/PostProcessor.h" +#include "oops/generic/gc99.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" #include "oops/util/DateTime.h" @@ -20,6 +25,38 @@ #include "soca/State/State.h" namespace gdasapp { + // Create a simple mask based on a Gaussian function + void gaussianMask(const soca::Geometry & geom, soca::Increment & gaussIncr, + eckit::LocalConfiguration conf) { + // Get the 2D grid + std::vector lats; + std::vector lons; + bool halo = true; + geom.latlon(lats, lons, halo); + + // Prepare fieldset from increment + atlas::FieldSet gaussIncrFs; + gaussIncr.toFieldSet(gaussIncrFs); + + // Get the GC99 parameters from config + double amp = conf.getDouble("amplitude"); + double scale = conf.getDouble("length scale"); + const atlas::PointLonLat p0(conf.getDouble("lon"), conf.getDouble("lat")); + + // Recompute weights + for (auto & field : gaussIncrFs) { + oops::Log::info() << "---------- Field name: " << field.name() << std::endl; + auto view = atlas::array::make_view(field); + for (int jnode = 0; jnode < field.shape(0); ++jnode) { + atlas::PointLonLat p1(lons[jnode], lats[jnode]); + double d = atlas::util::Earth::distance(p0, p1)/1000.0; + for (int jlevel = 0; jlevel < field.shape(1); ++jlevel) { + view(jnode, jlevel) += amp * oops::gc99(d/scale); + } + } + } + gaussIncr.fromFieldSet(gaussIncrFs); + } class SocaHybridWeights : public oops::Application { public: @@ -45,8 +82,7 @@ namespace gdasapp { socaVars += socaOcnVars; /// Read the background - // TODO(guillaume): Use the ice extent to set the weights ... no clue if this is - // possible at this level + // TODO(guillaume): Use the ice extent to set the weights soca::State socaBkg(geom, socaVars, dt); const eckit::LocalConfiguration socaBkgConfig(fullConfig, "background"); socaBkg.read(socaBkgConfig); @@ -70,6 +106,17 @@ namespace gdasapp { /// Create fields of weights for the ocean soca::Increment socaOcnHW(geom, socaOcnVars, dt); socaOcnHW.ones(); + + /// Apply localized gaussians to the weights + eckit::LocalConfiguration localWeightsConfigs(fullConfig, "weights.ocean local weights"); + std::vector localWeightsList = + localWeightsConfigs.getSubConfigurations(); + for (auto & conf : localWeightsList) { + gaussianMask(geom, socaOcnHW, conf); + oops::Log::info() << "Local weights for socaOcnHW: " << std::endl << conf << std::endl; + oops::Log::info() << socaOcnHW << std::endl; + } + socaOcnHW *= wOcean; oops::Log::info() << "socaOcnHW: " << std::endl << socaOcnHW << std::endl; socaOcnHW.write(socaHWOutConfig); From 00201eff3e7491f79ae764a36c9a368f5e1e9885 Mon Sep 17 00:00:00 2001 From: NicholasEsposito-NOAA <62616739+nicholasesposito@users.noreply.github.com> Date: Thu, 2 Nov 2023 12:45:05 -0400 Subject: [PATCH 08/14] Feature/nick e adpsfc prepbufr (#702) * working sfcshp * initial changes. no py add yet * add adpsfc py * some updates * update README * get rid of sys.appendpath * finally works again with json * coding norms hopefully pass now * more coding norms * more coding norms * one line wont work * logger * few logger updates * ok now should be 100 * change logger filename line * print -> loggers * Update bufr2ioda_adpsfc_prepbufr.py heightOfStation, stationElevation, var order, obstype * zobqm added * update zob elv * zob elv confusion * Oba -> Obs * readme change * rm readme, add to run_bufr2ioda.py * rm whitespace * rm more whitespace * rm run_bufr2ioda_adpsfc.sh - not needed * NESDIS removed. logger -> debug --------- Co-authored-by: Nicholas Esposito Co-authored-by: Nicholas Esposito Co-authored-by: Nicholas Esposito Co-authored-by: Nicholas Esposito --- .../bufr2ioda/bufr2ioda_adpsfc_prepbufr.json | 12 + .../bufr2ioda/bufr2ioda_adpsfc_prepbufr.py | 347 ++++++++++++++++++ ush/ioda/bufr2ioda/gen_bufr2ioda_json.py | 8 +- ush/ioda/bufr2ioda/run_bufr2ioda.py | 2 +- 4 files changed, 367 insertions(+), 2 deletions(-) create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.json create mode 100755 ush/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.py diff --git a/parm/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.json b/parm/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.json new file mode 100644 index 000000000..bdde19135 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.json @@ -0,0 +1,12 @@ +{ + "data_format" : "prepbufr", + "subsets" : [ "ADPSFC" ], + "source" : "prepBUFR", + "data_type" : "ADPSFC", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "data_description" : "ADPSFC_prepbufr", + "data_provider" : "U.S. NOAA" +} diff --git a/ush/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.py b/ush/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.py new file mode 100755 index 000000000..a5700574a --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.py @@ -0,0 +1,347 @@ +#!/usr/bin/env python3 +# (C) Copyright 2023 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import sys +import numpy as np +import os +import argparse +import math +import calendar +import time +from datetime import datetime +import json +from pyiodaconv import bufr +from collections import namedtuple +from pyioda import ioda_obs_space as ioda_ospace +from wxflow import Logger + + +def Compute_dateTime(cycleTimeSinceEpoch, dhr): + + dhr = np.int64(dhr*3600) + dateTime = dhr + cycleTimeSinceEpoch + + return dateTime + + +def bufr_to_ioda(config, logger): + + subsets = config["subsets"] + logger.debug(f"Checking subsets = {subsets}") + + # Get parameters from configuration + data_format = config["data_format"] + source = config["source"] + data_type = config["data_type"] + data_description = config["data_description"] + data_provider = config["data_provider"] + cycle_type = config["cycle_type"] + cycle_datetime = config["cycle_datetime"] + dump_dir = config["dump_directory"] + ioda_dir = config["ioda_directory"] + cycle = config["cycle_datetime"] + + # Get derived parameters + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + + reference_time = datetime.strptime(cycle, "%Y%m%d%H") + reference_time = reference_time.strftime("%Y-%m-%dT%H:%M:%SZ") + reference_time_full = f"{yyyymmdd}{hh}00" + + logger.debug(f"reference_time = {reference_time}") + + # General informaton + converter = 'BUFR to IODA Converter' + platform_description = 'SFCSHP data from prepBUFR format' + + bufrfile = f"{cycle_type}.t{hh}z.{data_format}" + DATA_PATH = os.path.join(dump_dir, f"{cycle_type}.{yyyymmdd}", + str(hh), bufrfile) + + logger.debug(f"The DATA_PATH is: {DATA_PATH}") + + # ============================================ + # Make the QuerySet for all the data we want + # ============================================ + start_time = time.time() + + logger.debug('Making QuerySet ...') + q = bufr.QuerySet(subsets) + + # ObsType + q.add('observationType', '*/TYP') + + # MetaData + q.add('stationIdentification', '*/SID') + q.add('latitude', '*/YOB') + q.add('longitude', '*/XOB') + q.add('obsTimeMinusCycleTime', '*/DHR') + q.add('heightOfStation', '*/Z___INFO/Z__EVENT{1}/ZOB') + q.add('pressure', '*/P___INFO/P__EVENT{1}/POB') + +# # Quality Infomation (Quality Indicator) + q.add('qualityMarkerStationPressure', '*/P___INFO/P__EVENT{1}/PQM') + q.add('qualityMarkerStationElevation', '*/Z___INFO/Z__EVENT{1}/ZQM') + + # ObsValue + q.add('stationPressure', '*/P___INFO/P__EVENT{1}/POB') + q.add('stationElevation', '*/ELV') + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for making QuerySet: {running_time} seconds") + + # ============================================================== + # Open the BUFR file and execute the QuerySet to get ResultSet + # Use the ResultSet returned to get numpy arrays of the data + # ============================================================== + start_time = time.time() + + logger.debug(f"Executing QuerySet to get ResultSet ...") + with bufr.File(DATA_PATH) as f: + r = f.execute(q) + + logger.debug(" ... Executing QuerySet: get ObsType ...") + # ObsType + typ = r.get('observationType') + + logger.debug(" ... Executing QuerySet: get MetaData ...") + # MetaData + sid = r.get('stationIdentification') + lat = r.get('latitude') + lon = r.get('longitude') + lon[lon > 180] -= 360 + zob = r.get('heightOfStation', type='float') + pressure = r.get('pressure') + pressure *= 100 + + logger.debug(f" ... Executing QuerySet: get QualityMarker information ...") + # Quality Information + pobqm = r.get('qualityMarkerStationPressure') + zobqm = r.get('qualityMarkerStationElevation') + + logger.debug(f" ... Executing QuerySet: get ObsValue ...") + # ObsValue + elv = r.get('stationElevation', type='float') + pob = r.get('stationPressure') + pob *= 100 + + logger.debug(f" ... Executing QuerySet: get dateTime ...") + # DateTime: seconds since Epoch time + # IODA has no support for numpy datetime arrays dtype=datetime64[s] + dhr = r.get('obsTimeMinusCycleTime', type='int64') + + logger.debug(f" ... Executing QuerySet: Done!") + + logger.debug(f" ... Executing QuerySet: Check BUFR variable generic \ + dimension and type ...") + # Check BUFR variable generic dimension and type + logger.debug(f" typ shape = {typ.shape}") + logger.debug(f" sid shape = {sid.shape}") + logger.debug(f" dhr shape = {dhr.shape}") + logger.debug(f" lat shape = {lat.shape}") + logger.debug(f" lon shape = {lon.shape}") + logger.debug(f" zob shape = {zob.shape}") + logger.debug(f" pressure shape = {pressure.shape}") + + logger.debug(f" pobqm shape = {pobqm.shape}") + logger.debug(f" zobqm shape = {zobqm.shape}") + + logger.debug(f" elv shape = {elv.shape}") + logger.debug(f" pob shape = {pob.shape}") + + logger.debug(f" dhr type = {dhr.shape}") + + logger.debug(f" sid type = {sid.dtype}") + logger.debug(f" dhr type = {dhr.dtype}") + logger.debug(f" lat type = {lat.dtype}") + logger.debug(f" lon type = {lon.dtype}") + logger.debug(f" zob type = {zob.dtype}") + logger.debug(f" typ type = {typ.dtype}") + logger.debug(f" pressure type = {pressure.dtype}") + + logger.debug(f" pobqm type = {pobqm.dtype}") + logger.debug(f" zobqm type = {zobqm.dtype}") + + logger.debug(f" elv type = {elv.dtype}") + logger.debug(f" pob type = {pob.dtype}") + + logger.debug(f" dhr type = {dhr.dtype}") + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for executing QuerySet to get ResultSet: \ + {running_time} seconds") + + # ========================= + # Create derived variables + # ========================= + start_time = time.time() + + logger.debug(f"Creating derived variables - dateTime ...") + + cycleTimeSinceEpoch = np.int64(calendar.timegm(time.strptime( + reference_time_full, '%Y%m%d%H%M'))) + dateTime = Compute_dateTime(cycleTimeSinceEpoch, dhr) + + logger.debug(f" Check derived variables type ... ") + logger.debug(f" dateTime shape = {dateTime.shape}") + logger.debug(f" dateTime type = {dateTime.dtype}") + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for creating derived variables: \ + {running_time} seconds") + + # ===================================== + # Create IODA ObsSpace + # Write IODA output + # ===================================== + + # Create the dimensions + dims = {'Location': np.arange(0, lat.shape[0])} + + iodafile = f"{cycle_type}.t{hh}z.{data_type}.{data_format}.nc" + OUTPUT_PATH = os.path.join(ioda_dir, iodafile) + logger.debug(f" ... ... Create OUTPUT file: {OUTPUT_PATH}") + + path, fname = os.path.split(OUTPUT_PATH) + if path and not os.path.exists(path): + os.makedirs(path) + + obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) + + # Create Global attributes + logger.debug(f" ... ... Create global attributes") + + obsspace.write_attr('Converter', converter) + obsspace.write_attr('source', source) + obsspace.write_attr('sourceFiles', bufrfile) + obsspace.write_attr('dataProviderOrigin', data_provider) + obsspace.write_attr('description', data_description) + obsspace.write_attr('datetimeReference', reference_time) + obsspace.write_attr('datetimeRange', + [str(min(dateTime)), str(max(dateTime))]) + obsspace.write_attr('platformLongDescription', platform_description) + + # Create IODA variables + logger.debug(f" ... ... Create variables: name, type, units, & attributes") + + # Observation Type - Station Elevation + obsspace.create_var('ObsType/stationElevation', dtype=typ.dtype, + fillval=typ.fill_value) \ + .write_attr('long_name', 'Station Elevation Observation Type') \ + .write_data(typ) + + # Observation Type - Station Pressure + obsspace.create_var('ObsType/stationPressure', dtype=typ.dtype, + fillval=typ.fill_value) \ + .write_attr('long_name', 'Station Pressure Observation Type') \ + .write_data(typ) + + # Longitude + obsspace.create_var('MetaData/longitude', dtype=lon.dtype, + fillval=lon.fill_value) \ + .write_attr('units', 'degrees_east') \ + .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ + .write_attr('long_name', 'Longitude') \ + .write_data(lon) + + # Latitude + obsspace.create_var('MetaData/latitude', dtype=lat.dtype, + fillval=lat.fill_value) \ + .write_attr('units', 'degrees_north') \ + .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Latitude') \ + .write_data(lat) + + # Datetime + obsspace.create_var('MetaData/dateTime', dtype=dateTime.dtype, + fillval=dateTime.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'Datetime') \ + .write_data(dateTime) + + # Station Identification + obsspace.create_var('MetaData/stationIdentification', dtype=sid.dtype, + fillval=sid.fill_value) \ + .write_attr('long_name', 'Station Identification') \ + .write_data(sid) + + # Height Of Station + obsspace.create_var('MetaData/heightOfStation', dtype=zob.dtype, + fillval=zob.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Height Of Station') \ + .write_data(zob) + + # Pressure + obsspace.create_var('MetaData/pressure', dtype=pressure.dtype, + fillval=pressure.fill_value) \ + .write_attr('units', 'Pa') \ + .write_attr('long_name', 'Pressure') \ + .write_data(pressure) + + # QualityMarker - Station Elevation + obsspace.create_var('QualityMarker/stationElevation', dtype=zobqm.dtype, + fillval=zobqm.fill_value) \ + .write_attr('long_name', 'Station Elevation Quality Marker') \ + .write_data(zobqm) + + # QualityMarker - Station Pressure + obsspace.create_var('QualityMarker/stationPressure', dtype=pobqm.dtype, + fillval=pobqm.fill_value) \ + .write_attr('long_name', 'Station Pressure Quality Marker') \ + .write_data(pobqm) + + # Station Elevation + obsspace.create_var('ObsValue/stationElevation', dtype=elv.dtype, + fillval=elv.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Station Elevation') \ + .write_data(elv) + + # Station Pressure + obsspace.create_var('ObsValue/stationPressure', dtype=pob.dtype, + fillval=pob.fill_value) \ + .write_attr('units', 'Pa') \ + .write_attr('long_name', 'Station Pressure') \ + .write_data(pob) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for splitting and output IODA: \ + {running_time} seconds") + + logger.debug("All Done!") + + +if __name__ == '__main__': + + start_time = time.time() + + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, + help='Input JSON configuration', + required=True) + parser.add_argument('-v', '--verbose', + help='print debug logging information', + action='store_true') + args = parser.parse_args() + + log_level = 'DEBUG' if args.verbose else 'INFO' + logger = Logger('bufr2ioda_adpsfc_prepbufr.py', level=log_level, + colored_log=True) + + with open(args.config, "r") as json_file: + config = json.load(json_file) + + bufr_to_ioda(config, logger) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Total running time: {running_time} seconds") diff --git a/ush/ioda/bufr2ioda/gen_bufr2ioda_json.py b/ush/ioda/bufr2ioda/gen_bufr2ioda_json.py index 617a8b024..0748e20f6 100755 --- a/ush/ioda/bufr2ioda/gen_bufr2ioda_json.py +++ b/ush/ioda/bufr2ioda/gen_bufr2ioda_json.py @@ -6,7 +6,8 @@ import argparse import json import os -from wxflow import Logger, parse_j2yaml +from wxflow import Logger, parse_j2yaml, cast_strdict_as_dtypedict +from wxflow import add_to_datetime, to_timedelta # Initialize root logger logger = Logger('gen_bufr2ioda_json.py', level='INFO', colored_log=True) @@ -28,4 +29,9 @@ def gen_bufr_json(config, template, output): parser.add_argument('-t', '--template', type=str, help='Input JSON template', required=True) parser.add_argument('-o', '--output', type=str, help='Output JSON file', required=True) args = parser.parse_args() + # get the config from your environment + config = cast_strdict_as_dtypedict(os.environ) + # we need to add in current cycle from PDYcyc + config['current_cycle'] = add_to_datetime(config['PDY'], to_timedelta(f"{config['cyc']}H")) + # call the parsing function gen_bufr_json(config, args.template, args.output) diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index d7a7bb130..63071d838 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -30,7 +30,7 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): } # Specify observation types to be processed by a script - BUFR_py = ["satwind_amv_goes", "satwind_scat", "adpupa_prepbufr"] + BUFR_py = ["satwind_amv_goes", "satwind_scat", "adpupa_prepbufr", "adpsfc_prepbufr"] for obtype in BUFR_py: logger.info(f"Convert {obtype}...") From 19701ac72b804e66f5eb66f7c916b0091ae61ddd Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Thu, 2 Nov 2023 15:58:26 -0400 Subject: [PATCH 09/14] Update conventional station pressure (conv_ps) for testing and end-to-end (#698) * Add a temporary hack of setting data window length for conv_ps. Due to error inflation for duplicate observations and the JEDI window length definition, the window length in the script has to be relaxed so that all observations from the 6-hour obs data will be accounted for. * Remove comments * Add assignment based on GSI error table, and update error inflation for pressure check. * Move filters two spacings left to line up with the filter header. * Add conv_ps.yaml under config for end-to-end test * Fix typo in comments * Fix typo * modify comments * Modify comments * Add and set window shift to true for the conventional difference in data window between GSI and JEDI. * Add logics to set window shift parameter * Add handling for tests with and without QC * Remove unwated comments * Remove debugging lines --------- Co-authored-by: Cory Martin --- parm/atm/obs/config/conv_ps.yaml | 295 ++++++++++++++++ parm/atm/obs/testing/conv_ps.yaml | 455 +++++++++++++++---------- parm/atm/obs/testing/conv_ps_noqc.yaml | 93 ++++- ush/ufoeval/run_ufo_hofx_test.sh | 4 +- ush/ufoeval/test_yamls.sh | 19 +- 5 files changed, 670 insertions(+), 196 deletions(-) create mode 100644 parm/atm/obs/config/conv_ps.yaml diff --git a/parm/atm/obs/config/conv_ps.yaml b/parm/atm/obs/config/conv_ps.yaml new file mode 100644 index 000000000..8fa4ab185 --- /dev/null +++ b/parm/atm/obs/config/conv_ps.yaml @@ -0,0 +1,295 @@ +obs space: + name: surface_ps + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)sondes.{{ current_cycle | to_YMDH }}.nc4 + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_sondes_{{ current_cycle | to_YMDH }}.nc4 + io pool: + max pool size: 1 + simulated variables: [stationPressure] + +obs operator: + name: SfcPCorrected + variables: + - name: stationPressure + da_psfc_scheme: GSI + station_altitude: height + geovar_sfc_geomz: surface_altitude + geovar_geomz: geopotential_height + +obs prior filters: +# Initial Error Assignments for SFC Observations +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [181] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [80000, 75000, 70000, 65000, 60000, 55000 ] + errors: [110, 120, 120, 120, 120, 1.0e+11] + +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [187] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [85000, 80000, 75000, 70000, 65000, 60000, 55000 ] + errors: [ 120, 140, 140, 140, 140, 140, 1.0e+11] + +# Initial Error Assignments for SFCSHIP Observations +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [180] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [60000, 55000 ] + errors: [ 130, 1.0e+11] + +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [183] + action: + name: assign error + error parameter: 1.0e+11 + +# Initial Error Assignments for Radiosonde +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [120] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [80000, 75000, 70000, 65000, 60000, 55000 ] + errors: [ 110, 120, 120, 120, 120, 1.0e+11] + +obs post filters: +# Observation range sanity check +- filter: Bounds Check + filter variables: + - name: stationPressure + minvalue: 37499.0 + maxvalue: 106999.0 + action: + name: reject + +# Reject all ObsType 183 +- filter: RejectList + where: + - variable: + name: ObsType/stationPressure + is_in: 183 + +# Reject surface pressure below 500 hPa +- filter: Bounds Check + filter variables: + - name: stationPressure + minvalue: 50000.00 + action: + name: reject + +- filter: RejectList + where: + - variable: + name: PreQC/stationPressure + is_in: 4-15 + +# Inflate obs error based on obs type +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: PreQC/stationPressure + is_in: 3, 7 + action: + name: inflate error + inflation factor: 1.2 + +# Calculate obs error inflation factors for duplicated observations at the same location +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorDuplicateCheck/stationPressure + type: float + function: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: false + variable: stationPressure + +# Reduce effective observation error based on obs type and subtype +# In this case: reduce effective obs error for buoy +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: + name: ObsType/stationPressure + is_in: 180 + - variable: + name: ObsSubType/stationPressure + is_in: 0 + action: + name: inflate error + inflation factor: 0.7 + +# Reduce original observation error based on obs type and subtype +# In this case: reduce original obs error for buoy +- filter: Variable Assignment + where: + - variable: + name: ObsType/stationPressure + is_in: 180 + - variable: + name: ObsSubType/stationPressure + is_in: 0 + assignments: + - name: ObsError/stationPressure + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsError/stationPressure + coefs: [0.7] + +# Calculate obs error inflation factors for large discrepancies between model and observations +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorSfcPressure/stationPressure + type: float + function: + name: ObsFunction/ObsErrorFactorSfcPressure + options: + geovar_sfc_geomz: surface_altitude + geovar_geomz: geopotential_height + station_altitude: height + +# Inflate surface pressure observation based on discrepancies between +# model and observations due to terrian +- filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSfcPressure/stationPressure + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/Innovation + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsValue/stationPressure + - name: HofX/stationPressure + coefs: [1, -1] + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBoundSfcPressure1 + type: float + function: + name: ObsFunction/ObsErrorBoundConventional + options: + obsvar: stationPressure + obserr_bound_min: 100 + obserr_bound_max: 300 + obserr_bound_factor: 5.0 + +- filter: Background Check + filter variables: + - name: stationPressure + where: + - variable: PreQC/stationPressure + is_not_in: 3 + function absolute threshold: + - name: DerivedMetaData/ObsErrorBoundSfcPressure1 + action: + name: reject + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBoundSfcPressure2 + type: float + function: + name: ObsFunction/ObsErrorBoundConventional + options: + obsvar: stationPressure + obserr_bound_min: 100 + obserr_bound_max: 300 + obserr_bound_factor: 3.5 + +- filter: Background Check + filter variables: + - name: stationPressure + where: + - variable: PreQC/stationPressure + is_in: 3 + function absolute threshold: + - name: DerivedMetaData/ObsErrorBoundSfcPressure2 + action: + name: reject + +# Inflate obs error based on duplicate check +- filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsErrorFactorDuplicateCheck/stationPressure + +# Reject data based on PreUseFlag (usage in GSI) +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: PreUseFlag/stationPressure + is_not_in: 0, 1 + action: + name: reject + +# End of Filters + diff --git a/parm/atm/obs/testing/conv_ps.yaml b/parm/atm/obs/testing/conv_ps.yaml index f43bc9f41..20a449a8a 100644 --- a/parm/atm/obs/testing/conv_ps.yaml +++ b/parm/atm/obs/testing/conv_ps.yaml @@ -8,8 +8,8 @@ obs space: engine: type: H5File obsfile: !ENV conv_ps_diag_${CDATE}.nc4 - overwrite: true simulated variables: [stationPressure] + geovals: filename: !ENV conv_ps_geoval_${CDATE}.nc4 @@ -22,191 +22,276 @@ obs operator: geovar_sfc_geomz: surface_altitude geovar_geomz: geopotential_height +obs prior filters: +# Initial Error Assignments for SFC Observations +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [181] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [80000, 75000, 70000, 65000, 60000, 55000 ] + errors: [110, 120, 120, 120, 120, 1.0e+11] + +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [187] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [85000, 80000, 75000, 70000, 65000, 60000, 55000 ] + errors: [ 120, 140, 140, 140, 140, 140, 1.0e+11] + +# Initial Error Assignments for SFCSHIP Observations +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [180] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [60000, 55000 ] + errors: [ 130, 1.0e+11] + +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [183] + action: + name: assign error + error parameter: 1.0e+11 + +# Initial Error Assignments for Radiosonde +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [120] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [80000, 75000, 70000, 65000, 60000, 55000 ] + errors: [ 110, 120, 120, 120, 120, 1.0e+11] + obs post filters: - # Observation range sanity check - - filter: Bounds Check - filter variables: - - name: stationPressure - minvalue: 37499.0 - maxvalue: 106999.0 - action: - name: reject - - # Reject all ObsType 183 - - filter: RejectList - where: - - variable: - name: ObsType/stationPressure - is_in: 183 - - # Reject surface pressure below 500 hPa - - filter: Bounds Check - filter variables: - - name: stationPressure - minvalue: 50000.00 - action: - name: reject - - - filter: RejectList - where: - - variable: - name: PreQC/stationPressure - is_in: 4-15 - - # Inflate obs error based on obs type - - filter: Perform Action - filter variables: - - name: stationPressure - where: - - variable: PreQC/stationPressure - is_in: 3, 7 - action: - name: inflate error - inflation factor: 1.2 - - # Calculate obs error inflation factors for duplicated observations at the same location - - filter: Variable Assignment - assignments: - - name: ObsErrorFactorDuplicateCheck/stationPressure - type: float - function: - name: ObsFunction/ObsErrorFactorDuplicateCheck - options: - use_air_pressure: false - variable: stationPressure - - # Reduce effective observation error based on obs type and subtype - # In this case: reduce effective obs error for buoy - - filter: Perform Action - filter variables: - - name: stationPressure - where: - - variable: - name: ObsType/stationPressure - is_in: 180 - - variable: - name: ObsSubType/stationPressure - is_in: 0 - action: - name: inflate error - inflation factor: 0.7 - - # Reduce original observation error based on obs type and subtype - # In this case: reduce original obs error for buoy - - filter: Variable Assignment - where: - - variable: - name: ObsType/stationPressure - is_in: 180 - - variable: - name: ObsSubType/stationPressure - is_in: 0 - assignments: - - name: ObsError/stationPressure - type: float - function: - name: ObsFunction/Arithmetic - options: - variables: - - name: ObsError/stationPressure - coefs: [0.7] - - # Calculate obs error inflation factors for large discrepancies between model and observations - - filter: Variable Assignment - assignments: - - name: ObsErrorFactorSfcPressure/stationPressure - type: float - function: - name: ObsFunction/ObsErrorFactorSfcPressure - options: - geovar_sfc_geomz: surface_altitude - - # Inflate surface pressure observation based on discrepancies between - # model and observations due to terrian - - filter: Perform Action - filter variables: - - name: stationPressure - action: - name: inflate error - inflation variable: - name: ObsErrorFactorSfcPressure/stationPressure - - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/Innovation - type: float - function: - name: ObsFunction/Arithmetic - options: - variables: - - name: ObsValue/stationPressure - - name: HofX/stationPressure - coefs: [1, -1] - - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ObsErrorBoundSfcPressure1 - type: float - function: - name: ObsFunction/ObsErrorBoundConventional - options: - obsvar: stationPressure - obserr_bound_min: 100 - obserr_bound_max: 300 - obserr_bound_factor: 5.0 - - - filter: Background Check - filter variables: - - name: stationPressure - where: - - variable: PreQC/stationPressure - is_not_in: 3 - function absolute threshold: - - name: DerivedMetaData/ObsErrorBoundSfcPressure1 - action: - name: reject - - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ObsErrorBoundSfcPressure2 - type: float - function: - name: ObsFunction/ObsErrorBoundConventional - options: - obsvar: stationPressure - obserr_bound_min: 100 - obserr_bound_max: 300 - obserr_bound_factor: 3.5 - - - filter: Background Check - filter variables: - - name: stationPressure - where: - - variable: PreQC/stationPressure - is_in: 3 - function absolute threshold: - - name: DerivedMetaData/ObsErrorBoundSfcPressure2 - action: - name: reject - - # Inflate obs error based on duplicate check - - filter: Perform Action - filter variables: - - name: stationPressure - action: - name: inflate error - inflation variable: - name: ObsErrorFactorDuplicateCheck/stationPressure - - # Reject data based on PreUseFlag (usage in GSI) - - filter: Perform Action - filter variables: - - name: stationPressure - where: - - variable: PreUseFlag/stationPressure - is_not_in: 0, 1 - action: - name: reject +# Observation range sanity check +- filter: Bounds Check + filter variables: + - name: stationPressure + minvalue: 37499.0 + maxvalue: 106999.0 + action: + name: reject + +# Reject all ObsType 183 +- filter: RejectList + where: + - variable: + name: ObsType/stationPressure + is_in: 183 + +# Reject surface pressure below 500 hPa +- filter: Bounds Check + filter variables: + - name: stationPressure + minvalue: 50000.00 + action: + name: reject + +- filter: RejectList + where: + - variable: + name: PreQC/stationPressure + is_in: 4-15 + +# Inflate obs error based on obs type +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: PreQC/stationPressure + is_in: 3, 7 + action: + name: inflate error + inflation factor: 1.2 + +# Calculate obs error inflation factors for duplicated observations at the same location +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorDuplicateCheck/stationPressure + type: float + function: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: false + variable: stationPressure + +# Reduce effective observation error based on obs type and subtype +# In this case: reduce effective obs error for buoy +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: + name: ObsType/stationPressure + is_in: 180 + - variable: + name: ObsSubType/stationPressure + is_in: 0 + action: + name: inflate error + inflation factor: 0.7 + +# Reduce original observation error based on obs type and subtype +# In this case: reduce original obs error for buoy +- filter: Variable Assignment + where: + - variable: + name: ObsType/stationPressure + is_in: 180 + - variable: + name: ObsSubType/stationPressure + is_in: 0 + assignments: + - name: ObsError/stationPressure + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsError/stationPressure + coefs: [0.7] + +# Calculate obs error inflation factors for large discrepancies between model and observations +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorSfcPressure/stationPressure + type: float + function: + name: ObsFunction/ObsErrorFactorSfcPressure + options: + geovar_sfc_geomz: surface_altitude + geovar_geomz: geopotential_height + station_altitude: height + +# Inflate surface pressure observation based on discrepancies between +# model and observations due to terrian +- filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSfcPressure/stationPressure + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/Innovation + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsValue/stationPressure + - name: HofX/stationPressure + coefs: [1, -1] + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBoundSfcPressure1 + type: float + function: + name: ObsFunction/ObsErrorBoundConventional + options: + obsvar: stationPressure + obserr_bound_min: 100 + obserr_bound_max: 300 + obserr_bound_factor: 5.0 + +- filter: Background Check + filter variables: + - name: stationPressure + where: + - variable: PreQC/stationPressure + is_not_in: 3 + function absolute threshold: + - name: DerivedMetaData/ObsErrorBoundSfcPressure1 + action: + name: reject + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBoundSfcPressure2 + type: float + function: + name: ObsFunction/ObsErrorBoundConventional + options: + obsvar: stationPressure + obserr_bound_min: 100 + obserr_bound_max: 300 + obserr_bound_factor: 3.5 + +- filter: Background Check + filter variables: + - name: stationPressure + where: + - variable: PreQC/stationPressure + is_in: 3 + function absolute threshold: + - name: DerivedMetaData/ObsErrorBoundSfcPressure2 + action: + name: reject + +# Inflate obs error based on duplicate check +- filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsErrorFactorDuplicateCheck/stationPressure + +# Reject data based on PreUseFlag (usage in GSI) +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: PreUseFlag/stationPressure + is_not_in: 0, 1 + action: + name: reject +# End of Filters passedBenchmark: 85378 diff --git a/parm/atm/obs/testing/conv_ps_noqc.yaml b/parm/atm/obs/testing/conv_ps_noqc.yaml index 44a6ef4c8..8d469729f 100644 --- a/parm/atm/obs/testing/conv_ps_noqc.yaml +++ b/parm/atm/obs/testing/conv_ps_noqc.yaml @@ -8,8 +8,8 @@ obs space: engine: type: H5File obsfile: !ENV conv_ps_diag_${CDATE}.nc4 - overwrite: true simulated variables: [stationPressure] + geovals: filename: !ENV conv_ps_geoval_${CDATE}.nc4 @@ -22,10 +22,89 @@ obs operator: geovar_sfc_geomz: surface_altitude geovar_geomz: geopotential_height -vector ref: GsiHofXBc -tolerance: 1.e-4 -#linear obs operator test: -# coef TL: 0.1 -# tolerance TL: 1.0e-13 -# tolerance AD: 1.0e-11 +obs prior filters: +# Initial Error Assignments for SFC Observations +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [181] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [80000, 75000, 70000, 65000, 60000, 55000 ] + errors: [110, 120, 120, 120, 120, 1.0e+11] + +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [187] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [85000, 80000, 75000, 70000, 65000, 60000, 55000 ] + errors: [ 120, 140, 140, 140, 140, 140, 1.0e+11] + +# Initial Error Assignments for SFCSHIP Observations +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [180] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [60000, 55000 ] + errors: [ 130, 1.0e+11] + +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [183] + action: + name: assign error + error parameter: 1.0e+11 + + # Initial Error Assignments for Radiosonde +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [120] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [80000, 75000, 70000, 65000, 60000, 55000 ] + errors: [ 110, 120, 120, 120, 120, 1.0e+11] + +passedBenchmark: 92824 # total: 92842; missing: 18 +#vector ref: GsiHofXBc +#tolerance: 1.e-4 diff --git a/ush/ufoeval/run_ufo_hofx_test.sh b/ush/ufoeval/run_ufo_hofx_test.sh index ccc043bd4..230966284 100755 --- a/ush/ufoeval/run_ufo_hofx_test.sh +++ b/ush/ufoeval/run_ufo_hofx_test.sh @@ -33,6 +33,7 @@ run_filtering=YES run_eva=YES eva_stats_only=NO keep_output=NO +window_shift=False while getopts "c:hsxq" opt; do case $opt in @@ -99,7 +100,6 @@ if [ $run_filtering == NO ]; then else yamlpath=$GDASApp/parm/atm/obs/testing/${obtype}.yaml fi - exename=test_ObsFilters.x #-------------- Do not modify below this line ---------------- @@ -178,11 +178,13 @@ export OPREFIX=gdas.t${cyc}z export APREFIX=gdas.t${cyc}z export GPREFIX=gdas.t${gcyc}z +if [ $obtype == "conv_ps" ]; then window_shift=True; fi cat > $workdir/temp.yaml << EOF window begin: '{{ WINDOW_BEGIN | to_isotime }}' window end: '{{ WINDOW_END | to_isotime }}' observations: - !INC $yamlpath +window shift: ${window_shift} EOF $GDASApp/ush/genYAML --input $workdir/temp.yaml --output $workdir/${obtype}_${cycle}.yaml diff --git a/ush/ufoeval/test_yamls.sh b/ush/ufoeval/test_yamls.sh index 7e98d80e2..d46056bd8 100755 --- a/ush/ufoeval/test_yamls.sh +++ b/ush/ufoeval/test_yamls.sh @@ -13,10 +13,23 @@ if [ $? -ne 0 ]; then exit 1 fi -for file in `ls ../../parm/atm/obs/testing/*yaml`; do +# Process tests wiht QC +for file in `find ../../parm/atm/obs/testing/*.yaml -type f -not -name "*noqc*"`; do basefile=${file##*/} - inst="${basefile%.*}" - ./run_ufo_hofx_test.sh -x $inst > $WORKDIR/$inst.log 2> $WORKDIR/$inst.err + obtype="${basefile%.*}" + ./run_ufo_hofx_test.sh -x $obtype > $WORKDIR/$obtype.log 2> $WORKDIR/$obtype.err + if [ $? == 0 ]; then + echo $basefile Passes \(yay!\) + else + echo $basefile Fails \(boo!\) + fi +done + +# Process tests without QC (HofX + Observation error assignment) +for file in `ls ../../parm/atm/obs/testing/*_noqc.yaml`; do + basefile=${file##*/} + obtype="${basefile%_noqc.*}" + ./run_ufo_hofx_test.sh -x -q $obtype > $WORKDIR/${obtype}_noqc.log 2> $WORKDIR/${obtype}_noqc.err if [ $? == 0 ]; then echo $basefile Passes \(yay!\) else From a0616fdac531d778ef6d90bbe538bd43b3b36b0c Mon Sep 17 00:00:00 2001 From: NicholasEsposito-NOAA <62616739+nicholasesposito@users.noreply.github.com> Date: Thu, 2 Nov 2023 16:43:51 -0400 Subject: [PATCH 10/14] Add sfcship prepBUFR conversion (#703) * working sfcshp * rm some comments * small cleanup * update README * get rid of sys.path.append * now works again * logger and code norms * remove some spaces * 1 more thing * rm some remaining prints * reference time print * Update bufr2ioda_sfcshp_prepbufr.py height, var changes * some mods * some elv zob updates * aesthetic * reame chamge * changes * some small changes * revert and fix coding norms * rm unneeded files * json update * run fixes --------- Co-authored-by: Nicholas Esposito Co-authored-by: Nicholas Esposito Co-authored-by: Nicholas Esposito Co-authored-by: Nicholas Esposito Co-authored-by: Cory Martin --- .../bufr2ioda/bufr2ioda_sfcshp_prepbufr.json | 12 + .../bufr2ioda/bufr2ioda_sfcshp_prepbufr.py | 515 ++++++++++++++++++ ush/ioda/bufr2ioda/run_bufr2ioda.py | 3 +- 3 files changed, 529 insertions(+), 1 deletion(-) create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.json create mode 100755 ush/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.py diff --git a/parm/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.json b/parm/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.json new file mode 100644 index 000000000..4bdc8f781 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.json @@ -0,0 +1,12 @@ +{ + "data_format" : "prepbufr", + "subsets" : [ "SFCSHP" ], + "source" : "prepBUFR", + "data_type" : "SFCSHP", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "data_description" : "SFCSHP_prepbufr", + "data_provider" : "U.S. NOAA" +} diff --git a/ush/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.py b/ush/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.py new file mode 100755 index 000000000..47207a3e9 --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.py @@ -0,0 +1,515 @@ +#!/usr/bin/env python3 +# (C) Copyright 2023 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import sys +import numpy as np +import os +import argparse +import math +import calendar +import time +from datetime import datetime +import json +from pyiodaconv import bufr +from collections import namedtuple +from pyioda import ioda_obs_space as ioda_ospace +from wxflow import Logger + + +def Compute_dateTime(cycleTimeSinceEpoch, dhr): + + dhr = np.int64(dhr*3600) + dateTime = dhr + cycleTimeSinceEpoch + + return dateTime + + +def bufr_to_ioda(config, logger): + + subsets = config["subsets"] + logger.debug(f"Checking subsets = {subsets}") + + # Get parameters from configuration + data_format = config["data_format"] + source = config["source"] + data_type = config["data_type"] + data_description = config["data_description"] + data_provider = config["data_provider"] + cycle_type = config["cycle_type"] + cycle_datetime = config["cycle_datetime"] + dump_dir = config["dump_directory"] + ioda_dir = config["ioda_directory"] + cycle = config["cycle_datetime"] + + # Get derived parameters + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + + reference_time = datetime.strptime(cycle, "%Y%m%d%H") + reference_time = reference_time.strftime("%Y-%m-%dT%H:%M:%SZ") + reference_time_full = f"{yyyymmdd}{hh}00" + + logger.debug(f"Reference time = {reference_time}") + + # General informaton + converter = 'BUFR to IODA Converter' + platform_description = 'SFCSHP data from prepBUFR format' + + bufrfile = f"{cycle_type}.t{hh}z.{data_format}" + DATA_PATH = os.path.join(dump_dir, f"{cycle_type}.{yyyymmdd}", + str(hh), bufrfile) + + logger.debug(f"The DATA_PATH is: {DATA_PATH}") + + # ============================================ + # Make the QuerySet for all the data we want + # ============================================ + start_time = time.time() + + logger.debug("Making QuerySet ...") + q = bufr.QuerySet(subsets) + + # ObsType + q.add('observationType', '*/TYP') + + # MetaData + q.add('stationIdentification', '*/SID') + q.add('latitude', '*/YOB') + q.add('longitude', '*/XOB') + q.add('obsTimeMinusCycleTime', '*/DHR') + q.add('heightOfStation', '*/Z___INFO/Z__EVENT{1}/ZOB') + q.add('pressure', '*/P___INFO/P__EVENT{1}/POB') + q.add('temperatureEventCode', '*/T___INFO/T__EVENT{1}/TPC') + +# # Quality Infomation (Quality Indicator) + q.add('qualityMarkerStationElevation', '*/Z___INFO/Z__EVENT{1}/ZQM') + q.add('qualityMarkerStationPressure', '*/P___INFO/P__EVENT{1}/PQM') + q.add('qualityMarkerAirTemperature', '*/T___INFO/T__EVENT{1}/TQM') + q.add('qualityMarkerSpecificHumidity', '*/Q___INFO/Q__EVENT{1}/QQM') + q.add('qualityMarkerWindNorthward', '*/W___INFO/W__EVENT{1}/WQM') + q.add('qualityMarkerSeaSurfaceTemperature', '*/SST_INFO/SSTEVENT{1}/SSTQM') + + # ObsValue + q.add('stationElevation', '*/ELV') + q.add('stationPressure', '*/P___INFO/P__EVENT{1}/POB') + q.add('airTemperature', '*/T___INFO/T__EVENT{1}/TOB') + q.add('specificHumidity', '*/Q___INFO/Q__EVENT{1}/QOB') + q.add('windNorthward', '*/W___INFO/W__EVENT{1}/VOB') + q.add('windEastward', '*/W___INFO/W__EVENT{1}/UOB') + q.add('seaSurfaceTemperature', '*/SST_INFO/SSTEVENT{1}/SST1') + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for making QuerySet: {running_time} seconds") + + # ============================================================== + # Open the BUFR file and execute the QuerySet to get ResultSet + # Use the ResultSet returned to get numpy arrays of the data + # ============================================================== + start_time = time.time() + + logger.debug(f"Executing QuerySet to get ResultSet ...") + with bufr.File(DATA_PATH) as f: + r = f.execute(q) + + logger.debug(f" ... Executing QuerySet: get metadata: basic ...") + # ObsType + logger.debug(" ... Executing QuerySet: get ObsType ...") + typ = r.get('observationType') + + # MetaData + logger.debug(" ... Executing QuerySet: MetaData ...") + sid = r.get('stationIdentification') + lat = r.get('latitude') + lon = r.get('longitude') + lon[lon > 180] -= 360 + zob = r.get('heightOfStation', type='float') + pressure = r.get('pressure') + pressure *= 100 + tpc = r.get('temperatureEventCode') + + # Quality Information + logger.debug(f" ... Executing QuerySet: QualityMarker ...") + zobqm = r.get('qualityMarkerStationElevation') + pobqm = r.get('qualityMarkerStationPressure') + tobqm = r.get('qualityMarkerAirTemperature') + tsenqm = np.full(tobqm.shape[0], tobqm.fill_value) + tsenqm = np.where(((tpc >= 1) & (tpc < 8)), tobqm, tsenqm) + tvoqm = np.full(tobqm.shape[0], tobqm.fill_value) + tvoqm = np.where((tpc == 8), tobqm, tvoqm) + qobqm = r.get('qualityMarkerSpecificHumidity') + wobqm = r.get('qualityMarkerWindNorthward') + sstqm = r.get('qualityMarkerSeaSurfaceTemperature') + + logger.debug(f" ... Executing QuerySet: ObsValue ...") + # ObsValue + elv = r.get('stationElevation', type='float') + pob = r.get('stationPressure') + pob *= 100 + tob = r.get('airTemperature') + tob += 273.15 + tsen = np.full(tob.shape[0], tob.fill_value) + tsen = np.where(((tpc >= 1) & (tpc < 8)), tob, tsen) + tvo = np.full(tob.shape[0], tob.fill_value) + tvo = np.where((tpc == 8), tob, tvo) + qob = r.get('specificHumidity', type='float') + qob *= 0.000001 + uob = r.get('windEastward') + vob = r.get('windNorthward') + sst1 = r.get('seaSurfaceTemperature') + + logger.debug(f" ... Executing QuerySet: get datatime: observation time ...") + # DateTime: seconds since Epoch time + # IODA has no support for numpy datetime arrays dtype=datetime64[s] + dhr = r.get('obsTimeMinusCycleTime', type='int64') + + logger.debug(f" ... Executing QuerySet: Done!") + + logger.debug(f" ... Executing QuerySet: Check BUFR variable generic \ + dimension and type ...") + # Check BUFR variable generic dimension and type + logger.debug(f" typ shape = {typ.shape}") + logger.debug(f" sid shape = {sid.shape}") + logger.debug(f" dhr shape = {dhr.shape}") + logger.debug(f" lat shape = {lat.shape}") + logger.debug(f" lon shape = {lon.shape}") + logger.debug(f" zob shape = {zob.shape}") + logger.debug(f" pressure shape = {pressure.shape}") + logger.debug(f" tpc shape = {tpc.shape}") + + logger.debug(f" zobqm shape = {zobqm.shape}") + logger.debug(f" pobqm shape = {pobqm.shape}") + logger.debug(f" tobqm shape = {tobqm.shape}") + logger.debug(f" tsenqm shape = {tsenqm.shape}") + logger.debug(f" tvoqm shape = {tvoqm.shape}") + logger.debug(f" qobqm shape = {qobqm.shape}") + logger.debug(f" wobqm shape = {wobqm.shape}") + logger.debug(f" sstqm shape = {sstqm.shape}") + + logger.debug(f" elv shape = {elv.shape}") + logger.debug(f" pob shape = {pob.shape}") + logger.debug(f" tob shape = {tob.shape}") + logger.debug(f" tsen shape = {tsen.shape}") + logger.debug(f" tvo shape = {tvo.shape}") + logger.debug(f" qob shape = {qob.shape}") + logger.debug(f" uob shape = {uob.shape}") + logger.debug(f" vob shape = {vob.shape}") + logger.debug(f" sst1 shape = {sst1.shape}") + + logger.debug(f" typ type = {typ.dtype}") + logger.debug(f" sid type = {sid.dtype}") + logger.debug(f" dhr type = {dhr.dtype}") + logger.debug(f" lat type = {lat.dtype}") + logger.debug(f" lon type = {lon.dtype}") + logger.debug(f" zob type = {zob.dtype}") + logger.debug(f" pressure type = {pressure.dtype}") + logger.debug(f" tpc type = {tpc.dtype}") + + logger.debug(f" pobqm type = {pobqm.dtype}") + logger.debug(f" tobqm type = {tobqm.dtype}") + logger.debug(f" tsenqm type = {tsenqm.dtype}") + logger.debug(f" tvoqm type = {tvoqm.dtype}") + logger.debug(f" qobqm type = {qobqm.dtype}") + logger.debug(f" wobqm type = {wobqm.dtype}") + logger.debug(f" sstqm type = {sstqm.dtype}") + + logger.debug(f" elv type = {elv.dtype}") + logger.debug(f" pob type = {pob.dtype}") + logger.debug(f" tob type = {tob.dtype}") + logger.debug(f" tsen type = {tsen.dtype}") + logger.debug(f" tvo type = {tvo.dtype}") + logger.debug(f" qob type = {qob.dtype}") + logger.debug(f" uob type = {uob.dtype}") + logger.debug(f" vob type = {vob.dtype}") + logger.debug(f" sst1 type = {sst1.dtype}") + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for executing QuerySet to get ResultSet: \ + {running_time} seconds") + + # ========================= + # Create derived variables + # ========================= + start_time = time.time() + + logger.debug(f"Creating derived variables - dateTime ...") + + cycleTimeSinceEpoch = np.int64(calendar.timegm(time.strptime( + reference_time_full, '%Y%m%d%H%M'))) + dateTime = Compute_dateTime(cycleTimeSinceEpoch, dhr) + + logger.debug(f" Check derived variables type ... ") + logger.debug(f" dateTime shape = {dateTime.shape}") + logger.debug(f" dateTime type = {dateTime.dtype}") + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for creating derived variables: {running_time} \ + seconds") + + # ===================================== + # Create IODA ObsSpace + # Write IODA output + # ===================================== + + # Create the dimensions + dims = {'Location': np.arange(0, lat.shape[0])} + + iodafile = f"{cycle_type}.t{hh}z.{data_type}.{data_format}.nc" + OUTPUT_PATH = os.path.join(ioda_dir, iodafile) + logger.debug(f" ... ... Create OUTPUT file: {OUTPUT_PATH}") + + path, fname = os.path.split(OUTPUT_PATH) + if path and not os.path.exists(path): + os.makedirs(path) + + obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) + + # Create Global attributes + logger.debug(f" ... ... Create global attributes") + + obsspace.write_attr('Converter', converter) + obsspace.write_attr('source', source) + obsspace.write_attr('sourceFiles', bufrfile) + obsspace.write_attr('dataProviderOrigin', data_provider) + obsspace.write_attr('description', data_description) + obsspace.write_attr('datetimeReference', reference_time) + obsspace.write_attr('datetimeRange', + [str(min(dateTime)), str(max(dateTime))]) + obsspace.write_attr('platformLongDescription', platform_description) + + # Create IODA variables + logger.debug(f" ... ... Create variables: name, type, units, & attributes") + + # ObsType: stationElevation + obsspace.create_var('ObsType/stationElevation', dtype=typ.dtype, + fillval=typ.fill_value) \ + .write_attr('long_name', 'Station Elevation Observation Type') \ + .write_data(typ) + + # ObsType: stationPressure + obsspace.create_var('ObsType/stationPressure', dtype=typ.dtype, + fillval=typ.fill_value) \ + .write_attr('long_name', 'Station Pressure Observation Type') \ + .write_data(typ) + + # ObsType: air Temperature + obsspace.create_var('ObsType/airTemperature', dtype=typ.dtype, + fillval=typ.fill_value) \ + .write_attr('long_name', 'Air Temperature Observation Type') \ + .write_data(typ) + + # ObsType: virtual Temperature + obsspace.create_var('ObsType/virtualTemperature', dtype=typ.dtype, + fillval=typ.fill_value) \ + .write_attr('long_name', 'Virtual Temperature Observation Type') \ + .write_data(typ) + + # ObsType: Specific Humidity + obsspace.create_var('ObsType/specificHumidity', dtype=typ.dtype, + fillval=typ.fill_value) \ + .write_attr('long_name', 'Specific Humidity Observation Type') \ + .write_data(typ) + + # ObsType: wind Eastward + obsspace.create_var('ObsType/windEastward', dtype=typ.dtype, + fillval=typ.fill_value) \ + .write_attr('long_name', 'Wind Eastward Observation Type') \ + .write_data(typ) + + # ObsType: wind Northward + obsspace.create_var('ObsType/windNorthward', dtype=typ.dtype, + fillval=typ.fill_value) \ + .write_attr('long_name', 'Wind Northward Observation Type') \ + .write_data(typ) + + # ObsType: Sea Surface Temperature + obsspace.create_var('ObsType/seaSurfaceTemperature', dtype=typ.dtype, + fillval=typ.fill_value) \ + .write_attr('long_name', 'Sea Surface Temperature Observation Type') \ + .write_data(typ) + + # Longitude + obsspace.create_var('MetaData/longitude', dtype=lon.dtype, + fillval=lon.fill_value) \ + .write_attr('units', 'degrees_east') \ + .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ + .write_attr('long_name', 'Longitude') \ + .write_data(lon) + + # Latitude + obsspace.create_var('MetaData/latitude', dtype=lat.dtype, + fillval=lat.fill_value) \ + .write_attr('units', 'degrees_north') \ + .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Latitude') \ + .write_data(lat) + + # Datetime + obsspace.create_var('MetaData/dateTime', dtype=dateTime.dtype, + fillval=dateTime.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'Datetime') \ + .write_data(dateTime) + + # Station Identification + obsspace.create_var('MetaData/stationIdentification', dtype=sid.dtype, + fillval=sid.fill_value) \ + .write_attr('long_name', 'Station Identification') \ + .write_data(sid) + + # Station Elevation + obsspace.create_var('MetaData/heightOfStation', dtype=zob.dtype, + fillval=zob.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Height Of Station') \ + .write_data(zob) + + # Pressure + obsspace.create_var('MetaData/pressure', dtype=pressure.dtype, + fillval=pressure.fill_value) \ + .write_attr('units', 'Pa') \ + .write_attr('long_name', 'Pressure') \ + .write_data(pressure) + + # Temperature Event Code + obsspace.create_var('MetaData/temperatureEventCode', dtype=tpc.dtype, + fillval=tpc.fill_value) \ + .write_attr('long_name', 'Temperature Event Code') \ + .write_data(tpc) + + # Quality Marker: Station Pressure + obsspace.create_var('QualityMarker/stationPressure', dtype=pobqm.dtype, + fillval=pobqm.fill_value) \ + .write_attr('long_name', 'Station Pressure Quality Marker') \ + .write_data(pobqm) + + # Quality Marker: Air Temperature + obsspace.create_var('QualityMarker/airTemperature', dtype=tobqm.dtype, + fillval=tobqm.fill_value) \ + .write_attr('long_name', 'Air Temperature Quality Marker') \ + .write_data(tsenqm) + + # Quality Marker: Virtual Temperature + obsspace.create_var('QualityMarker/virtualTemperature', dtype=tobqm.dtype, + fillval=tobqm.fill_value) \ + .write_attr('long_name', 'Virtual Temperature Quality Marker') \ + .write_data(tvoqm) + + # Quality Marker: Specific Humidity + obsspace.create_var('QualityMarker/specificHumidity', dtype=qobqm.dtype, + fillval=qobqm.fill_value) \ + .write_attr('long_name', 'Specific Humidity Quality Marker') \ + .write_data(qobqm) + + # Quality Marker: Eastward Wind + obsspace.create_var('QualityMarker/windEastward', dtype=wobqm.dtype, + fillval=wobqm.fill_value) \ + .write_attr('long_name', 'Eastward Wind Quality Marker') \ + .write_data(wobqm) + + # Quality Marker: Northward Wind + obsspace.create_var('QualityMarker/windNorthward', dtype=wobqm.dtype, + fillval=wobqm.fill_value) \ + .write_attr('long_name', 'Northward Wind Quality Marker') \ + .write_data(wobqm) + + # Quality Marker: Sea Surface Temperature + obsspace.create_var('QualityMarker/seaSurfaceTemperature', + dtype=sstqm.dtype, fillval=sstqm.fill_value) \ + .write_attr('long_name', 'Sea Surface Temperature Quality Marker') \ + .write_data(sstqm) + + # Station Elevation + obsspace.create_var('ObsValue/stationElevation', dtype=elv.dtype, + fillval=elv.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Station Elevation') \ + .write_data(elv) + + # Station Pressure + obsspace.create_var('ObsValue/stationPressure', dtype=pob.dtype, + fillval=pob.fill_value) \ + .write_attr('units', 'Pa') \ + .write_attr('long_name', 'Station Pressure') \ + .write_data(pob) + + # Air Temperature + obsspace.create_var('ObsValue/airTemperature', dtype=tob.dtype, + fillval=tob.fill_value) \ + .write_attr('units', 'K') \ + .write_attr('long_name', 'Air Temperature') \ + .write_data(tsen) + + # Virtual Temperature + obsspace.create_var('ObsValue/virtualTemperature', dtype=tob.dtype, + fillval=tob.fill_value) \ + .write_attr('units', 'K') \ + .write_attr('long_name', 'Virtual Temperature') \ + .write_data(tvo) + + # Specific Humidity + obsspace.create_var('ObsValue/specificHumidity', dtype=qob.dtype, + fillval=qob.fill_value) \ + .write_attr('units', 'kg kg-1') \ + .write_attr('long_name', 'Specific Humidity') \ + .write_data(qob) + + # Eastward Wind + obsspace.create_var('ObsValue/windEastward', dtype=uob.dtype, + fillval=uob.fill_value) \ + .write_attr('units', 'm s-1') \ + .write_attr('long_name', 'Eastward Wind') \ + .write_data(uob) + + # Northward Wind + obsspace.create_var('ObsValue/windNorthward', dtype=vob.dtype, + fillval=vob.fill_value) \ + .write_attr('units', 'm s-1') \ + .write_attr('long_name', 'Northward Wind') \ + .write_data(vob) + + # Sea Surface Temperature + obsspace.create_var('ObsValue/seaSurfaceTemperature', dtype=sst1.dtype, + fillval=sst1.fill_value) \ + .write_attr('units', 'K') \ + .write_attr('long_name', 'Sea Surface Temperature') \ + .write_data(sst1) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for splitting and output IODA: \ + {running_time} seconds") + + logger.debug("All Done!") + + +if __name__ == '__main__': + + start_time = time.time() + + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, + help='Input JSON configuration', + required=True) + parser.add_argument('-v', '--verbose', + help='print debug logging information', + action='store_true') + args = parser.parse_args() + + log_level = 'DEBUG' if args.verbose else 'INFO' + logger = Logger('bufr2ioda_sfcshp_prepbufr.py', level=log_level, + colored_log=True) + + with open(args.config, "r") as json_file: + config = json.load(json_file) + + bufr_to_ioda(config, logger) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Total running time: {running_time} seconds") diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index 63071d838..da01dd131 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -30,7 +30,8 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): } # Specify observation types to be processed by a script - BUFR_py = ["satwind_amv_goes", "satwind_scat", "adpupa_prepbufr", "adpsfc_prepbufr"] + + BUFR_py = ["satwind_amv_goes", "satwind_scat", "adpupa_prepbufr", "adpsfc_prepbufr", "sfcshp_prepbufr"] for obtype in BUFR_py: logger.info(f"Convert {obtype}...") From a88e6e9ad1f3bdfea1aa8e23a7252f2248060f9c Mon Sep 17 00:00:00 2001 From: Shastri Paturi Date: Fri, 3 Nov 2023 15:47:59 +0000 Subject: [PATCH 11/14] insitu Obsyamls for monthly tank data (#704) * insitu yamls for subpfl:argo&glider;tesac & bathy (monthly tank) * updated obs_list.yaml with in situ yamls * removed extra comment lines in obs_list.yaml --- parm/soca/obs/config/salt_profile_argo.yaml | 31 +++++++++++++++++++ parm/soca/obs/config/salt_profile_glider.yaml | 31 +++++++++++++++++++ parm/soca/obs/config/salt_profile_tesac.yaml | 31 +++++++++++++++++++ parm/soca/obs/config/temp_profile_argo.yaml | 28 +++++++++++++++++ parm/soca/obs/config/temp_profile_bathy.yaml | 28 +++++++++++++++++ parm/soca/obs/config/temp_profile_glider.yaml | 28 +++++++++++++++++ parm/soca/obs/config/temp_profile_tesac.yaml | 28 +++++++++++++++++ parm/soca/obs/obs_list.yaml | 7 +++++ 8 files changed, 212 insertions(+) create mode 100644 parm/soca/obs/config/salt_profile_argo.yaml create mode 100644 parm/soca/obs/config/salt_profile_glider.yaml create mode 100644 parm/soca/obs/config/salt_profile_tesac.yaml create mode 100644 parm/soca/obs/config/temp_profile_argo.yaml create mode 100644 parm/soca/obs/config/temp_profile_bathy.yaml create mode 100644 parm/soca/obs/config/temp_profile_glider.yaml create mode 100644 parm/soca/obs/config/temp_profile_tesac.yaml diff --git a/parm/soca/obs/config/salt_profile_argo.yaml b/parm/soca/obs/config/salt_profile_argo.yaml new file mode 100644 index 000000000..9f3e2521a --- /dev/null +++ b/parm/soca/obs/config/salt_profile_argo.yaml @@ -0,0 +1,31 @@ +obs space: + name: salt_profile_argo + obsdatain: + engine: + type: H5File + obsfile: !ENV ${DATA}/obs/${OPREFIX}salt_profile_argo.${PDY}${cyc}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ${DATA}/diags/salt_profile_argo.${PDY}${cyc}.nc4 + simulated variables: [salinity] + io pool: + max pool size: 1 +obs operator: + name: VertInterp + observation alias file: ./obsop_name_map.yaml + vertical coordinate: sea_water_depth + observation vertical coordinate: depth + interpolation method: linear +obs error: + covariance model: diagonal + +obs filter: + # Passivate obs where ocean fraction is > 90% + - filter: Domain Check + action: + name: passivate + where: + - variable: {name: GeoVaLs/sea_area_fraction} + maxvalue: 0.9 + diff --git a/parm/soca/obs/config/salt_profile_glider.yaml b/parm/soca/obs/config/salt_profile_glider.yaml new file mode 100644 index 000000000..a1504886b --- /dev/null +++ b/parm/soca/obs/config/salt_profile_glider.yaml @@ -0,0 +1,31 @@ +obs space: + name: salt_profile_glider + obsdatain: + engine: + type: H5File + obsfile: !ENV ${DATA}/obs/${OPREFIX}salt_profile_glider.${PDY}${cyc}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ${DATA}/diags/salt_profile_glider.${PDY}${cyc}.nc4 + simulated variables: [salinity] + io pool: + max pool size: 1 +obs operator: + name: VertInterp + observation alias file: ./obsop_name_map.yaml + vertical coordinate: sea_water_depth + observation vertical coordinate: depth + interpolation method: linear +obs error: + covariance model: diagonal + +obs filter: + # Passivate obs where ocean fraction is > 90% + - filter: Domain Check + action: + name: passivate + where: + - variable: {name: GeoVaLs/sea_area_fraction} + maxvalue: 0.9 + diff --git a/parm/soca/obs/config/salt_profile_tesac.yaml b/parm/soca/obs/config/salt_profile_tesac.yaml new file mode 100644 index 000000000..53cc4496e --- /dev/null +++ b/parm/soca/obs/config/salt_profile_tesac.yaml @@ -0,0 +1,31 @@ +obs space: + name: salt_profile_tesac + obsdatain: + engine: + type: H5File + obsfile: !ENV ${DATA}/obs/${OPREFIX}salt_profile_tesac.${PDY}${cyc}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ${DATA}/diags/salt_profile_tesac.${PDY}${cyc}.nc4 + simulated variables: [salinity] + io pool: + max pool size: 1 +obs operator: + name: VertInterp + observation alias file: ./obsop_name_map.yaml + vertical coordinate: sea_water_depth + observation vertical coordinate: depth + interpolation method: linear +obs error: + covariance model: diagonal + +obs filter: + # Passivate obs where ocean fraction is > 90% + - filter: Domain Check + action: + name: passivate + where: + - variable: {name: GeoVaLs/sea_area_fraction} + maxvalue: 0.9 + diff --git a/parm/soca/obs/config/temp_profile_argo.yaml b/parm/soca/obs/config/temp_profile_argo.yaml new file mode 100644 index 000000000..ff1b8ce9d --- /dev/null +++ b/parm/soca/obs/config/temp_profile_argo.yaml @@ -0,0 +1,28 @@ +obs space: + name: temp_profile_argo + obsdatain: + engine: + type: H5File + obsfile: !ENV ${DATA}/obs/${OPREFIX}temp_profile_argo.${PDY}${cyc}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ${DATA}/diags/temp_profile_argo.${PDY}${cyc}.nc4 + simulated variables: [waterTemperature] + io pool: + max pool size: 1 +obs operator: + name: InsituTemperature + +obs error: + covariance model: diagonal + +obs filters: + # Passivate obs where ocean fraction is > 90% + - filter: Domain Check + action: + name: passivate + where: + - variable: {name: GeoVaLs/sea_area_fraction} + maxvalue: 0.9 + diff --git a/parm/soca/obs/config/temp_profile_bathy.yaml b/parm/soca/obs/config/temp_profile_bathy.yaml new file mode 100644 index 000000000..90c0569ea --- /dev/null +++ b/parm/soca/obs/config/temp_profile_bathy.yaml @@ -0,0 +1,28 @@ +obs space: + name: temp_profile_bathy + obsdatain: + engine: + type: H5File + obsfile: !ENV ${DATA}/obs/${OPREFIX}temp_profile_bathy.${PDY}${cyc}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ${DATA}/diags/temp_profile_bathy.${PDY}${cyc}.nc4 + simulated variables: [waterTemperature] + io pool: + max pool size: 1 +obs operator: + name: InsituTemperature + +obs error: + covariance model: diagonal + +obs filters: + # Passivate obs where ocean fraction is > 90% + - filter: Domain Check + action: + name: passivate + where: + - variable: {name: GeoVaLs/sea_area_fraction} + maxvalue: 0.9 + diff --git a/parm/soca/obs/config/temp_profile_glider.yaml b/parm/soca/obs/config/temp_profile_glider.yaml new file mode 100644 index 000000000..721fccafb --- /dev/null +++ b/parm/soca/obs/config/temp_profile_glider.yaml @@ -0,0 +1,28 @@ +obs space: + name: temp_profile_glider + obsdatain: + engine: + type: H5File + obsfile: !ENV ${DATA}/obs/${OPREFIX}temp_profile_glider.${PDY}${cyc}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ${DATA}/diags/temp_profile_glider.${PDY}${cyc}.nc4 + simulated variables: [waterTemperature] + io pool: + max pool size: 1 +obs operator: + name: InsituTemperature + +obs error: + covariance model: diagonal + +obs filters: + # Passivate obs where ocean fraction is > 90% + - filter: Domain Check + action: + name: passivate + where: + - variable: {name: GeoVaLs/sea_area_fraction} + maxvalue: 0.9 + diff --git a/parm/soca/obs/config/temp_profile_tesac.yaml b/parm/soca/obs/config/temp_profile_tesac.yaml new file mode 100644 index 000000000..8cbe84bd6 --- /dev/null +++ b/parm/soca/obs/config/temp_profile_tesac.yaml @@ -0,0 +1,28 @@ +obs space: + name: temp_profile_tesac + obsdatain: + engine: + type: H5File + obsfile: !ENV ${DATA}/obs/${OPREFIX}temp_profile_tesac.${PDY}${cyc}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ${DATA}/diags/temp_profile_tesac.${PDY}${cyc}.nc4 + simulated variables: [waterTemperature] + io pool: + max pool size: 1 +obs operator: + name: InsituTemperature + +obs error: + covariance model: diagonal + +obs filters: + # Passivate obs where ocean fraction is > 90% + - filter: Domain Check + action: + name: passivate + where: + - variable: {name: GeoVaLs/sea_area_fraction} + maxvalue: 0.9 + diff --git a/parm/soca/obs/obs_list.yaml b/parm/soca/obs/obs_list.yaml index d710f3461..89b2659a1 100644 --- a/parm/soca/obs/obs_list.yaml +++ b/parm/soca/obs/obs_list.yaml @@ -13,3 +13,10 @@ observers: #- !INC ${OBS_YAML_DIR}/icec_ssmis_f17_south.yaml #- !INC ${OBS_YAML_DIR}/icec_ssmis_f18_north.yaml - !INC ${OBS_YAML_DIR}/icec_ssmis_f18_south.yaml +#- !INC ${OBS_YAML_DIR}/temp_profile_argo.yaml +#- !INC ${OBS_YAML_DIR}/temp_profile_bathy.yaml +#- !INC ${OBS_YAML_DIR}/temp_profile_glider.yaml +#- !INC ${OBS_YAML_DIR}/temp_profile_tesac.yaml +#- !INC ${OBS_YAML_DIR}/salt_profile_argo.yaml +#- !INC ${OBS_YAML_DIR}/salt_profile_glider.yaml +#- !INC ${OBS_YAML_DIR}/salt_profile_tesac.yaml \ No newline at end of file From 645e644e45be9b004a793ed98483d6487c159433 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA <58948505+AndrewEichmann-NOAA@users.noreply.github.com> Date: Fri, 3 Nov 2023 15:41:09 -0400 Subject: [PATCH 12/14] tell JGLOBAL_PREP_OCEAN_OBS ctest location of ex-script in GDASApp (#706) --- test/soca/gw/run_jjobs.yaml.test | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/soca/gw/run_jjobs.yaml.test b/test/soca/gw/run_jjobs.yaml.test index f773a8c79..f908c841f 100644 --- a/test/soca/gw/run_jjobs.yaml.test +++ b/test/soca/gw/run_jjobs.yaml.test @@ -33,6 +33,9 @@ gw environement: OOPS_DEBUG: 1 OMP_NUM_THREADS: 1 + run scripts: + GDASPREPOCNOBSPY: @HOMEgfs@/sorc/gdas.cd/scripts/exglobal_prep_ocean_obs.py + setup_expt config: base: DO_JEDIATMVAR: "NO" @@ -51,6 +54,8 @@ setup_expt config: SABER_BLOCKS_YAML: @HOMEgfs@/sorc/gdas.cd/parm/soca/berror/saber_blocks.yaml NICAS_RESOL: 1 NICAS_GRID_SIZE: 150 + prepoceanobs: + SOCA_OBS_LIST: @HOMEgfs@/sorc/gdas.cd/parm/soca/obs/obs_list_small.yaml job options: account: da-cpu From c501fed4ec0607ddec36854705628ae5d0096bcc Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Fri, 3 Nov 2023 16:06:19 -0400 Subject: [PATCH 13/14] Replace "MetaData/height" with "MetaData/stationElevation" (#707) Co-authored-by: Cory Martin --- test/land/letkfoi_land.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/land/letkfoi_land.yaml b/test/land/letkfoi_land.yaml index c9069f0e8..98cd3d171 100644 --- a/test/land/letkfoi_land.yaml +++ b/test/land/letkfoi_land.yaml @@ -64,7 +64,7 @@ observations: where: - minvalue: '-999.0' variable: - name: MetaData/height + name: MetaData/stationElevation - filter: Domain Check where: - maxvalue: '1.5' From 0d600e9c412db0f4c16849a8df89eb915f1caddc Mon Sep 17 00:00:00 2001 From: Andrew Collard <40322596+ADCollard@users.noreply.github.com> Date: Mon, 6 Nov 2023 08:35:17 -0500 Subject: [PATCH 14/14] Add error compare plot (#709) --- ush/eva/jedi_gsi_compare_conv.yaml | 55 ++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/ush/eva/jedi_gsi_compare_conv.yaml b/ush/eva/jedi_gsi_compare_conv.yaml index 02293a7b0..c82a0dbd9 100644 --- a/ush/eva/jedi_gsi_compare_conv.yaml +++ b/ush/eva/jedi_gsi_compare_conv.yaml @@ -107,6 +107,24 @@ transforms: for: variable: *variables + # Obs Error that passed QC for JEDI + - transform: accept where + new name: experiment::EffectiveErrorQc::${variable} + starting field: experiment::EffectiveError::${variable} + where: + - experiment::EffectiveQC::${variable} == 0 + for: + variable: *variables + + # ObsError that passed QC for GSI + - transform: accept where + new name: experiment::GsiFinalObsErrorQc::${variable} + starting field: experiment::GsiFinalObsError::${variable} + where: + - experiment::GsiEffectiveQC::${variable} == 0 + for: + variable: *variables + graphics: # ---------- Scatter Plots ---------- @@ -276,6 +294,7 @@ graphics: markersize: 5 color: 'red' label: 'hofxdiff vs pressure' + do_linear_regression: False statistics: fields: - field_name: experiment::HofXDiff::${variable} @@ -354,6 +373,42 @@ graphics: markersize: 5 color: 'red' label: 'errordiff vs pressure' + do_linear_regression: False + # Error comparison as a function of pressure + - batch figure: + variables: *variables + @CHANNELSKEY@ + figure: + layout: [1,1] + title: 'JEDI-GSI Comparison (Passed QC) | @NAME@ @CYCLE@ | ${variable_title}' + output name: observation_scatter_plots/errorcomp_vs_pressure_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png + plots: + - add_xlabel: 'Observation Error' + add_ylabel: 'Observation Pressure' + add_grid: + add_legend: + loc: 'lower left' + layers: + - type: Scatter + x: + variable: experiment::GsiFinalObsErrorQc::${variable} + y: + variable: experiment::MetaData::pressure + @CHANNELKEY@ + markersize: 3 + color: 'red' + label: 'GSI Error' + do_linear_regression: False + - type: Scatter + x: + variable: experiment::EffectiveErrorQc::${variable} + y: + variable: experiment::MetaData::pressure + @CHANNELKEY@ + markersize: 5 + color: 'green' + label: 'JEDI Error' + do_linear_regression: False # ---------- Histograms ---------- # Histogram of h(x) difference