From bcdd58c70e2c6b95c3b6c3cca569e2d25aa0b7a5 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Tue, 22 Aug 2023 15:39:00 -0400 Subject: [PATCH 01/29] working sfcshp --- .../bufr2ioda/bufr2ioda_sfcshp_prepbufr.json | 12 + ush/ioda/bufr2ioda/README_whoever | 9 + .../bufr2ioda/bufr2ioda_sfcshp_prepbufr.py | 426 ++++++++++++++++++ ush/ioda/bufr2ioda/run_bufr2ioda_sfcshp.sh | 79 ++++ 4 files changed, 526 insertions(+) create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.json create mode 100644 ush/ioda/bufr2ioda/README_whoever create mode 100644 ush/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.py create mode 100755 ush/ioda/bufr2ioda/run_bufr2ioda_sfcshp.sh diff --git a/parm/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.json b/parm/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.json new file mode 100644 index 000000000..4ec6272ae --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.json @@ -0,0 +1,12 @@ +{ + "data_format" : "prepbufr", + "subsets" : [ "SFCSHP" ], + "source" : "prepBUFR", + "data_type" : "SFCSHP", + "cycle_type" : "{{ DUMP }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "data_description" : "SFCSHP_prepbufr", + "data_provider" : "U.S. NOAA/NESDIS" +} diff --git a/ush/ioda/bufr2ioda/README_whoever b/ush/ioda/bufr2ioda/README_whoever new file mode 100644 index 000000000..3850067f1 --- /dev/null +++ b/ush/ioda/bufr2ioda/README_whoever @@ -0,0 +1,9 @@ +Needed to make a sfcshp-specific run_bufr2ioda.sh script because I needed to change a variable. + +To run: use command: +./run_bufr2ioda.sh 20210801 gdas /work/noaa/rstprod/dump/ /work2/noaa/da/nesposito/GDASApp_20230822/parm/ioda/bufr2ioda/ /work2/noaa/da/nesposito/NewConv/api_convert/json/json_outputs/ + +Outputs (and plots) are in: +/work2/noaa/da/nesposito/NewConv/api_convert/json/json_outputs + + diff --git a/ush/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.py b/ush/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.py new file mode 100644 index 000000000..00514b955 --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.py @@ -0,0 +1,426 @@ +# (C) Copyright 2023 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import sys +sys.path.append('/work2/noaa/da/nesposito/ioda-bundle_20230712/build/lib/python3.9/') +sys.path.append('/work2/noaa/da/nesposito/ioda-bundle_20230712/build/lib/python3.9/pyiodaconv/') +sys.path.append('/work2/noaa/da/nesposito/ioda-bundle_20230712/build/lib/python3.9/pyioda/') + +import numpy as np +import os +import argparse +import math +import calendar +import time +from datetime import datetime +import json +from pyiodaconv import bufr +from collections import namedtuple +from pyioda import ioda_obs_space as ioda_ospace + +def Compute_dateTime(cycleTimeSinceEpoch, dhr): + + dhr = np.int64(dhr*3600) + dateTime = dhr + cycleTimeSinceEpoch + + return dateTime + +def bufr_to_ioda(config): + + subsets = config["subsets"] + + # Get parameters from configuration + data_format = config["data_format"] + source = config["source"] + data_type = config["data_type"] + data_description = config["data_description"] + data_provider = config["data_provider"] + cycle_type = config["cycle_type"] + cycle_datetime = config["cycle_datetime"] + dump_dir = config["dump_directory"] + ioda_dir = config["ioda_directory"] + cycle = config["cycle_datetime"] + + # Get derived parameters + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + + reference_time = datetime.strptime(cycle, "%Y%m%d%H") + reference_time = reference_time.strftime("%Y-%m-%dT%H:%M:%SZ") + reference_time_full = f"{yyyymmdd}{hh}00" + + # General informaton + converter = 'BUFR to IODA Converter' + platform_description = 'SFCSHP data from prepBUFR format' + + bufrfile = f"{cycle_type}.t{hh}z.{data_format}" + DATA_PATH = os.path.join(dump_dir, f"{cycle_type}.{yyyymmdd}", str(hh), bufrfile) + + print("The DATA_PATH is: "+str(DATA_PATH)) + + # ============================================ + # Make the QuerySet for all the data we want + # ============================================ + start_time = time.time() + + print('Making QuerySet ...') + q = bufr.QuerySet(subsets) + + # MetaData + q.add('stationIdentification', '*/SID') + q.add('latitude', '*/YOB') + q.add('longitude', '*/XOB') + q.add('obsTimeMinusCycleTime', '*/DHR') + q.add('stationElevation', '*/ELV') + q.add('observationType', '*/TYP') + q.add('pressure', '*/P___INFO/P__EVENT{1}/POB') + q.add('temperatureEventCode', '*/T___INFO/T__EVENT{1}/TPC') + +# # Quality Infomation (Quality Indicator) + q.add('qualityMarkerStationPressure', '*/P___INFO/P__EVENT{1}/PQM') + q.add('qualityMarkerAirTemperature', '*/T___INFO/T__EVENT{1}/TQM') + q.add('qualityMarkerSpecificHumidity', '*/Q___INFO/Q__EVENT{1}/QQM') + q.add('qualityMarkerWindNorthward', '*/W___INFO/W__EVENT{1}/WQM') + q.add('qualityMarkerSeaSurfaceTemperature','*/SST_INFO/SSTEVENT{1}/SSTQM') + + # ObsValue + q.add('stationPressure', '*/P___INFO/P__EVENT{1}/POB') + q.add('airTemperature', '*/T___INFO/T__EVENT{1}/TOB') + q.add('virtualTemperature', '*/T___INFO/TVO') + q.add('specificHumidity', '*/Q___INFO/Q__EVENT{1}/QOB') + q.add('windNorthward', '*/W___INFO/W__EVENT{1}/VOB') + q.add('windEastward', '*/W___INFO/W__EVENT{1}/UOB') + q.add('seaSurfaceTemperature', '*/SST_INFO/SSTEVENT{1}/SST1') + + end_time = time.time() + running_time = end_time - start_time + print('Running time for making QuerySet : ', running_time, 'seconds') + + # ============================================================== + # Open the BUFR file and execute the QuerySet to get ResultSet + # Use the ResultSet returned to get numpy arrays of the data + # ============================================================== + start_time = time.time() + + print('Executing QuerySet to get ResultSet ...') + with bufr.File(DATA_PATH) as f: +# print("hello") + r = f.execute(q) + + + print(' ... Executing QuerySet: get metadata: basic ...') + # MetaData + sid = r.get('stationIdentification') + lat = r.get('latitude') + lon = r.get('longitude') + lon[lon>180] -= 360 + elv = r.get('stationElevation', type='float') + typ = r.get('observationType') + pressure = r.get('pressure') + pressure *= 100 + tpc = r.get('temperatureEventCode') + + print(' ... Executing QuerySet: get QualityMarker information ...') + # Quality Information + pqm = r.get('qualityMarkerStationPressure') + tqm = r.get('qualityMarkerAirTemperature') + qqm = r.get('qualityMarkerSpecificHumidity') + wqm = r.get('qualityMarkerWindNorthward') + sstqm = r.get('qualityMarkerSeaSurfaceTemperature') + + print(' ... Executing QuerySet: get obsvalue: stationPressure ...') + # ObsValue + pob = r.get('stationPressure') + pob *= 100 + tob = r.get('airTemperature') + tob += 273.15 + tvo = r.get('virtualTemperature') + tvo += 273.15 + qob = r.get('specificHumidity', type='float') + qob *= 0.000001 + uob = r.get('windEastward') + vob = r.get('windNorthward') + sst1 = r.get('seaSurfaceTemperature') + + print(' ... Executing QuerySet: get datatime: observation time ...') + # DateTime: seconds since Epoch time + # IODA has no support for numpy datetime arrays dtype=datetime64[s] + dhr = r.get('obsTimeMinusCycleTime', type='int64') + + print(' ... Executing QuerySet: Done!') + + print(' ... Executing QuerySet: Check BUFR variable generic dimension and type ...') + # Check BUFR variable generic dimension and type + print(' sid shape = ', sid.shape) + print(' dhr shape = ', dhr.shape) + print(' lat shape = ', lat.shape) + print(' lon shape = ', lon.shape) + print(' elv shape = ', elv.shape) + print(' typ shape = ', typ.shape) + print(' pressure shape = ', pressure.shape) + print(' tpc shape = ', tpc.shape) + + print(' pqm shape = ', pqm.shape) + print(' tqm shape = ', tqm.shape) + print(' qqm shape = ', qqm.shape) + print(' wqm shape = ', wqm.shape) + print(' sstqm shape = ', sstqm.shape) + + print(' pob shape = ', pob.shape) + print(' tob shape = ', pob.shape) + print(' tvo shape = ', tvo.shape) + print(' qob shape = ', qob.shape) + print(' uob shape = ', uob.shape) + print(' vob shape = ', vob.shape) + print(' sst1 shape = ', sst1.shape) + + print(' sid type = ', sid.dtype) + print(' dhr type = ', dhr.dtype) + print(' lat type = ', lat.dtype) + print(' lon type = ', lon.dtype) + print(' elv type = ', elv.dtype) + print(' typ type = ', typ.dtype) + print(' pressure type = ', pressure.dtype) + print(' tpc type = ', tpc.dtype) + + print(' pqm type = ', pqm.dtype) + print(' tqm type = ', tqm.dtype) + print(' qqm type = ', qqm.dtype) + print(' wqm type = ', wqm.dtype) + print(' sstqm type = ', sstqm.dtype) + + print(' pob type = ', pob.dtype) + print(' tob type = ', tob.dtype) + print(' tvo type = ', tvo.dtype) + print(' qob type = ', qob.dtype) + print(' uob type = ', uob.dtype) + print(' vob type = ', vob.dtype) + print(' sst1 type = ', sst1.dtype) + + end_time = time.time() + running_time = end_time - start_time + print('Running time for executing QuerySet to get ResultSet : ', running_time, 'seconds') + + # ========================= + # Create derived variables + # ========================= + start_time = time.time() + + print('Creating derived variables - dateTime ...') + + cycleTimeSinceEpoch = np.int64(calendar.timegm(time.strptime(reference_time_full, '%Y%m%d%H%M'))) + dateTime = Compute_dateTime(cycleTimeSinceEpoch, dhr) + + print(' Check drived variables type ... ') + print(' dateTime shape = ', dateTime.shape) + print(' dateTime type = ', dateTime.dtype) + + end_time = time.time() + running_time = end_time - start_time + print('Running time for creating derived variables : ', running_time, 'seconds') + + # ===================================== + # Create IODA ObsSpace + # Write IODA output + # ===================================== + + # Create the dimensions + dims = { + 'Location' : np.arange(0, lat.shape[0]), + } + + + iodafile = f"{cycle_type}.t{hh}z.{data_type}.{data_format}.nc" + OUTPUT_PATH = os.path.join(ioda_dir, iodafile) + print(" ... ... Create OUTPUT file: ", OUTPUT_PATH) + + path, fname = os.path.split(OUTPUT_PATH) + if path and not os.path.exists(path): + os.makedirs(path) + + obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) + + # Create Global attributes + print(' ... ... Create global attributes') + + obsspace.write_attr('Converter', converter) + obsspace.write_attr('source', source) + obsspace.write_attr('sourceFiles', bufrfile) + obsspace.write_attr('dataProviderOrigin', data_provider) + obsspace.write_attr('description', data_description) + obsspace.write_attr('datetimeReference', reference_time) + obsspace.write_attr('datetimeRange', [str(min(dateTime)), str(max(dateTime))]) + obsspace.write_attr('platformLongDescription', platform_description) + + # Create IODA variables + print(' ... ... Create variables: name, type, units, and attributes') + # Longitude + obsspace.create_var('MetaData/longitude', dtype=lon.dtype, fillval=lon.fill_value) \ + .write_attr('units', 'degrees_east') \ + .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ + .write_attr('long_name', 'Longitude') \ + .write_data(lon) + + # Latitude + obsspace.create_var('MetaData/latitude', dtype=lat.dtype, fillval=lat.fill_value) \ + .write_attr('units', 'degrees_north') \ + .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Latitude') \ + .write_data(lat) + + # Datetime + obsspace.create_var('MetaData/dateTime', dtype=dateTime.dtype, fillval=dateTime.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'Datetime') \ + .write_data(dateTime) + + # Station Identification + obsspace.create_var('MetaData/stationIdentification', dtype=sid.dtype, fillval=sid.fill_value) \ + .write_attr('units', '1') \ + .write_attr('long_name', 'Station Identification') \ + .write_data(sid) + + # Station Elevation + obsspace.create_var('MetaData/stationElevation', dtype=elv.dtype, fillval=elv.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Station Elevation') \ + .write_data(elv) + + # Observation Type + obsspace.create_var('MetaData/observationType', dtype=typ.dtype, fillval=typ.fill_value) \ + .write_attr('long_name', 'Observation Type') \ + .write_data(typ) + + # Pressure + obsspace.create_var('MetaData/pressure', dtype=pressure.dtype, fillval=pressure.fill_value) \ + .write_attr('units', 'Pa') \ + .write_attr('long_name', 'Pressure') \ + .write_data(pressure) + + # Temperature Event Code + obsspace.create_var('MetaData/temperatureEventCode', dtype=tpc.dtype, fillval=tpc.fill_value) \ + .write_attr('long_name', 'Temperature Event Code') \ + .write_data(typ) + + # Quality Marker: Station Pressure + obsspace.create_var('QualityMarker/stationPressure', dtype=pqm.dtype, fillval=pqm.fill_value) \ + .write_attr('long_name', 'Station Pressure Quality Marker') \ + .write_data(pqm) + + # Quality Marker: Air Temperature + obsspace.create_var('QualityMarker/airTemperature', dtype=tqm.dtype, fillval=tqm.fill_value) \ + .write_attr('long_name', 'Air Temperature Quality Marker') \ + .write_data(tqm) + + # Quality Marker: Specific Humidity + obsspace.create_var('QualityMarker/specificHumidity', dtype=qqm.dtype, fillval=qqm.fill_value) \ + .write_attr('long_name', 'Specific Humidity Quality Marker') \ + .write_data(qqm) + + # Quality Marker: Northward Wind + obsspace.create_var('QualityMarker/windNorthward', dtype=wqm.dtype, fillval=wqm.fill_value) \ + .write_attr('long_name', 'Northward Wind Quality Marker') \ + .write_data(wqm) + + # Quality Marker: Sea Surface Temperature + obsspace.create_var('QualityMarker/seaSurfaceTemperature', dtype=sstqm.dtype, fillval=sstqm.fill_value) \ + .write_attr('long_name', 'Sea Surface Temperature Quality Marker') \ + .write_data(sstqm) + + # Station Pressure + obsspace.create_var('ObsValue/pressure', dtype=pob.dtype, fillval=pob.fill_value) \ + .write_attr('units', 'Pa') \ + .write_attr('long_name', 'Station Pressure') \ + .write_data(pob) + + # Air Temperature + obsspace.create_var('ObsValue/airTemperature', dtype=tob.dtype, fillval=tob.fill_value) \ + .write_attr('units', 'K') \ + .write_attr('long_name', 'Air Temperature') \ + .write_data(tob) + + # Virtual Temperature + obsspace.create_var('ObsValue/virtualTemperature', dtype=tvo.dtype, fillval=tvo.fill_value) \ + .write_attr('units', 'K') \ + .write_attr('long_name', 'Virtual Temperature') \ + .write_data(tvo) + + # Specific Humidity + obsspace.create_var('ObsValue/specificHumidity', dtype=qob.dtype, fillval=qob.fill_value) \ + .write_attr('units', 'kg kg-1') \ + .write_attr('long_name', 'Specific Humidity') \ + .write_data(qob) + + # Eastward Wind + obsspace.create_var('ObsValue/windEastward', dtype=uob.dtype, fillval=uob.fill_value) \ + .write_attr('units', 'm s-1') \ + .write_attr('long_name', 'Eastward Wind') \ + .write_data(uob) + + # Northward Wind + obsspace.create_var('ObsValue/windNorthward', dtype=vob.dtype, fillval=vob.fill_value) \ + .write_attr('units', 'm s-1') \ + .write_attr('long_name', 'Northward Wind') \ + .write_data(vob) + + # Sea Surface Temperature + obsspace.create_var('ObsValue/seaSurfaceTemperature', dtype=sst1.dtype, fillval=sst1.fill_value) \ + .write_attr('units', 'K') \ + .write_attr('long_name', 'Sea Surface Temperature') \ + .write_data(sst1) + + + end_time = time.time() + running_time = end_time - start_time + print('Running time for splitting and output IODA:', running_time, 'seconds') + + print("All Done!") + +if __name__ == '__main__': + + start_time = time.time() + + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, help='Input JSON configuration', required=True) + parser.add_argument('-v', '--verbose', help='print debug logging information', + action='store_true') + args = parser.parse_args() + + + with open(args.config, "r") as json_file: + config = json.load(json_file) + + bufr_to_ioda(config) +###### +# parser = argparse.ArgumentParser() +# description=( +# 'Reads NCEP PREPBUFR formated ADPsurface input files' +# ' convert into IODA formatted output files. ' +# ) +# +## DATA_PATH = './testinput/gdas.t12z.adpsfc.prepbufr' +## DATA_PATH = '/scratch1/NCEPDEV/da/Nicholas.Esposito/JEDI/pyapi_fromOrion_20230627/testinput/2021080100/SFCSHP.prepbufr' +# DATA_PATH = '/work2/noaa/da/nesposito/NewConv/api_convert/testinput/2021080100/SFCSHP.prepbufr' +## OUTPUT_PATH = './testrun/prepbufr_adpsfc_api.nc' +## OUTPUT_PATH = '/scratch1/NCEPDEV/da/Nicholas.Esposito/JEDI/pyapi_fromOrion_20230627/testoutput/2021080100/sfcshp_v2.nc' +# OUTPUT_PATH = '/work2/noaa/da/nesposito/NewConv/api_convert/testoutput/2021080100/sfcshp_v2.nc' +# +# optional = parser.add_argument_group(title='optional arguments') +# optional.add_argument('-f', '--filename', type=str, default=DATA_PATH, +# dest='filename', help='sfcshp prepbufr file name') +# optional.add_argument('-o', '--output', type=str, default=OUTPUT_PATH, +# dest='output', help='output filename') +# optional.add_argument('-d', '--date', type=str, default='202108010000', +# dest='date', metavar='YYYYmmddHHMM', help='analysis cycle date') +# +# args = parser.parse_args() +# +# bufr_to_ioda(args.filename, args.output, args.date) +##################### + + + end_time = time.time() + running_time = end_time - start_time + print('Total running time: ', running_time, 'seconds') diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda_sfcshp.sh b/ush/ioda/bufr2ioda/run_bufr2ioda_sfcshp.sh new file mode 100755 index 000000000..121141f0a --- /dev/null +++ b/ush/ioda/bufr2ioda/run_bufr2ioda_sfcshp.sh @@ -0,0 +1,79 @@ +#!/bin/bash +#-------------------------------------------------------------------------------------- +# run_bufr2ioda.sh +# This driver script will: +# - generate config files from templates for each BUFR file +# - use bufr2ioda python scripts or executable and produce output IODA files +# usage: +# run_bufr2ioda.sh YYYYMMDDHH $DMPDIR /path/to/templates $COM_OBS +# +#-------------------------------------------------------------------------------------- +if [[ $# -ne 5 ]] ; then + echo "usage:" + echo " $0 YYYYMMDDHH gdas|gfs /path/to/files.bufr_d/ /path/to/templates /path/to/output.ioda/" + exit 1 +fi + +# some of these need exported to be picked up by the python script below +# input parameters +CDATE=${CDATE:-$1} +export DUMP=${RUN:-$2} +export DMPDIR=${DMPDIR:-$3} +config_template_dir=${config_template_dir:-$4} +export COM_OBS=${COM_OBS:-$5} + +# derived parameters +export PDY=$(echo $CDATE | cut -c1-8) +export cyc=$(echo $CDATE | cut -c9-10) + +# get gdasapp root directory +readonly DIR_ROOT=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )/../../.." && pwd -P) +BUFR2IODA=$DIR_ROOT/build/bin/bufr2ioda.x +USH_IODA=$DIR_ROOT/ush/ioda/bufr2ioda +BUFRYAMLGEN=$USH_IODA/gen_bufr2ioda_yaml.py +BUFRJSONGEN=$USH_IODA/gen_bufr2ioda_json.py + +# create output directory if it doesn't exist +mkdir -p $COM_OBS +if [ $? -ne 0 ]; then + echo "cannot make $COM_OBS" + exit 1 +fi + +# add to pythonpath the necessary libraries +PYIODALIB=`echo $DIR_ROOT/build/lib/python3.?` +export PYTHONPATH=${PYIODALIB}:${PYTHONPATH} + +#--------------------------------------------------------- +# for some BUFR files, we will use python scripts +# and for others, bufr2ioda.x + +#--------------------------- +#----- python and json ----- +# first specify what observation types will be processed by a script +BUFR_py="sfcshp_prepbufr" + +for obtype in $BUFR_py; do + # this loop assumes that there is a python script and template with the same name + echo "Processing ${obtype}..." + + # first generate a JSON from the template + ${BUFRJSONGEN} -t ${config_template_dir}/bufr2ioda_${obtype}.json -o ${COM_OBS}/${obtype}_${PDY}${cyc}.json + + # now use the converter script for the ob type + python3 $USH_IODA/bufr2ioda_${obtype}.py -c ${COM_OBS}/${obtype}_${PDY}${cyc}.json + + # check if converter was successful + if [ $? == 0 ]; then + # remove JSON file + rm -rf ${COM_OBS}/${obtype}_${PDY}${cyc}.json + else + # warn and keep the JSON file + echo "Problem running converter script for ${obtype}" + fi +done + +#---------------------------- +#---- bufr2ioda and yaml ---- +# need to ask Emily which will be YAML based +BUFR_yaml="" From 90fd9959eee1c8bd8ff58de4a35a11f7a0e7e7e9 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Tue, 22 Aug 2023 15:54:03 -0400 Subject: [PATCH 02/29] initial changes. no py add yet --- ...fr.json => bufr2ioda_adpsfc_prepbufr.json} | 6 +- .../bufr2ioda/bufr2ioda_sfcshp_prepbufr.py | 426 ------------------ ...ioda_sfcshp.sh => run_bufr2ioda_adpsfc.sh} | 2 +- 3 files changed, 4 insertions(+), 430 deletions(-) rename parm/ioda/bufr2ioda/{bufr2ioda_sfcshp_prepbufr.json => bufr2ioda_adpsfc_prepbufr.json} (72%) delete mode 100644 ush/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.py rename ush/ioda/bufr2ioda/{run_bufr2ioda_sfcshp.sh => run_bufr2ioda_adpsfc.sh} (99%) diff --git a/parm/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.json b/parm/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.json similarity index 72% rename from parm/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.json rename to parm/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.json index 4ec6272ae..e2fb31bb7 100644 --- a/parm/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.json +++ b/parm/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.json @@ -1,12 +1,12 @@ { "data_format" : "prepbufr", - "subsets" : [ "SFCSHP" ], + "subsets" : [ "ADPSFC" ], "source" : "prepBUFR", - "data_type" : "SFCSHP", + "data_type" : "ADPSFC", "cycle_type" : "{{ DUMP }}", "cycle_datetime" : "{{ current_cycle | to_YMDH }}", "dump_directory" : "{{ DMPDIR }}", "ioda_directory" : "{{ COM_OBS }}", - "data_description" : "SFCSHP_prepbufr", + "data_description" : "ADPSFC_prepbufr", "data_provider" : "U.S. NOAA/NESDIS" } diff --git a/ush/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.py b/ush/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.py deleted file mode 100644 index 00514b955..000000000 --- a/ush/ioda/bufr2ioda/bufr2ioda_sfcshp_prepbufr.py +++ /dev/null @@ -1,426 +0,0 @@ -# (C) Copyright 2023 NOAA/NWS/NCEP/EMC -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -import sys -sys.path.append('/work2/noaa/da/nesposito/ioda-bundle_20230712/build/lib/python3.9/') -sys.path.append('/work2/noaa/da/nesposito/ioda-bundle_20230712/build/lib/python3.9/pyiodaconv/') -sys.path.append('/work2/noaa/da/nesposito/ioda-bundle_20230712/build/lib/python3.9/pyioda/') - -import numpy as np -import os -import argparse -import math -import calendar -import time -from datetime import datetime -import json -from pyiodaconv import bufr -from collections import namedtuple -from pyioda import ioda_obs_space as ioda_ospace - -def Compute_dateTime(cycleTimeSinceEpoch, dhr): - - dhr = np.int64(dhr*3600) - dateTime = dhr + cycleTimeSinceEpoch - - return dateTime - -def bufr_to_ioda(config): - - subsets = config["subsets"] - - # Get parameters from configuration - data_format = config["data_format"] - source = config["source"] - data_type = config["data_type"] - data_description = config["data_description"] - data_provider = config["data_provider"] - cycle_type = config["cycle_type"] - cycle_datetime = config["cycle_datetime"] - dump_dir = config["dump_directory"] - ioda_dir = config["ioda_directory"] - cycle = config["cycle_datetime"] - - # Get derived parameters - yyyymmdd = cycle[0:8] - hh = cycle[8:10] - - reference_time = datetime.strptime(cycle, "%Y%m%d%H") - reference_time = reference_time.strftime("%Y-%m-%dT%H:%M:%SZ") - reference_time_full = f"{yyyymmdd}{hh}00" - - # General informaton - converter = 'BUFR to IODA Converter' - platform_description = 'SFCSHP data from prepBUFR format' - - bufrfile = f"{cycle_type}.t{hh}z.{data_format}" - DATA_PATH = os.path.join(dump_dir, f"{cycle_type}.{yyyymmdd}", str(hh), bufrfile) - - print("The DATA_PATH is: "+str(DATA_PATH)) - - # ============================================ - # Make the QuerySet for all the data we want - # ============================================ - start_time = time.time() - - print('Making QuerySet ...') - q = bufr.QuerySet(subsets) - - # MetaData - q.add('stationIdentification', '*/SID') - q.add('latitude', '*/YOB') - q.add('longitude', '*/XOB') - q.add('obsTimeMinusCycleTime', '*/DHR') - q.add('stationElevation', '*/ELV') - q.add('observationType', '*/TYP') - q.add('pressure', '*/P___INFO/P__EVENT{1}/POB') - q.add('temperatureEventCode', '*/T___INFO/T__EVENT{1}/TPC') - -# # Quality Infomation (Quality Indicator) - q.add('qualityMarkerStationPressure', '*/P___INFO/P__EVENT{1}/PQM') - q.add('qualityMarkerAirTemperature', '*/T___INFO/T__EVENT{1}/TQM') - q.add('qualityMarkerSpecificHumidity', '*/Q___INFO/Q__EVENT{1}/QQM') - q.add('qualityMarkerWindNorthward', '*/W___INFO/W__EVENT{1}/WQM') - q.add('qualityMarkerSeaSurfaceTemperature','*/SST_INFO/SSTEVENT{1}/SSTQM') - - # ObsValue - q.add('stationPressure', '*/P___INFO/P__EVENT{1}/POB') - q.add('airTemperature', '*/T___INFO/T__EVENT{1}/TOB') - q.add('virtualTemperature', '*/T___INFO/TVO') - q.add('specificHumidity', '*/Q___INFO/Q__EVENT{1}/QOB') - q.add('windNorthward', '*/W___INFO/W__EVENT{1}/VOB') - q.add('windEastward', '*/W___INFO/W__EVENT{1}/UOB') - q.add('seaSurfaceTemperature', '*/SST_INFO/SSTEVENT{1}/SST1') - - end_time = time.time() - running_time = end_time - start_time - print('Running time for making QuerySet : ', running_time, 'seconds') - - # ============================================================== - # Open the BUFR file and execute the QuerySet to get ResultSet - # Use the ResultSet returned to get numpy arrays of the data - # ============================================================== - start_time = time.time() - - print('Executing QuerySet to get ResultSet ...') - with bufr.File(DATA_PATH) as f: -# print("hello") - r = f.execute(q) - - - print(' ... Executing QuerySet: get metadata: basic ...') - # MetaData - sid = r.get('stationIdentification') - lat = r.get('latitude') - lon = r.get('longitude') - lon[lon>180] -= 360 - elv = r.get('stationElevation', type='float') - typ = r.get('observationType') - pressure = r.get('pressure') - pressure *= 100 - tpc = r.get('temperatureEventCode') - - print(' ... Executing QuerySet: get QualityMarker information ...') - # Quality Information - pqm = r.get('qualityMarkerStationPressure') - tqm = r.get('qualityMarkerAirTemperature') - qqm = r.get('qualityMarkerSpecificHumidity') - wqm = r.get('qualityMarkerWindNorthward') - sstqm = r.get('qualityMarkerSeaSurfaceTemperature') - - print(' ... Executing QuerySet: get obsvalue: stationPressure ...') - # ObsValue - pob = r.get('stationPressure') - pob *= 100 - tob = r.get('airTemperature') - tob += 273.15 - tvo = r.get('virtualTemperature') - tvo += 273.15 - qob = r.get('specificHumidity', type='float') - qob *= 0.000001 - uob = r.get('windEastward') - vob = r.get('windNorthward') - sst1 = r.get('seaSurfaceTemperature') - - print(' ... Executing QuerySet: get datatime: observation time ...') - # DateTime: seconds since Epoch time - # IODA has no support for numpy datetime arrays dtype=datetime64[s] - dhr = r.get('obsTimeMinusCycleTime', type='int64') - - print(' ... Executing QuerySet: Done!') - - print(' ... Executing QuerySet: Check BUFR variable generic dimension and type ...') - # Check BUFR variable generic dimension and type - print(' sid shape = ', sid.shape) - print(' dhr shape = ', dhr.shape) - print(' lat shape = ', lat.shape) - print(' lon shape = ', lon.shape) - print(' elv shape = ', elv.shape) - print(' typ shape = ', typ.shape) - print(' pressure shape = ', pressure.shape) - print(' tpc shape = ', tpc.shape) - - print(' pqm shape = ', pqm.shape) - print(' tqm shape = ', tqm.shape) - print(' qqm shape = ', qqm.shape) - print(' wqm shape = ', wqm.shape) - print(' sstqm shape = ', sstqm.shape) - - print(' pob shape = ', pob.shape) - print(' tob shape = ', pob.shape) - print(' tvo shape = ', tvo.shape) - print(' qob shape = ', qob.shape) - print(' uob shape = ', uob.shape) - print(' vob shape = ', vob.shape) - print(' sst1 shape = ', sst1.shape) - - print(' sid type = ', sid.dtype) - print(' dhr type = ', dhr.dtype) - print(' lat type = ', lat.dtype) - print(' lon type = ', lon.dtype) - print(' elv type = ', elv.dtype) - print(' typ type = ', typ.dtype) - print(' pressure type = ', pressure.dtype) - print(' tpc type = ', tpc.dtype) - - print(' pqm type = ', pqm.dtype) - print(' tqm type = ', tqm.dtype) - print(' qqm type = ', qqm.dtype) - print(' wqm type = ', wqm.dtype) - print(' sstqm type = ', sstqm.dtype) - - print(' pob type = ', pob.dtype) - print(' tob type = ', tob.dtype) - print(' tvo type = ', tvo.dtype) - print(' qob type = ', qob.dtype) - print(' uob type = ', uob.dtype) - print(' vob type = ', vob.dtype) - print(' sst1 type = ', sst1.dtype) - - end_time = time.time() - running_time = end_time - start_time - print('Running time for executing QuerySet to get ResultSet : ', running_time, 'seconds') - - # ========================= - # Create derived variables - # ========================= - start_time = time.time() - - print('Creating derived variables - dateTime ...') - - cycleTimeSinceEpoch = np.int64(calendar.timegm(time.strptime(reference_time_full, '%Y%m%d%H%M'))) - dateTime = Compute_dateTime(cycleTimeSinceEpoch, dhr) - - print(' Check drived variables type ... ') - print(' dateTime shape = ', dateTime.shape) - print(' dateTime type = ', dateTime.dtype) - - end_time = time.time() - running_time = end_time - start_time - print('Running time for creating derived variables : ', running_time, 'seconds') - - # ===================================== - # Create IODA ObsSpace - # Write IODA output - # ===================================== - - # Create the dimensions - dims = { - 'Location' : np.arange(0, lat.shape[0]), - } - - - iodafile = f"{cycle_type}.t{hh}z.{data_type}.{data_format}.nc" - OUTPUT_PATH = os.path.join(ioda_dir, iodafile) - print(" ... ... Create OUTPUT file: ", OUTPUT_PATH) - - path, fname = os.path.split(OUTPUT_PATH) - if path and not os.path.exists(path): - os.makedirs(path) - - obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) - - # Create Global attributes - print(' ... ... Create global attributes') - - obsspace.write_attr('Converter', converter) - obsspace.write_attr('source', source) - obsspace.write_attr('sourceFiles', bufrfile) - obsspace.write_attr('dataProviderOrigin', data_provider) - obsspace.write_attr('description', data_description) - obsspace.write_attr('datetimeReference', reference_time) - obsspace.write_attr('datetimeRange', [str(min(dateTime)), str(max(dateTime))]) - obsspace.write_attr('platformLongDescription', platform_description) - - # Create IODA variables - print(' ... ... Create variables: name, type, units, and attributes') - # Longitude - obsspace.create_var('MetaData/longitude', dtype=lon.dtype, fillval=lon.fill_value) \ - .write_attr('units', 'degrees_east') \ - .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ - .write_attr('long_name', 'Longitude') \ - .write_data(lon) - - # Latitude - obsspace.create_var('MetaData/latitude', dtype=lat.dtype, fillval=lat.fill_value) \ - .write_attr('units', 'degrees_north') \ - .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ - .write_attr('long_name', 'Latitude') \ - .write_data(lat) - - # Datetime - obsspace.create_var('MetaData/dateTime', dtype=dateTime.dtype, fillval=dateTime.fill_value) \ - .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ - .write_attr('long_name', 'Datetime') \ - .write_data(dateTime) - - # Station Identification - obsspace.create_var('MetaData/stationIdentification', dtype=sid.dtype, fillval=sid.fill_value) \ - .write_attr('units', '1') \ - .write_attr('long_name', 'Station Identification') \ - .write_data(sid) - - # Station Elevation - obsspace.create_var('MetaData/stationElevation', dtype=elv.dtype, fillval=elv.fill_value) \ - .write_attr('units', 'm') \ - .write_attr('long_name', 'Station Elevation') \ - .write_data(elv) - - # Observation Type - obsspace.create_var('MetaData/observationType', dtype=typ.dtype, fillval=typ.fill_value) \ - .write_attr('long_name', 'Observation Type') \ - .write_data(typ) - - # Pressure - obsspace.create_var('MetaData/pressure', dtype=pressure.dtype, fillval=pressure.fill_value) \ - .write_attr('units', 'Pa') \ - .write_attr('long_name', 'Pressure') \ - .write_data(pressure) - - # Temperature Event Code - obsspace.create_var('MetaData/temperatureEventCode', dtype=tpc.dtype, fillval=tpc.fill_value) \ - .write_attr('long_name', 'Temperature Event Code') \ - .write_data(typ) - - # Quality Marker: Station Pressure - obsspace.create_var('QualityMarker/stationPressure', dtype=pqm.dtype, fillval=pqm.fill_value) \ - .write_attr('long_name', 'Station Pressure Quality Marker') \ - .write_data(pqm) - - # Quality Marker: Air Temperature - obsspace.create_var('QualityMarker/airTemperature', dtype=tqm.dtype, fillval=tqm.fill_value) \ - .write_attr('long_name', 'Air Temperature Quality Marker') \ - .write_data(tqm) - - # Quality Marker: Specific Humidity - obsspace.create_var('QualityMarker/specificHumidity', dtype=qqm.dtype, fillval=qqm.fill_value) \ - .write_attr('long_name', 'Specific Humidity Quality Marker') \ - .write_data(qqm) - - # Quality Marker: Northward Wind - obsspace.create_var('QualityMarker/windNorthward', dtype=wqm.dtype, fillval=wqm.fill_value) \ - .write_attr('long_name', 'Northward Wind Quality Marker') \ - .write_data(wqm) - - # Quality Marker: Sea Surface Temperature - obsspace.create_var('QualityMarker/seaSurfaceTemperature', dtype=sstqm.dtype, fillval=sstqm.fill_value) \ - .write_attr('long_name', 'Sea Surface Temperature Quality Marker') \ - .write_data(sstqm) - - # Station Pressure - obsspace.create_var('ObsValue/pressure', dtype=pob.dtype, fillval=pob.fill_value) \ - .write_attr('units', 'Pa') \ - .write_attr('long_name', 'Station Pressure') \ - .write_data(pob) - - # Air Temperature - obsspace.create_var('ObsValue/airTemperature', dtype=tob.dtype, fillval=tob.fill_value) \ - .write_attr('units', 'K') \ - .write_attr('long_name', 'Air Temperature') \ - .write_data(tob) - - # Virtual Temperature - obsspace.create_var('ObsValue/virtualTemperature', dtype=tvo.dtype, fillval=tvo.fill_value) \ - .write_attr('units', 'K') \ - .write_attr('long_name', 'Virtual Temperature') \ - .write_data(tvo) - - # Specific Humidity - obsspace.create_var('ObsValue/specificHumidity', dtype=qob.dtype, fillval=qob.fill_value) \ - .write_attr('units', 'kg kg-1') \ - .write_attr('long_name', 'Specific Humidity') \ - .write_data(qob) - - # Eastward Wind - obsspace.create_var('ObsValue/windEastward', dtype=uob.dtype, fillval=uob.fill_value) \ - .write_attr('units', 'm s-1') \ - .write_attr('long_name', 'Eastward Wind') \ - .write_data(uob) - - # Northward Wind - obsspace.create_var('ObsValue/windNorthward', dtype=vob.dtype, fillval=vob.fill_value) \ - .write_attr('units', 'm s-1') \ - .write_attr('long_name', 'Northward Wind') \ - .write_data(vob) - - # Sea Surface Temperature - obsspace.create_var('ObsValue/seaSurfaceTemperature', dtype=sst1.dtype, fillval=sst1.fill_value) \ - .write_attr('units', 'K') \ - .write_attr('long_name', 'Sea Surface Temperature') \ - .write_data(sst1) - - - end_time = time.time() - running_time = end_time - start_time - print('Running time for splitting and output IODA:', running_time, 'seconds') - - print("All Done!") - -if __name__ == '__main__': - - start_time = time.time() - - parser = argparse.ArgumentParser() - parser.add_argument('-c', '--config', type=str, help='Input JSON configuration', required=True) - parser.add_argument('-v', '--verbose', help='print debug logging information', - action='store_true') - args = parser.parse_args() - - - with open(args.config, "r") as json_file: - config = json.load(json_file) - - bufr_to_ioda(config) -###### -# parser = argparse.ArgumentParser() -# description=( -# 'Reads NCEP PREPBUFR formated ADPsurface input files' -# ' convert into IODA formatted output files. ' -# ) -# -## DATA_PATH = './testinput/gdas.t12z.adpsfc.prepbufr' -## DATA_PATH = '/scratch1/NCEPDEV/da/Nicholas.Esposito/JEDI/pyapi_fromOrion_20230627/testinput/2021080100/SFCSHP.prepbufr' -# DATA_PATH = '/work2/noaa/da/nesposito/NewConv/api_convert/testinput/2021080100/SFCSHP.prepbufr' -## OUTPUT_PATH = './testrun/prepbufr_adpsfc_api.nc' -## OUTPUT_PATH = '/scratch1/NCEPDEV/da/Nicholas.Esposito/JEDI/pyapi_fromOrion_20230627/testoutput/2021080100/sfcshp_v2.nc' -# OUTPUT_PATH = '/work2/noaa/da/nesposito/NewConv/api_convert/testoutput/2021080100/sfcshp_v2.nc' -# -# optional = parser.add_argument_group(title='optional arguments') -# optional.add_argument('-f', '--filename', type=str, default=DATA_PATH, -# dest='filename', help='sfcshp prepbufr file name') -# optional.add_argument('-o', '--output', type=str, default=OUTPUT_PATH, -# dest='output', help='output filename') -# optional.add_argument('-d', '--date', type=str, default='202108010000', -# dest='date', metavar='YYYYmmddHHMM', help='analysis cycle date') -# -# args = parser.parse_args() -# -# bufr_to_ioda(args.filename, args.output, args.date) -##################### - - - end_time = time.time() - running_time = end_time - start_time - print('Total running time: ', running_time, 'seconds') diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda_sfcshp.sh b/ush/ioda/bufr2ioda/run_bufr2ioda_adpsfc.sh similarity index 99% rename from ush/ioda/bufr2ioda/run_bufr2ioda_sfcshp.sh rename to ush/ioda/bufr2ioda/run_bufr2ioda_adpsfc.sh index 121141f0a..4f5126043 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda_sfcshp.sh +++ b/ush/ioda/bufr2ioda/run_bufr2ioda_adpsfc.sh @@ -51,7 +51,7 @@ export PYTHONPATH=${PYIODALIB}:${PYTHONPATH} #--------------------------- #----- python and json ----- # first specify what observation types will be processed by a script -BUFR_py="sfcshp_prepbufr" +BUFR_py="adpsfc_prepbufr" for obtype in $BUFR_py; do # this loop assumes that there is a python script and template with the same name From 08df4dce512151cf801db023f9c951de80305ad2 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Tue, 22 Aug 2023 16:09:30 -0400 Subject: [PATCH 03/29] add adpsfc py --- .../bufr2ioda/bufr2ioda_adpsfc_prepbufr.py | 285 ++++++++++++++++++ 1 file changed, 285 insertions(+) create mode 100644 ush/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.py diff --git a/ush/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.py b/ush/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.py new file mode 100644 index 000000000..469e666c9 --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.py @@ -0,0 +1,285 @@ +# (C) Copyright 2023 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import sys + +sys.path.append('/work2/noaa/da/nesposito/ioda-bundle_20230712/build/lib/python3.9/') +sys.path.append('/work2/noaa/da/nesposito/ioda-bundle_20230712/build/lib/python3.9/pyiodaconv/') +sys.path.append('/work2/noaa/da/nesposito/ioda-bundle_20230712/build/lib/python3.9/pyioda/') + +import numpy as np +import os +import argparse +import math +import calendar +import time +from datetime import datetime +import json +from pyiodaconv import bufr +from collections import namedtuple +from pyioda import ioda_obs_space as ioda_ospace + +def Compute_dateTime(cycleTimeSinceEpoch, dhr): + + dhr = np.int64(dhr*3600) + dateTime = dhr + cycleTimeSinceEpoch + + return dateTime + +def bufr_to_ioda(config): + + + subsets = config["subsets"] + + # Get parameters from configuration + data_format = config["data_format"] + source = config["source"] + data_type = config["data_type"] + data_description = config["data_description"] + data_provider = config["data_provider"] + cycle_type = config["cycle_type"] + cycle_datetime = config["cycle_datetime"] + dump_dir = config["dump_directory"] + ioda_dir = config["ioda_directory"] + cycle = config["cycle_datetime"] + + # Get derived parameters + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + + reference_time = datetime.strptime(cycle, "%Y%m%d%H") + reference_time = reference_time.strftime("%Y-%m-%dT%H:%M:%SZ") + reference_time_full = f"{yyyymmdd}{hh}00" + + # General informaton + converter = 'BUFR to IODA Converter' + platform_description = 'SFCSHP data from prepBUFR format' + + bufrfile = f"{cycle_type}.t{hh}z.{data_format}" + DATA_PATH = os.path.join(dump_dir, f"{cycle_type}.{yyyymmdd}", str(hh), bufrfile) + + print("The DATA_PATH is: "+str(DATA_PATH)) + + # ============================================ + # Make the QuerySet for all the data we want + # ============================================ + start_time = time.time() + + print('Making QuerySet ...') + q = bufr.QuerySet(subsets) + + # MetaData + q.add('stationIdentification', '*/SID') + q.add('latitude', '*/YOB') + q.add('longitude', '*/XOB') + q.add('obsTimeMinusCycleTime', '*/DHR') + q.add('stationElevation', '*/ELV') + q.add('observationType', '*/TYP') + q.add('pressure', '*/P___INFO/P__EVENT{1}/POB') + +# # Quality Infomation (Quality Indicator) + q.add('qualityMarkerStationPressure', '*/P___INFO/P__EVENT{1}/PQM') + + # ObsValue + q.add('stationPressure', '*/P___INFO/P__EVENT{1}/POB') + + end_time = time.time() + running_time = end_time - start_time + print('Running time for making QuerySet : ', running_time, 'seconds') + + # ============================================================== + # Open the BUFR file and execute the QuerySet to get ResultSet + # Use the ResultSet returned to get numpy arrays of the data + # ============================================================== + start_time = time.time() + + print('Executing QuerySet to get ResultSet ...') + with bufr.File(DATA_PATH) as f: + r = f.execute(q) + + print(' ... Executing QuerySet: get metadata: basic ...') + # MetaData + sid = r.get('stationIdentification') + lat = r.get('latitude') + lon = r.get('longitude') + lon[lon>180] -=360 + elv = r.get('stationElevation') + typ = r.get('observationType') + pressure = r.get('pressure') + pressure *= 100 + + print(' ... Executing QuerySet: get QualityMarker information ...') + # Quality Information + pobqm = r.get('qualityMarkerStationPressure') + + print(' ... Executing QuerySet: get obsvalue: stationPressure ...') + # ObsValue + pob = r.get('stationPressure') + pob *= 100 + + print(' ... Executing QuerySet: get datatime: observation time ...') + # DateTime: seconds since Epoch time + # IODA has no support for numpy datetime arrays dtype=datetime64[s] + dhr = r.get('obsTimeMinusCycleTime', type='int64') + + print(' ... Executing QuerySet: Done!') + + print(' ... Executing QuerySet: Check BUFR variable generic dimension and type ...') + # Check BUFR variable generic dimension and type + print(' sid shape = ', sid.shape) + print(' dhr shape = ', dhr.shape) + print(' lat shape = ', lat.shape) + print(' lon shape = ', lon.shape) + print(' elv shape = ', elv.shape) + print(' typ shape = ', typ.shape) + print(' pressure shape = ', pressure.shape) + + print(' pobqm shape = ', pobqm.shape) + print(' pob shape = ', pob.shape) + + print(' sid type = ', sid.dtype) + print(' dhr type = ', dhr.dtype) + print(' lat type = ', lat.dtype) + print(' lon type = ', lon.dtype) + print(' elv type = ', elv.dtype) + print(' typ type = ', typ.dtype) + print(' pressure type = ', pressure.dtype) + + print(' pobqm type = ', pobqm.dtype) + print(' pob type = ', pob.dtype) + + end_time = time.time() + running_time = end_time - start_time + print('Running time for executing QuerySet to get ResultSet : ', running_time, 'seconds') + + # ========================= + # Create derived variables + # ========================= + start_time = time.time() + + print('Creating derived variables - dateTime ...') + + cycleTimeSinceEpoch = np.int64(calendar.timegm(time.strptime(date, '%Y%m%d%H%M'))) + dateTime = Compute_dateTime(cycleTimeSinceEpoch, dhr) + + print(' Check derived variables type ... ') + print(' dateTime shape = ', dateTime.shape) + print(' dateTime type = ', dateTime.dtype) + + end_time = time.time() + running_time = end_time - start_time + print('Running time for creating derived variables : ', running_time, 'seconds') + + # ===================================== + # Create IODA ObsSpace + # Write IODA output + # ===================================== + + # Create the dimensions + dims = { + 'Location' : np.arange(0, lat.shape[0]), + } + + iodafile = f"{cycle_type}.t{hh}z.{data_type}.{data_format}.nc" + OUTPUT_PATH = os.path.join(ioda_dir, iodafile) + print(" ... ... Create OUTPUT file: ", OUTPUT_PATH) + + path, fname = os.path.split(OUTPUT_PATH) + if path and not os.path.exists(path): + os.makedirs(path) + + obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) + + # Create Global attributes + print(' ... ... Create global attributes') + + obsspace.write_attr('Converter', converter) + obsspace.write_attr('source', source) + obsspace.write_attr('sourceFiles', bufrfile) + obsspace.write_attr('dataProviderOrigin', data_provider) + obsspace.write_attr('description', data_description) + obsspace.write_attr('datetimeReference', reference_time) + obsspace.write_attr('datetimeRange', [str(min(dateTime)), str(max(dateTime))]) + obsspace.write_attr('platformLongDescription', platform_description) + + # Create IODA variables + print(' ... ... Create variables: name, type, units, and attributes') + # Longitude + obsspace.create_var('MetaData/longitude', dtype=lon.dtype, fillval=lon.fill_value) \ + .write_attr('units', 'degrees_east') \ + .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ + .write_attr('long_name', 'Longitude') \ + .write_data(lon) + + # Latitude + obsspace.create_var('MetaData/latitude', dtype=lat.dtype, fillval=lat.fill_value) \ + .write_attr('units', 'degrees_north') \ + .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Latitude') \ + .write_data(lat) + + # Datetime + obsspace.create_var('MetaData/dateTime', dtype=dateTime.dtype, fillval=dateTime.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'Datetime') \ + .write_data(dateTime) + + # Station Identification + obsspace.create_var('MetaData/stationIdentification', dtype=sid.dtype, fillval=sid.fill_value) \ + .write_attr('long_name', 'Station Identification') \ + .write_data(sid) + + # Station Elevation + obsspace.create_var('MetaData/stationElevation', dtype=elv.dtype, fillval=elv.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Station Elevation') \ + .write_data(elv) + + # Observation Type + obsspace.create_var('MetaData/observationType', dtype=typ.dtype, fillval=typ.fill_value) \ + .write_attr('long_name', 'Observation Type') \ + .write_data(typ) + + # Pressure + obsspace.create_var('MetaData/pressure', dtype=pressure.dtype, fillval=pressure.fill_value) \ + .write_attr('units', 'Pa') \ + .write_attr('long_name', 'Pressure') \ + .write_data(pressure) + + # Quality: Percent Confidence - Quality Information Without Forecast + obsspace.create_var('QualityMarker/stationPressure', dtype=pobqm.dtype, fillval=pobqm.fill_value) \ + .write_attr('long_name', 'Station Pressure Quality Marker') \ + .write_data(pobqm) + + # Station Pressure + obsspace.create_var('ObsValue/pressure', dtype=pob.dtype, fillval=pob.fill_value) \ + .write_attr('units', 'Pa') \ + .write_attr('long_name', 'Station Pressure') \ + .write_data(pob) + + end_time = time.time() + running_time = end_time - start_time + print('Running time for splitting and output IODA:', running_time, 'seconds') + + print("All Done!") + +if __name__ == '__main__': + + start_time = time.time() + + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, help='Input JSON configuration', required=True) + parser.add_argument('-v', '--verbose', help='print debug logging information', + action='store_true') + args = parser.parse_args() + + with open(args.config, "r") as json_file: + config = json.load(json_file) + + bufr_to_ioda(config) + + end_time = time.time() + running_time = end_time - start_time + print('Total running time: ', running_time, 'seconds') From 785f8c372354bc2cb3d1898d3d2e5d88b14c7164 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Tue, 22 Aug 2023 16:19:26 -0400 Subject: [PATCH 04/29] some updates --- ush/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.py b/ush/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.py index 469e666c9..a07a0ed2a 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.py @@ -4,7 +4,6 @@ # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. import sys - sys.path.append('/work2/noaa/da/nesposito/ioda-bundle_20230712/build/lib/python3.9/') sys.path.append('/work2/noaa/da/nesposito/ioda-bundle_20230712/build/lib/python3.9/pyiodaconv/') sys.path.append('/work2/noaa/da/nesposito/ioda-bundle_20230712/build/lib/python3.9/pyioda/') @@ -161,7 +160,7 @@ def bufr_to_ioda(config): print('Creating derived variables - dateTime ...') - cycleTimeSinceEpoch = np.int64(calendar.timegm(time.strptime(date, '%Y%m%d%H%M'))) + cycleTimeSinceEpoch = np.int64(calendar.timegm(time.strptime(reference_time_full, '%Y%m%d%H%M'))) dateTime = Compute_dateTime(cycleTimeSinceEpoch, dhr) print(' Check derived variables type ... ') From afc857be9f86a9c7ca1cde49a8f346655e7045d9 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Tue, 22 Aug 2023 16:33:56 -0400 Subject: [PATCH 05/29] update README --- ush/ioda/bufr2ioda/README_whoever | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/ioda/bufr2ioda/README_whoever b/ush/ioda/bufr2ioda/README_whoever index 3850067f1..48e889040 100644 --- a/ush/ioda/bufr2ioda/README_whoever +++ b/ush/ioda/bufr2ioda/README_whoever @@ -1,7 +1,7 @@ Needed to make a sfcshp-specific run_bufr2ioda.sh script because I needed to change a variable. To run: use command: -./run_bufr2ioda.sh 20210801 gdas /work/noaa/rstprod/dump/ /work2/noaa/da/nesposito/GDASApp_20230822/parm/ioda/bufr2ioda/ /work2/noaa/da/nesposito/NewConv/api_convert/json/json_outputs/ +./run_bufr2ioda_adpsfc.sh 20210801 gdas /work/noaa/rstprod/dump/ /work2/noaa/da/nesposito/GDASApp_20230822/parm/ioda/bufr2ioda/ /work2/noaa/da/nesposito/NewConv/api_convert/json/json_outputs/ Outputs (and plots) are in: /work2/noaa/da/nesposito/NewConv/api_convert/json/json_outputs From 48371951c8d89329f6d6a8ebe044e9303e113039 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Tue, 22 Aug 2023 16:39:14 -0400 Subject: [PATCH 06/29] get rid of sys.appendpath --- ush/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.py b/ush/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.py index a07a0ed2a..11a9fc06e 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.py @@ -4,10 +4,6 @@ # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. import sys -sys.path.append('/work2/noaa/da/nesposito/ioda-bundle_20230712/build/lib/python3.9/') -sys.path.append('/work2/noaa/da/nesposito/ioda-bundle_20230712/build/lib/python3.9/pyiodaconv/') -sys.path.append('/work2/noaa/da/nesposito/ioda-bundle_20230712/build/lib/python3.9/pyioda/') - import numpy as np import os import argparse From b619e3b23cbbe8787fbf11a6d878ace4a3acb565 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 7 Sep 2023 13:54:48 -0400 Subject: [PATCH 07/29] initial adds --- .../bufr2ioda/bufr2ioda_adpsfc_prepbufr.json | 12 - parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.json | 28 + .../bufr2ioda/bufr2ioda_adpsfc_prepbufr.py | 280 -------- ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py | 633 ++++++++++++++++++ ...2ioda_adpsfc.sh => run_bufr2ioda_gpsro.sh} | 2 +- 5 files changed, 662 insertions(+), 293 deletions(-) delete mode 100644 parm/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.json create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.json delete mode 100644 ush/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.py create mode 100644 ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py rename ush/ioda/bufr2ioda/{run_bufr2ioda_adpsfc.sh => run_bufr2ioda_gpsro.sh} (99%) diff --git a/parm/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.json b/parm/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.json deleted file mode 100644 index e2fb31bb7..000000000 --- a/parm/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "data_format" : "prepbufr", - "subsets" : [ "ADPSFC" ], - "source" : "prepBUFR", - "data_type" : "ADPSFC", - "cycle_type" : "{{ DUMP }}", - "cycle_datetime" : "{{ current_cycle | to_YMDH }}", - "dump_directory" : "{{ DMPDIR }}", - "ioda_directory" : "{{ COM_OBS }}", - "data_description" : "ADPSFC_prepbufr", - "data_provider" : "U.S. NOAA/NESDIS" -} diff --git a/parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.json b/parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.json new file mode 100644 index 000000000..aaee14982 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.json @@ -0,0 +1,28 @@ +{ + "data_format" : "bufr_d", + "data_type" : "gpsro", + "cycle_type" : "{{ DUMP }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "subsets" : [ "GPS-RO" ], + "data_description" : "MSG TYPE 003-010 NESDIS GPS-RO", + "data_provider" : "U.S. NOAA/NESDIS", + "sensor_satellite_info" : [ + { "sensor_name" : "GRAS", "sensor_full_name" : "GNSS Receiver for Atmospheric Sounding", "sensor_id" : 202, "satellite_name" : "MetOp-B", "satellite_full_name" : "Meteorological Operational satellite B", "satellite_id" : 3}, + { "sensor_name" : "GRAS", "sensor_full_name" : "GNSS Receiver for Atmospheric Sounding", "sensor_id" : 202, "satellite_name" : "MetOp-A", "satellite_full_name" : "Meteorological Operational satellite A", "satellite_id" : 4}, + { "sensor_name" : "GRAS", "sensor_full_name" : "GNSS Receiver for Atmospheric Sounding", "sensor_id" : 202, "satellite_name" : "MetOp-C", "satellite_full_name" : "Meteorological Operational satellite C", "satellite_id" : 5}, + { "sensor_name" : "IGOR", "sensor_full_name" : "Integrated GPS and Occultation Receiver", "sensor_id" : 103, "satellite_name" : "TerraSAR-X", "satellite_full_name" : "X-band TerraSAR satellite", "satellite_id" : 42}, + { "sensor_name" : "IGOR", "sensor_full_name" : "Integrated GPS and Occultation Receiver", "sensor_id" : 103, "satellite_name" : "TanDEM-X", "satellite_full_name" : "TerraSAR-X add-on for Digital Elevation Measurement", "satellite_id" : 43}, + { "sensor_name" : "IGOR", "sensor_full_name" : "Integrated GPS and Occultation Receiver", "sensor_id" : 103, "satellite_name" : "PAZ", "satellite_full_name" : "Paz satellite", "satellite_id" : 44}, + { "sensor_name" :"CION", "sensor_full_name" :"CICERO Instrument for GNSS-RO ", "sensor_id" : 526, "satellite_name" : "CICERO-1 OP1", "satellite_full_name" : "Community Initiative for Cellular Earth Remote Observation OP1", "satellite_id" : 265}, + { "sensor_name" : "Tri-G", "sensor_full_name" : "Triple-G", "sensor_id" : 104, "satellite_name" : "COSMIC-2 E1", "satellite_full_name" : "Constellation Observing System for Meteorology, Ionosphere, and Climate-2 E1", "satellite_id" : 750}, + { "sensor_name" : "Tri-G", "sensor_full_name" : "Triple-G", "sensor_id" : 104, "satellite_name" : "COSMIC-2 E2", "satellite_full_name" : "Constellation Observing System for Meteorology, Ionosphere, and Climate-2 E2", "satellite_id" : 751}, + { "sensor_name" : "Tri-G", "sensor_full_name" : "Triple-G", "sensor_id" : 104, "satellite_name" : "COSMIC-2 E3", "satellite_full_name" : "Constellation Observing System for Meteorology, Ionosphere, and Climate-2 E3", "satellite_id" : 752}, + { "sensor_name" : "Tri-G", "sensor_full_name" : "Triple-G", "sensor_id" : 104, "satellite_name" : "COSMIC-2 E4", "satellite_full_name" : "Constellation Observing System for Meteorology, Ionosphere, and Climate-2 E4", "satellite_id" : 753}, + { "sensor_name" : "Tri-G", "sensor_full_name" : "Triple-G", "sensor_id" : 104, "satellite_name" : "COSMIC-2 E5", "satellite_full_name" : "Constellation Observing System for Meteorology, Ionosphere, and Climate-2 E5", "satellite_id" : 754}, + { "sensor_name" : "Tri-G", "sensor_full_name" : "Triple-G", "sensor_id" : 104, "satellite_name" : "COSMIC-2 E6", "satellite_full_name" : "Constellation Observing System for Meteorology, Ionosphere, and Climate-2 E6", "satellite_id" : 755}, + { "sensor_name" : "Tri-G", "sensor_full_name" :"Triple-G", "sensor_id" : 104, "satellite_name" : "GRACE C", "satellite_full_name" : "Gravity Recovery and Climate Experiment Follow-On C", "satellite_id" : 803}, + { "sensor_name" : "IGOR", "sensor_full_name" : "Integrated GPS and Occultation Receiver", "sensor_id" : 103, "satellite_name" : "KOMPSAT-9", "satellite_full_name" : "Korean Multi-Purpose Satellite", "satellite_id" : 825} + ] +} diff --git a/ush/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.py b/ush/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.py deleted file mode 100644 index 11a9fc06e..000000000 --- a/ush/ioda/bufr2ioda/bufr2ioda_adpsfc_prepbufr.py +++ /dev/null @@ -1,280 +0,0 @@ -# (C) Copyright 2023 NOAA/NWS/NCEP/EMC -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -import sys -import numpy as np -import os -import argparse -import math -import calendar -import time -from datetime import datetime -import json -from pyiodaconv import bufr -from collections import namedtuple -from pyioda import ioda_obs_space as ioda_ospace - -def Compute_dateTime(cycleTimeSinceEpoch, dhr): - - dhr = np.int64(dhr*3600) - dateTime = dhr + cycleTimeSinceEpoch - - return dateTime - -def bufr_to_ioda(config): - - - subsets = config["subsets"] - - # Get parameters from configuration - data_format = config["data_format"] - source = config["source"] - data_type = config["data_type"] - data_description = config["data_description"] - data_provider = config["data_provider"] - cycle_type = config["cycle_type"] - cycle_datetime = config["cycle_datetime"] - dump_dir = config["dump_directory"] - ioda_dir = config["ioda_directory"] - cycle = config["cycle_datetime"] - - # Get derived parameters - yyyymmdd = cycle[0:8] - hh = cycle[8:10] - - reference_time = datetime.strptime(cycle, "%Y%m%d%H") - reference_time = reference_time.strftime("%Y-%m-%dT%H:%M:%SZ") - reference_time_full = f"{yyyymmdd}{hh}00" - - # General informaton - converter = 'BUFR to IODA Converter' - platform_description = 'SFCSHP data from prepBUFR format' - - bufrfile = f"{cycle_type}.t{hh}z.{data_format}" - DATA_PATH = os.path.join(dump_dir, f"{cycle_type}.{yyyymmdd}", str(hh), bufrfile) - - print("The DATA_PATH is: "+str(DATA_PATH)) - - # ============================================ - # Make the QuerySet for all the data we want - # ============================================ - start_time = time.time() - - print('Making QuerySet ...') - q = bufr.QuerySet(subsets) - - # MetaData - q.add('stationIdentification', '*/SID') - q.add('latitude', '*/YOB') - q.add('longitude', '*/XOB') - q.add('obsTimeMinusCycleTime', '*/DHR') - q.add('stationElevation', '*/ELV') - q.add('observationType', '*/TYP') - q.add('pressure', '*/P___INFO/P__EVENT{1}/POB') - -# # Quality Infomation (Quality Indicator) - q.add('qualityMarkerStationPressure', '*/P___INFO/P__EVENT{1}/PQM') - - # ObsValue - q.add('stationPressure', '*/P___INFO/P__EVENT{1}/POB') - - end_time = time.time() - running_time = end_time - start_time - print('Running time for making QuerySet : ', running_time, 'seconds') - - # ============================================================== - # Open the BUFR file and execute the QuerySet to get ResultSet - # Use the ResultSet returned to get numpy arrays of the data - # ============================================================== - start_time = time.time() - - print('Executing QuerySet to get ResultSet ...') - with bufr.File(DATA_PATH) as f: - r = f.execute(q) - - print(' ... Executing QuerySet: get metadata: basic ...') - # MetaData - sid = r.get('stationIdentification') - lat = r.get('latitude') - lon = r.get('longitude') - lon[lon>180] -=360 - elv = r.get('stationElevation') - typ = r.get('observationType') - pressure = r.get('pressure') - pressure *= 100 - - print(' ... Executing QuerySet: get QualityMarker information ...') - # Quality Information - pobqm = r.get('qualityMarkerStationPressure') - - print(' ... Executing QuerySet: get obsvalue: stationPressure ...') - # ObsValue - pob = r.get('stationPressure') - pob *= 100 - - print(' ... Executing QuerySet: get datatime: observation time ...') - # DateTime: seconds since Epoch time - # IODA has no support for numpy datetime arrays dtype=datetime64[s] - dhr = r.get('obsTimeMinusCycleTime', type='int64') - - print(' ... Executing QuerySet: Done!') - - print(' ... Executing QuerySet: Check BUFR variable generic dimension and type ...') - # Check BUFR variable generic dimension and type - print(' sid shape = ', sid.shape) - print(' dhr shape = ', dhr.shape) - print(' lat shape = ', lat.shape) - print(' lon shape = ', lon.shape) - print(' elv shape = ', elv.shape) - print(' typ shape = ', typ.shape) - print(' pressure shape = ', pressure.shape) - - print(' pobqm shape = ', pobqm.shape) - print(' pob shape = ', pob.shape) - - print(' sid type = ', sid.dtype) - print(' dhr type = ', dhr.dtype) - print(' lat type = ', lat.dtype) - print(' lon type = ', lon.dtype) - print(' elv type = ', elv.dtype) - print(' typ type = ', typ.dtype) - print(' pressure type = ', pressure.dtype) - - print(' pobqm type = ', pobqm.dtype) - print(' pob type = ', pob.dtype) - - end_time = time.time() - running_time = end_time - start_time - print('Running time for executing QuerySet to get ResultSet : ', running_time, 'seconds') - - # ========================= - # Create derived variables - # ========================= - start_time = time.time() - - print('Creating derived variables - dateTime ...') - - cycleTimeSinceEpoch = np.int64(calendar.timegm(time.strptime(reference_time_full, '%Y%m%d%H%M'))) - dateTime = Compute_dateTime(cycleTimeSinceEpoch, dhr) - - print(' Check derived variables type ... ') - print(' dateTime shape = ', dateTime.shape) - print(' dateTime type = ', dateTime.dtype) - - end_time = time.time() - running_time = end_time - start_time - print('Running time for creating derived variables : ', running_time, 'seconds') - - # ===================================== - # Create IODA ObsSpace - # Write IODA output - # ===================================== - - # Create the dimensions - dims = { - 'Location' : np.arange(0, lat.shape[0]), - } - - iodafile = f"{cycle_type}.t{hh}z.{data_type}.{data_format}.nc" - OUTPUT_PATH = os.path.join(ioda_dir, iodafile) - print(" ... ... Create OUTPUT file: ", OUTPUT_PATH) - - path, fname = os.path.split(OUTPUT_PATH) - if path and not os.path.exists(path): - os.makedirs(path) - - obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) - - # Create Global attributes - print(' ... ... Create global attributes') - - obsspace.write_attr('Converter', converter) - obsspace.write_attr('source', source) - obsspace.write_attr('sourceFiles', bufrfile) - obsspace.write_attr('dataProviderOrigin', data_provider) - obsspace.write_attr('description', data_description) - obsspace.write_attr('datetimeReference', reference_time) - obsspace.write_attr('datetimeRange', [str(min(dateTime)), str(max(dateTime))]) - obsspace.write_attr('platformLongDescription', platform_description) - - # Create IODA variables - print(' ... ... Create variables: name, type, units, and attributes') - # Longitude - obsspace.create_var('MetaData/longitude', dtype=lon.dtype, fillval=lon.fill_value) \ - .write_attr('units', 'degrees_east') \ - .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ - .write_attr('long_name', 'Longitude') \ - .write_data(lon) - - # Latitude - obsspace.create_var('MetaData/latitude', dtype=lat.dtype, fillval=lat.fill_value) \ - .write_attr('units', 'degrees_north') \ - .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ - .write_attr('long_name', 'Latitude') \ - .write_data(lat) - - # Datetime - obsspace.create_var('MetaData/dateTime', dtype=dateTime.dtype, fillval=dateTime.fill_value) \ - .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ - .write_attr('long_name', 'Datetime') \ - .write_data(dateTime) - - # Station Identification - obsspace.create_var('MetaData/stationIdentification', dtype=sid.dtype, fillval=sid.fill_value) \ - .write_attr('long_name', 'Station Identification') \ - .write_data(sid) - - # Station Elevation - obsspace.create_var('MetaData/stationElevation', dtype=elv.dtype, fillval=elv.fill_value) \ - .write_attr('units', 'm') \ - .write_attr('long_name', 'Station Elevation') \ - .write_data(elv) - - # Observation Type - obsspace.create_var('MetaData/observationType', dtype=typ.dtype, fillval=typ.fill_value) \ - .write_attr('long_name', 'Observation Type') \ - .write_data(typ) - - # Pressure - obsspace.create_var('MetaData/pressure', dtype=pressure.dtype, fillval=pressure.fill_value) \ - .write_attr('units', 'Pa') \ - .write_attr('long_name', 'Pressure') \ - .write_data(pressure) - - # Quality: Percent Confidence - Quality Information Without Forecast - obsspace.create_var('QualityMarker/stationPressure', dtype=pobqm.dtype, fillval=pobqm.fill_value) \ - .write_attr('long_name', 'Station Pressure Quality Marker') \ - .write_data(pobqm) - - # Station Pressure - obsspace.create_var('ObsValue/pressure', dtype=pob.dtype, fillval=pob.fill_value) \ - .write_attr('units', 'Pa') \ - .write_attr('long_name', 'Station Pressure') \ - .write_data(pob) - - end_time = time.time() - running_time = end_time - start_time - print('Running time for splitting and output IODA:', running_time, 'seconds') - - print("All Done!") - -if __name__ == '__main__': - - start_time = time.time() - - parser = argparse.ArgumentParser() - parser.add_argument('-c', '--config', type=str, help='Input JSON configuration', required=True) - parser.add_argument('-v', '--verbose', help='print debug logging information', - action='store_true') - args = parser.parse_args() - - with open(args.config, "r") as json_file: - config = json.load(json_file) - - bufr_to_ioda(config) - - end_time = time.time() - running_time = end_time - start_time - print('Total running time: ', running_time, 'seconds') diff --git a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py new file mode 100644 index 000000000..92a95f416 --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py @@ -0,0 +1,633 @@ +# (C) Copyright 2023 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import sys + +sys.path.append('/work2/noaa/da/nesposito/ioda-bundle_20230712/build/lib/python3.9/') +sys.path.append('/work2/noaa/da/nesposito/ioda-bundle_20230712/build/lib/python3.9/pyiodaconv/') +sys.path.append('/work2/noaa/da/nesposito/ioda-bundle_20230712/build/lib/python3.9/pyioda/') + +import os +import argparse +import json +import numpy as np +import numpy.ma as ma +import math +import calendar +import time +import datetime +from pyiodaconv import bufr +from collections import namedtuple +from pyioda import ioda_obs_space as ioda_ospace + +# ==================================================================== +# GPS-RO BUFR dump file +#===================================================================== +# NC003010 | GPS-RO +# ==================================================================== + +def Compute_Grid_Location(degrees): + + rad = np.deg2rad(degrees) + + return rad + +def Compute_imph(impp, elrc, geodu): + + imph = impp - elrc - geodu + + return imph + +def bufr_to_ioda(config): + + subsets = config["subsets"] + print("NE subsets", subsets) + # ========================================= + # Get parameters from configuration + # ========================================= + data_format = config["data_format"] + data_type = config["data_type"] + data_description = config["data_description"] + data_provider = config["data_provider"] + cycle_type = config["cycle_type"] + dump_dir = config["dump_directory"] + ioda_dir = config["ioda_directory"] + cycle = config["cycle_datetime"] + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + + print("NE data_type", data_type) + + sensor_satellite_info_array = config["sensor_satellite_info"] +# print("NE", sensor_satellite_info_array) +# sensor_name = sensor_satellite_info_array["sensor_name"] +# print("NE sensorname: ",sensor_name) +# sensor_full_name = config["sensor_satellite_info"]["sensor_full_name"] +# sensor_id = config["sensor_satellite_info"]["sensor_id"] +# satellite_name = config["sensor_satellite_info"]["satellite_name"] +# satellite_full_name = config["sensor_satellite_info"]["satellite_full_name"] +# satellite_id = config["sensor_satellite_info"]["satellite_id"] + + + bufrfile = f"{cycle_type}.t{hh}z.{data_type}.tm00.{data_format}" + DATA_PATH = os.path.join(dump_dir, f"{cycle_type}.{yyyymmdd}", str(hh), 'atmos', bufrfile) + + # =================================================== + # Define instrument sensor and satellite information + # =================================================== + +# print('Instrument: Sensor/Platform Specification...') +# +# print('Instrument Info ... ') +# print(sensor_satellite_info_array) +# print('Total number of instrument = ', len(sensor_satellite_info_array)) +# +# for sats_sensors in sensor_satellite_info_array: +# print('Sensor Name = ', sensor_satellite_info_array.sensor_name) +# print('Sensor Full Name = ', sensor_satellite_info_array.sensor_full_name) +# print('Sensor ID = ', sensor_satellite_info_array.sensor_id) +# print('Satellite Name = ', sensor_satellite_info_array.satellite_name) +# print('Satellite Full Name = ', sensor_satellite_info_array.satellite_full_name) +# print('Satellite ID = ', sensor_satellite_info_array.satellite_id) +# +# print('Sensor Name = ', sensor_satellite_info_array[0].sensor_name) +# print('Sensor Full Name = ', sensor_satellite_info_array[0].sensor_full_name) +# print('Sensor ID = ', sensor_satellite_info_array[0].sensor_id) +# print('Satellite Name = ', sensor_satellite_info_array[0].satellite_name) +# print('Satellite Full Name = ', sensor_satellite_info_array[0].satellite_full_name) +# print('Satellite ID = ', sensor_satellite_info_array[0].satellite_id) + + # ============================================ + # Make the QuerySet for all the data we want + # ============================================ + start_time = time.time() + + print('Making QuerySet ...') + q = bufr.QuerySet(subsets) + + # MetaData + q.add('latitude', '*/ROSEQ1/CLATH' ) + q.add('longitude', '*/ROSEQ1/CLONH' ) + q.add('gridLatitude', '*/ROSEQ1/CLATH' ) + q.add('gridLongitude', '*/ROSEQ1/CLONH' ) + q.add('year', '*/YEAR' ) + q.add('year2', '*/YEAR' ) + q.add('month', '*/MNTH' ) + q.add('day', '*/DAYS' ) + q.add('hour', '*/HOUR' ) + q.add('minute', '*/MINU' ) + q.add('second', '*/SECO' ) + q.add('satelliteIdentifier', '*/SAID' ) + q.add('satelliteInstrument', '*/SIID' ) + q.add('satelliteConstellationRO', '*/SCLF' ) + q.add('platformTransmitterId', '*/PTID' ) + q.add('earthRadiusCurvature', '*/ELRC' ) + #q.add('observationSequenceNum', '*/SEQNUM' ) + q.add('geoidUndulation', '*/GEODU' ) + q.add('height', '*/ROSEQ3/HEIT' ) + q.add('impactHeightRO_roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/IMPP' ) + q.add('impactHeightRO_roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/IMPP' ) + q.add('impactParameterRO_roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/IMPP' ) + q.add('impactParameterRO_roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/IMPP' ) + q.add('frequency__roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/MEFR' ) + q.add('frequency__roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/MEFR' ) + q.add('pccf', '*/ROSEQ1/PCCF' ) + q.add('percentConfidence', '*/ROSEQ3/PCCF' ) + q.add('sensorAzimuthAngle', '*/BEARAZ' ) + + # Processing Center + q.add('dataProviderOrigin', '*/OGCE' ) + + # Quality Information + q.add('qualityFlags', '*/QFRO' ) + q.add('satelliteAscendingFlag', '*/QFRO' ) + + # ObsValue + q.add('bendingAngle_roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/BNDA[1]' ) + q.add('bendingAngle_roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/BNDA[1]' ) + q.add('atmosphericRefractivity', '*/ROSEQ3/ARFR[1]' ) + + # ObsError + q.add('obsErrorBendingAngle1', '*/ROSEQ1/ROSEQ2{1}/BNDA[2]') + q.add('obsErrorBendingAngle3', '*/ROSEQ1/ROSEQ2{3}/BNDA[2]') + q.add('obsErrorAtmosphericRefractivity', '*/ROSEQ3/ARFR[2]') + + # ObsType + q.add('obsTypeBendingAngle', '*/SAID' ) + + end_time = time.time() + running_time = end_time - start_time + print('Running time for making QuerySet : ', running_time, 'seconds') + + # ============================================================== + # Open the BUFR file and execute the QuerySet to get ResultSet + # Use the ResultSet returned to get numpy arrays of the data + # ============================================================== + start_time = time.time() + + print('Executing QuerySet to get ResultSet ...') + with bufr.File(DATA_PATH) as f: + r = f.execute(q) + + print(' ... Executing QuerySet: get metadata: basic ...') + # MetaData + clath = r.get('latitude', 'latitude') + clonh = r.get('longitude', 'latitude') + gclath = r.get('gridLatitude', 'latitude') + gclonh = r.get('gridLongitude', 'latitude') + year = r.get('year', 'latitude') + year2 = r.get('year2') + mnth = r.get('month', 'latitude') + days = r.get('day', 'latitude') + hour = r.get('hour', 'latitude') + minu = r.get('minute', 'latitude') + seco = r.get('second', 'latitude') + said = r.get('satelliteIdentifier', 'latitude') + siid = r.get('satelliteInstrument', 'latitude') + sclf = r.get('satelliteConstellationRO', 'latitude') + ptid = r.get('platformTransmitterId', 'latitude') + elrc = r.get('earthRadiusCurvature', 'latitude') + #seqnum = r.get('observationSequenceNum') + geodu = r.get('geoidUndulation', 'latitude') + heit = r.get('height', 'height', type='float') + impp1 = r.get('impactParameterRO_roseq2repl1', 'latitude') + impp3 = r.get('impactParameterRO_roseq2repl3', 'latitude') + imph1 = r.get('impactHeightRO_roseq2repl1', 'latitude') + imph3 = r.get('impactHeightRO_roseq2repl3', 'latitude') + mefr1 = r.get('frequency__roseq2repl1', 'latitude') + mefr3 = r.get('frequency__roseq2repl3', 'latitude') + pccf = r.get('pccf', 'latitude') + ref_pccf = r.get('percentConfidence', 'height' ) + bearaz = r.get('sensorAzimuthAngle', 'latitude') + + print(' ... Executing QuerySet: get metadata: processing center ...') + # Processing Center + ogce = r.get('dataProviderOrigin', 'latitude') + + print(' ... Executing QuerySet: get metadata: data quality information ...') + # Quality Information + qfro = r.get('qualityFlags', 'latitude') + satasc = r.get('satelliteAscendingFlag', 'latitude' ) + + print(' ... Executing QuerySet: get obsvalue: Bending Angle ...') + # ObsValue + # Bending Angle + bnda1 = r.get('bendingAngle_roseq2repl1', 'latitude') + bnda3 = r.get('bendingAngle_roseq2repl3', 'latitude') + arfr = r.get('atmosphericRefractivity', 'height') + + # ObsError + # Bending Angle + oebnda1 = r.get('obsErrorBendingAngle1', 'latitude') + oebnda3 = r.get('obsErrorBendingAngle3', 'latitude') + oearfr = r.get('obsErrorAtmosphericRefractivity','height') + + # ObsType + # Bending Angle + otbnda = r.get('obsTypeBendingAngle', 'latitude') + otarfr = r.get('obsTypeBendingAngle', 'latitude') + + print(' ... Executing QuerySet: get datatime: observation time ...') + # DateTime: seconds since Epoch time + # IODA has no support for numpy datetime arrays dtype=datetime64[s] + timestamp = r.get_datetime('year','month','day','hour','minute','second','latitude').astype(np.int64) + +###NICKE SEQNUM would happen around here +#For seqnum +# seqnum=[] +#    qq=1 +#    for i in range(len(elrc)): +#       if elrc[i] != elrc[i+1]: +#          qq = qq+1 +# seqnum.append(qq[i]) +# check length. should be 666000 +##### SOMEHTING LIKE THAT + + + print(' ... Executing QuerySet: Done!') + + print(' ... Executing QuerySet: Check BUFR variable generic dimension and type ...') + # Check BUFR variable generic dimension and type + print(' clath shape,type = ', clath.shape, clath.dtype) + print(' clonh shape,type = ', clonh.shape, clonh.dtype) + print(' gclath shape,type = ', gclath.shape, gclath.dtype) + print(' gclonh shape,type = ', gclonh.shape, gclonh.dtype) + print(' year shape,type = ', year.shape, year.dtype) + print(' mnth shape,type = ', mnth.shape, mnth.dtype) + print(' days shape,type = ', days.shape, days.dtype) + print(' hour shape,type = ', hour.shape, hour.dtype) + print(' minu shape,type = ', minu.shape, minu.dtype) + print(' seco shape,type = ', seco.shape, seco.dtype) + print(' said shape,type = ', said.shape, said.dtype) + print(' siid shape,type = ', siid.shape, siid.dtype) + print(' sclf shape,type = ', sclf.shape, sclf.dtype) + print(' ptid shape,type = ', ptid.shape, ptid.dtype) + print(' elrc shape,type = ', elrc.shape, elrc.dtype) + print(' ogce shape,type = ', ogce.shape, ogce.dtype) + print(' geodu shape,type = ', geodu.shape, geodu.dtype) + print(' heit shape,type = ', heit.shape, heit.dtype) + print(' impp1 shape,type = ', impp1.shape, impp1.dtype) + print(' impp3 shape,type = ', impp3.shape, impp3.dtype) + print(' imph1 shape,type = ', imph1.shape, imph1.dtype) + print(' imph3 shape,type = ', imph3.shape, imph3.dtype) + print(' mefr1 shape,type = ', mefr1.shape, mefr1.dtype) + print(' mefr3 shape,type = ', mefr3.shape, mefr3.dtype) + print(' pccf shape,type = ', pccf.shape, pccf.dtype) + print(' ref_pccf shape,type = ', ref_pccf.shape, ref_pccf.dtype) + print(' bearaz shape,type = ', bearaz.shape, bearaz.dtype) + + print(' ogce shape,type = ', ogce.shape, ogce.dtype) + + print(' qfro shape,type = ', qfro.shape, qfro.dtype) + print(' satasc shape,type = ', satasc.shape, satasc.dtype) + + print(' bnda1 shape,type = ', bnda1.shape, bnda1.dtype) + print(' bnda3 shape,type = ', bnda3.shape, bnda3.dtype) + print(' arfr shape,type = ', arfr.shape, arfr.dtype) + + print(' oebnda1 shape,type = ', oebnda1.shape, oebnda1.dtype) + print(' oebnda3 shape,type = ', oebnda3.shape, oebnda3.dtype) + print(' oearfr shape,type = ', arfr.shape, arfr.dtype) + + print(' otbnda shape,type = ', otbnda.shape, otbnda.dtype) + + end_time = time.time() + running_time = end_time - start_time + print('Running time for executing QuerySet to get ResultSet : ', running_time, 'seconds') + + # ========================= + # Create derived variables + # ========================= + start_time = time.time() + + print('Creating derived variables - Grid Latitude / Longitude ...') + gclonh = Compute_Grid_Location(gclonh) + gclath = Compute_Grid_Location(gclath) + + print(' gclonh shape,type = ', gclonh.shape, gclonh.dtype) + print(' gclath shape,type = ', gclath.shape, gclath.dtype) + print(' gclonh min/max = ', gclonh.min(), gclonh.max()) + print(' gclath min/max = ', gclath.min(), gclath.max()) + + print('Creating derived variables - imph ...') + imph1 = Compute_imph(impp1, elrc, geodu) + imph3 = Compute_imph(impp3, elrc, geodu) + + print(' imph1 shape,type = ', imph1.shape, imph1.dtype) + print(' imph3 shape,type = ', imph3.shape, imph3.dtype) + print(' imph1 min/max = ', imph1.min(), imph1.max()) + print(' imph3 min/max = ', imph3.min(), imph3.max()) + + end_time = time.time() + running_time = end_time - start_time + print('Running time for creating derived variables : ', running_time, 'seconds') + + # ===================================== + # Split output based on satellite id + # Create IODA ObsSpace + # Write IODA output + # ===================================== + print('Split data based on satellite id, Create IODA ObsSpace and Write IODA output') + + # Find unique satellite identifiers in data to process + unique_satids = np.unique(said) + print(' ... Number of Unique satellite identifiers: ', len(unique_satids)) + print(' ... Unique satellite identifiers: ', unique_satids) + + print(' ... Loop through unique satellite identifier ... : ', unique_satids) + for sat in unique_satids.tolist(): + print("Processing output for said: ", sat) + start_time = time.time() + + # Find matched instrument from instrument namedtuple + matched = list(filter(lambda instrument: instrument.satellite_id == sat, instruments)) + print(' ... sat = ', sat) + print(' ... instruments = ', instruments) + print(' ... Matched instrument = ', matched) + satellite_id = list(map(lambda instrument: instrument.satellite_id, matched)) + sensor_id = list(map(lambda instrument: instrument.sensor_id, matched)) + satellite_name = list(map(lambda instrument: instrument.satellite_name, matched)) + sensor_name = list(map(lambda instrument: instrument.sensor_name, matched)) + print(' ... Matched satellite_id = ', satellite_id) + print(' ... Matched satellite = ', satellite_name) + print(' ... Matched sensor = ', sensor_name) + satinst = sensor_name[0].lower().replace(" ","_")+'_'+satellite_name[0].lower().replace(" ","_") + + print(' ... Split data for', satinst, 'satid = ', sat) + + # Define a boolean mask to subset data from the original data object + mask = said == sat + # MetaData + clonh2 = clonh[mask] + clath2 = clath[mask] + gclonh2 = gclonh[mask] + gclath2 = gclath[mask] + timestamp2 = timestamp[mask] + said2 = said[mask] + siid2 = siid[mask] + sclf2 = sclf[mask] + ptid2 = ptid[mask] + elrc2 = elrc[mask] + geodu2 = geodu[mask] + heit2 = heit[mask] + impp1_2 = impp1[mask] + impp3_2 = impp3[mask] + imph1_2 = imph1[mask] + imph3_2 = imph3[mask] + mefr1_2 = mefr1[mask] + mefr3_2 = mefr3[mask] + pccf2 = pccf[mask] + ref_pccf2 = ref_pccf[mask] + bearaz2 = bearaz[mask] + + # Processing Center + ogce2 = ogce[mask] + + # QC Info + qfro2 = qfro[mask] + satasc2 = satasc[mask] + + # ObsValue + bnda1_2 = bnda1[mask] + bnda3_2 = bnda3[mask] + arfr2 = arfr[mask] + + # ObsError + oebnda1_2 = oebnda1[mask] + oebnda3_2 = oebnda3[mask] + oearfr2 = arfr[mask] + + # ObsType + otbnda2 = otbnda[mask] + + # Choose bnda, mefr, impp, imph + if ((sat == 44) or (sat == 825)): + bnda2 = bnda1_2 + mefr2 = mefr1_2 + impp2 = impp1_2 + imph2 = imph1_2 + oebnda2 = oebnda1_2 + else: + bnda2 = bnda3_2 + mefr2 = mefr3_2 + impp2 = impp3_2 + imph2 = imph3_2 + oebnda2 = oebnda3_2 + + # Check unique observation time + unique_timestamp2 = np.unique(timestamp2) + print(' ... Number of Unique observation timestamp: ', len(unique_timestamp2)) + print(' ... Unique observation timestamp: ', unique_timestamp2) + + print(' ... Create ObsSpae for', satinst, 'satid = ', sat) + + # ===================================== + # Create IODA ObsSpace + # Write IODA output + # ===================================== + + # Write the data to an IODA file + OUTPUT_PATH = '/work2/noaa/da/nesposito/NewConv/api_convert/work/testrun/gdas.t00z.gpsro_'+satinst+'.tm00.nc' + path, fname = os.path.split(OUTPUT_PATH) + if path and not os.path.exists(path): + os.makedirs(path) + + # Create the dimensions + dims = { + 'Location' : np.arange(0, clath2.shape[0]), + } + + # Create IODA ObsSpace + obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) + + # Create Global attributes + print(' ... ... Create global attributes') + obsspace.write_attr('sensor', sensor_id) + obsspace.write_attr('platform', satellite_id) + obsspace.write_attr('dataProviderOrigin', 'U.S. NOAA/NESDIS') + obsspace.write_attr('description', 'MSG TYPE 003-010 NESDIS GPS-RO') + + # Create IODA variables + print(' ... ... Create variables: name, type, units, and attributes') + # Longitude + obsspace.create_var('MetaData/longitude', dtype=clonh2.dtype, fillval=clonh2.fill_value) \ + .write_attr('units', 'degrees_east') \ + .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ + .write_attr('long_name', 'Longitude') \ + .write_data(clonh2) + + # Latitude + obsspace.create_var('MetaData/latitude', dtype=clath.dtype, fillval=clath2.fill_value) \ + .write_attr('units', 'degrees_north') \ + .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Latitude') \ + .write_data(clath2) + + # Grid Longitude + obsspace.create_var('MetaData/gridLongitude', dtype=gclonh2.dtype, fillval=gclonh2.fill_value) \ + .write_attr('units', 'radians') \ + .write_attr('valid_range', np.array([-3.14159265, 3.14159265], dtype=np.float32)) \ + .write_attr('long_name', 'Grid Longitude') \ + .write_data(gclonh2) + + # Grid Latitude + obsspace.create_var('MetaData/gridLatitude', dtype=gclath.dtype, fillval=gclath2.fill_value) \ + .write_attr('units', 'radians') \ + .write_attr('valid_range', np.array([-1.570796325, 1.570796325], dtype=np.float32)) \ + .write_attr('long_name', 'Grid Latitude') \ + .write_data(gclath2) + + # Datetime + obsspace.create_var('MetaData/dateTime', dtype=np.int64, fillval=timestamp2.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'Datetime') \ + .write_data(timestamp2) + + # Satellite Identifier + obsspace.create_var('MetaData/satelliteIdentifier', dtype=said2.dtype, fillval=said2.fill_value) \ + .write_attr('long_name', 'Satellite Identifier') \ + .write_data(said2) + + # Satellite Instrument + obsspace.create_var('MetaData/satelliteInstrument', dtype=siid2.dtype, fillval=siid2.fill_value) \ + .write_attr('long_name', 'Satellite Instrument') \ + .write_data(siid2) + + # Satellite Constellation RO + obsspace.create_var('MetaData/satelliteConstellationRO', dtype=sclf2.dtype, fillval=sclf2.fill_value) \ + .write_attr('long_name', 'Satellite Constellation RO') \ + .write_data(sclf2) + + # Platform Transmitter ID + obsspace.create_var('MetaData/platformTransmitterId', dtype=ptid2.dtype, fillval=ptid2.fill_value) \ + .write_attr('long_name', 'Platform Transmitter Id') \ + .write_data(ptid2) + + # Earth Radius Curvature + obsspace.create_var('MetaData/earthRadiusCurvature', dtype=elrc2.dtype, fillval=elrc2.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Earth Radius of Curvature') \ + .write_data(elrc2) + + # Geoid Undulation + obsspace.create_var('MetaData/geoidUndulation', dtype=geodu2.dtype, fillval=geodu2.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Geoid Undulation') \ + .write_data(geodu2) + + # Height + obsspace.create_var('MetaData/height', dtype=heit2.dtype, fillval=heit2.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Height' ) \ + .write_data(heit2) + + # Impact Parameter RO + obsspace.create_var('MetaData/impactParameterRO', dtype=impp2.dtype, fillval=impp2.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Impact Parameter RO') \ + .write_data(impp2) + + # Impact Height RO + obsspace.create_var('MetaData/impactHeightRO', dtype=imph2.dtype, fillval=imph2.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Impact Height RO') \ + .write_data(imph2) + + # Impact Height RO + obsspace.create_var('MetaData/frequency', dtype=mefr2.dtype, fillval=mefr2.fill_value) \ + .write_attr('units', 'Hz') \ + .write_attr('long_name', 'Frequency') \ + .write_data(imph2) + + # PCCF Percent Confidence + obsspace.create_var('MetaData/pccf', dtype=pccf2.dtype, fillval=pccf2.fill_value) \ + .write_attr('units', '%') \ + .write_attr('long_name', 'Percent Confidence') \ + .write_data(pccf2) + + # PCCF Ref Percent Confidence + obsspace.create_var('MetaData/percentConfidence', dtype=ref_pccf2.dtype, fillval=ref_pccf2.fill_value) \ + .write_attr('units', '%') \ + .write_attr('long_name', 'Ref Percent Confidence') \ + .write_data(ref_pccf2) + + # Azimuth Angle + obsspace.create_var('MetaData/sensorAzimuthAngle', dtype=bearaz2.dtype, fillval=bearaz2.fill_value) \ + .write_attr('units', 'degree') \ + .write_attr('long_name', 'Percent Confidence') \ + .write_data(pccf2) + + # Data Provider + obsspace.create_var('MetaData/dataProviderOrigin', dtype=ogce2.dtype, fillval=ogce2.fill_value) \ + .write_attr('long_name', 'Identification of Originating/Generating Center') \ + .write_data(ogce2) + + # Quality: Quality Flags + obsspace.create_var('MetaData/qualityFlags', dtype=qfro2.dtype, fillval=qfro2.fill_value) \ + .write_attr('long_name', 'Quality Flags') \ + .write_data(qfro2) + + # Quality: Satellite Ascending Flag + obsspace.create_var('MetaData/satelliteAscendingFlag', dtype=satasc2.dtype, fillval=satasc2.fill_value) \ + .write_attr('long_name', 'Satellite Ascending Flag') \ + .write_data(satasc2) + + # ObsValue: Bending Angle + obsspace.create_var('ObsValue/bendingAngle', dtype=bnda2.dtype, fillval=bnda2.fill_value) \ + .write_attr('units', 'radians') \ + .write_attr('long_name', 'Bending Angle') \ + .write_data(bnda2) + + # ObsValue: Atmospheric Refractivity + obsspace.create_var('ObsValue/atmosphericRefractivity', dtype=arfr2.dtype, fillval=arfr2.fill_value) \ + .write_attr('units', 'N-units') \ + .write_attr('long_name', 'Atmospheric Refractivity ObsError') \ + .write_data(arfr2) + + # ObsValue: Bending Angle + obsspace.create_var('ObsError/bendingAngle', dtype=oebnda2.dtype, fillval=oebnda2.fill_value) \ + .write_attr('units', 'radians') \ + .write_attr('long_name', 'Bending Angle Obs Error') \ + .write_data(oebnda2) + + # ObsError: Atmospheric Refractivity + obsspace.create_var('ObsError/atmosphericRefractivity', dtype=oearfr2.dtype, fillval=oearfr2.fill_value) \ + .write_attr('units', 'N-units') \ + .write_attr('long_name', 'Atmospheric Refractivity ObsError') \ + .write_data(oearfr2) + + # ObsType + obsspace.create_var('ObsType/BendingAngle', dtype=otbnda2.dtype, fillval=otbnda2.fill_value) \ + .write_attr('long_name', 'Bending Angle ObsType') \ + .write_data(otbnda2) + + + end_time = time.time() + running_time = end_time - start_time + print('Running time for splitting and output IODA for', satinst, running_time, 'seconds') + + print("All Done!") + +if __name__ == '__main__': + + start_time = time.time() + + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, help='Input JSON configuration', required=True) + parser.add_argument('-v', '--verbose', help='print debug logging information', + action='store_true') + args = parser.parse_args() + + with open(args.config, "r") as json_file: + config = json.load(json_file) + + bufr_to_ioda(config) + + end_time = time.time() + running_time = end_time - start_time + print('Total running time: ', running_time, 'seconds') + + + diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda_adpsfc.sh b/ush/ioda/bufr2ioda/run_bufr2ioda_gpsro.sh similarity index 99% rename from ush/ioda/bufr2ioda/run_bufr2ioda_adpsfc.sh rename to ush/ioda/bufr2ioda/run_bufr2ioda_gpsro.sh index 4f5126043..12b53af93 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda_adpsfc.sh +++ b/ush/ioda/bufr2ioda/run_bufr2ioda_gpsro.sh @@ -51,7 +51,7 @@ export PYTHONPATH=${PYIODALIB}:${PYTHONPATH} #--------------------------- #----- python and json ----- # first specify what observation types will be processed by a script -BUFR_py="adpsfc_prepbufr" +BUFR_py="gpsro_bufr" for obtype in $BUFR_py; do # this loop assumes that there is a python script and template with the same name From d7da3bb057d8dafd1ad1284dd57d5ecd586fa78b Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 7 Sep 2023 14:08:22 -0400 Subject: [PATCH 08/29] readme update --- ush/ioda/bufr2ioda/README_whoever | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/ioda/bufr2ioda/README_whoever b/ush/ioda/bufr2ioda/README_whoever index 48e889040..744d69184 100644 --- a/ush/ioda/bufr2ioda/README_whoever +++ b/ush/ioda/bufr2ioda/README_whoever @@ -1,7 +1,7 @@ Needed to make a sfcshp-specific run_bufr2ioda.sh script because I needed to change a variable. To run: use command: -./run_bufr2ioda_adpsfc.sh 20210801 gdas /work/noaa/rstprod/dump/ /work2/noaa/da/nesposito/GDASApp_20230822/parm/ioda/bufr2ioda/ /work2/noaa/da/nesposito/NewConv/api_convert/json/json_outputs/ +./run_bufr2ioda_gpsro.sh 20210801 gdas /work/noaa/rstprod/dump/ /work2/noaa/da/nesposito/GDASApp_20230822/parm/ioda/bufr2ioda/ /work2/noaa/da/nesposito/NewConv/api_convert/json/json_outputs/ Outputs (and plots) are in: /work2/noaa/da/nesposito/NewConv/api_convert/json/json_outputs From a5cd48601da3adb686ce7ad10480f43b9770ae62 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Tue, 12 Sep 2023 19:26:48 -0400 Subject: [PATCH 09/29] re-update again with json changes --- parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.json | 2 +- ush/ioda/bufr2ioda/README_whoever | 11 +++++++---- ush/ioda/bufr2ioda/gen_bufr2ioda_json.py | 8 +++++++- ...bufr2ioda_gpsro.sh => run_bufr2ioda_gpsro_bufr.sh} | 5 ++++- 4 files changed, 19 insertions(+), 7 deletions(-) rename ush/ioda/bufr2ioda/{run_bufr2ioda_gpsro.sh => run_bufr2ioda_gpsro_bufr.sh} (93%) diff --git a/parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.json b/parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.json index aaee14982..142478e3a 100644 --- a/parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.json +++ b/parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.json @@ -1,7 +1,7 @@ { "data_format" : "bufr_d", "data_type" : "gpsro", - "cycle_type" : "{{ DUMP }}", + "cycle_type" : "{{ RUN }}", "cycle_datetime" : "{{ current_cycle | to_YMDH }}", "dump_directory" : "{{ DMPDIR }}", "ioda_directory" : "{{ COM_OBS }}", diff --git a/ush/ioda/bufr2ioda/README_whoever b/ush/ioda/bufr2ioda/README_whoever index 744d69184..1070c3a47 100644 --- a/ush/ioda/bufr2ioda/README_whoever +++ b/ush/ioda/bufr2ioda/README_whoever @@ -1,9 +1,12 @@ -Needed to make a sfcshp-specific run_bufr2ioda.sh script because I needed to change a variable. + +Load the modules for the gdasapp first To run: use command: -./run_bufr2ioda_gpsro.sh 20210801 gdas /work/noaa/rstprod/dump/ /work2/noaa/da/nesposito/GDASApp_20230822/parm/ioda/bufr2ioda/ /work2/noaa/da/nesposito/NewConv/api_convert/json/json_outputs/ -Outputs (and plots) are in: -/work2/noaa/da/nesposito/NewConv/api_convert/json/json_outputs +#This works +./run_bufr2ioda_gpsro_bufr.sh 2021080100 gdas /work/noaa/rstprod/dump/ /work2/noaa/da/nesposito/GDASApp_20230907/parm/ioda/bufr2ioda/ /work2/noaa/da/nesposito/NewConv/api_convert/json/json_gdasapp +Outputs (and plots) are in: +/work2/noaa/da/nesposito/NewConv/api_convert/json/json_gdasapp + diff --git a/ush/ioda/bufr2ioda/gen_bufr2ioda_json.py b/ush/ioda/bufr2ioda/gen_bufr2ioda_json.py index 617a8b024..0748e20f6 100755 --- a/ush/ioda/bufr2ioda/gen_bufr2ioda_json.py +++ b/ush/ioda/bufr2ioda/gen_bufr2ioda_json.py @@ -6,7 +6,8 @@ import argparse import json import os -from wxflow import Logger, parse_j2yaml +from wxflow import Logger, parse_j2yaml, cast_strdict_as_dtypedict +from wxflow import add_to_datetime, to_timedelta # Initialize root logger logger = Logger('gen_bufr2ioda_json.py', level='INFO', colored_log=True) @@ -28,4 +29,9 @@ def gen_bufr_json(config, template, output): parser.add_argument('-t', '--template', type=str, help='Input JSON template', required=True) parser.add_argument('-o', '--output', type=str, help='Output JSON file', required=True) args = parser.parse_args() + # get the config from your environment + config = cast_strdict_as_dtypedict(os.environ) + # we need to add in current cycle from PDYcyc + config['current_cycle'] = add_to_datetime(config['PDY'], to_timedelta(f"{config['cyc']}H")) + # call the parsing function gen_bufr_json(config, args.template, args.output) diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda_gpsro.sh b/ush/ioda/bufr2ioda/run_bufr2ioda_gpsro_bufr.sh similarity index 93% rename from ush/ioda/bufr2ioda/run_bufr2ioda_gpsro.sh rename to ush/ioda/bufr2ioda/run_bufr2ioda_gpsro_bufr.sh index 12b53af93..977b67ca5 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda_gpsro.sh +++ b/ush/ioda/bufr2ioda/run_bufr2ioda_gpsro_bufr.sh @@ -17,7 +17,7 @@ fi # some of these need exported to be picked up by the python script below # input parameters CDATE=${CDATE:-$1} -export DUMP=${RUN:-$2} +export RUN=${RUN:-$2} export DMPDIR=${DMPDIR:-$3} config_template_dir=${config_template_dir:-$4} export COM_OBS=${COM_OBS:-$5} @@ -59,9 +59,12 @@ for obtype in $BUFR_py; do # first generate a JSON from the template ${BUFRJSONGEN} -t ${config_template_dir}/bufr2ioda_${obtype}.json -o ${COM_OBS}/${obtype}_${PDY}${cyc}.json +# ${BUFRJSONGEN} -t ${config_template_dir}/bufr2ioda_${obtype}.json -c bufr2ioda_${obtype}.json -o ${COM_OBS}/${obtype}_${PDY}${cyc}.json + echo "1 done" # now use the converter script for the ob type python3 $USH_IODA/bufr2ioda_${obtype}.py -c ${COM_OBS}/${obtype}_${PDY}${cyc}.json + echo "2 done" # check if converter was successful if [ $? == 0 ]; then From 2ff7c9ed1ceae2d485ba0be9734792a056775b28 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Wed, 13 Sep 2023 10:18:27 -0400 Subject: [PATCH 10/29] split works --- parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.json | 2 +- ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py | 537 +++++++++--------- 2 files changed, 270 insertions(+), 269 deletions(-) diff --git a/parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.json b/parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.json index 142478e3a..ca73714eb 100644 --- a/parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.json +++ b/parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.json @@ -5,7 +5,7 @@ "cycle_datetime" : "{{ current_cycle | to_YMDH }}", "dump_directory" : "{{ DMPDIR }}", "ioda_directory" : "{{ COM_OBS }}", - "subsets" : [ "GPS-RO" ], + "subsets" : [ "NC003010" ], "data_description" : "MSG TYPE 003-010 NESDIS GPS-RO", "data_provider" : "U.S. NOAA/NESDIS", "sensor_satellite_info" : [ diff --git a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py index 92a95f416..c9cf27ebc 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py @@ -61,8 +61,9 @@ def bufr_to_ioda(config): print("NE data_type", data_type) sensor_satellite_info_array = config["sensor_satellite_info"] -# print("NE", sensor_satellite_info_array) -# sensor_name = sensor_satellite_info_array["sensor_name"] + print("NE", sensor_satellite_info_array) +# sensor_name = config["sensor_name"]#["satellite_id"] +# print("NE SENSOR NAME", sensor_name) # print("NE sensorname: ",sensor_name) # sensor_full_name = config["sensor_satellite_info"]["sensor_full_name"] # sensor_id = config["sensor_satellite_info"]["sensor_id"] @@ -265,7 +266,6 @@ def bufr_to_ioda(config): print(' sclf shape,type = ', sclf.shape, sclf.dtype) print(' ptid shape,type = ', ptid.shape, ptid.dtype) print(' elrc shape,type = ', elrc.shape, elrc.dtype) - print(' ogce shape,type = ', ogce.shape, ogce.dtype) print(' geodu shape,type = ', geodu.shape, geodu.dtype) print(' heit shape,type = ', heit.shape, heit.dtype) print(' impp1 shape,type = ', impp1.shape, impp1.dtype) @@ -341,272 +341,273 @@ def bufr_to_ioda(config): print("Processing output for said: ", sat) start_time = time.time() - # Find matched instrument from instrument namedtuple - matched = list(filter(lambda instrument: instrument.satellite_id == sat, instruments)) - print(' ... sat = ', sat) - print(' ... instruments = ', instruments) - print(' ... Matched instrument = ', matched) - satellite_id = list(map(lambda instrument: instrument.satellite_id, matched)) - sensor_id = list(map(lambda instrument: instrument.sensor_id, matched)) - satellite_name = list(map(lambda instrument: instrument.satellite_name, matched)) - sensor_name = list(map(lambda instrument: instrument.sensor_name, matched)) - print(' ... Matched satellite_id = ', satellite_id) - print(' ... Matched satellite = ', satellite_name) - print(' ... Matched sensor = ', sensor_name) - satinst = sensor_name[0].lower().replace(" ","_")+'_'+satellite_name[0].lower().replace(" ","_") - - print(' ... Split data for', satinst, 'satid = ', sat) - - # Define a boolean mask to subset data from the original data object - mask = said == sat - # MetaData - clonh2 = clonh[mask] - clath2 = clath[mask] - gclonh2 = gclonh[mask] - gclath2 = gclath[mask] - timestamp2 = timestamp[mask] - said2 = said[mask] - siid2 = siid[mask] - sclf2 = sclf[mask] - ptid2 = ptid[mask] - elrc2 = elrc[mask] - geodu2 = geodu[mask] - heit2 = heit[mask] - impp1_2 = impp1[mask] - impp3_2 = impp3[mask] - imph1_2 = imph1[mask] - imph3_2 = imph3[mask] - mefr1_2 = mefr1[mask] - mefr3_2 = mefr3[mask] - pccf2 = pccf[mask] - ref_pccf2 = ref_pccf[mask] - bearaz2 = bearaz[mask] - - # Processing Center - ogce2 = ogce[mask] - - # QC Info - qfro2 = qfro[mask] - satasc2 = satasc[mask] - - # ObsValue - bnda1_2 = bnda1[mask] - bnda3_2 = bnda3[mask] - arfr2 = arfr[mask] - - # ObsError - oebnda1_2 = oebnda1[mask] - oebnda3_2 = oebnda3[mask] - oearfr2 = arfr[mask] - - # ObsType - otbnda2 = otbnda[mask] - - # Choose bnda, mefr, impp, imph - if ((sat == 44) or (sat == 825)): - bnda2 = bnda1_2 - mefr2 = mefr1_2 - impp2 = impp1_2 - imph2 = imph1_2 - oebnda2 = oebnda1_2 - else: - bnda2 = bnda3_2 - mefr2 = mefr3_2 - impp2 = impp3_2 - imph2 = imph3_2 - oebnda2 = oebnda3_2 - - # Check unique observation time - unique_timestamp2 = np.unique(timestamp2) - print(' ... Number of Unique observation timestamp: ', len(unique_timestamp2)) - print(' ... Unique observation timestamp: ', unique_timestamp2) - - print(' ... Create ObsSpae for', satinst, 'satid = ', sat) - - # ===================================== - # Create IODA ObsSpace - # Write IODA output - # ===================================== - - # Write the data to an IODA file - OUTPUT_PATH = '/work2/noaa/da/nesposito/NewConv/api_convert/work/testrun/gdas.t00z.gpsro_'+satinst+'.tm00.nc' - path, fname = os.path.split(OUTPUT_PATH) - if path and not os.path.exists(path): - os.makedirs(path) - - # Create the dimensions - dims = { - 'Location' : np.arange(0, clath2.shape[0]), - } + # Find matched sensor_satellite_info from sensor_satellite_info namedtuple + matched = False + for sensor_satellite_info in sensor_satellite_info_array: + if (sensor_satellite_info["satellite_id"] == sat): + matched = True + sensor_id = sensor_satellite_info["sensor_id"] + sensor_full_name = sensor_satellite_info["sensor_full_name"] + sensor_id = sensor_satellite_info["sensor_id"] + satellite_name = sensor_satellite_info["satellite_name"] + satellite_full_name = sensor_satellite_info["satellite_full_name"] + satellite_id = sensor_satellite_info["satellite_id"] + + + if matched: + + print(' ... Split data for satid = ', sat) + + # Define a boolean mask to subset data from the original data object + mask = said == sat + # MetaData + clonh2 = clonh[mask] + clath2 = clath[mask] + gclonh2 = gclonh[mask] + gclath2 = gclath[mask] + timestamp2 = timestamp[mask] + said2 = said[mask] + siid2 = siid[mask] + sclf2 = sclf[mask] + ptid2 = ptid[mask] + elrc2 = elrc[mask] + geodu2 = geodu[mask] + heit2 = heit[mask] + impp1_2 = impp1[mask] + impp3_2 = impp3[mask] + imph1_2 = imph1[mask] + imph3_2 = imph3[mask] + mefr1_2 = mefr1[mask] + mefr3_2 = mefr3[mask] + pccf2 = pccf[mask] + ref_pccf2 = ref_pccf[mask] + bearaz2 = bearaz[mask] + + # Processing Center + ogce2 = ogce[mask] + + # QC Info + qfro2 = qfro[mask] + satasc2 = satasc[mask] + + # ObsValue + bnda1_2 = bnda1[mask] + bnda3_2 = bnda3[mask] + arfr2 = arfr[mask] + + # ObsError + oebnda1_2 = oebnda1[mask] + oebnda3_2 = oebnda3[mask] + oearfr2 = arfr[mask] + + # ObsType + otbnda2 = otbnda[mask] + + # Choose bnda, mefr, impp, imph + if ((sat == 44) or (sat == 825)): + bnda2 = bnda1_2 + mefr2 = mefr1_2 + impp2 = impp1_2 + imph2 = imph1_2 + oebnda2 = oebnda1_2 + else: + bnda2 = bnda3_2 + mefr2 = mefr3_2 + impp2 = impp3_2 + imph2 = imph3_2 + oebnda2 = oebnda3_2 + + # Check unique observation time + unique_timestamp2 = np.unique(timestamp2) + print(' ... Number of Unique observation timestamp: ', len(unique_timestamp2)) + print(' ... Unique observation timestamp: ', unique_timestamp2) + + print(' ... Create ObsSpae for satid = ', sat) + + # ===================================== + # Create IODA ObsSpace + # Write IODA output + # ===================================== - # Create IODA ObsSpace - obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) - - # Create Global attributes - print(' ... ... Create global attributes') - obsspace.write_attr('sensor', sensor_id) - obsspace.write_attr('platform', satellite_id) - obsspace.write_attr('dataProviderOrigin', 'U.S. NOAA/NESDIS') - obsspace.write_attr('description', 'MSG TYPE 003-010 NESDIS GPS-RO') - - # Create IODA variables - print(' ... ... Create variables: name, type, units, and attributes') - # Longitude - obsspace.create_var('MetaData/longitude', dtype=clonh2.dtype, fillval=clonh2.fill_value) \ - .write_attr('units', 'degrees_east') \ - .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ - .write_attr('long_name', 'Longitude') \ - .write_data(clonh2) - - # Latitude - obsspace.create_var('MetaData/latitude', dtype=clath.dtype, fillval=clath2.fill_value) \ - .write_attr('units', 'degrees_north') \ - .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ - .write_attr('long_name', 'Latitude') \ - .write_data(clath2) - - # Grid Longitude - obsspace.create_var('MetaData/gridLongitude', dtype=gclonh2.dtype, fillval=gclonh2.fill_value) \ - .write_attr('units', 'radians') \ - .write_attr('valid_range', np.array([-3.14159265, 3.14159265], dtype=np.float32)) \ - .write_attr('long_name', 'Grid Longitude') \ - .write_data(gclonh2) - - # Grid Latitude - obsspace.create_var('MetaData/gridLatitude', dtype=gclath.dtype, fillval=gclath2.fill_value) \ - .write_attr('units', 'radians') \ - .write_attr('valid_range', np.array([-1.570796325, 1.570796325], dtype=np.float32)) \ - .write_attr('long_name', 'Grid Latitude') \ - .write_data(gclath2) - - # Datetime - obsspace.create_var('MetaData/dateTime', dtype=np.int64, fillval=timestamp2.fill_value) \ - .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ - .write_attr('long_name', 'Datetime') \ - .write_data(timestamp2) - - # Satellite Identifier - obsspace.create_var('MetaData/satelliteIdentifier', dtype=said2.dtype, fillval=said2.fill_value) \ - .write_attr('long_name', 'Satellite Identifier') \ - .write_data(said2) - - # Satellite Instrument - obsspace.create_var('MetaData/satelliteInstrument', dtype=siid2.dtype, fillval=siid2.fill_value) \ - .write_attr('long_name', 'Satellite Instrument') \ - .write_data(siid2) - - # Satellite Constellation RO - obsspace.create_var('MetaData/satelliteConstellationRO', dtype=sclf2.dtype, fillval=sclf2.fill_value) \ - .write_attr('long_name', 'Satellite Constellation RO') \ - .write_data(sclf2) - - # Platform Transmitter ID - obsspace.create_var('MetaData/platformTransmitterId', dtype=ptid2.dtype, fillval=ptid2.fill_value) \ - .write_attr('long_name', 'Platform Transmitter Id') \ - .write_data(ptid2) - - # Earth Radius Curvature - obsspace.create_var('MetaData/earthRadiusCurvature', dtype=elrc2.dtype, fillval=elrc2.fill_value) \ - .write_attr('units', 'm') \ - .write_attr('long_name', 'Earth Radius of Curvature') \ - .write_data(elrc2) - - # Geoid Undulation - obsspace.create_var('MetaData/geoidUndulation', dtype=geodu2.dtype, fillval=geodu2.fill_value) \ - .write_attr('units', 'm') \ - .write_attr('long_name', 'Geoid Undulation') \ - .write_data(geodu2) - - # Height - obsspace.create_var('MetaData/height', dtype=heit2.dtype, fillval=heit2.fill_value) \ - .write_attr('units', 'm') \ - .write_attr('long_name', 'Height' ) \ - .write_data(heit2) - - # Impact Parameter RO - obsspace.create_var('MetaData/impactParameterRO', dtype=impp2.dtype, fillval=impp2.fill_value) \ - .write_attr('units', 'm') \ - .write_attr('long_name', 'Impact Parameter RO') \ - .write_data(impp2) - - # Impact Height RO - obsspace.create_var('MetaData/impactHeightRO', dtype=imph2.dtype, fillval=imph2.fill_value) \ - .write_attr('units', 'm') \ - .write_attr('long_name', 'Impact Height RO') \ - .write_data(imph2) - - # Impact Height RO - obsspace.create_var('MetaData/frequency', dtype=mefr2.dtype, fillval=mefr2.fill_value) \ - .write_attr('units', 'Hz') \ - .write_attr('long_name', 'Frequency') \ - .write_data(imph2) - - # PCCF Percent Confidence - obsspace.create_var('MetaData/pccf', dtype=pccf2.dtype, fillval=pccf2.fill_value) \ - .write_attr('units', '%') \ - .write_attr('long_name', 'Percent Confidence') \ - .write_data(pccf2) - - # PCCF Ref Percent Confidence - obsspace.create_var('MetaData/percentConfidence', dtype=ref_pccf2.dtype, fillval=ref_pccf2.fill_value) \ - .write_attr('units', '%') \ - .write_attr('long_name', 'Ref Percent Confidence') \ - .write_data(ref_pccf2) - - # Azimuth Angle - obsspace.create_var('MetaData/sensorAzimuthAngle', dtype=bearaz2.dtype, fillval=bearaz2.fill_value) \ - .write_attr('units', 'degree') \ - .write_attr('long_name', 'Percent Confidence') \ - .write_data(pccf2) - - # Data Provider - obsspace.create_var('MetaData/dataProviderOrigin', dtype=ogce2.dtype, fillval=ogce2.fill_value) \ - .write_attr('long_name', 'Identification of Originating/Generating Center') \ - .write_data(ogce2) - - # Quality: Quality Flags - obsspace.create_var('MetaData/qualityFlags', dtype=qfro2.dtype, fillval=qfro2.fill_value) \ - .write_attr('long_name', 'Quality Flags') \ - .write_data(qfro2) - - # Quality: Satellite Ascending Flag - obsspace.create_var('MetaData/satelliteAscendingFlag', dtype=satasc2.dtype, fillval=satasc2.fill_value) \ - .write_attr('long_name', 'Satellite Ascending Flag') \ - .write_data(satasc2) - - # ObsValue: Bending Angle - obsspace.create_var('ObsValue/bendingAngle', dtype=bnda2.dtype, fillval=bnda2.fill_value) \ - .write_attr('units', 'radians') \ - .write_attr('long_name', 'Bending Angle') \ - .write_data(bnda2) - - # ObsValue: Atmospheric Refractivity - obsspace.create_var('ObsValue/atmosphericRefractivity', dtype=arfr2.dtype, fillval=arfr2.fill_value) \ - .write_attr('units', 'N-units') \ - .write_attr('long_name', 'Atmospheric Refractivity ObsError') \ - .write_data(arfr2) - - # ObsValue: Bending Angle - obsspace.create_var('ObsError/bendingAngle', dtype=oebnda2.dtype, fillval=oebnda2.fill_value) \ - .write_attr('units', 'radians') \ - .write_attr('long_name', 'Bending Angle Obs Error') \ - .write_data(oebnda2) - - # ObsError: Atmospheric Refractivity - obsspace.create_var('ObsError/atmosphericRefractivity', dtype=oearfr2.dtype, fillval=oearfr2.fill_value) \ - .write_attr('units', 'N-units') \ - .write_attr('long_name', 'Atmospheric Refractivity ObsError') \ - .write_data(oearfr2) - - # ObsType - obsspace.create_var('ObsType/BendingAngle', dtype=otbnda2.dtype, fillval=otbnda2.fill_value) \ - .write_attr('long_name', 'Bending Angle ObsType') \ - .write_data(otbnda2) - - - end_time = time.time() - running_time = end_time - start_time - print('Running time for splitting and output IODA for', satinst, running_time, 'seconds') + # Write the data to an IODA file + OUTPUT_PATH = '/work2/noaa/da/nesposito/NewConv/api_convert/json/json_gdasapp/gdas.t00z.gpsro_'+str(sat)+'.tm00.nc' + path, fname = os.path.split(OUTPUT_PATH) + if path and not os.path.exists(path): + os.makedirs(path) + + # Create the dimensions + dims = { + 'Location' : np.arange(0, clath2.shape[0]), + } + + # Create IODA ObsSpace + obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) + + # Create Global attributes + print(' ... ... Create global attributes') + obsspace.write_attr('sensor', sensor_id) + obsspace.write_attr('platform', satellite_id) + obsspace.write_attr('dataProviderOrigin', 'U.S. NOAA/NESDIS') + obsspace.write_attr('description', 'MSG TYPE 003-010 NESDIS GPS-RO') + + # Create IODA variables + print(' ... ... Create variables: name, type, units, and attributes') + # Longitude + obsspace.create_var('MetaData/longitude', dtype=clonh2.dtype, fillval=clonh2.fill_value) \ + .write_attr('units', 'degrees_east') \ + .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ + .write_attr('long_name', 'Longitude') \ + .write_data(clonh2) + + # Latitude + obsspace.create_var('MetaData/latitude', dtype=clath.dtype, fillval=clath2.fill_value) \ + .write_attr('units', 'degrees_north') \ + .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Latitude') \ + .write_data(clath2) + + # Grid Longitude + obsspace.create_var('MetaData/gridLongitude', dtype=gclonh2.dtype, fillval=gclonh2.fill_value) \ + .write_attr('units', 'radians') \ + .write_attr('valid_range', np.array([-3.14159265, 3.14159265], dtype=np.float32)) \ + .write_attr('long_name', 'Grid Longitude') \ + .write_data(gclonh2) + + # Grid Latitude + obsspace.create_var('MetaData/gridLatitude', dtype=gclath.dtype, fillval=gclath2.fill_value) \ + .write_attr('units', 'radians') \ + .write_attr('valid_range', np.array([-1.570796325, 1.570796325], dtype=np.float32)) \ + .write_attr('long_name', 'Grid Latitude') \ + .write_data(gclath2) + + # Datetime + obsspace.create_var('MetaData/dateTime', dtype=np.int64, fillval=timestamp2.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'Datetime') \ + .write_data(timestamp2) + + # Satellite Identifier + obsspace.create_var('MetaData/satelliteIdentifier', dtype=said2.dtype, fillval=said2.fill_value) \ + .write_attr('long_name', 'Satellite Identifier') \ + .write_data(said2) + + # Satellite Instrument + obsspace.create_var('MetaData/satelliteInstrument', dtype=siid2.dtype, fillval=siid2.fill_value) \ + .write_attr('long_name', 'Satellite Instrument') \ + .write_data(siid2) + + # Satellite Constellation RO + obsspace.create_var('MetaData/satelliteConstellationRO', dtype=sclf2.dtype, fillval=sclf2.fill_value) \ + .write_attr('long_name', 'Satellite Constellation RO') \ + .write_data(sclf2) + + # Platform Transmitter ID + obsspace.create_var('MetaData/platformTransmitterId', dtype=ptid2.dtype, fillval=ptid2.fill_value) \ + .write_attr('long_name', 'Platform Transmitter Id') \ + .write_data(ptid2) + + # Earth Radius Curvature + obsspace.create_var('MetaData/earthRadiusCurvature', dtype=elrc2.dtype, fillval=elrc2.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Earth Radius of Curvature') \ + .write_data(elrc2) + + # Geoid Undulation + obsspace.create_var('MetaData/geoidUndulation', dtype=geodu2.dtype, fillval=geodu2.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Geoid Undulation') \ + .write_data(geodu2) + + # Height + obsspace.create_var('MetaData/height', dtype=heit2.dtype, fillval=heit2.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Height' ) \ + .write_data(heit2) + + # Impact Parameter RO + obsspace.create_var('MetaData/impactParameterRO', dtype=impp2.dtype, fillval=impp2.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Impact Parameter RO') \ + .write_data(impp2) + + # Impact Height RO + obsspace.create_var('MetaData/impactHeightRO', dtype=imph2.dtype, fillval=imph2.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Impact Height RO') \ + .write_data(imph2) + + # Impact Height RO + obsspace.create_var('MetaData/frequency', dtype=mefr2.dtype, fillval=mefr2.fill_value) \ + .write_attr('units', 'Hz') \ + .write_attr('long_name', 'Frequency') \ + .write_data(imph2) + + # PCCF Percent Confidence + obsspace.create_var('MetaData/pccf', dtype=pccf2.dtype, fillval=pccf2.fill_value) \ + .write_attr('units', '%') \ + .write_attr('long_name', 'Percent Confidence') \ + .write_data(pccf2) + + # PCCF Ref Percent Confidence + obsspace.create_var('MetaData/percentConfidence', dtype=ref_pccf2.dtype, fillval=ref_pccf2.fill_value) \ + .write_attr('units', '%') \ + .write_attr('long_name', 'Ref Percent Confidence') \ + .write_data(ref_pccf2) + + # Azimuth Angle + obsspace.create_var('MetaData/sensorAzimuthAngle', dtype=bearaz2.dtype, fillval=bearaz2.fill_value) \ + .write_attr('units', 'degree') \ + .write_attr('long_name', 'Percent Confidence') \ + .write_data(pccf2) + + # Data Provider + obsspace.create_var('MetaData/dataProviderOrigin', dtype=ogce2.dtype, fillval=ogce2.fill_value) \ + .write_attr('long_name', 'Identification of Originating/Generating Center') \ + .write_data(ogce2) + + # Quality: Quality Flags + obsspace.create_var('MetaData/qualityFlags', dtype=qfro2.dtype, fillval=qfro2.fill_value) \ + .write_attr('long_name', 'Quality Flags') \ + .write_data(qfro2) + + # Quality: Satellite Ascending Flag + obsspace.create_var('MetaData/satelliteAscendingFlag', dtype=satasc2.dtype, fillval=satasc2.fill_value) \ + .write_attr('long_name', 'Satellite Ascending Flag') \ + .write_data(satasc2) + + # ObsValue: Bending Angle + obsspace.create_var('ObsValue/bendingAngle', dtype=bnda2.dtype, fillval=bnda2.fill_value) \ + .write_attr('units', 'radians') \ + .write_attr('long_name', 'Bending Angle') \ + .write_data(bnda2) + + # ObsValue: Atmospheric Refractivity + obsspace.create_var('ObsValue/atmosphericRefractivity', dtype=arfr2.dtype, fillval=arfr2.fill_value) \ + .write_attr('units', 'N-units') \ + .write_attr('long_name', 'Atmospheric Refractivity ObsError') \ + .write_data(arfr2) + + # ObsValue: Bending Angle + obsspace.create_var('ObsError/bendingAngle', dtype=oebnda2.dtype, fillval=oebnda2.fill_value) \ + .write_attr('units', 'radians') \ + .write_attr('long_name', 'Bending Angle Obs Error') \ + .write_data(oebnda2) + + # ObsError: Atmospheric Refractivity + obsspace.create_var('ObsError/atmosphericRefractivity', dtype=oearfr2.dtype, fillval=oearfr2.fill_value) \ + .write_attr('units', 'N-units') \ + .write_attr('long_name', 'Atmospheric Refractivity ObsError') \ + .write_data(oearfr2) + + # ObsType + obsspace.create_var('ObsType/BendingAngle', dtype=otbnda2.dtype, fillval=otbnda2.fill_value) \ + .write_attr('long_name', 'Bending Angle ObsType') \ + .write_data(otbnda2) + + + end_time = time.time() + running_time = end_time - start_time + print('Running time for splitting and output IODA for', str(sat), running_time, 'seconds') print("All Done!") From 56eba96194c1421b20ee4e81d77a20226721a4e3 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 14 Sep 2023 11:57:01 -0400 Subject: [PATCH 11/29] undo obserror and add attributes --- ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py | 117 ++++++++------------- 1 file changed, 45 insertions(+), 72 deletions(-) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py index c9cf27ebc..baf3e61f5 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py @@ -58,48 +58,11 @@ def bufr_to_ioda(config): yyyymmdd = cycle[0:8] hh = cycle[8:10] - print("NE data_type", data_type) - sensor_satellite_info_array = config["sensor_satellite_info"] - print("NE", sensor_satellite_info_array) -# sensor_name = config["sensor_name"]#["satellite_id"] -# print("NE SENSOR NAME", sensor_name) -# print("NE sensorname: ",sensor_name) -# sensor_full_name = config["sensor_satellite_info"]["sensor_full_name"] -# sensor_id = config["sensor_satellite_info"]["sensor_id"] -# satellite_name = config["sensor_satellite_info"]["satellite_name"] -# satellite_full_name = config["sensor_satellite_info"]["satellite_full_name"] -# satellite_id = config["sensor_satellite_info"]["satellite_id"] - bufrfile = f"{cycle_type}.t{hh}z.{data_type}.tm00.{data_format}" DATA_PATH = os.path.join(dump_dir, f"{cycle_type}.{yyyymmdd}", str(hh), 'atmos', bufrfile) - # =================================================== - # Define instrument sensor and satellite information - # =================================================== - -# print('Instrument: Sensor/Platform Specification...') -# -# print('Instrument Info ... ') -# print(sensor_satellite_info_array) -# print('Total number of instrument = ', len(sensor_satellite_info_array)) -# -# for sats_sensors in sensor_satellite_info_array: -# print('Sensor Name = ', sensor_satellite_info_array.sensor_name) -# print('Sensor Full Name = ', sensor_satellite_info_array.sensor_full_name) -# print('Sensor ID = ', sensor_satellite_info_array.sensor_id) -# print('Satellite Name = ', sensor_satellite_info_array.satellite_name) -# print('Satellite Full Name = ', sensor_satellite_info_array.satellite_full_name) -# print('Satellite ID = ', sensor_satellite_info_array.satellite_id) -# -# print('Sensor Name = ', sensor_satellite_info_array[0].sensor_name) -# print('Sensor Full Name = ', sensor_satellite_info_array[0].sensor_full_name) -# print('Sensor ID = ', sensor_satellite_info_array[0].sensor_id) -# print('Satellite Name = ', sensor_satellite_info_array[0].satellite_name) -# print('Satellite Full Name = ', sensor_satellite_info_array[0].satellite_full_name) -# print('Satellite ID = ', sensor_satellite_info_array[0].satellite_id) - # ============================================ # Make the QuerySet for all the data we want # ============================================ @@ -123,7 +86,7 @@ def bufr_to_ioda(config): q.add('satelliteIdentifier', '*/SAID' ) q.add('satelliteInstrument', '*/SIID' ) q.add('satelliteConstellationRO', '*/SCLF' ) - q.add('platformTransmitterId', '*/PTID' ) + q.add('satelliteTransmitterId', '*/PTID' ) q.add('earthRadiusCurvature', '*/ELRC' ) #q.add('observationSequenceNum', '*/SEQNUM' ) q.add('geoidUndulation', '*/GEODU' ) @@ -151,12 +114,13 @@ def bufr_to_ioda(config): q.add('atmosphericRefractivity', '*/ROSEQ3/ARFR[1]' ) # ObsError - q.add('obsErrorBendingAngle1', '*/ROSEQ1/ROSEQ2{1}/BNDA[2]') - q.add('obsErrorBendingAngle3', '*/ROSEQ1/ROSEQ2{3}/BNDA[2]') + q.add('obsErrorBendingAngle1', '*/ROSEQ1/ROSEQ2{1}/BNDA[2]') + q.add('obsErrorBendingAngle3', '*/ROSEQ1/ROSEQ2{3}/BNDA[2]') q.add('obsErrorAtmosphericRefractivity', '*/ROSEQ3/ARFR[2]') # ObsType - q.add('obsTypeBendingAngle', '*/SAID' ) + q.add('obsTypeBendingAngle', '*/SAID' ) + q.add('obsTypeAtmosphericRefractivity', '*/SAID' ) end_time = time.time() running_time = end_time - start_time @@ -188,7 +152,7 @@ def bufr_to_ioda(config): said = r.get('satelliteIdentifier', 'latitude') siid = r.get('satelliteInstrument', 'latitude') sclf = r.get('satelliteConstellationRO', 'latitude') - ptid = r.get('platformTransmitterId', 'latitude') + ptid = r.get('satelliteTransmitterId', 'latitude') elrc = r.get('earthRadiusCurvature', 'latitude') #seqnum = r.get('observationSequenceNum') geodu = r.get('geoidUndulation', 'latitude') @@ -221,14 +185,14 @@ def bufr_to_ioda(config): # ObsError # Bending Angle - oebnda1 = r.get('obsErrorBendingAngle1', 'latitude') - oebnda3 = r.get('obsErrorBendingAngle3', 'latitude') - oearfr = r.get('obsErrorAtmosphericRefractivity','height') + bndaoe1 = r.get('obsErrorBendingAngle1', 'latitude') + bndaoe3 = r.get('obsErrorBendingAngle3', 'latitude') + arfroe = r.get('obsErrorAtmosphericRefractivity','height') # ObsType # Bending Angle - otbnda = r.get('obsTypeBendingAngle', 'latitude') - otarfr = r.get('obsTypeBendingAngle', 'latitude') + bndaot = r.get('obsTypeBendingAngle', 'latitude') + arfrot = r.get('obsTypeBendingAngle', 'latitude') print(' ... Executing QuerySet: get datatime: observation time ...') # DateTime: seconds since Epoch time @@ -287,11 +251,11 @@ def bufr_to_ioda(config): print(' bnda3 shape,type = ', bnda3.shape, bnda3.dtype) print(' arfr shape,type = ', arfr.shape, arfr.dtype) - print(' oebnda1 shape,type = ', oebnda1.shape, oebnda1.dtype) - print(' oebnda3 shape,type = ', oebnda3.shape, oebnda3.dtype) - print(' oearfr shape,type = ', arfr.shape, arfr.dtype) + print(' bndaoe1 shape,type = ', bndaoe1.shape, bndaoe1.dtype) + print(' bndaoe3 shape,type = ', bndaoe3.shape, bndaoe3.dtype) + print(' arfroe shape,type = ', arfr.shape, arfr.dtype) - print(' otbnda shape,type = ', otbnda.shape, otbnda.dtype) + print(' bndaot shape,type = ', bndaot.shape, bndaot.dtype) end_time = time.time() running_time = end_time - start_time @@ -348,10 +312,10 @@ def bufr_to_ioda(config): matched = True sensor_id = sensor_satellite_info["sensor_id"] sensor_full_name = sensor_satellite_info["sensor_full_name"] - sensor_id = sensor_satellite_info["sensor_id"] - satellite_name = sensor_satellite_info["satellite_name"] - satellite_full_name = sensor_satellite_info["satellite_full_name"] + sensor_name = sensor_satellite_info["sensor_name"] satellite_id = sensor_satellite_info["satellite_id"] + satellite_full_name = sensor_satellite_info["satellite_full_name"] + satellite_name = sensor_satellite_info["satellite_name"] if matched: @@ -396,12 +360,13 @@ def bufr_to_ioda(config): arfr2 = arfr[mask] # ObsError - oebnda1_2 = oebnda1[mask] - oebnda3_2 = oebnda3[mask] - oearfr2 = arfr[mask] + bndaoe1_2 = bndaoe1[mask] + bndaoe3_2 = bndaoe3[mask] + arfroe2 = arfr[mask] # ObsType - otbnda2 = otbnda[mask] + bndaot2 = bndaot[mask] + arfrot2 = arfrot[mask] # Choose bnda, mefr, impp, imph if ((sat == 44) or (sat == 825)): @@ -409,13 +374,13 @@ def bufr_to_ioda(config): mefr2 = mefr1_2 impp2 = impp1_2 imph2 = imph1_2 - oebnda2 = oebnda1_2 + bndaoe2 = bndaoe1_2 else: bnda2 = bnda3_2 mefr2 = mefr3_2 impp2 = impp3_2 imph2 = imph3_2 - oebnda2 = oebnda3_2 + bndaoe2 = bndaoe3_2 # Check unique observation time unique_timestamp2 = np.unique(timestamp2) @@ -446,7 +411,11 @@ def bufr_to_ioda(config): # Create Global attributes print(' ... ... Create global attributes') obsspace.write_attr('sensor', sensor_id) - obsspace.write_attr('platform', satellite_id) + obsspace.write_attr('satellite', satellite_id) + obsspace.write_attr('sensorName', sensor_name) + obsspace.write_attr('satelliteName', satellite_name) + obsspace.write_attr('sensorFullName', sensor_full_name) + obsspace.write_attr('satelliteFullName', satellite_full_name) obsspace.write_attr('dataProviderOrigin', 'U.S. NOAA/NESDIS') obsspace.write_attr('description', 'MSG TYPE 003-010 NESDIS GPS-RO') @@ -501,9 +470,9 @@ def bufr_to_ioda(config): .write_attr('long_name', 'Satellite Constellation RO') \ .write_data(sclf2) - # Platform Transmitter ID - obsspace.create_var('MetaData/platformTransmitterId', dtype=ptid2.dtype, fillval=ptid2.fill_value) \ - .write_attr('long_name', 'Platform Transmitter Id') \ + # Satellite Transmitter ID + obsspace.create_var('MetaData/satelliteTransmitterId', dtype=ptid2.dtype, fillval=ptid2.fill_value) \ + .write_attr('long_name', 'Satellite Transmitter Id') \ .write_data(ptid2) # Earth Radius Curvature @@ -587,23 +556,27 @@ def bufr_to_ioda(config): .write_attr('long_name', 'Atmospheric Refractivity ObsError') \ .write_data(arfr2) - # ObsValue: Bending Angle - obsspace.create_var('ObsError/bendingAngle', dtype=oebnda2.dtype, fillval=oebnda2.fill_value) \ + # ObsError: Bending Angle + obsspace.create_var('ObsError/bendingAngle', dtype=bndaoe2.dtype, fillval=bndaoe2.fill_value) \ .write_attr('units', 'radians') \ .write_attr('long_name', 'Bending Angle Obs Error') \ - .write_data(oebnda2) + .write_data(bndaoe2) # ObsError: Atmospheric Refractivity - obsspace.create_var('ObsError/atmosphericRefractivity', dtype=oearfr2.dtype, fillval=oearfr2.fill_value) \ + obsspace.create_var('ObsError/atmosphericRefractivity', dtype=arfroe2.dtype, fillval=arfroe2.fill_value) \ .write_attr('units', 'N-units') \ .write_attr('long_name', 'Atmospheric Refractivity ObsError') \ - .write_data(oearfr2) + .write_data(arfroe2) # ObsType - obsspace.create_var('ObsType/BendingAngle', dtype=otbnda2.dtype, fillval=otbnda2.fill_value) \ + obsspace.create_var('ObsType/BendingAngle', dtype=bndaot2.dtype, fillval=bndaot2.fill_value) \ .write_attr('long_name', 'Bending Angle ObsType') \ - .write_data(otbnda2) - + .write_data(bndaot2) + + # ObsType: Atmospheric Refractivity + obsspace.create_var('ObsType/atmosphericRefractivity', dtype=arfrot2.dtype, fillval=arfrot2.fill_value) \ + .write_attr('long_name', 'Atmospheric Refractivity ObsType') \ + .write_data(arfrot2) end_time = time.time() running_time = end_time - start_time From e590ad5810b96d8d8081cb2d0c7c77860fd69ee5 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 14 Sep 2023 13:16:20 -0400 Subject: [PATCH 12/29] imph should have been mefr --- ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py index baf3e61f5..44eefe24a 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py @@ -509,7 +509,7 @@ def bufr_to_ioda(config): obsspace.create_var('MetaData/frequency', dtype=mefr2.dtype, fillval=mefr2.fill_value) \ .write_attr('units', 'Hz') \ .write_attr('long_name', 'Frequency') \ - .write_data(imph2) + .write_data(mefr2) # PCCF Percent Confidence obsspace.create_var('MetaData/pccf', dtype=pccf2.dtype, fillval=pccf2.fill_value) \ From 63d4ae6909d93da61d39a5df447a2ea5972b165b Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 14 Sep 2023 15:12:39 -0400 Subject: [PATCH 13/29] update attributes, some clean up --- .../bufr2ioda_gpsro_bufr_combined.json | 11 + ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py | 13 +- .../bufr2ioda_gpsro_bufr_combined.py | 523 ++++++++++++++++++ .../bufr2ioda/run_bufr2ioda_gpsro_bufr.sh | 3 +- 4 files changed, 546 insertions(+), 4 deletions(-) create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.json create mode 100644 ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py diff --git a/parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.json b/parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.json new file mode 100644 index 000000000..0d5dcea88 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.json @@ -0,0 +1,11 @@ +{ + "data_format" : "bufr_d", + "data_type" : "gpsro", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "subsets" : [ "NC003010" ], + "data_description" : "MSG TYPE 003-010 NESDIS GPS-RO: MetOp-A GRAS, MetOp-B GRAS, MetOp-C GRAS, TerraSAR-X IGOR, TanDEM-X IGOR, PAZ IGOR, CICERO-1 OP1 CION, COSMIC-2 E1-E6 Tri-G", + "data_provider" : "U.S. NOAA/NESDIS", +} diff --git a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py index 44eefe24a..f93c88d9d 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py @@ -289,10 +289,10 @@ def bufr_to_ioda(config): print('Running time for creating derived variables : ', running_time, 'seconds') # ===================================== - # Split output based on satellite id # Create IODA ObsSpace # Write IODA output # ===================================== + print('Split data based on satellite id, Create IODA ObsSpace and Write IODA output') # Find unique satellite identifiers in data to process @@ -410,14 +410,21 @@ def bufr_to_ioda(config): # Create Global attributes print(' ... ... Create global attributes') + obsspace.write_attr('data_format', data_format) + obsspace.write_attr('data_type', data_type) + obsspace.write_attr('subsets', subsets) + obsspace.write_attr('cycle_type', cycle_type) + obsspace.write_attr('cycle_datetime', cycle) + obsspace.write_attr('dataProviderOrigin', data_provider) + obsspace.write_attr('converter', os.path.basename(__file__)) + + obsspace.write_attr('description', data_description) obsspace.write_attr('sensor', sensor_id) obsspace.write_attr('satellite', satellite_id) obsspace.write_attr('sensorName', sensor_name) obsspace.write_attr('satelliteName', satellite_name) obsspace.write_attr('sensorFullName', sensor_full_name) obsspace.write_attr('satelliteFullName', satellite_full_name) - obsspace.write_attr('dataProviderOrigin', 'U.S. NOAA/NESDIS') - obsspace.write_attr('description', 'MSG TYPE 003-010 NESDIS GPS-RO') # Create IODA variables print(' ... ... Create variables: name, type, units, and attributes') diff --git a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py new file mode 100644 index 000000000..cd080ae59 --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py @@ -0,0 +1,523 @@ +# (C) Copyright 2023 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import sys + +sys.path.append('/work2/noaa/da/nesposito/ioda-bundle_20230712/build/lib/python3.9/') +sys.path.append('/work2/noaa/da/nesposito/ioda-bundle_20230712/build/lib/python3.9/pyiodaconv/') +sys.path.append('/work2/noaa/da/nesposito/ioda-bundle_20230712/build/lib/python3.9/pyioda/') + +import os +import argparse +import json +import numpy as np +import numpy.ma as ma +import math +import calendar +import time +import datetime +from pyiodaconv import bufr +from collections import namedtuple +from pyioda import ioda_obs_space as ioda_ospace + +# ==================================================================== +# GPS-RO BUFR dump file +#===================================================================== +# NC003010 | GPS-RO +# ==================================================================== + +def Compute_Grid_Location(degrees): + + rad = np.deg2rad(degrees) + + return rad + +def Compute_imph(impp, elrc, geodu): + + imph = impp - elrc - geodu + + return imph + +def bufr_to_ioda(config): + + subsets = config["subsets"] + # ========================================= + # Get parameters from configuration + # ========================================= + data_format = config["data_format"] + data_type = config["data_type"] + data_description = config["data_description"] + data_provider = config["data_provider"] + cycle_type = config["cycle_type"] + dump_dir = config["dump_directory"] + ioda_dir = config["ioda_directory"] + cycle = config["cycle_datetime"] + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + + bufrfile = f"{cycle_type}.t{hh}z.{data_type}.tm00.{data_format}" + DATA_PATH = os.path.join(dump_dir, f"{cycle_type}.{yyyymmdd}", str(hh), 'atmos', bufrfile) + + # ============================================ + # Make the QuerySet for all the data we want + # ============================================ + start_time = time.time() + + print('Making QuerySet ...') + q = bufr.QuerySet(subsets) + + # MetaData + q.add('latitude', '*/ROSEQ1/CLATH' ) + q.add('longitude', '*/ROSEQ1/CLONH' ) + q.add('gridLatitude', '*/ROSEQ1/CLATH' ) + q.add('gridLongitude', '*/ROSEQ1/CLONH' ) + q.add('year', '*/YEAR' ) + q.add('year2', '*/YEAR' ) + q.add('month', '*/MNTH' ) + q.add('day', '*/DAYS' ) + q.add('hour', '*/HOUR' ) + q.add('minute', '*/MINU' ) + q.add('second', '*/SECO' ) + q.add('satelliteIdentifier', '*/SAID' ) + q.add('satelliteInstrument', '*/SIID' ) + q.add('satelliteConstellationRO', '*/SCLF' ) + q.add('satelliteTransmitterId', '*/PTID' ) + q.add('earthRadiusCurvature', '*/ELRC' ) + #q.add('observationSequenceNum', '*/SEQNUM' ) + q.add('geoidUndulation', '*/GEODU' ) + q.add('height', '*/ROSEQ3/HEIT' ) + q.add('impactHeightRO_roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/IMPP' ) + q.add('impactHeightRO_roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/IMPP' ) + q.add('impactParameterRO_roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/IMPP' ) + q.add('impactParameterRO_roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/IMPP' ) + q.add('frequency__roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/MEFR' ) + q.add('frequency__roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/MEFR' ) + q.add('pccf', '*/ROSEQ1/PCCF' ) + q.add('percentConfidence', '*/ROSEQ3/PCCF' ) + q.add('sensorAzimuthAngle', '*/BEARAZ' ) + + # Processing Center + q.add('dataProviderOrigin', '*/OGCE' ) + + # Quality Information + q.add('qualityFlags', '*/QFRO' ) + q.add('satelliteAscendingFlag', '*/QFRO' ) + + # ObsValue + q.add('bendingAngle_roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/BNDA[1]' ) + q.add('bendingAngle_roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/BNDA[1]' ) + q.add('atmosphericRefractivity', '*/ROSEQ3/ARFR[1]' ) + + # ObsError + q.add('obsErrorBendingAngle1', '*/ROSEQ1/ROSEQ2{1}/BNDA[2]') + q.add('obsErrorBendingAngle3', '*/ROSEQ1/ROSEQ2{3}/BNDA[2]') + q.add('obsErrorAtmosphericRefractivity', '*/ROSEQ3/ARFR[2]') + + # ObsType + q.add('obsTypeBendingAngle', '*/SAID' ) + q.add('obsTypeAtmosphericRefractivity', '*/SAID' ) + + end_time = time.time() + running_time = end_time - start_time + print('Running time for making QuerySet : ', running_time, 'seconds') + + # ============================================================== + # Open the BUFR file and execute the QuerySet to get ResultSet + # Use the ResultSet returned to get numpy arrays of the data + # ============================================================== + start_time = time.time() + + print('Executing QuerySet to get ResultSet ...') + with bufr.File(DATA_PATH) as f: + r = f.execute(q) + + print(' ... Executing QuerySet: get metadata: basic ...') + # MetaData + clath = r.get('latitude', 'latitude') + clonh = r.get('longitude', 'latitude') + gclath = r.get('gridLatitude', 'latitude') + gclonh = r.get('gridLongitude', 'latitude') + year = r.get('year', 'latitude') + year2 = r.get('year2') + mnth = r.get('month', 'latitude') + days = r.get('day', 'latitude') + hour = r.get('hour', 'latitude') + minu = r.get('minute', 'latitude') + seco = r.get('second', 'latitude') + said = r.get('satelliteIdentifier', 'latitude') + siid = r.get('satelliteInstrument', 'latitude') + sclf = r.get('satelliteConstellationRO', 'latitude') + ptid = r.get('satelliteTransmitterId', 'latitude') + elrc = r.get('earthRadiusCurvature', 'latitude') + #seqnum = r.get('observationSequenceNum') + geodu = r.get('geoidUndulation', 'latitude') + heit = r.get('height', 'height', type='float') + impp1 = r.get('impactParameterRO_roseq2repl1', 'latitude') + impp3 = r.get('impactParameterRO_roseq2repl3', 'latitude') + imph1 = r.get('impactHeightRO_roseq2repl1', 'latitude') + imph3 = r.get('impactHeightRO_roseq2repl3', 'latitude') + mefr1 = r.get('frequency__roseq2repl1', 'latitude') + mefr3 = r.get('frequency__roseq2repl3', 'latitude') + pccf = r.get('pccf', 'latitude') + ref_pccf = r.get('percentConfidence', 'height' ) + bearaz = r.get('sensorAzimuthAngle', 'latitude') + + print(' ... Executing QuerySet: get metadata: processing center ...') + # Processing Center + ogce = r.get('dataProviderOrigin', 'latitude') + + print(' ... Executing QuerySet: get metadata: data quality information ...') + # Quality Information + qfro = r.get('qualityFlags', 'latitude') + satasc = r.get('satelliteAscendingFlag', 'latitude' ) + + print(' ... Executing QuerySet: get obsvalue: Bending Angle ...') + # ObsValue + # Bending Angle + bnda1 = r.get('bendingAngle_roseq2repl1', 'latitude') + bnda3 = r.get('bendingAngle_roseq2repl3', 'latitude') + arfr = r.get('atmosphericRefractivity', 'height') + + # ObsError + # Bending Angle + bndaoe1 = r.get('obsErrorBendingAngle1', 'latitude') + bndaoe3 = r.get('obsErrorBendingAngle3', 'latitude') + arfroe = r.get('obsErrorAtmosphericRefractivity','height') + + # ObsType + # Bending Angle + bndaot = r.get('obsTypeBendingAngle', 'latitude') + arfrot = r.get('obsTypeBendingAngle', 'latitude') + + print(' ... Executing QuerySet: get datatime: observation time ...') + # DateTime: seconds since Epoch time + # IODA has no support for numpy datetime arrays dtype=datetime64[s] + timestamp = r.get_datetime('year','month','day','hour','minute','second','latitude').astype(np.int64) + +###NICKE SEQNUM would happen around here +#For seqnum +# seqnum=[] +#    qq=1 +#    for i in range(len(elrc)): +#       if elrc[i] != elrc[i+1]: +#          qq = qq+1 +# seqnum.append(qq[i]) +# check length. should be 666000 +##### SOMEHTING LIKE THAT + + + print(' ... Executing QuerySet: Done!') + + print(' ... Executing QuerySet: Check BUFR variable generic dimension and type ...') + # Check BUFR variable generic dimension and type + print(' clath shape,type = ', clath.shape, clath.dtype) + print(' clonh shape,type = ', clonh.shape, clonh.dtype) + print(' gclath shape,type = ', gclath.shape, gclath.dtype) + print(' gclonh shape,type = ', gclonh.shape, gclonh.dtype) + print(' year shape,type = ', year.shape, year.dtype) + print(' mnth shape,type = ', mnth.shape, mnth.dtype) + print(' days shape,type = ', days.shape, days.dtype) + print(' hour shape,type = ', hour.shape, hour.dtype) + print(' minu shape,type = ', minu.shape, minu.dtype) + print(' seco shape,type = ', seco.shape, seco.dtype) + print(' said shape,type = ', said.shape, said.dtype) + print(' siid shape,type = ', siid.shape, siid.dtype) + print(' sclf shape,type = ', sclf.shape, sclf.dtype) + print(' ptid shape,type = ', ptid.shape, ptid.dtype) + print(' elrc shape,type = ', elrc.shape, elrc.dtype) + print(' geodu shape,type = ', geodu.shape, geodu.dtype) + print(' heit shape,type = ', heit.shape, heit.dtype) + print(' impp1 shape,type = ', impp1.shape, impp1.dtype) + print(' impp3 shape,type = ', impp3.shape, impp3.dtype) + print(' imph1 shape,type = ', imph1.shape, imph1.dtype) + print(' imph3 shape,type = ', imph3.shape, imph3.dtype) + print(' mefr1 shape,type = ', mefr1.shape, mefr1.dtype) + print(' mefr3 shape,type = ', mefr3.shape, mefr3.dtype) + print(' pccf shape,type = ', pccf.shape, pccf.dtype) + print(' ref_pccf shape,type = ', ref_pccf.shape, ref_pccf.dtype) + print(' bearaz shape,type = ', bearaz.shape, bearaz.dtype) + + print(' ogce shape,type = ', ogce.shape, ogce.dtype) + + print(' qfro shape,type = ', qfro.shape, qfro.dtype) + print(' satasc shape,type = ', satasc.shape, satasc.dtype) + + print(' bnda1 shape,type = ', bnda1.shape, bnda1.dtype) + print(' bnda3 shape,type = ', bnda3.shape, bnda3.dtype) + print(' arfr shape,type = ', arfr.shape, arfr.dtype) + + print(' bndaoe1 shape,type = ', bndaoe1.shape, bndaoe1.dtype) + print(' bndaoe3 shape,type = ', bndaoe3.shape, bndaoe3.dtype) + print(' arfroe shape,type = ', arfr.shape, arfr.dtype) + + print(' bndaot shape,type = ', bndaot.shape, bndaot.dtype) + + end_time = time.time() + running_time = end_time - start_time + print('Running time for executing QuerySet to get ResultSet : ', running_time, 'seconds') + + # ========================= + # Create derived variables + # ========================= + start_time = time.time() + + print('Creating derived variables - Grid Latitude / Longitude ...') + gclonh = Compute_Grid_Location(gclonh) + gclath = Compute_Grid_Location(gclath) + + print(' gclonh shape,type = ', gclonh.shape, gclonh.dtype) + print(' gclath shape,type = ', gclath.shape, gclath.dtype) + print(' gclonh min/max = ', gclonh.min(), gclonh.max()) + print(' gclath min/max = ', gclath.min(), gclath.max()) + + print('Creating derived variables - imph ...') + imph1 = Compute_imph(impp1, elrc, geodu) + imph3 = Compute_imph(impp3, elrc, geodu) + + print(' imph1 shape,type = ', imph1.shape, imph1.dtype) + print(' imph3 shape,type = ', imph3.shape, imph3.dtype) + print(' imph1 min/max = ', imph1.min(), imph1.max()) + print(' imph3 min/max = ', imph3.min(), imph3.max()) + + print('Editing some derived variables if SAID is not 44 or 825') + for i in range(len(said)): + if (said[i] != 44) or (said[i] != 825): + bnda1[i] = bnda3[i] + mefr1[i] = mefr3[i] + impp1[i] = impp3[i] + imph1[i] = imph3[i] + bndaoe1[i] = bndaoe3[i] + + print(' new bnda1 shape, type, min, max', bnda1.shape, bnda1.dtype, bnda1.min(), bnda1.max()) + print(' new mefr1 shape, type, min, max', mefr1.shape, mefr1.dtype, mefr1.min(), mefr1.max()) + print(' new impp1 shape, type, min, max', impp1.shape, impp1.dtype, impp1.min(), impp1.max()) + print(' new imph1 shape, type, min, max', imph1.shape, imph1.dtype, imph1.min(), imph1.max()) + print(' new bndaoe1 shape, type, min, max', bndaoe1.shape, bndaoe1.dtype, bndaoe1.min(), bndaoe1.max()) + + end_time = time.time() + running_time = end_time - start_time + print('Running time for creating derived variables : ', running_time, 'seconds') + + # ===================================== + # Create IODA ObsSpace + # Write IODA output + # ===================================== + + # Find unique satellite identifiers in data to process + unique_satids = np.unique(said) + print(' ... Number of Unique satellite identifiers: ', len(unique_satids)) + print(' ... Unique satellite identifiers: ', unique_satids) + + # Write the data to an IODA file + OUTPUT_PATH = '/work2/noaa/da/nesposito/NewConv/api_convert/json/json_gdasapp/gdas.t00z.gpsro.tm00.nc' + path, fname = os.path.split(OUTPUT_PATH) + if path and not os.path.exists(path): + os.makedirs(path) + + # Create the dimensions + dims = { + 'Location' : np.arange(0, clath.shape[0]), + } + + # Create IODA ObsSpace + obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) + + # Create Global attributes + print(' ... ... Create global attributes') + obsspace.write_attr('data_format', data_format) + obsspace.write_attr('data_type', data_type) + obsspace.write_attr('subsets', subsets) + obsspace.write_attr('cycle_type', cycle_type) + obsspace.write_attr('cycle_datetime', cycle) + obsspace.write_attr('dataProviderOrigin', data_provider) + obsspace.write_attr('description', data_description) + obsspace.write_attr('converter', os.path.basename(__file__)) + + # Create IODA variables + print(' ... ... Create variables: name, type, units, and attributes') + # Longitude + obsspace.create_var('MetaData/longitude', dtype=clonh.dtype, fillval=clonh.fill_value) \ + .write_attr('units', 'degrees_east') \ + .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ + .write_attr('long_name', 'Longitude') \ + .write_data(clonh) + + # Latitude + obsspace.create_var('MetaData/latitude', dtype=clath.dtype, fillval=clath.fill_value) \ + .write_attr('units', 'degrees_north') \ + .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Latitude') \ + .write_data(clath) + + # Grid Longitude + obsspace.create_var('MetaData/gridLongitude', dtype=gclonh.dtype, fillval=gclonh.fill_value) \ + .write_attr('units', 'radians') \ + .write_attr('valid_range', np.array([-3.14159265, 3.14159265], dtype=np.float32)) \ + .write_attr('long_name', 'Grid Longitude') \ + .write_data(gclonh) + + # Grid Latitude + obsspace.create_var('MetaData/gridLatitude', dtype=gclath.dtype, fillval=gclath.fill_value) \ + .write_attr('units', 'radians') \ + .write_attr('valid_range', np.array([-1.570796325, 1.570796325], dtype=np.float32)) \ + .write_attr('long_name', 'Grid Latitude') \ + .write_data(gclath) + + # Datetime + obsspace.create_var('MetaData/dateTime', dtype=np.int64, fillval=timestamp.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'Datetime') \ + .write_data(timestamp) + + # Satellite Identifier + obsspace.create_var('MetaData/satelliteIdentifier', dtype=said.dtype, fillval=said.fill_value) \ + .write_attr('long_name', 'Satellite Identifier') \ + .write_data(said) + + # Satellite Instrument + obsspace.create_var('MetaData/satelliteInstrument', dtype=siid.dtype, fillval=siid.fill_value) \ + .write_attr('long_name', 'Satellite Instrument') \ + .write_data(siid) + + # Satellite Constellation RO + obsspace.create_var('MetaData/satelliteConstellationRO', dtype=sclf.dtype, fillval=sclf.fill_value) \ + .write_attr('long_name', 'Satellite Constellation RO') \ + .write_data(sclf) + + # Satellite Transmitter ID + obsspace.create_var('MetaData/satelliteTransmitterId', dtype=ptid.dtype, fillval=ptid.fill_value) \ + .write_attr('long_name', 'Satellite Transmitter Id') \ + .write_data(ptid) + + # Earth Radius Curvature + obsspace.create_var('MetaData/earthRadiusCurvature', dtype=elrc.dtype, fillval=elrc.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Earth Radius of Curvature') \ + .write_data(elrc) + + # Geoid Undulation + obsspace.create_var('MetaData/geoidUndulation', dtype=geodu.dtype, fillval=geodu.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Geoid Undulation') \ + .write_data(geodu) + + # Height + obsspace.create_var('MetaData/height', dtype=heit.dtype, fillval=heit.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Height' ) \ + .write_data(heit) + + # Impact Parameter RO + obsspace.create_var('MetaData/impactParameterRO', dtype=impp1.dtype, fillval=impp1.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Impact Parameter RO') \ + .write_data(impp1) + + # Impact Height RO + obsspace.create_var('MetaData/impactHeightRO', dtype=imph1.dtype, fillval=imph1.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Impact Height RO') \ + .write_data(imph1) + + # Impact Height RO + obsspace.create_var('MetaData/frequency', dtype=mefr1.dtype, fillval=mefr1.fill_value) \ + .write_attr('units', 'Hz') \ + .write_attr('long_name', 'Frequency') \ + .write_data(mefr1) + + # PCCF Percent Confidence + obsspace.create_var('MetaData/pccf', dtype=pccf.dtype, fillval=pccf.fill_value) \ + .write_attr('units', '%') \ + .write_attr('long_name', 'Percent Confidence') \ + .write_data(pccf) + + # PCCF Ref Percent Confidence + obsspace.create_var('MetaData/percentConfidence', dtype=ref_pccf.dtype, fillval=ref_pccf.fill_value) \ + .write_attr('units', '%') \ + .write_attr('long_name', 'Ref Percent Confidence') \ + .write_data(ref_pccf) + + # Azimuth Angle + obsspace.create_var('MetaData/sensorAzimuthAngle', dtype=bearaz.dtype, fillval=bearaz.fill_value) \ + .write_attr('units', 'degree') \ + .write_attr('long_name', 'Percent Confidence') \ + .write_data(pccf) + + # Data Provider + obsspace.create_var('MetaData/dataProviderOrigin', dtype=ogce.dtype, fillval=ogce.fill_value) \ + .write_attr('long_name', 'Identification of Originating/Generating Center') \ + .write_data(ogce) + + # Quality: Quality Flags + obsspace.create_var('MetaData/qualityFlags', dtype=qfro.dtype, fillval=qfro.fill_value) \ + .write_attr('long_name', 'Quality Flags') \ + .write_data(qfro) + + # Quality: Satellite Ascending Flag + obsspace.create_var('MetaData/satelliteAscendingFlag', dtype=satasc.dtype, fillval=satasc.fill_value) \ + .write_attr('long_name', 'Satellite Ascending Flag') \ + .write_data(satasc) + + # ObsValue: Bending Angle + obsspace.create_var('ObsValue/bendingAngle', dtype=bnda1.dtype, fillval=bnda1.fill_value) \ + .write_attr('units', 'radians') \ + .write_attr('long_name', 'Bending Angle') \ + .write_data(bnda1) + + # ObsValue: Atmospheric Refractivity + obsspace.create_var('ObsValue/atmosphericRefractivity', dtype=arfr.dtype, fillval=arfr.fill_value) \ + .write_attr('units', 'N-units') \ + .write_attr('long_name', 'Atmospheric Refractivity ObsError') \ + .write_data(arfr) + + # ObsError: Bending Angle + obsspace.create_var('ObsError/bendingAngle', dtype=bndaoe1.dtype, fillval=bndaoe1.fill_value) \ + .write_attr('units', 'radians') \ + .write_attr('long_name', 'Bending Angle Obs Error') \ + .write_data(bndaoe1) + + # ObsError: Atmospheric Refractivity + obsspace.create_var('ObsError/atmosphericRefractivity', dtype=arfroe.dtype, fillval=arfroe.fill_value) \ + .write_attr('units', 'N-units') \ + .write_attr('long_name', 'Atmospheric Refractivity ObsError') \ + .write_data(arfroe) + + # ObsType: Bending Angle + obsspace.create_var('ObsType/BendingAngle', dtype=bndaot.dtype, fillval=bndaot.fill_value) \ + .write_attr('long_name', 'Bending Angle ObsType') \ + .write_data(bndaot) + + # ObsType: Atmospheric Refractivity + obsspace.create_var('ObsType/atmosphericRefractivity', dtype=arfrot.dtype, fillval=arfrot.fill_value) \ + .write_attr('long_name', 'Atmospheric Refractivity ObsType') \ + .write_data(arfrot) + + end_time = time.time() + running_time = end_time - start_time + print('Running time for splitting and output IODA for gpsro bufr ', running_time, 'seconds') + + print("All Done!") + +if __name__ == '__main__': + + start_time = time.time() + + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, help='Input JSON configuration', required=True) + parser.add_argument('-v', '--verbose', help='print debug logging information', + action='store_true') + args = parser.parse_args() + + with open(args.config, "r") as json_file: + config = json.load(json_file) + + bufr_to_ioda(config) + + end_time = time.time() + running_time = end_time - start_time + print('Total running time: ', running_time, 'seconds') + + + diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda_gpsro_bufr.sh b/ush/ioda/bufr2ioda/run_bufr2ioda_gpsro_bufr.sh index 977b67ca5..0623dfb9e 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda_gpsro_bufr.sh +++ b/ush/ioda/bufr2ioda/run_bufr2ioda_gpsro_bufr.sh @@ -51,7 +51,8 @@ export PYTHONPATH=${PYIODALIB}:${PYTHONPATH} #--------------------------- #----- python and json ----- # first specify what observation types will be processed by a script -BUFR_py="gpsro_bufr" +#BUFR_py="gpsro_bufr" +BUFR_py="gpsro_bufr_combined" for obtype in $BUFR_py; do # this loop assumes that there is a python script and template with the same name From c672ebe6d573aa7e0dbe67eb61af9c7cd8bbc19d Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 14 Sep 2023 15:39:38 -0400 Subject: [PATCH 14/29] fix bearaz --- ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py index cd080ae59..bc0a6269b 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py @@ -443,7 +443,7 @@ def bufr_to_ioda(config): obsspace.create_var('MetaData/sensorAzimuthAngle', dtype=bearaz.dtype, fillval=bearaz.fill_value) \ .write_attr('units', 'degree') \ .write_attr('long_name', 'Percent Confidence') \ - .write_data(pccf) + .write_data(bearaz) # Data Provider obsspace.create_var('MetaData/dataProviderOrigin', dtype=ogce.dtype, fillval=ogce.fill_value) \ From 23e6434b4c33339d8e3877e6189e9e4148189009 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 14 Sep 2023 15:40:07 -0400 Subject: [PATCH 15/29] fix bearaz 2 other file --- ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py index f93c88d9d..c4ad17214 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py @@ -534,7 +534,7 @@ def bufr_to_ioda(config): obsspace.create_var('MetaData/sensorAzimuthAngle', dtype=bearaz2.dtype, fillval=bearaz2.fill_value) \ .write_attr('units', 'degree') \ .write_attr('long_name', 'Percent Confidence') \ - .write_data(pccf2) + .write_data(bearaz2) # Data Provider obsspace.create_var('MetaData/dataProviderOrigin', dtype=ogce2.dtype, fillval=ogce2.fill_value) \ From bcbaf3be48cf47c0cb93a7015c015f57ea812f10 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 14 Sep 2023 16:13:19 -0400 Subject: [PATCH 16/29] fix Compute_Grid_Location so it doesnt run missing vals --- ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py | 7 +++++-- ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py index c4ad17214..554b861c6 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py @@ -30,9 +30,12 @@ def Compute_Grid_Location(degrees): - rad = np.deg2rad(degrees) + for i in range(len(degrees)): + if degrees[i] < 180 and degrees[i] > -180 : + degrees[i] = np.deg2rad(degrees[i]) + rad = degrees - return rad + return rad def Compute_imph(impp, elrc, geodu): diff --git a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py index bc0a6269b..8792881db 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py @@ -30,8 +30,11 @@ def Compute_Grid_Location(degrees): - rad = np.deg2rad(degrees) - + for i in range(len(degrees)): + if degrees[i] < 180 and degrees[i] > -180 : + degrees[i] = np.deg2rad(degrees[i]) + rad = degrees + return rad def Compute_imph(impp, elrc, geodu): From 598623af153c1be5aea351e35a354f20af0fe66b Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Mon, 18 Sep 2023 17:19:23 -0400 Subject: [PATCH 17/29] stationIdentification derivation fix --- .../bufr2ioda_gpsro_bufr_combined.py | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py index 8792881db..2ad5afa2d 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py @@ -28,6 +28,15 @@ # NC003010 | GPS-RO # ==================================================================== +def Derive_stationIdentification(said,ptid,stid): + stid = stid.astype('str') + for i in range(len(said)): + stid[i] = str(said[i]).zfill(4)+str(ptid[i]).zfill(4) + print('NE STID: ', str(said[i]).zfill(4), str(ptid[i]).zfill(4)) + print("NE i STID: ", i, stid[i]) + + return stid + def Compute_Grid_Location(degrees): for i in range(len(degrees)): @@ -149,6 +158,7 @@ def bufr_to_ioda(config): hour = r.get('hour', 'latitude') minu = r.get('minute', 'latitude') seco = r.get('second', 'latitude') + stid = r.get('satelliteIdentifier', 'latitude') said = r.get('satelliteIdentifier', 'latitude') siid = r.get('satelliteInstrument', 'latitude') sclf = r.get('satelliteConstellationRO', 'latitude') @@ -161,9 +171,9 @@ def bufr_to_ioda(config): impp3 = r.get('impactParameterRO_roseq2repl3', 'latitude') imph1 = r.get('impactHeightRO_roseq2repl1', 'latitude') imph3 = r.get('impactHeightRO_roseq2repl3', 'latitude') - mefr1 = r.get('frequency__roseq2repl1', 'latitude') - mefr3 = r.get('frequency__roseq2repl3', 'latitude') - pccf = r.get('pccf', 'latitude') + mefr1 = r.get('frequency__roseq2repl1', 'latitude', type='float') + mefr3 = r.get('frequency__roseq2repl3', 'latitude', type='float') + pccf = r.get('pccf', 'latitude', type='float') ref_pccf = r.get('percentConfidence', 'height' ) bearaz = r.get('sensorAzimuthAngle', 'latitude') @@ -225,6 +235,7 @@ def bufr_to_ioda(config): print(' hour shape,type = ', hour.shape, hour.dtype) print(' minu shape,type = ', minu.shape, minu.dtype) print(' seco shape,type = ', seco.shape, seco.dtype) + print(' stid shape,type = ', stid.shape, stid.dtype) print(' said shape,type = ', said.shape, said.dtype) print(' siid shape,type = ', siid.shape, siid.dtype) print(' sclf shape,type = ', sclf.shape, sclf.dtype) @@ -266,6 +277,12 @@ def bufr_to_ioda(config): # ========================= start_time = time.time() + print('Creating derived variables - stationIdentification') + stid = Derive_stationIdentification(said,ptid,stid) + + print(' stid shape,type = ', stid.shape, stid.dtype) + print(' stid[0] = ', stid[0]) + print('Creating derived variables - Grid Latitude / Longitude ...') gclonh = Compute_Grid_Location(gclonh) gclath = Compute_Grid_Location(gclath) @@ -373,6 +390,11 @@ def bufr_to_ioda(config): .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ .write_attr('long_name', 'Datetime') \ .write_data(timestamp) + + # Station Identification + obsspace.create_var('MetaData/stationIdentification', dtype=stid.dtype, fillval=stid.fill_value) \ + .write_attr('long_name', 'Station Identification') \ + .write_data(stid) # Satellite Identifier obsspace.create_var('MetaData/satelliteIdentifier', dtype=said.dtype, fillval=said.fill_value) \ From c33227b6502f5b260c95c0ced86ae540524e9a86 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Tue, 19 Sep 2023 12:01:42 -0400 Subject: [PATCH 18/29] combined removes some prints. split makes changes just made to comined --- ...r.json => bufr2ioda_gpsro_bufr_split.json} | 0 .../bufr2ioda_gpsro_bufr_combined.py | 2 -- ..._bufr.py => bufr2ioda_gpsro_bufr_split.py} | 28 ++++++++++++++++--- .../bufr2ioda/run_bufr2ioda_gpsro_bufr.sh | 4 +-- 4 files changed, 26 insertions(+), 8 deletions(-) rename parm/ioda/bufr2ioda/{bufr2ioda_gpsro_bufr.json => bufr2ioda_gpsro_bufr_split.json} (100%) rename ush/ioda/bufr2ioda/{bufr2ioda_gpsro_bufr.py => bufr2ioda_gpsro_bufr_split.py} (96%) diff --git a/parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.json b/parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_split.json similarity index 100% rename from parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.json rename to parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_split.json diff --git a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py index 2ad5afa2d..d226d82cb 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py @@ -32,8 +32,6 @@ def Derive_stationIdentification(said,ptid,stid): stid = stid.astype('str') for i in range(len(said)): stid[i] = str(said[i]).zfill(4)+str(ptid[i]).zfill(4) - print('NE STID: ', str(said[i]).zfill(4), str(ptid[i]).zfill(4)) - print("NE i STID: ", i, stid[i]) return stid diff --git a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_split.py similarity index 96% rename from ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py rename to ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_split.py index 554b861c6..0a68a4703 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_split.py @@ -28,6 +28,14 @@ # NC003010 | GPS-RO # ==================================================================== +def Derive_stationIdentification(said,ptid,stid): + stid = stid.astype('str') + for i in range(len(said)): + stid[i] = str(said[i]).zfill(4)+str(ptid[i]).zfill(4) + + return stid + + def Compute_Grid_Location(degrees): for i in range(len(degrees)): @@ -46,7 +54,6 @@ def Compute_imph(impp, elrc, geodu): def bufr_to_ioda(config): subsets = config["subsets"] - print("NE subsets", subsets) # ========================================= # Get parameters from configuration # ========================================= @@ -152,6 +159,7 @@ def bufr_to_ioda(config): hour = r.get('hour', 'latitude') minu = r.get('minute', 'latitude') seco = r.get('second', 'latitude') + stid = r.get('satelliteIdentifier', 'latitude') said = r.get('satelliteIdentifier', 'latitude') siid = r.get('satelliteInstrument', 'latitude') sclf = r.get('satelliteConstellationRO', 'latitude') @@ -164,9 +172,9 @@ def bufr_to_ioda(config): impp3 = r.get('impactParameterRO_roseq2repl3', 'latitude') imph1 = r.get('impactHeightRO_roseq2repl1', 'latitude') imph3 = r.get('impactHeightRO_roseq2repl3', 'latitude') - mefr1 = r.get('frequency__roseq2repl1', 'latitude') - mefr3 = r.get('frequency__roseq2repl3', 'latitude') - pccf = r.get('pccf', 'latitude') + mefr1 = r.get('frequency__roseq2repl1', 'latitude', type='float') + mefr3 = r.get('frequency__roseq2repl3', 'latitude', type='float') + pccf = r.get('pccf', 'latitude', type='float') ref_pccf = r.get('percentConfidence', 'height' ) bearaz = r.get('sensorAzimuthAngle', 'latitude') @@ -228,6 +236,7 @@ def bufr_to_ioda(config): print(' hour shape,type = ', hour.shape, hour.dtype) print(' minu shape,type = ', minu.shape, minu.dtype) print(' seco shape,type = ', seco.shape, seco.dtype) + print(' stid shape,type = ', stid.shape, stid.dtype) print(' said shape,type = ', said.shape, said.dtype) print(' siid shape,type = ', siid.shape, siid.dtype) print(' sclf shape,type = ', sclf.shape, sclf.dtype) @@ -269,6 +278,11 @@ def bufr_to_ioda(config): # ========================= start_time = time.time() + print('Creating derived variables - stationIdentification') + stid = Derive_stationIdentification(said,ptid,stid) + + print(' stid shape,type = ', stid.shape, stid.dtype) + print('Creating derived variables - Grid Latitude / Longitude ...') gclonh = Compute_Grid_Location(gclonh) gclath = Compute_Grid_Location(gclath) @@ -333,6 +347,7 @@ def bufr_to_ioda(config): gclonh2 = gclonh[mask] gclath2 = gclath[mask] timestamp2 = timestamp[mask] + stid2 = stid[mask] said2 = said[mask] siid2 = siid[mask] sclf2 = sclf[mask] @@ -465,6 +480,11 @@ def bufr_to_ioda(config): .write_attr('long_name', 'Datetime') \ .write_data(timestamp2) + # Satellite Identifier + obsspace.create_var('MetaData/stationIdentification', dtype=stid2.dtype, fillval=stid2.fill_value) \ + .write_attr('long_name', 'Station Identification') \ + .write_data(stid2) + # Satellite Identifier obsspace.create_var('MetaData/satelliteIdentifier', dtype=said2.dtype, fillval=said2.fill_value) \ .write_attr('long_name', 'Satellite Identifier') \ diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda_gpsro_bufr.sh b/ush/ioda/bufr2ioda/run_bufr2ioda_gpsro_bufr.sh index 0623dfb9e..7cc5296b6 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda_gpsro_bufr.sh +++ b/ush/ioda/bufr2ioda/run_bufr2ioda_gpsro_bufr.sh @@ -51,8 +51,8 @@ export PYTHONPATH=${PYIODALIB}:${PYTHONPATH} #--------------------------- #----- python and json ----- # first specify what observation types will be processed by a script -#BUFR_py="gpsro_bufr" -BUFR_py="gpsro_bufr_combined" +BUFR_py="gpsro_bufr_split" +#BUFR_py="gpsro_bufr_combined" for obtype in $BUFR_py; do # this loop assumes that there is a python script and template with the same name From 3b10c2830b20b74e36c6de71c83fe2b7f928011d Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 21 Sep 2023 10:06:39 -0400 Subject: [PATCH 19/29] gridlat/lon change. tried to get float to work for some var --- .../bufr2ioda_gpsro_bufr_combined.py | 45 ++++++++++++------- .../bufr2ioda/run_bufr2ioda_gpsro_bufr.sh | 5 +-- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py index d226d82cb..8db9520f5 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py @@ -29,6 +29,7 @@ # ==================================================================== def Derive_stationIdentification(said,ptid,stid): + stid = stid.astype('str') for i in range(len(said)): stid[i] = str(said[i]).zfill(4)+str(ptid[i]).zfill(4) @@ -38,15 +39,21 @@ def Derive_stationIdentification(said,ptid,stid): def Compute_Grid_Location(degrees): for i in range(len(degrees)): - if degrees[i] < 180 and degrees[i] > -180 : - degrees[i] = np.deg2rad(degrees[i]) + if degrees[i] <= 360 and degrees[i] >= -180 : + degrees[i] = np.deg2rad(degrees[i]) +# else: +# degrees[i] = np.float32(0) rad = degrees return rad -def Compute_imph(impp, elrc, geodu): +def Compute_imph(impp, elrc):#, geodu, imph): - imph = impp - elrc - geodu + imph=(impp-elrc).astype(np.float32) +# for i in range(len(impp)): +## imph = imph.astype('float') +# imph[i]=impp[i]-elrc[i]#-geodu[i] +## print("NE i impp elrc imph", i, impp[i], elrc[i], imph[i]) return imph @@ -98,8 +105,8 @@ def bufr_to_ioda(config): #q.add('observationSequenceNum', '*/SEQNUM' ) q.add('geoidUndulation', '*/GEODU' ) q.add('height', '*/ROSEQ3/HEIT' ) - q.add('impactHeightRO_roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/IMPP' ) - q.add('impactHeightRO_roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/IMPP' ) +# q.add('impactHeightRO_roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/IMPP' ) +# q.add('impactHeightRO_roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/IMPP' ) q.add('impactParameterRO_roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/IMPP' ) q.add('impactParameterRO_roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/IMPP' ) q.add('frequency__roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/MEFR' ) @@ -164,14 +171,14 @@ def bufr_to_ioda(config): elrc = r.get('earthRadiusCurvature', 'latitude') #seqnum = r.get('observationSequenceNum') geodu = r.get('geoidUndulation', 'latitude') - heit = r.get('height', 'height', type='float') + heit = r.get('height', 'height', type='float32').astype(np.float32) impp1 = r.get('impactParameterRO_roseq2repl1', 'latitude') impp3 = r.get('impactParameterRO_roseq2repl3', 'latitude') - imph1 = r.get('impactHeightRO_roseq2repl1', 'latitude') - imph3 = r.get('impactHeightRO_roseq2repl3', 'latitude') - mefr1 = r.get('frequency__roseq2repl1', 'latitude', type='float') - mefr3 = r.get('frequency__roseq2repl3', 'latitude', type='float') - pccf = r.get('pccf', 'latitude', type='float') +# imph1 = r.get('impactHeightRO_roseq2repl1', 'latitude') +# imph3 = r.get('impactHeightRO_roseq2repl3', 'latitude') + mefr1 = r.get('frequency__roseq2repl1', 'latitude', type='float32').astype(np.float32) + mefr3 = r.get('frequency__roseq2repl3', 'latitude', type='float32').astype(np.float32) + pccf = r.get('pccf', 'latitude', type='float32').astype(np.float32) ref_pccf = r.get('percentConfidence', 'height' ) bearaz = r.get('sensorAzimuthAngle', 'latitude') @@ -243,8 +250,8 @@ def bufr_to_ioda(config): print(' heit shape,type = ', heit.shape, heit.dtype) print(' impp1 shape,type = ', impp1.shape, impp1.dtype) print(' impp3 shape,type = ', impp3.shape, impp3.dtype) - print(' imph1 shape,type = ', imph1.shape, imph1.dtype) - print(' imph3 shape,type = ', imph3.shape, imph3.dtype) +# print(' imph1 shape,type = ', imph1.shape, imph1.dtype) +# print(' imph3 shape,type = ', imph3.shape, imph3.dtype) print(' mefr1 shape,type = ', mefr1.shape, mefr1.dtype) print(' mefr3 shape,type = ', mefr3.shape, mefr3.dtype) print(' pccf shape,type = ', pccf.shape, pccf.dtype) @@ -291,8 +298,8 @@ def bufr_to_ioda(config): print(' gclath min/max = ', gclath.min(), gclath.max()) print('Creating derived variables - imph ...') - imph1 = Compute_imph(impp1, elrc, geodu) - imph3 = Compute_imph(impp3, elrc, geodu) + imph1 = Compute_imph(impp1, elrc) + imph3 = Compute_imph(impp3, elrc) print(' imph1 shape,type = ', imph1.shape, imph1.dtype) print(' imph3 shape,type = ', imph3.shape, imph3.dtype) @@ -352,6 +359,11 @@ def bufr_to_ioda(config): obsspace.write_attr('dataProviderOrigin', data_provider) obsspace.write_attr('description', data_description) obsspace.write_attr('converter', os.path.basename(__file__)) + + + for i in range(len(elrc)): + if i > 0 and i < 10: + print("NE elrc 0-10", elrc[i]) # Create IODA variables print(' ... ... Create variables: name, type, units, and attributes') @@ -415,6 +427,7 @@ def bufr_to_ioda(config): .write_data(ptid) # Earth Radius Curvature +# obsspace.create_var('MetaData/earthRadiusCurvature', dtype=elrc.dtype, fillval=elrc.fill_value) \ obsspace.create_var('MetaData/earthRadiusCurvature', dtype=elrc.dtype, fillval=elrc.fill_value) \ .write_attr('units', 'm') \ .write_attr('long_name', 'Earth Radius of Curvature') \ diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda_gpsro_bufr.sh b/ush/ioda/bufr2ioda/run_bufr2ioda_gpsro_bufr.sh index 7cc5296b6..2184cf84c 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda_gpsro_bufr.sh +++ b/ush/ioda/bufr2ioda/run_bufr2ioda_gpsro_bufr.sh @@ -51,8 +51,8 @@ export PYTHONPATH=${PYIODALIB}:${PYTHONPATH} #--------------------------- #----- python and json ----- # first specify what observation types will be processed by a script -BUFR_py="gpsro_bufr_split" -#BUFR_py="gpsro_bufr_combined" +#BUFR_py="gpsro_bufr_split" +BUFR_py="gpsro_bufr_combined" for obtype in $BUFR_py; do # this loop assumes that there is a python script and template with the same name @@ -60,7 +60,6 @@ for obtype in $BUFR_py; do # first generate a JSON from the template ${BUFRJSONGEN} -t ${config_template_dir}/bufr2ioda_${obtype}.json -o ${COM_OBS}/${obtype}_${PDY}${cyc}.json -# ${BUFRJSONGEN} -t ${config_template_dir}/bufr2ioda_${obtype}.json -c bufr2ioda_${obtype}.json -o ${COM_OBS}/${obtype}_${PDY}${cyc}.json echo "1 done" # now use the converter script for the ob type From 9913bf3dc644957ff859c3a14e6d667951d7f71f Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 21 Sep 2023 16:46:54 -0400 Subject: [PATCH 20/29] logger, code norm 1 --- .../bufr2ioda_gpsro_bufr_combined.py | 619 ++++++++++-------- 1 file changed, 329 insertions(+), 290 deletions(-) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py index 8db9520f5..ebb24efb0 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py @@ -4,11 +4,6 @@ # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. import sys - -sys.path.append('/work2/noaa/da/nesposito/ioda-bundle_20230712/build/lib/python3.9/') -sys.path.append('/work2/noaa/da/nesposito/ioda-bundle_20230712/build/lib/python3.9/pyiodaconv/') -sys.path.append('/work2/noaa/da/nesposito/ioda-bundle_20230712/build/lib/python3.9/pyioda/') - import os import argparse import json @@ -16,50 +11,57 @@ import numpy.ma as ma import math import calendar -import time +import time import datetime from pyiodaconv import bufr from collections import namedtuple from pyioda import ioda_obs_space as ioda_ospace +from wxflow import Logger # ==================================================================== # GPS-RO BUFR dump file -#===================================================================== +# ===================================================================== # NC003010 | GPS-RO # ==================================================================== -def Derive_stationIdentification(said,ptid,stid): + +def Derive_stationIdentification(said, ptid, stid): stid = stid.astype('str') for i in range(len(said)): - stid[i] = str(said[i]).zfill(4)+str(ptid[i]).zfill(4) + stid[i] = str(said[i]).zfill(4)+str(ptid[i]).zfill(4) return stid + def Compute_Grid_Location(degrees): for i in range(len(degrees)): - if degrees[i] <= 360 and degrees[i] >= -180 : - degrees[i] = np.deg2rad(degrees[i]) + if degrees[i] <= 360 and degrees[i] >= -180: + degrees[i] = np.deg2rad(degrees[i]) # else: -# degrees[i] = np.float32(0) +# degrees[i] = np.float32(0) rad = degrees - - return rad -def Compute_imph(impp, elrc):#, geodu, imph): + return rad - imph=(impp-elrc).astype(np.float32) + +def Compute_imph(impp, elrc): # , geodu, imph): + + imph = (impp - elrc).astype(np.float32) # for i in range(len(impp)): -## imph = imph.astype('float') +# # imph = imph.astype('float') # imph[i]=impp[i]-elrc[i]#-geodu[i] -## print("NE i impp elrc imph", i, impp[i], elrc[i], imph[i]) +# # print("NE i impp elrc imph", i, impp[i], elrc[i], imph[i]) return imph -def bufr_to_ioda(config): + +def bufr_to_ioda(config, logger): subsets = config["subsets"] + logger.debug(f"Checking subsets = {subsets}") + # ========================================= # Get parameters from configuration # ========================================= @@ -75,70 +77,71 @@ def bufr_to_ioda(config): hh = cycle[8:10] bufrfile = f"{cycle_type}.t{hh}z.{data_type}.tm00.{data_format}" - DATA_PATH = os.path.join(dump_dir, f"{cycle_type}.{yyyymmdd}", str(hh), 'atmos', bufrfile) + DATA_PATH = os.path.join(dump_dir, f"{cycle_type}.{yyyymmdd}", str(hh), + 'atmos', bufrfile) # ============================================ # Make the QuerySet for all the data we want # ============================================ start_time = time.time() - print('Making QuerySet ...') + logger.info(f"Making QuerySet ...") q = bufr.QuerySet(subsets) # MetaData - q.add('latitude', '*/ROSEQ1/CLATH' ) - q.add('longitude', '*/ROSEQ1/CLONH' ) - q.add('gridLatitude', '*/ROSEQ1/CLATH' ) - q.add('gridLongitude', '*/ROSEQ1/CLONH' ) - q.add('year', '*/YEAR' ) - q.add('year2', '*/YEAR' ) - q.add('month', '*/MNTH' ) - q.add('day', '*/DAYS' ) - q.add('hour', '*/HOUR' ) - q.add('minute', '*/MINU' ) - q.add('second', '*/SECO' ) - q.add('satelliteIdentifier', '*/SAID' ) - q.add('satelliteInstrument', '*/SIID' ) - q.add('satelliteConstellationRO', '*/SCLF' ) - q.add('satelliteTransmitterId', '*/PTID' ) - q.add('earthRadiusCurvature', '*/ELRC' ) - #q.add('observationSequenceNum', '*/SEQNUM' ) - q.add('geoidUndulation', '*/GEODU' ) - q.add('height', '*/ROSEQ3/HEIT' ) -# q.add('impactHeightRO_roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/IMPP' ) -# q.add('impactHeightRO_roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/IMPP' ) - q.add('impactParameterRO_roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/IMPP' ) - q.add('impactParameterRO_roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/IMPP' ) - q.add('frequency__roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/MEFR' ) - q.add('frequency__roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/MEFR' ) - q.add('pccf', '*/ROSEQ1/PCCF' ) - q.add('percentConfidence', '*/ROSEQ3/PCCF' ) - q.add('sensorAzimuthAngle', '*/BEARAZ' ) + q.add('latitude', '*/ROSEQ1/CLATH') + q.add('longitude', '*/ROSEQ1/CLONH') + q.add('gridLatitude', '*/ROSEQ1/CLATH') + q.add('gridLongitude', '*/ROSEQ1/CLONH') + q.add('year', '*/YEAR') + q.add('year2', '*/YEAR') + q.add('month', '*/MNTH') + q.add('day', '*/DAYS') + q.add('hour', '*/HOUR') + q.add('minute', '*/MINU') + q.add('second', '*/SECO') + q.add('satelliteIdentifier', '*/SAID') + q.add('satelliteInstrument', '*/SIID') + q.add('satelliteConstellationRO', '*/SCLF') + q.add('satelliteTransmitterId', '*/PTID') + q.add('earthRadiusCurvature', '*/ELRC') +# q.add('observationSequenceNum', '*/SEQNUM') + q.add('geoidUndulation', '*/GEODU') + q.add('height', '*/ROSEQ3/HEIT') +# q.add('impactHeightRO_roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/IMPP') +# q.add('impactHeightRO_roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/IMPP') + q.add('impactParameterRO_roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/IMPP') + q.add('impactParameterRO_roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/IMPP') + q.add('frequency__roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/MEFR') + q.add('frequency__roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/MEFR') + q.add('pccf', '*/ROSEQ1/PCCF') + q.add('percentConfidence', '*/ROSEQ3/PCCF') + q.add('sensorAzimuthAngle', '*/BEARAZ') # Processing Center - q.add('dataProviderOrigin', '*/OGCE' ) + q.add('dataProviderOrigin', '*/OGCE') # Quality Information - q.add('qualityFlags', '*/QFRO' ) - q.add('satelliteAscendingFlag', '*/QFRO' ) + q.add('qualityFlags', '*/QFRO') + q.add('satelliteAscendingFlag', '*/QFRO') # ObsValue - q.add('bendingAngle_roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/BNDA[1]' ) - q.add('bendingAngle_roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/BNDA[1]' ) - q.add('atmosphericRefractivity', '*/ROSEQ3/ARFR[1]' ) + q.add('bendingAngle_roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/BNDA[1]') + q.add('bendingAngle_roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/BNDA[1]') + q.add('atmosphericRefractivity', '*/ROSEQ3/ARFR[1]') # ObsError - q.add('obsErrorBendingAngle1', '*/ROSEQ1/ROSEQ2{1}/BNDA[2]') - q.add('obsErrorBendingAngle3', '*/ROSEQ1/ROSEQ2{3}/BNDA[2]') + q.add('obsErrorBendingAngle1', '*/ROSEQ1/ROSEQ2{1}/BNDA[2]') + q.add('obsErrorBendingAngle3', '*/ROSEQ1/ROSEQ2{3}/BNDA[2]') q.add('obsErrorAtmosphericRefractivity', '*/ROSEQ3/ARFR[2]') # ObsType - q.add('obsTypeBendingAngle', '*/SAID' ) - q.add('obsTypeAtmosphericRefractivity', '*/SAID' ) + q.add('obsTypeBendingAngle', '*/SAID') + q.add('obsTypeAtmosphericRefractivity', '*/SAID') end_time = time.time() running_time = end_time - start_time - print('Running time for making QuerySet : ', running_time, 'seconds') + logger.info(f"Running time for making QuerySet: {running_time} seconds") # ============================================================== # Open the BUFR file and execute the QuerySet to get ResultSet @@ -146,184 +149,186 @@ def bufr_to_ioda(config): # ============================================================== start_time = time.time() - print('Executing QuerySet to get ResultSet ...') + logger.info(f"Executing QuerySet to get ResultSet ...") with bufr.File(DATA_PATH) as f: - r = f.execute(q) - - print(' ... Executing QuerySet: get metadata: basic ...') + r = f.execute(q) + + logger.info(f" ... Executing QuerySet: get MetaData: basic ...") # MetaData - clath = r.get('latitude', 'latitude') - clonh = r.get('longitude', 'latitude') - gclath = r.get('gridLatitude', 'latitude') - gclonh = r.get('gridLongitude', 'latitude') - year = r.get('year', 'latitude') - year2 = r.get('year2') - mnth = r.get('month', 'latitude') - days = r.get('day', 'latitude') - hour = r.get('hour', 'latitude') - minu = r.get('minute', 'latitude') - seco = r.get('second', 'latitude') - stid = r.get('satelliteIdentifier', 'latitude') - said = r.get('satelliteIdentifier', 'latitude') - siid = r.get('satelliteInstrument', 'latitude') - sclf = r.get('satelliteConstellationRO', 'latitude') - ptid = r.get('satelliteTransmitterId', 'latitude') - elrc = r.get('earthRadiusCurvature', 'latitude') - #seqnum = r.get('observationSequenceNum') - geodu = r.get('geoidUndulation', 'latitude') - heit = r.get('height', 'height', type='float32').astype(np.float32) - impp1 = r.get('impactParameterRO_roseq2repl1', 'latitude') - impp3 = r.get('impactParameterRO_roseq2repl3', 'latitude') -# imph1 = r.get('impactHeightRO_roseq2repl1', 'latitude') -# imph3 = r.get('impactHeightRO_roseq2repl3', 'latitude') - mefr1 = r.get('frequency__roseq2repl1', 'latitude', type='float32').astype(np.float32) - mefr3 = r.get('frequency__roseq2repl3', 'latitude', type='float32').astype(np.float32) - pccf = r.get('pccf', 'latitude', type='float32').astype(np.float32) - ref_pccf = r.get('percentConfidence', 'height' ) - bearaz = r.get('sensorAzimuthAngle', 'latitude') - - print(' ... Executing QuerySet: get metadata: processing center ...') + clath = r.get('latitude', 'latitude') + clonh = r.get('longitude', 'latitude') + gclath = r.get('gridLatitude', 'latitude') + gclonh = r.get('gridLongitude', 'latitude') + year = r.get('year', 'latitude') + year2 = r.get('year2') + mnth = r.get('month', 'latitude') + days = r.get('day', 'latitude') + hour = r.get('hour', 'latitude') + minu = r.get('minute', 'latitude') + seco = r.get('second', 'latitude') + stid = r.get('satelliteIdentifier', 'latitude') + said = r.get('satelliteIdentifier', 'latitude') + siid = r.get('satelliteInstrument', 'latitude') + sclf = r.get('satelliteConstellationRO', 'latitude') + ptid = r.get('satelliteTransmitterId', 'latitude') + elrc = r.get('earthRadiusCurvature', 'latitude') +# seqnum = r.get('observationSequenceNum') + geodu = r.get('geoidUndulation', 'latitude') + heit = r.get('height', 'height', type='float32').astype(np.float32) + impp1 = r.get('impactParameterRO_roseq2repl1', 'latitude') + impp3 = r.get('impactParameterRO_roseq2repl3', 'latitude') +# imph1 = r.get('impactHeightRO_roseq2repl1', 'latitude') +# imph3 = r.get('impactHeightRO_roseq2repl3', 'latitude') + mefr1 = r.get('frequency__roseq2repl1', 'latitude', + type='float32').astype(np.float32) + mefr3 = r.get('frequency__roseq2repl3', 'latitude', + type='float32').astype(np.float32) + pccf = r.get('pccf', 'latitude', type='float32').astype(np.float32) + ref_pccf = r.get('percentConfidence', 'height') + bearaz = r.get('sensorAzimuthAngle', 'latitude') + + logger.info(f" ... Executing QuerySet: get MetaData: processing center...") # Processing Center - ogce = r.get('dataProviderOrigin', 'latitude') + ogce = r.get('dataProviderOrigin', 'latitude') - print(' ... Executing QuerySet: get metadata: data quality information ...') - # Quality Information - qfro = r.get('qualityFlags', 'latitude') - satasc = r.get('satelliteAscendingFlag', 'latitude' ) + logger.info(f" ... Executing QuerySet: get metadata: data quality \ + information ...") + # Quality Information + qfro = r.get('qualityFlags', 'latitude') + satasc = r.get('satelliteAscendingFlag', 'latitude') - print(' ... Executing QuerySet: get obsvalue: Bending Angle ...') + logger.info(f" ... Executing QuerySet: get ObsValue: Bending Angle ...") # ObsValue # Bending Angle - bnda1 = r.get('bendingAngle_roseq2repl1', 'latitude') - bnda3 = r.get('bendingAngle_roseq2repl3', 'latitude') - arfr = r.get('atmosphericRefractivity', 'height') + bnda1 = r.get('bendingAngle_roseq2repl1', 'latitude') + bnda3 = r.get('bendingAngle_roseq2repl3', 'latitude') + arfr = r.get('atmosphericRefractivity', 'height') # ObsError # Bending Angle - bndaoe1 = r.get('obsErrorBendingAngle1', 'latitude') - bndaoe3 = r.get('obsErrorBendingAngle3', 'latitude') - arfroe = r.get('obsErrorAtmosphericRefractivity','height') + bndaoe1 = r.get('obsErrorBendingAngle1', 'latitude') + bndaoe3 = r.get('obsErrorBendingAngle3', 'latitude') + arfroe = r.get('obsErrorAtmosphericRefractivity', 'height') # ObsType # Bending Angle - bndaot = r.get('obsTypeBendingAngle', 'latitude') - arfrot = r.get('obsTypeBendingAngle', 'latitude') + bndaot = r.get('obsTypeBendingAngle', 'latitude') + arfrot = r.get('obsTypeBendingAngle', 'latitude') - print(' ... Executing QuerySet: get datatime: observation time ...') + logger.info(f" ... Executing QuerySet: get datatime: observation time ...") # DateTime: seconds since Epoch time # IODA has no support for numpy datetime arrays dtype=datetime64[s] - timestamp = r.get_datetime('year','month','day','hour','minute','second','latitude').astype(np.int64) - -###NICKE SEQNUM would happen around here -#For seqnum -# seqnum=[] -#    qq=1 -#    for i in range(len(elrc)): -#       if elrc[i] != elrc[i+1]: -#          qq = qq+1 -# seqnum.append(qq[i]) -# check length. should be 666000 -##### SOMEHTING LIKE THAT + timestamp = r.get_datetime('year', 'month', 'day', 'hour', 'minute', + 'second', 'latitude').astype(np.int64) + logger.info(f" ... Executing QuerySet: Done!") - print(' ... Executing QuerySet: Done!') - - print(' ... Executing QuerySet: Check BUFR variable generic dimension and type ...') + logger.info(f" ... Executing QuerySet: Check BUFR variable generic \ + dimension and type ...") # Check BUFR variable generic dimension and type - print(' clath shape,type = ', clath.shape, clath.dtype) - print(' clonh shape,type = ', clonh.shape, clonh.dtype) - print(' gclath shape,type = ', gclath.shape, gclath.dtype) - print(' gclonh shape,type = ', gclonh.shape, gclonh.dtype) - print(' year shape,type = ', year.shape, year.dtype) - print(' mnth shape,type = ', mnth.shape, mnth.dtype) - print(' days shape,type = ', days.shape, days.dtype) - print(' hour shape,type = ', hour.shape, hour.dtype) - print(' minu shape,type = ', minu.shape, minu.dtype) - print(' seco shape,type = ', seco.shape, seco.dtype) - print(' stid shape,type = ', stid.shape, stid.dtype) - print(' said shape,type = ', said.shape, said.dtype) - print(' siid shape,type = ', siid.shape, siid.dtype) - print(' sclf shape,type = ', sclf.shape, sclf.dtype) - print(' ptid shape,type = ', ptid.shape, ptid.dtype) - print(' elrc shape,type = ', elrc.shape, elrc.dtype) - print(' geodu shape,type = ', geodu.shape, geodu.dtype) - print(' heit shape,type = ', heit.shape, heit.dtype) - print(' impp1 shape,type = ', impp1.shape, impp1.dtype) - print(' impp3 shape,type = ', impp3.shape, impp3.dtype) -# print(' imph1 shape,type = ', imph1.shape, imph1.dtype) -# print(' imph3 shape,type = ', imph3.shape, imph3.dtype) - print(' mefr1 shape,type = ', mefr1.shape, mefr1.dtype) - print(' mefr3 shape,type = ', mefr3.shape, mefr3.dtype) - print(' pccf shape,type = ', pccf.shape, pccf.dtype) - print(' ref_pccf shape,type = ', ref_pccf.shape, ref_pccf.dtype) - print(' bearaz shape,type = ', bearaz.shape, bearaz.dtype) - - print(' ogce shape,type = ', ogce.shape, ogce.dtype) - - print(' qfro shape,type = ', qfro.shape, qfro.dtype) - print(' satasc shape,type = ', satasc.shape, satasc.dtype) - - print(' bnda1 shape,type = ', bnda1.shape, bnda1.dtype) - print(' bnda3 shape,type = ', bnda3.shape, bnda3.dtype) - print(' arfr shape,type = ', arfr.shape, arfr.dtype) - - print(' bndaoe1 shape,type = ', bndaoe1.shape, bndaoe1.dtype) - print(' bndaoe3 shape,type = ', bndaoe3.shape, bndaoe3.dtype) - print(' arfroe shape,type = ', arfr.shape, arfr.dtype) - - print(' bndaot shape,type = ', bndaot.shape, bndaot.dtype) + logger.info(f" clath shape, type = {clath.shape}, {clath.dtype}") + logger.info(f" clonh shape, type = {clonh.shape}, {clonh.dtype}") + logger.info(f" gclath shape, type = {gclath.shape}, {gclath.dtype}") + logger.info(f" gclonh shape, type = {gclonh.shape}, {gclonh.dtype}") + logger.info(f" year shape, type = {year.shape}, {year.dtype}") + logger.info(f" mnth shape, type = {mnth.shape}, {mnth.dtype}") + logger.info(f" days shape, type = {days.shape}, {days.dtype}") + logger.info(f" hour shape, type = {hour.shape}, {hour.dtype}") + logger.info(f" minu shape, type = {minu.shape}, {minu.dtype}") + logger.info(f" seco shape, type = {seco.shape}, {seco.dtype}") + logger.info(f" stid shape, type = {stid.shape}, {stid.dtype}") + logger.info(f" said shape, type = {said.shape}, {said.dtype}") + logger.info(f" siid shape, type = {siid.shape}, {siid.dtype}") + logger.info(f" sclf shape, type = {sclf.shape}, {sclf.dtype}") + logger.info(f" ptid shape, type = {ptid.shape}, {ptid.dtype}") + logger.info(f" elrc shape, type = {elrc.shape}, {elrc.dtype}") + logger.info(f" geodu shape, type = {geodu.shape}, {geodu.dtype}") + logger.info(f" heit shape, type = {heit.shape}, {heit.dtype}") + logger.info(f" impp1 shape, type = {impp1.shape}, {impp1.dtype}") + logger.info(f" impp3 shape, type = {impp3.shape}, {impp3.dtype}") +# logger.info(f" imph1 shape, {type = {imph1.shape, imph1.dtype}") +# logger.info(f" imph3 shape, {type = {imph3.shape, imph3.dtype}") + logger.info(f" mefr1 shape, type = {mefr1.shape}, {mefr1.dtype}") + logger.info(f" mefr3 shape, type = {mefr3.shape}, {mefr3.dtype}") + logger.info(f" pccf shape, type = {pccf.shape}, {pccf.dtype}") + logger.info(f" ref_pccf shape, type = {ref_pccf.shape}, \ + {ref_pccf.dtype}") + logger.info(f" bearaz shape, type = {bearaz.shape}, {bearaz.dtype}") + + logger.info(f" ogce shape, type = {ogce.shape}, {ogce.dtype}") + + logger.info(f" qfro shape, type = {qfro.shape}, {qfro.dtype}") + logger.info(f" satasc shape, type = {satasc.shape}, {satasc.dtype}") + + logger.info(f" bnda1 shape, type = {bnda1.shape}, {bnda1.dtype}") + logger.info(f" bnda3 shape, type = {bnda3.shape}, {bnda3.dtype}") + logger.info(f" arfr shape, type = {arfr.shape}, {arfr.dtype}") + + logger.info(f" bndaoe1 shape, type = {bndaoe1.shape}, \ + {bndaoe1.dtype}") + logger.info(f" bndaoe3 shape, type = {bndaoe3.shape}, \ + {bndaoe3.dtype}") + logger.info(f" arfroe shape, type = {arfr.shape}, {arfr.dtype}") + + logger.info(f" bndaot shape, type = {bndaot.shape}, {bndaot.dtype}") end_time = time.time() running_time = end_time - start_time - print('Running time for executing QuerySet to get ResultSet : ', running_time, 'seconds') + logger.info(f"Running time for executing QuerySet to get ResultSet: \ + {running_time} seconds") # ========================= # Create derived variables # ========================= start_time = time.time() - print('Creating derived variables - stationIdentification') - stid = Derive_stationIdentification(said,ptid,stid) - - print(' stid shape,type = ', stid.shape, stid.dtype) - print(' stid[0] = ', stid[0]) + logger.info(f"Creating derived variables - stationIdentification") + stid = Derive_stationIdentification(said, ptid, stid) + + logger.info(f" stid shape,type = {stid.shape}, {stid.dtype}") - print('Creating derived variables - Grid Latitude / Longitude ...') + logger.info(f"Creating derived variables - Grid Latitude / Longitude ...") gclonh = Compute_Grid_Location(gclonh) gclath = Compute_Grid_Location(gclath) - print(' gclonh shape,type = ', gclonh.shape, gclonh.dtype) - print(' gclath shape,type = ', gclath.shape, gclath.dtype) - print(' gclonh min/max = ', gclonh.min(), gclonh.max()) - print(' gclath min/max = ', gclath.min(), gclath.max()) + logger.info(f" gclonh shape,type = {gclonh.shape}, {gclonh.dtype}") + logger.info(f" gclath shape,type = {gclath.shape}, {gclath.dtype}") + logger.info(f" gclonh min/max = {gclonh.min()}, {gclonh.max()}") + logger.info(f" gclath min/max = {gclath.min()}, {gclath.max()}") - print('Creating derived variables - imph ...') + logger.info(f"Creating derived variables - imph ...") imph1 = Compute_imph(impp1, elrc) imph3 = Compute_imph(impp3, elrc) - print(' imph1 shape,type = ', imph1.shape, imph1.dtype) - print(' imph3 shape,type = ', imph3.shape, imph3.dtype) - print(' imph1 min/max = ', imph1.min(), imph1.max()) - print(' imph3 min/max = ', imph3.min(), imph3.max()) + logger.info(f" imph1 shape,type = {imph1.shape}, {imph1.dtype}") + logger.info(f" imph3 shape,type = {imph3.shape}, {imph3.dtype}") + logger.info(f" imph1 min/max = {imph1.min()}, {imph1.max()}") + logger.info(f" imph3 min/max = {imph3.min()}, {imph3.max()}") - print('Editing some derived variables if SAID is not 44 or 825') + logger.info(f"Editing some derived variables if SAID is not 44 or 825") for i in range(len(said)): - if (said[i] != 44) or (said[i] != 825): - bnda1[i] = bnda3[i] - mefr1[i] = mefr3[i] - impp1[i] = impp3[i] - imph1[i] = imph3[i] - bndaoe1[i] = bndaoe3[i] - - print(' new bnda1 shape, type, min, max', bnda1.shape, bnda1.dtype, bnda1.min(), bnda1.max()) - print(' new mefr1 shape, type, min, max', mefr1.shape, mefr1.dtype, mefr1.min(), mefr1.max()) - print(' new impp1 shape, type, min, max', impp1.shape, impp1.dtype, impp1.min(), impp1.max()) - print(' new imph1 shape, type, min, max', imph1.shape, imph1.dtype, imph1.min(), imph1.max()) - print(' new bndaoe1 shape, type, min, max', bndaoe1.shape, bndaoe1.dtype, bndaoe1.min(), bndaoe1.max()) + if (said[i] != 44) or (said[i] != 825): + bnda1[i] = bnda3[i] + mefr1[i] = mefr3[i] + impp1[i] = impp3[i] + imph1[i] = imph3[i] + bndaoe1[i] = bndaoe3[i] + + logger.info(f" new bnda1 shape, type, min/max {bnda1.shape}, \ + {bnda1.dtype}, {bnda1.min()}, {bnda1.max()}") + logger.info(f" new mefr1 shape, type, min/max {mefr1.shape}, \ + {mefr1.dtype}, {mefr1.min()}, {mefr1.max()}") + logger.info(f" new impp1 shape, type, min/max {impp1.shape}, \ + {impp1.dtype}, {impp1.min()}, {impp1.max()}") + logger.info(f" new imph1 shape, type, min/max {imph1.shape}, \ + {imph1.dtype}, {imph1.min()}, {imph1.max()}") + logger.info(f" new bndaoe1 shape, type, min/max {bndaoe1.shape}, \ + {bndaoe1.dtype}, {bndaoe1.min()}, {bndaoe1.max()}") end_time = time.time() running_time = end_time - start_time - print('Running time for creating derived variables : ', running_time, 'seconds') + logger.info(f"Running time for creating derived variables: {running_time} \ + seconds") # ===================================== # Create IODA ObsSpace @@ -332,25 +337,26 @@ def bufr_to_ioda(config): # Find unique satellite identifiers in data to process unique_satids = np.unique(said) - print(' ... Number of Unique satellite identifiers: ', len(unique_satids)) - print(' ... Unique satellite identifiers: ', unique_satids) + logger.info(f" ... Number of Unique satellite identifiers: \ + {len(unique_satids)}") + logger.info(f" ... Unique satellite identifiers: {unique_satids}") + + # Create the dimensions + dims = {'Location': np.arange(0, clath.shape[0])} + + iodafile = f"{cycle_type}.t{hh}z.{data_type}.{data_format}.nc" + OUTPUT_PATH = os.path.join(ioda_dir, iodafile) + logger.info(f" ... ... Create OUTPUT file: {OUTPUT_PATH}") - # Write the data to an IODA file - OUTPUT_PATH = '/work2/noaa/da/nesposito/NewConv/api_convert/json/json_gdasapp/gdas.t00z.gpsro.tm00.nc' path, fname = os.path.split(OUTPUT_PATH) if path and not os.path.exists(path): os.makedirs(path) - - # Create the dimensions - dims = { - 'Location' : np.arange(0, clath.shape[0]), - } - - # Create IODA ObsSpace + + # Create IODA ObsSpace obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) - + # Create Global attributes - print(' ... ... Create global attributes') + logger.info(f" ... ... Create global attributes") obsspace.write_attr('data_format', data_format) obsspace.write_attr('data_type', data_type) obsspace.write_attr('subsets', subsets) @@ -360,200 +366,233 @@ def bufr_to_ioda(config): obsspace.write_attr('description', data_description) obsspace.write_attr('converter', os.path.basename(__file__)) - for i in range(len(elrc)): - if i > 0 and i < 10: - print("NE elrc 0-10", elrc[i]) - + if i > 0 and i < 10: + print("NE elrc 0-10", elrc[i]) + # Create IODA variables - print(' ... ... Create variables: name, type, units, and attributes') + logger.info(f" ... ... Create variables: name, type, units, & attributes") # Longitude - obsspace.create_var('MetaData/longitude', dtype=clonh.dtype, fillval=clonh.fill_value) \ + obsspace.create_var('MetaData/longitude', dtype=clonh.dtype, + fillval=clonh.fill_value) \ .write_attr('units', 'degrees_east') \ .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ .write_attr('long_name', 'Longitude') \ .write_data(clonh) - - # Latitude - obsspace.create_var('MetaData/latitude', dtype=clath.dtype, fillval=clath.fill_value) \ + + # Latitude + obsspace.create_var('MetaData/latitude', dtype=clath.dtype, + fillval=clath.fill_value) \ .write_attr('units', 'degrees_north') \ .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ .write_attr('long_name', 'Latitude') \ .write_data(clath) - + # Grid Longitude - obsspace.create_var('MetaData/gridLongitude', dtype=gclonh.dtype, fillval=gclonh.fill_value) \ + obsspace.create_var('MetaData/gridLongitude', dtype=gclonh.dtype, + fillval=gclonh.fill_value) \ .write_attr('units', 'radians') \ - .write_attr('valid_range', np.array([-3.14159265, 3.14159265], dtype=np.float32)) \ + .write_attr('valid_range', np.array([-3.14159265, 3.14159265], + dtype=np.float32)) \ .write_attr('long_name', 'Grid Longitude') \ .write_data(gclonh) - + # Grid Latitude - obsspace.create_var('MetaData/gridLatitude', dtype=gclath.dtype, fillval=gclath.fill_value) \ + obsspace.create_var('MetaData/gridLatitude', dtype=gclath.dtype, + fillval=gclath.fill_value) \ .write_attr('units', 'radians') \ - .write_attr('valid_range', np.array([-1.570796325, 1.570796325], dtype=np.float32)) \ + .write_attr('valid_range', np.array([-1.570796325, 1.570796325], + dtype=np.float32)) \ .write_attr('long_name', 'Grid Latitude') \ .write_data(gclath) - + # Datetime - obsspace.create_var('MetaData/dateTime', dtype=np.int64, fillval=timestamp.fill_value) \ + obsspace.create_var('MetaData/dateTime', dtype=np.int64, + fillval=timestamp.fill_value) \ .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ .write_attr('long_name', 'Datetime') \ .write_data(timestamp) # Station Identification - obsspace.create_var('MetaData/stationIdentification', dtype=stid.dtype, fillval=stid.fill_value) \ + obsspace.create_var('MetaData/stationIdentification', dtype=stid.dtype, + fillval=stid.fill_value) \ .write_attr('long_name', 'Station Identification') \ .write_data(stid) - - # Satellite Identifier - obsspace.create_var('MetaData/satelliteIdentifier', dtype=said.dtype, fillval=said.fill_value) \ + + # Satellite Identifier + obsspace.create_var('MetaData/satelliteIdentifier', dtype=said.dtype, + fillval=said.fill_value) \ .write_attr('long_name', 'Satellite Identifier') \ .write_data(said) - + # Satellite Instrument - obsspace.create_var('MetaData/satelliteInstrument', dtype=siid.dtype, fillval=siid.fill_value) \ + obsspace.create_var('MetaData/satelliteInstrument', dtype=siid.dtype, + fillval=siid.fill_value) \ .write_attr('long_name', 'Satellite Instrument') \ .write_data(siid) - + # Satellite Constellation RO - obsspace.create_var('MetaData/satelliteConstellationRO', dtype=sclf.dtype, fillval=sclf.fill_value) \ + obsspace.create_var('MetaData/satelliteConstellationRO', dtype=sclf.dtype, + fillval=sclf.fill_value) \ .write_attr('long_name', 'Satellite Constellation RO') \ .write_data(sclf) - + # Satellite Transmitter ID - obsspace.create_var('MetaData/satelliteTransmitterId', dtype=ptid.dtype, fillval=ptid.fill_value) \ + obsspace.create_var('MetaData/satelliteTransmitterId', dtype=ptid.dtype, + fillval=ptid.fill_value) \ .write_attr('long_name', 'Satellite Transmitter Id') \ .write_data(ptid) - + # Earth Radius Curvature -# obsspace.create_var('MetaData/earthRadiusCurvature', dtype=elrc.dtype, fillval=elrc.fill_value) \ - obsspace.create_var('MetaData/earthRadiusCurvature', dtype=elrc.dtype, fillval=elrc.fill_value) \ + obsspace.create_var('MetaData/earthRadiusCurvature', dtype=elrc.dtype, + fillval=elrc.fill_value) \ .write_attr('units', 'm') \ .write_attr('long_name', 'Earth Radius of Curvature') \ .write_data(elrc) - + # Geoid Undulation - obsspace.create_var('MetaData/geoidUndulation', dtype=geodu.dtype, fillval=geodu.fill_value) \ + obsspace.create_var('MetaData/geoidUndulation', dtype=geodu.dtype, + fillval=geodu.fill_value) \ .write_attr('units', 'm') \ .write_attr('long_name', 'Geoid Undulation') \ .write_data(geodu) - + # Height - obsspace.create_var('MetaData/height', dtype=heit.dtype, fillval=heit.fill_value) \ + obsspace.create_var('MetaData/height', dtype=heit.dtype, + fillval=heit.fill_value) \ .write_attr('units', 'm') \ - .write_attr('long_name', 'Height' ) \ - .write_data(heit) - - # Impact Parameter RO - obsspace.create_var('MetaData/impactParameterRO', dtype=impp1.dtype, fillval=impp1.fill_value) \ + .write_attr('long_name', 'Height') \ + .write_data(heit) + + # Impact Parameter RO + obsspace.create_var('MetaData/impactParameterRO', dtype=impp1.dtype, + fillval=impp1.fill_value) \ .write_attr('units', 'm') \ .write_attr('long_name', 'Impact Parameter RO') \ .write_data(impp1) - + # Impact Height RO - obsspace.create_var('MetaData/impactHeightRO', dtype=imph1.dtype, fillval=imph1.fill_value) \ + obsspace.create_var('MetaData/impactHeightRO', dtype=imph1.dtype, + fillval=imph1.fill_value) \ .write_attr('units', 'm') \ .write_attr('long_name', 'Impact Height RO') \ .write_data(imph1) - + # Impact Height RO - obsspace.create_var('MetaData/frequency', dtype=mefr1.dtype, fillval=mefr1.fill_value) \ + obsspace.create_var('MetaData/frequency', dtype=mefr1.dtype, + fillval=mefr1.fill_value) \ .write_attr('units', 'Hz') \ .write_attr('long_name', 'Frequency') \ .write_data(mefr1) - + # PCCF Percent Confidence - obsspace.create_var('MetaData/pccf', dtype=pccf.dtype, fillval=pccf.fill_value) \ + obsspace.create_var('MetaData/pccf', dtype=pccf.dtype, + fillval=pccf.fill_value) \ .write_attr('units', '%') \ .write_attr('long_name', 'Percent Confidence') \ .write_data(pccf) - + # PCCF Ref Percent Confidence - obsspace.create_var('MetaData/percentConfidence', dtype=ref_pccf.dtype, fillval=ref_pccf.fill_value) \ + obsspace.create_var('MetaData/percentConfidence', dtype=ref_pccf.dtype, + fillval=ref_pccf.fill_value) \ .write_attr('units', '%') \ .write_attr('long_name', 'Ref Percent Confidence') \ .write_data(ref_pccf) - + # Azimuth Angle - obsspace.create_var('MetaData/sensorAzimuthAngle', dtype=bearaz.dtype, fillval=bearaz.fill_value) \ + obsspace.create_var('MetaData/sensorAzimuthAngle', dtype=bearaz.dtype, + fillval=bearaz.fill_value) \ .write_attr('units', 'degree') \ .write_attr('long_name', 'Percent Confidence') \ .write_data(bearaz) - - # Data Provider - obsspace.create_var('MetaData/dataProviderOrigin', dtype=ogce.dtype, fillval=ogce.fill_value) \ - .write_attr('long_name', 'Identification of Originating/Generating Center') \ + + # Data Provider + obsspace.create_var('MetaData/dataProviderOrigin', dtype=ogce.dtype, + fillval=ogce.fill_value) \ + .write_attr('long_name', 'Identification of Originating Center') \ .write_data(ogce) - + # Quality: Quality Flags - obsspace.create_var('MetaData/qualityFlags', dtype=qfro.dtype, fillval=qfro.fill_value) \ + obsspace.create_var('MetaData/qualityFlags', dtype=qfro.dtype, + fillval=qfro.fill_value) \ .write_attr('long_name', 'Quality Flags') \ .write_data(qfro) - + # Quality: Satellite Ascending Flag - obsspace.create_var('MetaData/satelliteAscendingFlag', dtype=satasc.dtype, fillval=satasc.fill_value) \ + obsspace.create_var('MetaData/satelliteAscendingFlag', dtype=satasc.dtype, + fillval=satasc.fill_value) \ .write_attr('long_name', 'Satellite Ascending Flag') \ .write_data(satasc) - + # ObsValue: Bending Angle - obsspace.create_var('ObsValue/bendingAngle', dtype=bnda1.dtype, fillval=bnda1.fill_value) \ + obsspace.create_var('ObsValue/bendingAngle', dtype=bnda1.dtype, + fillval=bnda1.fill_value) \ .write_attr('units', 'radians') \ .write_attr('long_name', 'Bending Angle') \ .write_data(bnda1) - - # ObsValue: Atmospheric Refractivity - obsspace.create_var('ObsValue/atmosphericRefractivity', dtype=arfr.dtype, fillval=arfr.fill_value) \ + + # ObsValue: Atmospheric Refractivity + obsspace.create_var('ObsValue/atmosphericRefractivity', dtype=arfr.dtype, + fillval=arfr.fill_value) \ .write_attr('units', 'N-units') \ .write_attr('long_name', 'Atmospheric Refractivity ObsError') \ .write_data(arfr) - + # ObsError: Bending Angle - obsspace.create_var('ObsError/bendingAngle', dtype=bndaoe1.dtype, fillval=bndaoe1.fill_value) \ + obsspace.create_var('ObsError/bendingAngle', dtype=bndaoe1.dtype, + fillval=bndaoe1.fill_value) \ .write_attr('units', 'radians') \ .write_attr('long_name', 'Bending Angle Obs Error') \ .write_data(bndaoe1) - + # ObsError: Atmospheric Refractivity - obsspace.create_var('ObsError/atmosphericRefractivity', dtype=arfroe.dtype, fillval=arfroe.fill_value) \ + obsspace.create_var('ObsError/atmosphericRefractivity', dtype=arfroe.dtype, + fillval=arfroe.fill_value) \ .write_attr('units', 'N-units') \ .write_attr('long_name', 'Atmospheric Refractivity ObsError') \ .write_data(arfroe) - + # ObsType: Bending Angle - obsspace.create_var('ObsType/BendingAngle', dtype=bndaot.dtype, fillval=bndaot.fill_value) \ + obsspace.create_var('ObsType/BendingAngle', dtype=bndaot.dtype, + fillval=bndaot.fill_value) \ .write_attr('long_name', 'Bending Angle ObsType') \ .write_data(bndaot) - + # ObsType: Atmospheric Refractivity - obsspace.create_var('ObsType/atmosphericRefractivity', dtype=arfrot.dtype, fillval=arfrot.fill_value) \ + obsspace.create_var('ObsType/atmosphericRefractivity', dtype=arfrot.dtype, + fillval=arfrot.fill_value) \ .write_attr('long_name', 'Atmospheric Refractivity ObsType') \ - .write_data(arfrot) - + .write_data(arfrot) + end_time = time.time() running_time = end_time - start_time - print('Running time for splitting and output IODA for gpsro bufr ', running_time, 'seconds') + logger.info(f"Running time for splitting and output IODA for gpsro bufr: \ + {running_time} seconds") print("All Done!") + if __name__ == '__main__': start_time = time.time() parser = argparse.ArgumentParser() - parser.add_argument('-c', '--config', type=str, help='Input JSON configuration', required=True) - parser.add_argument('-v', '--verbose', help='print debug logging information', + parser.add_argument('-c', '--config', type=str, + help='Input JSON configuration', required=True) + parser.add_argument('-v', '--verbose', + help='print debug logging information', action='store_true') args = parser.parse_args() + log_level = 'DEBUG' if args.verbose else 'INFO' + logger = Logger('bufr2ioda_acft_profiles_prepbufr.py', level=log_level, + colored_log=True) + with open(args.config, "r") as json_file: config = json.load(json_file) - bufr_to_ioda(config) + bufr_to_ioda(config, logger) end_time = time.time() running_time = end_time - start_time - print('Total running time: ', running_time, 'seconds') - - - + logger.info(f"Total running time: {running_time} seconds") From a0079aceca4cddad30009aabc71338de18a4a10b Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 21 Sep 2023 17:10:27 -0400 Subject: [PATCH 21/29] rm split files to another branch --- .../bufr2ioda/bufr2ioda_gpsro_bufr_split.py | 637 ------------------ 1 file changed, 637 deletions(-) delete mode 100644 ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_split.py diff --git a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_split.py b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_split.py deleted file mode 100644 index 0a68a4703..000000000 --- a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_split.py +++ /dev/null @@ -1,637 +0,0 @@ -# (C) Copyright 2023 NOAA/NWS/NCEP/EMC -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -import sys - -sys.path.append('/work2/noaa/da/nesposito/ioda-bundle_20230712/build/lib/python3.9/') -sys.path.append('/work2/noaa/da/nesposito/ioda-bundle_20230712/build/lib/python3.9/pyiodaconv/') -sys.path.append('/work2/noaa/da/nesposito/ioda-bundle_20230712/build/lib/python3.9/pyioda/') - -import os -import argparse -import json -import numpy as np -import numpy.ma as ma -import math -import calendar -import time -import datetime -from pyiodaconv import bufr -from collections import namedtuple -from pyioda import ioda_obs_space as ioda_ospace - -# ==================================================================== -# GPS-RO BUFR dump file -#===================================================================== -# NC003010 | GPS-RO -# ==================================================================== - -def Derive_stationIdentification(said,ptid,stid): - stid = stid.astype('str') - for i in range(len(said)): - stid[i] = str(said[i]).zfill(4)+str(ptid[i]).zfill(4) - - return stid - - -def Compute_Grid_Location(degrees): - - for i in range(len(degrees)): - if degrees[i] < 180 and degrees[i] > -180 : - degrees[i] = np.deg2rad(degrees[i]) - rad = degrees - - return rad - -def Compute_imph(impp, elrc, geodu): - - imph = impp - elrc - geodu - - return imph - -def bufr_to_ioda(config): - - subsets = config["subsets"] - # ========================================= - # Get parameters from configuration - # ========================================= - data_format = config["data_format"] - data_type = config["data_type"] - data_description = config["data_description"] - data_provider = config["data_provider"] - cycle_type = config["cycle_type"] - dump_dir = config["dump_directory"] - ioda_dir = config["ioda_directory"] - cycle = config["cycle_datetime"] - yyyymmdd = cycle[0:8] - hh = cycle[8:10] - - sensor_satellite_info_array = config["sensor_satellite_info"] - - bufrfile = f"{cycle_type}.t{hh}z.{data_type}.tm00.{data_format}" - DATA_PATH = os.path.join(dump_dir, f"{cycle_type}.{yyyymmdd}", str(hh), 'atmos', bufrfile) - - # ============================================ - # Make the QuerySet for all the data we want - # ============================================ - start_time = time.time() - - print('Making QuerySet ...') - q = bufr.QuerySet(subsets) - - # MetaData - q.add('latitude', '*/ROSEQ1/CLATH' ) - q.add('longitude', '*/ROSEQ1/CLONH' ) - q.add('gridLatitude', '*/ROSEQ1/CLATH' ) - q.add('gridLongitude', '*/ROSEQ1/CLONH' ) - q.add('year', '*/YEAR' ) - q.add('year2', '*/YEAR' ) - q.add('month', '*/MNTH' ) - q.add('day', '*/DAYS' ) - q.add('hour', '*/HOUR' ) - q.add('minute', '*/MINU' ) - q.add('second', '*/SECO' ) - q.add('satelliteIdentifier', '*/SAID' ) - q.add('satelliteInstrument', '*/SIID' ) - q.add('satelliteConstellationRO', '*/SCLF' ) - q.add('satelliteTransmitterId', '*/PTID' ) - q.add('earthRadiusCurvature', '*/ELRC' ) - #q.add('observationSequenceNum', '*/SEQNUM' ) - q.add('geoidUndulation', '*/GEODU' ) - q.add('height', '*/ROSEQ3/HEIT' ) - q.add('impactHeightRO_roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/IMPP' ) - q.add('impactHeightRO_roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/IMPP' ) - q.add('impactParameterRO_roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/IMPP' ) - q.add('impactParameterRO_roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/IMPP' ) - q.add('frequency__roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/MEFR' ) - q.add('frequency__roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/MEFR' ) - q.add('pccf', '*/ROSEQ1/PCCF' ) - q.add('percentConfidence', '*/ROSEQ3/PCCF' ) - q.add('sensorAzimuthAngle', '*/BEARAZ' ) - - # Processing Center - q.add('dataProviderOrigin', '*/OGCE' ) - - # Quality Information - q.add('qualityFlags', '*/QFRO' ) - q.add('satelliteAscendingFlag', '*/QFRO' ) - - # ObsValue - q.add('bendingAngle_roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/BNDA[1]' ) - q.add('bendingAngle_roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/BNDA[1]' ) - q.add('atmosphericRefractivity', '*/ROSEQ3/ARFR[1]' ) - - # ObsError - q.add('obsErrorBendingAngle1', '*/ROSEQ1/ROSEQ2{1}/BNDA[2]') - q.add('obsErrorBendingAngle3', '*/ROSEQ1/ROSEQ2{3}/BNDA[2]') - q.add('obsErrorAtmosphericRefractivity', '*/ROSEQ3/ARFR[2]') - - # ObsType - q.add('obsTypeBendingAngle', '*/SAID' ) - q.add('obsTypeAtmosphericRefractivity', '*/SAID' ) - - end_time = time.time() - running_time = end_time - start_time - print('Running time for making QuerySet : ', running_time, 'seconds') - - # ============================================================== - # Open the BUFR file and execute the QuerySet to get ResultSet - # Use the ResultSet returned to get numpy arrays of the data - # ============================================================== - start_time = time.time() - - print('Executing QuerySet to get ResultSet ...') - with bufr.File(DATA_PATH) as f: - r = f.execute(q) - - print(' ... Executing QuerySet: get metadata: basic ...') - # MetaData - clath = r.get('latitude', 'latitude') - clonh = r.get('longitude', 'latitude') - gclath = r.get('gridLatitude', 'latitude') - gclonh = r.get('gridLongitude', 'latitude') - year = r.get('year', 'latitude') - year2 = r.get('year2') - mnth = r.get('month', 'latitude') - days = r.get('day', 'latitude') - hour = r.get('hour', 'latitude') - minu = r.get('minute', 'latitude') - seco = r.get('second', 'latitude') - stid = r.get('satelliteIdentifier', 'latitude') - said = r.get('satelliteIdentifier', 'latitude') - siid = r.get('satelliteInstrument', 'latitude') - sclf = r.get('satelliteConstellationRO', 'latitude') - ptid = r.get('satelliteTransmitterId', 'latitude') - elrc = r.get('earthRadiusCurvature', 'latitude') - #seqnum = r.get('observationSequenceNum') - geodu = r.get('geoidUndulation', 'latitude') - heit = r.get('height', 'height', type='float') - impp1 = r.get('impactParameterRO_roseq2repl1', 'latitude') - impp3 = r.get('impactParameterRO_roseq2repl3', 'latitude') - imph1 = r.get('impactHeightRO_roseq2repl1', 'latitude') - imph3 = r.get('impactHeightRO_roseq2repl3', 'latitude') - mefr1 = r.get('frequency__roseq2repl1', 'latitude', type='float') - mefr3 = r.get('frequency__roseq2repl3', 'latitude', type='float') - pccf = r.get('pccf', 'latitude', type='float') - ref_pccf = r.get('percentConfidence', 'height' ) - bearaz = r.get('sensorAzimuthAngle', 'latitude') - - print(' ... Executing QuerySet: get metadata: processing center ...') - # Processing Center - ogce = r.get('dataProviderOrigin', 'latitude') - - print(' ... Executing QuerySet: get metadata: data quality information ...') - # Quality Information - qfro = r.get('qualityFlags', 'latitude') - satasc = r.get('satelliteAscendingFlag', 'latitude' ) - - print(' ... Executing QuerySet: get obsvalue: Bending Angle ...') - # ObsValue - # Bending Angle - bnda1 = r.get('bendingAngle_roseq2repl1', 'latitude') - bnda3 = r.get('bendingAngle_roseq2repl3', 'latitude') - arfr = r.get('atmosphericRefractivity', 'height') - - # ObsError - # Bending Angle - bndaoe1 = r.get('obsErrorBendingAngle1', 'latitude') - bndaoe3 = r.get('obsErrorBendingAngle3', 'latitude') - arfroe = r.get('obsErrorAtmosphericRefractivity','height') - - # ObsType - # Bending Angle - bndaot = r.get('obsTypeBendingAngle', 'latitude') - arfrot = r.get('obsTypeBendingAngle', 'latitude') - - print(' ... Executing QuerySet: get datatime: observation time ...') - # DateTime: seconds since Epoch time - # IODA has no support for numpy datetime arrays dtype=datetime64[s] - timestamp = r.get_datetime('year','month','day','hour','minute','second','latitude').astype(np.int64) - -###NICKE SEQNUM would happen around here -#For seqnum -# seqnum=[] -#    qq=1 -#    for i in range(len(elrc)): -#       if elrc[i] != elrc[i+1]: -#          qq = qq+1 -# seqnum.append(qq[i]) -# check length. should be 666000 -##### SOMEHTING LIKE THAT - - - print(' ... Executing QuerySet: Done!') - - print(' ... Executing QuerySet: Check BUFR variable generic dimension and type ...') - # Check BUFR variable generic dimension and type - print(' clath shape,type = ', clath.shape, clath.dtype) - print(' clonh shape,type = ', clonh.shape, clonh.dtype) - print(' gclath shape,type = ', gclath.shape, gclath.dtype) - print(' gclonh shape,type = ', gclonh.shape, gclonh.dtype) - print(' year shape,type = ', year.shape, year.dtype) - print(' mnth shape,type = ', mnth.shape, mnth.dtype) - print(' days shape,type = ', days.shape, days.dtype) - print(' hour shape,type = ', hour.shape, hour.dtype) - print(' minu shape,type = ', minu.shape, minu.dtype) - print(' seco shape,type = ', seco.shape, seco.dtype) - print(' stid shape,type = ', stid.shape, stid.dtype) - print(' said shape,type = ', said.shape, said.dtype) - print(' siid shape,type = ', siid.shape, siid.dtype) - print(' sclf shape,type = ', sclf.shape, sclf.dtype) - print(' ptid shape,type = ', ptid.shape, ptid.dtype) - print(' elrc shape,type = ', elrc.shape, elrc.dtype) - print(' geodu shape,type = ', geodu.shape, geodu.dtype) - print(' heit shape,type = ', heit.shape, heit.dtype) - print(' impp1 shape,type = ', impp1.shape, impp1.dtype) - print(' impp3 shape,type = ', impp3.shape, impp3.dtype) - print(' imph1 shape,type = ', imph1.shape, imph1.dtype) - print(' imph3 shape,type = ', imph3.shape, imph3.dtype) - print(' mefr1 shape,type = ', mefr1.shape, mefr1.dtype) - print(' mefr3 shape,type = ', mefr3.shape, mefr3.dtype) - print(' pccf shape,type = ', pccf.shape, pccf.dtype) - print(' ref_pccf shape,type = ', ref_pccf.shape, ref_pccf.dtype) - print(' bearaz shape,type = ', bearaz.shape, bearaz.dtype) - - print(' ogce shape,type = ', ogce.shape, ogce.dtype) - - print(' qfro shape,type = ', qfro.shape, qfro.dtype) - print(' satasc shape,type = ', satasc.shape, satasc.dtype) - - print(' bnda1 shape,type = ', bnda1.shape, bnda1.dtype) - print(' bnda3 shape,type = ', bnda3.shape, bnda3.dtype) - print(' arfr shape,type = ', arfr.shape, arfr.dtype) - - print(' bndaoe1 shape,type = ', bndaoe1.shape, bndaoe1.dtype) - print(' bndaoe3 shape,type = ', bndaoe3.shape, bndaoe3.dtype) - print(' arfroe shape,type = ', arfr.shape, arfr.dtype) - - print(' bndaot shape,type = ', bndaot.shape, bndaot.dtype) - - end_time = time.time() - running_time = end_time - start_time - print('Running time for executing QuerySet to get ResultSet : ', running_time, 'seconds') - - # ========================= - # Create derived variables - # ========================= - start_time = time.time() - - print('Creating derived variables - stationIdentification') - stid = Derive_stationIdentification(said,ptid,stid) - - print(' stid shape,type = ', stid.shape, stid.dtype) - - print('Creating derived variables - Grid Latitude / Longitude ...') - gclonh = Compute_Grid_Location(gclonh) - gclath = Compute_Grid_Location(gclath) - - print(' gclonh shape,type = ', gclonh.shape, gclonh.dtype) - print(' gclath shape,type = ', gclath.shape, gclath.dtype) - print(' gclonh min/max = ', gclonh.min(), gclonh.max()) - print(' gclath min/max = ', gclath.min(), gclath.max()) - - print('Creating derived variables - imph ...') - imph1 = Compute_imph(impp1, elrc, geodu) - imph3 = Compute_imph(impp3, elrc, geodu) - - print(' imph1 shape,type = ', imph1.shape, imph1.dtype) - print(' imph3 shape,type = ', imph3.shape, imph3.dtype) - print(' imph1 min/max = ', imph1.min(), imph1.max()) - print(' imph3 min/max = ', imph3.min(), imph3.max()) - - end_time = time.time() - running_time = end_time - start_time - print('Running time for creating derived variables : ', running_time, 'seconds') - - # ===================================== - # Create IODA ObsSpace - # Write IODA output - # ===================================== - - print('Split data based on satellite id, Create IODA ObsSpace and Write IODA output') - - # Find unique satellite identifiers in data to process - unique_satids = np.unique(said) - print(' ... Number of Unique satellite identifiers: ', len(unique_satids)) - print(' ... Unique satellite identifiers: ', unique_satids) - - print(' ... Loop through unique satellite identifier ... : ', unique_satids) - for sat in unique_satids.tolist(): - print("Processing output for said: ", sat) - start_time = time.time() - - # Find matched sensor_satellite_info from sensor_satellite_info namedtuple - matched = False - for sensor_satellite_info in sensor_satellite_info_array: - if (sensor_satellite_info["satellite_id"] == sat): - matched = True - sensor_id = sensor_satellite_info["sensor_id"] - sensor_full_name = sensor_satellite_info["sensor_full_name"] - sensor_name = sensor_satellite_info["sensor_name"] - satellite_id = sensor_satellite_info["satellite_id"] - satellite_full_name = sensor_satellite_info["satellite_full_name"] - satellite_name = sensor_satellite_info["satellite_name"] - - - if matched: - - print(' ... Split data for satid = ', sat) - - # Define a boolean mask to subset data from the original data object - mask = said == sat - # MetaData - clonh2 = clonh[mask] - clath2 = clath[mask] - gclonh2 = gclonh[mask] - gclath2 = gclath[mask] - timestamp2 = timestamp[mask] - stid2 = stid[mask] - said2 = said[mask] - siid2 = siid[mask] - sclf2 = sclf[mask] - ptid2 = ptid[mask] - elrc2 = elrc[mask] - geodu2 = geodu[mask] - heit2 = heit[mask] - impp1_2 = impp1[mask] - impp3_2 = impp3[mask] - imph1_2 = imph1[mask] - imph3_2 = imph3[mask] - mefr1_2 = mefr1[mask] - mefr3_2 = mefr3[mask] - pccf2 = pccf[mask] - ref_pccf2 = ref_pccf[mask] - bearaz2 = bearaz[mask] - - # Processing Center - ogce2 = ogce[mask] - - # QC Info - qfro2 = qfro[mask] - satasc2 = satasc[mask] - - # ObsValue - bnda1_2 = bnda1[mask] - bnda3_2 = bnda3[mask] - arfr2 = arfr[mask] - - # ObsError - bndaoe1_2 = bndaoe1[mask] - bndaoe3_2 = bndaoe3[mask] - arfroe2 = arfr[mask] - - # ObsType - bndaot2 = bndaot[mask] - arfrot2 = arfrot[mask] - - # Choose bnda, mefr, impp, imph - if ((sat == 44) or (sat == 825)): - bnda2 = bnda1_2 - mefr2 = mefr1_2 - impp2 = impp1_2 - imph2 = imph1_2 - bndaoe2 = bndaoe1_2 - else: - bnda2 = bnda3_2 - mefr2 = mefr3_2 - impp2 = impp3_2 - imph2 = imph3_2 - bndaoe2 = bndaoe3_2 - - # Check unique observation time - unique_timestamp2 = np.unique(timestamp2) - print(' ... Number of Unique observation timestamp: ', len(unique_timestamp2)) - print(' ... Unique observation timestamp: ', unique_timestamp2) - - print(' ... Create ObsSpae for satid = ', sat) - - # ===================================== - # Create IODA ObsSpace - # Write IODA output - # ===================================== - - # Write the data to an IODA file - OUTPUT_PATH = '/work2/noaa/da/nesposito/NewConv/api_convert/json/json_gdasapp/gdas.t00z.gpsro_'+str(sat)+'.tm00.nc' - path, fname = os.path.split(OUTPUT_PATH) - if path and not os.path.exists(path): - os.makedirs(path) - - # Create the dimensions - dims = { - 'Location' : np.arange(0, clath2.shape[0]), - } - - # Create IODA ObsSpace - obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) - - # Create Global attributes - print(' ... ... Create global attributes') - obsspace.write_attr('data_format', data_format) - obsspace.write_attr('data_type', data_type) - obsspace.write_attr('subsets', subsets) - obsspace.write_attr('cycle_type', cycle_type) - obsspace.write_attr('cycle_datetime', cycle) - obsspace.write_attr('dataProviderOrigin', data_provider) - obsspace.write_attr('converter', os.path.basename(__file__)) - - obsspace.write_attr('description', data_description) - obsspace.write_attr('sensor', sensor_id) - obsspace.write_attr('satellite', satellite_id) - obsspace.write_attr('sensorName', sensor_name) - obsspace.write_attr('satelliteName', satellite_name) - obsspace.write_attr('sensorFullName', sensor_full_name) - obsspace.write_attr('satelliteFullName', satellite_full_name) - - # Create IODA variables - print(' ... ... Create variables: name, type, units, and attributes') - # Longitude - obsspace.create_var('MetaData/longitude', dtype=clonh2.dtype, fillval=clonh2.fill_value) \ - .write_attr('units', 'degrees_east') \ - .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ - .write_attr('long_name', 'Longitude') \ - .write_data(clonh2) - - # Latitude - obsspace.create_var('MetaData/latitude', dtype=clath.dtype, fillval=clath2.fill_value) \ - .write_attr('units', 'degrees_north') \ - .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ - .write_attr('long_name', 'Latitude') \ - .write_data(clath2) - - # Grid Longitude - obsspace.create_var('MetaData/gridLongitude', dtype=gclonh2.dtype, fillval=gclonh2.fill_value) \ - .write_attr('units', 'radians') \ - .write_attr('valid_range', np.array([-3.14159265, 3.14159265], dtype=np.float32)) \ - .write_attr('long_name', 'Grid Longitude') \ - .write_data(gclonh2) - - # Grid Latitude - obsspace.create_var('MetaData/gridLatitude', dtype=gclath.dtype, fillval=gclath2.fill_value) \ - .write_attr('units', 'radians') \ - .write_attr('valid_range', np.array([-1.570796325, 1.570796325], dtype=np.float32)) \ - .write_attr('long_name', 'Grid Latitude') \ - .write_data(gclath2) - - # Datetime - obsspace.create_var('MetaData/dateTime', dtype=np.int64, fillval=timestamp2.fill_value) \ - .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ - .write_attr('long_name', 'Datetime') \ - .write_data(timestamp2) - - # Satellite Identifier - obsspace.create_var('MetaData/stationIdentification', dtype=stid2.dtype, fillval=stid2.fill_value) \ - .write_attr('long_name', 'Station Identification') \ - .write_data(stid2) - - # Satellite Identifier - obsspace.create_var('MetaData/satelliteIdentifier', dtype=said2.dtype, fillval=said2.fill_value) \ - .write_attr('long_name', 'Satellite Identifier') \ - .write_data(said2) - - # Satellite Instrument - obsspace.create_var('MetaData/satelliteInstrument', dtype=siid2.dtype, fillval=siid2.fill_value) \ - .write_attr('long_name', 'Satellite Instrument') \ - .write_data(siid2) - - # Satellite Constellation RO - obsspace.create_var('MetaData/satelliteConstellationRO', dtype=sclf2.dtype, fillval=sclf2.fill_value) \ - .write_attr('long_name', 'Satellite Constellation RO') \ - .write_data(sclf2) - - # Satellite Transmitter ID - obsspace.create_var('MetaData/satelliteTransmitterId', dtype=ptid2.dtype, fillval=ptid2.fill_value) \ - .write_attr('long_name', 'Satellite Transmitter Id') \ - .write_data(ptid2) - - # Earth Radius Curvature - obsspace.create_var('MetaData/earthRadiusCurvature', dtype=elrc2.dtype, fillval=elrc2.fill_value) \ - .write_attr('units', 'm') \ - .write_attr('long_name', 'Earth Radius of Curvature') \ - .write_data(elrc2) - - # Geoid Undulation - obsspace.create_var('MetaData/geoidUndulation', dtype=geodu2.dtype, fillval=geodu2.fill_value) \ - .write_attr('units', 'm') \ - .write_attr('long_name', 'Geoid Undulation') \ - .write_data(geodu2) - - # Height - obsspace.create_var('MetaData/height', dtype=heit2.dtype, fillval=heit2.fill_value) \ - .write_attr('units', 'm') \ - .write_attr('long_name', 'Height' ) \ - .write_data(heit2) - - # Impact Parameter RO - obsspace.create_var('MetaData/impactParameterRO', dtype=impp2.dtype, fillval=impp2.fill_value) \ - .write_attr('units', 'm') \ - .write_attr('long_name', 'Impact Parameter RO') \ - .write_data(impp2) - - # Impact Height RO - obsspace.create_var('MetaData/impactHeightRO', dtype=imph2.dtype, fillval=imph2.fill_value) \ - .write_attr('units', 'm') \ - .write_attr('long_name', 'Impact Height RO') \ - .write_data(imph2) - - # Impact Height RO - obsspace.create_var('MetaData/frequency', dtype=mefr2.dtype, fillval=mefr2.fill_value) \ - .write_attr('units', 'Hz') \ - .write_attr('long_name', 'Frequency') \ - .write_data(mefr2) - - # PCCF Percent Confidence - obsspace.create_var('MetaData/pccf', dtype=pccf2.dtype, fillval=pccf2.fill_value) \ - .write_attr('units', '%') \ - .write_attr('long_name', 'Percent Confidence') \ - .write_data(pccf2) - - # PCCF Ref Percent Confidence - obsspace.create_var('MetaData/percentConfidence', dtype=ref_pccf2.dtype, fillval=ref_pccf2.fill_value) \ - .write_attr('units', '%') \ - .write_attr('long_name', 'Ref Percent Confidence') \ - .write_data(ref_pccf2) - - # Azimuth Angle - obsspace.create_var('MetaData/sensorAzimuthAngle', dtype=bearaz2.dtype, fillval=bearaz2.fill_value) \ - .write_attr('units', 'degree') \ - .write_attr('long_name', 'Percent Confidence') \ - .write_data(bearaz2) - - # Data Provider - obsspace.create_var('MetaData/dataProviderOrigin', dtype=ogce2.dtype, fillval=ogce2.fill_value) \ - .write_attr('long_name', 'Identification of Originating/Generating Center') \ - .write_data(ogce2) - - # Quality: Quality Flags - obsspace.create_var('MetaData/qualityFlags', dtype=qfro2.dtype, fillval=qfro2.fill_value) \ - .write_attr('long_name', 'Quality Flags') \ - .write_data(qfro2) - - # Quality: Satellite Ascending Flag - obsspace.create_var('MetaData/satelliteAscendingFlag', dtype=satasc2.dtype, fillval=satasc2.fill_value) \ - .write_attr('long_name', 'Satellite Ascending Flag') \ - .write_data(satasc2) - - # ObsValue: Bending Angle - obsspace.create_var('ObsValue/bendingAngle', dtype=bnda2.dtype, fillval=bnda2.fill_value) \ - .write_attr('units', 'radians') \ - .write_attr('long_name', 'Bending Angle') \ - .write_data(bnda2) - - # ObsValue: Atmospheric Refractivity - obsspace.create_var('ObsValue/atmosphericRefractivity', dtype=arfr2.dtype, fillval=arfr2.fill_value) \ - .write_attr('units', 'N-units') \ - .write_attr('long_name', 'Atmospheric Refractivity ObsError') \ - .write_data(arfr2) - - # ObsError: Bending Angle - obsspace.create_var('ObsError/bendingAngle', dtype=bndaoe2.dtype, fillval=bndaoe2.fill_value) \ - .write_attr('units', 'radians') \ - .write_attr('long_name', 'Bending Angle Obs Error') \ - .write_data(bndaoe2) - - # ObsError: Atmospheric Refractivity - obsspace.create_var('ObsError/atmosphericRefractivity', dtype=arfroe2.dtype, fillval=arfroe2.fill_value) \ - .write_attr('units', 'N-units') \ - .write_attr('long_name', 'Atmospheric Refractivity ObsError') \ - .write_data(arfroe2) - - # ObsType - obsspace.create_var('ObsType/BendingAngle', dtype=bndaot2.dtype, fillval=bndaot2.fill_value) \ - .write_attr('long_name', 'Bending Angle ObsType') \ - .write_data(bndaot2) - - # ObsType: Atmospheric Refractivity - obsspace.create_var('ObsType/atmosphericRefractivity', dtype=arfrot2.dtype, fillval=arfrot2.fill_value) \ - .write_attr('long_name', 'Atmospheric Refractivity ObsType') \ - .write_data(arfrot2) - - end_time = time.time() - running_time = end_time - start_time - print('Running time for splitting and output IODA for', str(sat), running_time, 'seconds') - - print("All Done!") - -if __name__ == '__main__': - - start_time = time.time() - - parser = argparse.ArgumentParser() - parser.add_argument('-c', '--config', type=str, help='Input JSON configuration', required=True) - parser.add_argument('-v', '--verbose', help='print debug logging information', - action='store_true') - args = parser.parse_args() - - with open(args.config, "r") as json_file: - config = json.load(json_file) - - bufr_to_ioda(config) - - end_time = time.time() - running_time = end_time - start_time - print('Total running time: ', running_time, 'seconds') - - - From bd0973ea64229d9b0c6068f8842bc7fd517c6b38 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Tue, 26 Sep 2023 14:13:14 -0400 Subject: [PATCH 22/29] remove some changes once elrc ncdump issues were understood --- ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py index ebb24efb0..58af6b71e 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py @@ -48,11 +48,7 @@ def Compute_Grid_Location(degrees): def Compute_imph(impp, elrc): # , geodu, imph): - imph = (impp - elrc).astype(np.float32) -# for i in range(len(impp)): -# # imph = imph.astype('float') -# imph[i]=impp[i]-elrc[i]#-geodu[i] -# # print("NE i impp elrc imph", i, impp[i], elrc[i], imph[i]) + imph = impp - elrc).astype(np.float32) return imph @@ -366,10 +362,6 @@ def bufr_to_ioda(config, logger): obsspace.write_attr('description', data_description) obsspace.write_attr('converter', os.path.basename(__file__)) - for i in range(len(elrc)): - if i > 0 and i < 10: - print("NE elrc 0-10", elrc[i]) - # Create IODA variables logger.info(f" ... ... Create variables: name, type, units, & attributes") # Longitude @@ -569,7 +561,7 @@ def bufr_to_ioda(config, logger): logger.info(f"Running time for splitting and output IODA for gpsro bufr: \ {running_time} seconds") - print("All Done!") + logger.info("All Done!") if __name__ == '__main__': From b30bd612e1cd18ae29d38d61bfebcbd6f4c7cba0 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Tue, 26 Sep 2023 14:16:27 -0400 Subject: [PATCH 23/29] remove ) --- ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py index 58af6b71e..bdb3242bb 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py @@ -48,7 +48,7 @@ def Compute_Grid_Location(degrees): def Compute_imph(impp, elrc): # , geodu, imph): - imph = impp - elrc).astype(np.float32) + imph = (impp - elrc).astype(np.float32) return imph From 12f90628af691ebde9f59ee3e47cf306cda5ef05 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Tue, 7 Nov 2023 16:02:00 -0500 Subject: [PATCH 24/29] newer updates --- ush/ioda/bufr2ioda/README_whoever | 12 --- .../bufr2ioda_gpsro_bufr_combined.py | 4 +- ush/ioda/bufr2ioda/run_bufr2ioda.py | 3 +- .../bufr2ioda/run_bufr2ioda_gpsro_bufr.sh | 82 ------------------- 4 files changed, 3 insertions(+), 98 deletions(-) delete mode 100644 ush/ioda/bufr2ioda/README_whoever mode change 100644 => 100755 ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py delete mode 100755 ush/ioda/bufr2ioda/run_bufr2ioda_gpsro_bufr.sh diff --git a/ush/ioda/bufr2ioda/README_whoever b/ush/ioda/bufr2ioda/README_whoever deleted file mode 100644 index 1070c3a47..000000000 --- a/ush/ioda/bufr2ioda/README_whoever +++ /dev/null @@ -1,12 +0,0 @@ - -Load the modules for the gdasapp first - -To run: use command: - -#This works -./run_bufr2ioda_gpsro_bufr.sh 2021080100 gdas /work/noaa/rstprod/dump/ /work2/noaa/da/nesposito/GDASApp_20230907/parm/ioda/bufr2ioda/ /work2/noaa/da/nesposito/NewConv/api_convert/json/json_gdasapp - - -Outputs (and plots) are in: -/work2/noaa/da/nesposito/NewConv/api_convert/json/json_gdasapp - diff --git a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py old mode 100644 new mode 100755 index bdb3242bb..d2158f300 --- a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py @@ -39,8 +39,6 @@ def Compute_Grid_Location(degrees): for i in range(len(degrees)): if degrees[i] <= 360 and degrees[i] >= -180: degrees[i] = np.deg2rad(degrees[i]) -# else: -# degrees[i] = np.float32(0) rad = degrees return rad @@ -162,7 +160,7 @@ def bufr_to_ioda(config, logger): hour = r.get('hour', 'latitude') minu = r.get('minute', 'latitude') seco = r.get('second', 'latitude') - stid = r.get('satelliteIdentifier', 'latitude') + stid = r.get('stationIdentification', 'latitude') said = r.get('satelliteIdentifier', 'latitude') siid = r.get('satelliteInstrument', 'latitude') sclf = r.get('satelliteConstellationRO', 'latitude') diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index da01dd131..3c1adc7bf 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -31,7 +31,8 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): # Specify observation types to be processed by a script - BUFR_py = ["satwind_amv_goes", "satwind_scat", "adpupa_prepbufr", "adpsfc_prepbufr", "sfcshp_prepbufr"] +# BUFR_py = ["satwind_amv_goes", "satwind_scat", "adpupa_prepbufr", "adpsfc_prepbufr", "sfcshp_prepbufr", "acft_profiles_prepbufr", "gpsro_bufr"] + BUFR_py = ["gpsro_bufr_combined"] for obtype in BUFR_py: logger.info(f"Convert {obtype}...") diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda_gpsro_bufr.sh b/ush/ioda/bufr2ioda/run_bufr2ioda_gpsro_bufr.sh deleted file mode 100755 index 2184cf84c..000000000 --- a/ush/ioda/bufr2ioda/run_bufr2ioda_gpsro_bufr.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash -#-------------------------------------------------------------------------------------- -# run_bufr2ioda.sh -# This driver script will: -# - generate config files from templates for each BUFR file -# - use bufr2ioda python scripts or executable and produce output IODA files -# usage: -# run_bufr2ioda.sh YYYYMMDDHH $DMPDIR /path/to/templates $COM_OBS -# -#-------------------------------------------------------------------------------------- -if [[ $# -ne 5 ]] ; then - echo "usage:" - echo " $0 YYYYMMDDHH gdas|gfs /path/to/files.bufr_d/ /path/to/templates /path/to/output.ioda/" - exit 1 -fi - -# some of these need exported to be picked up by the python script below -# input parameters -CDATE=${CDATE:-$1} -export RUN=${RUN:-$2} -export DMPDIR=${DMPDIR:-$3} -config_template_dir=${config_template_dir:-$4} -export COM_OBS=${COM_OBS:-$5} - -# derived parameters -export PDY=$(echo $CDATE | cut -c1-8) -export cyc=$(echo $CDATE | cut -c9-10) - -# get gdasapp root directory -readonly DIR_ROOT=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )/../../.." && pwd -P) -BUFR2IODA=$DIR_ROOT/build/bin/bufr2ioda.x -USH_IODA=$DIR_ROOT/ush/ioda/bufr2ioda -BUFRYAMLGEN=$USH_IODA/gen_bufr2ioda_yaml.py -BUFRJSONGEN=$USH_IODA/gen_bufr2ioda_json.py - -# create output directory if it doesn't exist -mkdir -p $COM_OBS -if [ $? -ne 0 ]; then - echo "cannot make $COM_OBS" - exit 1 -fi - -# add to pythonpath the necessary libraries -PYIODALIB=`echo $DIR_ROOT/build/lib/python3.?` -export PYTHONPATH=${PYIODALIB}:${PYTHONPATH} - -#--------------------------------------------------------- -# for some BUFR files, we will use python scripts -# and for others, bufr2ioda.x - -#--------------------------- -#----- python and json ----- -# first specify what observation types will be processed by a script -#BUFR_py="gpsro_bufr_split" -BUFR_py="gpsro_bufr_combined" - -for obtype in $BUFR_py; do - # this loop assumes that there is a python script and template with the same name - echo "Processing ${obtype}..." - - # first generate a JSON from the template - ${BUFRJSONGEN} -t ${config_template_dir}/bufr2ioda_${obtype}.json -o ${COM_OBS}/${obtype}_${PDY}${cyc}.json - echo "1 done" - - # now use the converter script for the ob type - python3 $USH_IODA/bufr2ioda_${obtype}.py -c ${COM_OBS}/${obtype}_${PDY}${cyc}.json - echo "2 done" - - # check if converter was successful - if [ $? == 0 ]; then - # remove JSON file - rm -rf ${COM_OBS}/${obtype}_${PDY}${cyc}.json - else - # warn and keep the JSON file - echo "Problem running converter script for ${obtype}" - fi -done - -#---------------------------- -#---- bufr2ioda and yaml ---- -# need to ask Emily which will be YAML based -BUFR_yaml="" From 574832c6a8fb3900f882b150e079b0bcdb6a70b1 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Wed, 8 Nov 2023 10:59:16 -0500 Subject: [PATCH 25/29] rm split_jason and fix str stid that broke somehow --- .../bufr2ioda/bufr2ioda_gpsro_bufr_split.json | 28 ------------------- .../bufr2ioda_gpsro_bufr_combined.py | 25 ++++++++--------- 2 files changed, 11 insertions(+), 42 deletions(-) delete mode 100644 parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_split.json diff --git a/parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_split.json b/parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_split.json deleted file mode 100644 index ca73714eb..000000000 --- a/parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_split.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "data_format" : "bufr_d", - "data_type" : "gpsro", - "cycle_type" : "{{ RUN }}", - "cycle_datetime" : "{{ current_cycle | to_YMDH }}", - "dump_directory" : "{{ DMPDIR }}", - "ioda_directory" : "{{ COM_OBS }}", - "subsets" : [ "NC003010" ], - "data_description" : "MSG TYPE 003-010 NESDIS GPS-RO", - "data_provider" : "U.S. NOAA/NESDIS", - "sensor_satellite_info" : [ - { "sensor_name" : "GRAS", "sensor_full_name" : "GNSS Receiver for Atmospheric Sounding", "sensor_id" : 202, "satellite_name" : "MetOp-B", "satellite_full_name" : "Meteorological Operational satellite B", "satellite_id" : 3}, - { "sensor_name" : "GRAS", "sensor_full_name" : "GNSS Receiver for Atmospheric Sounding", "sensor_id" : 202, "satellite_name" : "MetOp-A", "satellite_full_name" : "Meteorological Operational satellite A", "satellite_id" : 4}, - { "sensor_name" : "GRAS", "sensor_full_name" : "GNSS Receiver for Atmospheric Sounding", "sensor_id" : 202, "satellite_name" : "MetOp-C", "satellite_full_name" : "Meteorological Operational satellite C", "satellite_id" : 5}, - { "sensor_name" : "IGOR", "sensor_full_name" : "Integrated GPS and Occultation Receiver", "sensor_id" : 103, "satellite_name" : "TerraSAR-X", "satellite_full_name" : "X-band TerraSAR satellite", "satellite_id" : 42}, - { "sensor_name" : "IGOR", "sensor_full_name" : "Integrated GPS and Occultation Receiver", "sensor_id" : 103, "satellite_name" : "TanDEM-X", "satellite_full_name" : "TerraSAR-X add-on for Digital Elevation Measurement", "satellite_id" : 43}, - { "sensor_name" : "IGOR", "sensor_full_name" : "Integrated GPS and Occultation Receiver", "sensor_id" : 103, "satellite_name" : "PAZ", "satellite_full_name" : "Paz satellite", "satellite_id" : 44}, - { "sensor_name" :"CION", "sensor_full_name" :"CICERO Instrument for GNSS-RO ", "sensor_id" : 526, "satellite_name" : "CICERO-1 OP1", "satellite_full_name" : "Community Initiative for Cellular Earth Remote Observation OP1", "satellite_id" : 265}, - { "sensor_name" : "Tri-G", "sensor_full_name" : "Triple-G", "sensor_id" : 104, "satellite_name" : "COSMIC-2 E1", "satellite_full_name" : "Constellation Observing System for Meteorology, Ionosphere, and Climate-2 E1", "satellite_id" : 750}, - { "sensor_name" : "Tri-G", "sensor_full_name" : "Triple-G", "sensor_id" : 104, "satellite_name" : "COSMIC-2 E2", "satellite_full_name" : "Constellation Observing System for Meteorology, Ionosphere, and Climate-2 E2", "satellite_id" : 751}, - { "sensor_name" : "Tri-G", "sensor_full_name" : "Triple-G", "sensor_id" : 104, "satellite_name" : "COSMIC-2 E3", "satellite_full_name" : "Constellation Observing System for Meteorology, Ionosphere, and Climate-2 E3", "satellite_id" : 752}, - { "sensor_name" : "Tri-G", "sensor_full_name" : "Triple-G", "sensor_id" : 104, "satellite_name" : "COSMIC-2 E4", "satellite_full_name" : "Constellation Observing System for Meteorology, Ionosphere, and Climate-2 E4", "satellite_id" : 753}, - { "sensor_name" : "Tri-G", "sensor_full_name" : "Triple-G", "sensor_id" : 104, "satellite_name" : "COSMIC-2 E5", "satellite_full_name" : "Constellation Observing System for Meteorology, Ionosphere, and Climate-2 E5", "satellite_id" : 754}, - { "sensor_name" : "Tri-G", "sensor_full_name" : "Triple-G", "sensor_id" : 104, "satellite_name" : "COSMIC-2 E6", "satellite_full_name" : "Constellation Observing System for Meteorology, Ionosphere, and Climate-2 E6", "satellite_id" : 755}, - { "sensor_name" : "Tri-G", "sensor_full_name" :"Triple-G", "sensor_id" : 104, "satellite_name" : "GRACE C", "satellite_full_name" : "Gravity Recovery and Climate Experiment Follow-On C", "satellite_id" : 803}, - { "sensor_name" : "IGOR", "sensor_full_name" : "Integrated GPS and Occultation Receiver", "sensor_id" : 103, "satellite_name" : "KOMPSAT-9", "satellite_full_name" : "Korean Multi-Purpose Satellite", "satellite_id" : 825} - ] -} diff --git a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py index d2158f300..57bd72942 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # (C) Copyright 2023 NOAA/NWS/NCEP/EMC # # This software is licensed under the terms of the Apache Licence Version 2.0 @@ -25,12 +26,16 @@ # ==================================================================== -def Derive_stationIdentification(said, ptid, stid): +def Derive_stationIdentification(said, ptid): - stid = stid.astype('str') + stid=[] for i in range(len(said)): - stid[i] = str(said[i]).zfill(4)+str(ptid[i]).zfill(4) - + newval = str(said[i]).zfill(4)+str(ptid[i]).zfill(4) + stid.append(str(newval)) + stid = np.array(stid).astype(dtype='str') + stid = ma.array(stid) + ma.set_fill_value(stid, "") + return stid @@ -44,7 +49,7 @@ def Compute_Grid_Location(degrees): return rad -def Compute_imph(impp, elrc): # , geodu, imph): +def Compute_imph(impp, elrc): imph = (impp - elrc).astype(np.float32) @@ -102,8 +107,6 @@ def bufr_to_ioda(config, logger): # q.add('observationSequenceNum', '*/SEQNUM') q.add('geoidUndulation', '*/GEODU') q.add('height', '*/ROSEQ3/HEIT') -# q.add('impactHeightRO_roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/IMPP') -# q.add('impactHeightRO_roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/IMPP') q.add('impactParameterRO_roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/IMPP') q.add('impactParameterRO_roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/IMPP') q.add('frequency__roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/MEFR') @@ -160,7 +163,6 @@ def bufr_to_ioda(config, logger): hour = r.get('hour', 'latitude') minu = r.get('minute', 'latitude') seco = r.get('second', 'latitude') - stid = r.get('stationIdentification', 'latitude') said = r.get('satelliteIdentifier', 'latitude') siid = r.get('satelliteInstrument', 'latitude') sclf = r.get('satelliteConstellationRO', 'latitude') @@ -171,8 +173,6 @@ def bufr_to_ioda(config, logger): heit = r.get('height', 'height', type='float32').astype(np.float32) impp1 = r.get('impactParameterRO_roseq2repl1', 'latitude') impp3 = r.get('impactParameterRO_roseq2repl3', 'latitude') -# imph1 = r.get('impactHeightRO_roseq2repl1', 'latitude') -# imph3 = r.get('impactHeightRO_roseq2repl3', 'latitude') mefr1 = r.get('frequency__roseq2repl1', 'latitude', type='float32').astype(np.float32) mefr3 = r.get('frequency__roseq2repl3', 'latitude', @@ -230,7 +230,6 @@ def bufr_to_ioda(config, logger): logger.info(f" hour shape, type = {hour.shape}, {hour.dtype}") logger.info(f" minu shape, type = {minu.shape}, {minu.dtype}") logger.info(f" seco shape, type = {seco.shape}, {seco.dtype}") - logger.info(f" stid shape, type = {stid.shape}, {stid.dtype}") logger.info(f" said shape, type = {said.shape}, {said.dtype}") logger.info(f" siid shape, type = {siid.shape}, {siid.dtype}") logger.info(f" sclf shape, type = {sclf.shape}, {sclf.dtype}") @@ -240,8 +239,6 @@ def bufr_to_ioda(config, logger): logger.info(f" heit shape, type = {heit.shape}, {heit.dtype}") logger.info(f" impp1 shape, type = {impp1.shape}, {impp1.dtype}") logger.info(f" impp3 shape, type = {impp3.shape}, {impp3.dtype}") -# logger.info(f" imph1 shape, {type = {imph1.shape, imph1.dtype}") -# logger.info(f" imph3 shape, {type = {imph3.shape, imph3.dtype}") logger.info(f" mefr1 shape, type = {mefr1.shape}, {mefr1.dtype}") logger.info(f" mefr3 shape, type = {mefr3.shape}, {mefr3.dtype}") logger.info(f" pccf shape, type = {pccf.shape}, {pccf.dtype}") @@ -277,7 +274,7 @@ def bufr_to_ioda(config, logger): start_time = time.time() logger.info(f"Creating derived variables - stationIdentification") - stid = Derive_stationIdentification(said, ptid, stid) + stid = Derive_stationIdentification(said, ptid) logger.info(f" stid shape,type = {stid.shape}, {stid.dtype}") From 6d6593c4ffbc9250f0f14ab0b044ff91d4af97b0 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Wed, 8 Nov 2023 11:02:28 -0500 Subject: [PATCH 26/29] coding norms --- ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py index 57bd72942..718576ebb 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py @@ -28,14 +28,14 @@ def Derive_stationIdentification(said, ptid): - stid=[] + stid = [] for i in range(len(said)): newval = str(said[i]).zfill(4)+str(ptid[i]).zfill(4) stid.append(str(newval)) stid = np.array(stid).astype(dtype='str') stid = ma.array(stid) ma.set_fill_value(stid, "") - + return stid From 4ae6fc47f0dc19e5cecbc456e9c17b474c6785be Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Wed, 8 Nov 2023 11:17:51 -0500 Subject: [PATCH 27/29] run_bufr2ioda --- ush/ioda/bufr2ioda/run_bufr2ioda.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index 3c1adc7bf..669df04cf 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -31,8 +31,7 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): # Specify observation types to be processed by a script -# BUFR_py = ["satwind_amv_goes", "satwind_scat", "adpupa_prepbufr", "adpsfc_prepbufr", "sfcshp_prepbufr", "acft_profiles_prepbufr", "gpsro_bufr"] - BUFR_py = ["gpsro_bufr_combined"] + BUFR_py = ["satwind_amv_goes", "satwind_scat", "adpupa_prepbufr", "adpsfc_prepbufr", "sfcshp_prepbufr", "acft_profiles_prepbufr", "gpsro_bufr"] for obtype in BUFR_py: logger.info(f"Convert {obtype}...") From 0fe26b38bc33a4e0364e24d847559ef5d8416cec Mon Sep 17 00:00:00 2001 From: NicholasEsposito-NOAA <62616739+nicholasesposito@users.noreply.github.com> Date: Wed, 8 Nov 2023 11:19:01 -0500 Subject: [PATCH 28/29] Update bufr2ioda_gpsro_bufr_combined.json --- parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.json b/parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.json index 0d5dcea88..067f92c28 100644 --- a/parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.json +++ b/parm/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.json @@ -7,5 +7,5 @@ "ioda_directory" : "{{ COM_OBS }}", "subsets" : [ "NC003010" ], "data_description" : "MSG TYPE 003-010 NESDIS GPS-RO: MetOp-A GRAS, MetOp-B GRAS, MetOp-C GRAS, TerraSAR-X IGOR, TanDEM-X IGOR, PAZ IGOR, CICERO-1 OP1 CION, COSMIC-2 E1-E6 Tri-G", - "data_provider" : "U.S. NOAA/NESDIS", + "data_provider" : "U.S. NOAA", } From 99c5f490ef464510a6d2f27fe6a056cbf69f811f Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Wed, 8 Nov 2023 11:57:52 -0500 Subject: [PATCH 29/29] info -> debug --- .../bufr2ioda_gpsro_bufr_combined.py | 144 +++++++++--------- 1 file changed, 72 insertions(+), 72 deletions(-) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py index 718576ebb..f6db55893 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_gpsro_bufr_combined.py @@ -84,7 +84,7 @@ def bufr_to_ioda(config, logger): # ============================================ start_time = time.time() - logger.info(f"Making QuerySet ...") + logger.debug(f"Making QuerySet ...") q = bufr.QuerySet(subsets) # MetaData @@ -138,7 +138,7 @@ def bufr_to_ioda(config, logger): end_time = time.time() running_time = end_time - start_time - logger.info(f"Running time for making QuerySet: {running_time} seconds") + logger.debug(f"Running time for making QuerySet: {running_time} seconds") # ============================================================== # Open the BUFR file and execute the QuerySet to get ResultSet @@ -146,11 +146,11 @@ def bufr_to_ioda(config, logger): # ============================================================== start_time = time.time() - logger.info(f"Executing QuerySet to get ResultSet ...") + logger.debug(f"Executing QuerySet to get ResultSet ...") with bufr.File(DATA_PATH) as f: r = f.execute(q) - logger.info(f" ... Executing QuerySet: get MetaData: basic ...") + logger.debug(f" ... Executing QuerySet: get MetaData: basic ...") # MetaData clath = r.get('latitude', 'latitude') clonh = r.get('longitude', 'latitude') @@ -181,17 +181,17 @@ def bufr_to_ioda(config, logger): ref_pccf = r.get('percentConfidence', 'height') bearaz = r.get('sensorAzimuthAngle', 'latitude') - logger.info(f" ... Executing QuerySet: get MetaData: processing center...") + logger.debug(f" ... Executing QuerySet: get MetaData: processing center...") # Processing Center ogce = r.get('dataProviderOrigin', 'latitude') - logger.info(f" ... Executing QuerySet: get metadata: data quality \ + logger.debug(f" ... Executing QuerySet: get metadata: data quality \ information ...") # Quality Information qfro = r.get('qualityFlags', 'latitude') satasc = r.get('satelliteAscendingFlag', 'latitude') - logger.info(f" ... Executing QuerySet: get ObsValue: Bending Angle ...") + logger.debug(f" ... Executing QuerySet: get ObsValue: Bending Angle ...") # ObsValue # Bending Angle bnda1 = r.get('bendingAngle_roseq2repl1', 'latitude') @@ -209,63 +209,63 @@ def bufr_to_ioda(config, logger): bndaot = r.get('obsTypeBendingAngle', 'latitude') arfrot = r.get('obsTypeBendingAngle', 'latitude') - logger.info(f" ... Executing QuerySet: get datatime: observation time ...") + logger.debug(f" ... Executing QuerySet: get datatime: observation time ...") # DateTime: seconds since Epoch time # IODA has no support for numpy datetime arrays dtype=datetime64[s] timestamp = r.get_datetime('year', 'month', 'day', 'hour', 'minute', 'second', 'latitude').astype(np.int64) - logger.info(f" ... Executing QuerySet: Done!") + logger.debug(f" ... Executing QuerySet: Done!") - logger.info(f" ... Executing QuerySet: Check BUFR variable generic \ + logger.debug(f" ... Executing QuerySet: Check BUFR variable generic \ dimension and type ...") # Check BUFR variable generic dimension and type - logger.info(f" clath shape, type = {clath.shape}, {clath.dtype}") - logger.info(f" clonh shape, type = {clonh.shape}, {clonh.dtype}") - logger.info(f" gclath shape, type = {gclath.shape}, {gclath.dtype}") - logger.info(f" gclonh shape, type = {gclonh.shape}, {gclonh.dtype}") - logger.info(f" year shape, type = {year.shape}, {year.dtype}") - logger.info(f" mnth shape, type = {mnth.shape}, {mnth.dtype}") - logger.info(f" days shape, type = {days.shape}, {days.dtype}") - logger.info(f" hour shape, type = {hour.shape}, {hour.dtype}") - logger.info(f" minu shape, type = {minu.shape}, {minu.dtype}") - logger.info(f" seco shape, type = {seco.shape}, {seco.dtype}") - logger.info(f" said shape, type = {said.shape}, {said.dtype}") - logger.info(f" siid shape, type = {siid.shape}, {siid.dtype}") - logger.info(f" sclf shape, type = {sclf.shape}, {sclf.dtype}") - logger.info(f" ptid shape, type = {ptid.shape}, {ptid.dtype}") - logger.info(f" elrc shape, type = {elrc.shape}, {elrc.dtype}") - logger.info(f" geodu shape, type = {geodu.shape}, {geodu.dtype}") - logger.info(f" heit shape, type = {heit.shape}, {heit.dtype}") - logger.info(f" impp1 shape, type = {impp1.shape}, {impp1.dtype}") - logger.info(f" impp3 shape, type = {impp3.shape}, {impp3.dtype}") - logger.info(f" mefr1 shape, type = {mefr1.shape}, {mefr1.dtype}") - logger.info(f" mefr3 shape, type = {mefr3.shape}, {mefr3.dtype}") - logger.info(f" pccf shape, type = {pccf.shape}, {pccf.dtype}") - logger.info(f" ref_pccf shape, type = {ref_pccf.shape}, \ + logger.debug(f" clath shape, type = {clath.shape}, {clath.dtype}") + logger.debug(f" clonh shape, type = {clonh.shape}, {clonh.dtype}") + logger.debug(f" gclath shape, type = {gclath.shape}, {gclath.dtype}") + logger.debug(f" gclonh shape, type = {gclonh.shape}, {gclonh.dtype}") + logger.debug(f" year shape, type = {year.shape}, {year.dtype}") + logger.debug(f" mnth shape, type = {mnth.shape}, {mnth.dtype}") + logger.debug(f" days shape, type = {days.shape}, {days.dtype}") + logger.debug(f" hour shape, type = {hour.shape}, {hour.dtype}") + logger.debug(f" minu shape, type = {minu.shape}, {minu.dtype}") + logger.debug(f" seco shape, type = {seco.shape}, {seco.dtype}") + logger.debug(f" said shape, type = {said.shape}, {said.dtype}") + logger.debug(f" siid shape, type = {siid.shape}, {siid.dtype}") + logger.debug(f" sclf shape, type = {sclf.shape}, {sclf.dtype}") + logger.debug(f" ptid shape, type = {ptid.shape}, {ptid.dtype}") + logger.debug(f" elrc shape, type = {elrc.shape}, {elrc.dtype}") + logger.debug(f" geodu shape, type = {geodu.shape}, {geodu.dtype}") + logger.debug(f" heit shape, type = {heit.shape}, {heit.dtype}") + logger.debug(f" impp1 shape, type = {impp1.shape}, {impp1.dtype}") + logger.debug(f" impp3 shape, type = {impp3.shape}, {impp3.dtype}") + logger.debug(f" mefr1 shape, type = {mefr1.shape}, {mefr1.dtype}") + logger.debug(f" mefr3 shape, type = {mefr3.shape}, {mefr3.dtype}") + logger.debug(f" pccf shape, type = {pccf.shape}, {pccf.dtype}") + logger.debug(f" ref_pccf shape, type = {ref_pccf.shape}, \ {ref_pccf.dtype}") - logger.info(f" bearaz shape, type = {bearaz.shape}, {bearaz.dtype}") + logger.debug(f" bearaz shape, type = {bearaz.shape}, {bearaz.dtype}") - logger.info(f" ogce shape, type = {ogce.shape}, {ogce.dtype}") + logger.debug(f" ogce shape, type = {ogce.shape}, {ogce.dtype}") - logger.info(f" qfro shape, type = {qfro.shape}, {qfro.dtype}") - logger.info(f" satasc shape, type = {satasc.shape}, {satasc.dtype}") + logger.debug(f" qfro shape, type = {qfro.shape}, {qfro.dtype}") + logger.debug(f" satasc shape, type = {satasc.shape}, {satasc.dtype}") - logger.info(f" bnda1 shape, type = {bnda1.shape}, {bnda1.dtype}") - logger.info(f" bnda3 shape, type = {bnda3.shape}, {bnda3.dtype}") - logger.info(f" arfr shape, type = {arfr.shape}, {arfr.dtype}") + logger.debug(f" bnda1 shape, type = {bnda1.shape}, {bnda1.dtype}") + logger.debug(f" bnda3 shape, type = {bnda3.shape}, {bnda3.dtype}") + logger.debug(f" arfr shape, type = {arfr.shape}, {arfr.dtype}") - logger.info(f" bndaoe1 shape, type = {bndaoe1.shape}, \ + logger.debug(f" bndaoe1 shape, type = {bndaoe1.shape}, \ {bndaoe1.dtype}") - logger.info(f" bndaoe3 shape, type = {bndaoe3.shape}, \ + logger.debug(f" bndaoe3 shape, type = {bndaoe3.shape}, \ {bndaoe3.dtype}") - logger.info(f" arfroe shape, type = {arfr.shape}, {arfr.dtype}") + logger.debug(f" arfroe shape, type = {arfr.shape}, {arfr.dtype}") - logger.info(f" bndaot shape, type = {bndaot.shape}, {bndaot.dtype}") + logger.debug(f" bndaot shape, type = {bndaot.shape}, {bndaot.dtype}") end_time = time.time() running_time = end_time - start_time - logger.info(f"Running time for executing QuerySet to get ResultSet: \ + logger.debug(f"Running time for executing QuerySet to get ResultSet: \ {running_time} seconds") # ========================= @@ -273,30 +273,30 @@ def bufr_to_ioda(config, logger): # ========================= start_time = time.time() - logger.info(f"Creating derived variables - stationIdentification") + logger.debug(f"Creating derived variables - stationIdentification") stid = Derive_stationIdentification(said, ptid) - logger.info(f" stid shape,type = {stid.shape}, {stid.dtype}") + logger.debug(f" stid shape,type = {stid.shape}, {stid.dtype}") - logger.info(f"Creating derived variables - Grid Latitude / Longitude ...") + logger.debug(f"Creating derived variables - Grid Latitude / Longitude ...") gclonh = Compute_Grid_Location(gclonh) gclath = Compute_Grid_Location(gclath) - logger.info(f" gclonh shape,type = {gclonh.shape}, {gclonh.dtype}") - logger.info(f" gclath shape,type = {gclath.shape}, {gclath.dtype}") - logger.info(f" gclonh min/max = {gclonh.min()}, {gclonh.max()}") - logger.info(f" gclath min/max = {gclath.min()}, {gclath.max()}") + logger.debug(f" gclonh shape,type = {gclonh.shape}, {gclonh.dtype}") + logger.debug(f" gclath shape,type = {gclath.shape}, {gclath.dtype}") + logger.debug(f" gclonh min/max = {gclonh.min()}, {gclonh.max()}") + logger.debug(f" gclath min/max = {gclath.min()}, {gclath.max()}") - logger.info(f"Creating derived variables - imph ...") + logger.debug(f"Creating derived variables - imph ...") imph1 = Compute_imph(impp1, elrc) imph3 = Compute_imph(impp3, elrc) - logger.info(f" imph1 shape,type = {imph1.shape}, {imph1.dtype}") - logger.info(f" imph3 shape,type = {imph3.shape}, {imph3.dtype}") - logger.info(f" imph1 min/max = {imph1.min()}, {imph1.max()}") - logger.info(f" imph3 min/max = {imph3.min()}, {imph3.max()}") + logger.debug(f" imph1 shape,type = {imph1.shape}, {imph1.dtype}") + logger.debug(f" imph3 shape,type = {imph3.shape}, {imph3.dtype}") + logger.debug(f" imph1 min/max = {imph1.min()}, {imph1.max()}") + logger.debug(f" imph3 min/max = {imph3.min()}, {imph3.max()}") - logger.info(f"Editing some derived variables if SAID is not 44 or 825") + logger.debug(f"Editing some derived variables if SAID is not 44 or 825") for i in range(len(said)): if (said[i] != 44) or (said[i] != 825): bnda1[i] = bnda3[i] @@ -305,20 +305,20 @@ def bufr_to_ioda(config, logger): imph1[i] = imph3[i] bndaoe1[i] = bndaoe3[i] - logger.info(f" new bnda1 shape, type, min/max {bnda1.shape}, \ + logger.debug(f" new bnda1 shape, type, min/max {bnda1.shape}, \ {bnda1.dtype}, {bnda1.min()}, {bnda1.max()}") - logger.info(f" new mefr1 shape, type, min/max {mefr1.shape}, \ + logger.debug(f" new mefr1 shape, type, min/max {mefr1.shape}, \ {mefr1.dtype}, {mefr1.min()}, {mefr1.max()}") - logger.info(f" new impp1 shape, type, min/max {impp1.shape}, \ + logger.debug(f" new impp1 shape, type, min/max {impp1.shape}, \ {impp1.dtype}, {impp1.min()}, {impp1.max()}") - logger.info(f" new imph1 shape, type, min/max {imph1.shape}, \ + logger.debug(f" new imph1 shape, type, min/max {imph1.shape}, \ {imph1.dtype}, {imph1.min()}, {imph1.max()}") - logger.info(f" new bndaoe1 shape, type, min/max {bndaoe1.shape}, \ + logger.debug(f" new bndaoe1 shape, type, min/max {bndaoe1.shape}, \ {bndaoe1.dtype}, {bndaoe1.min()}, {bndaoe1.max()}") end_time = time.time() running_time = end_time - start_time - logger.info(f"Running time for creating derived variables: {running_time} \ + logger.debug(f"Running time for creating derived variables: {running_time} \ seconds") # ===================================== @@ -328,16 +328,16 @@ def bufr_to_ioda(config, logger): # Find unique satellite identifiers in data to process unique_satids = np.unique(said) - logger.info(f" ... Number of Unique satellite identifiers: \ + logger.debug(f" ... Number of Unique satellite identifiers: \ {len(unique_satids)}") - logger.info(f" ... Unique satellite identifiers: {unique_satids}") + logger.debug(f" ... Unique satellite identifiers: {unique_satids}") # Create the dimensions dims = {'Location': np.arange(0, clath.shape[0])} iodafile = f"{cycle_type}.t{hh}z.{data_type}.{data_format}.nc" OUTPUT_PATH = os.path.join(ioda_dir, iodafile) - logger.info(f" ... ... Create OUTPUT file: {OUTPUT_PATH}") + logger.debug(f" ... ... Create OUTPUT file: {OUTPUT_PATH}") path, fname = os.path.split(OUTPUT_PATH) if path and not os.path.exists(path): @@ -347,7 +347,7 @@ def bufr_to_ioda(config, logger): obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) # Create Global attributes - logger.info(f" ... ... Create global attributes") + logger.debug(f" ... ... Create global attributes") obsspace.write_attr('data_format', data_format) obsspace.write_attr('data_type', data_type) obsspace.write_attr('subsets', subsets) @@ -358,7 +358,7 @@ def bufr_to_ioda(config, logger): obsspace.write_attr('converter', os.path.basename(__file__)) # Create IODA variables - logger.info(f" ... ... Create variables: name, type, units, & attributes") + logger.debug(f" ... ... Create variables: name, type, units, & attributes") # Longitude obsspace.create_var('MetaData/longitude', dtype=clonh.dtype, fillval=clonh.fill_value) \ @@ -553,10 +553,10 @@ def bufr_to_ioda(config, logger): end_time = time.time() running_time = end_time - start_time - logger.info(f"Running time for splitting and output IODA for gpsro bufr: \ + logger.debug(f"Running time for splitting and output IODA for gpsro bufr: \ {running_time} seconds") - logger.info("All Done!") + logger.debug("All Done!") if __name__ == '__main__': @@ -582,4 +582,4 @@ def bufr_to_ioda(config, logger): end_time = time.time() running_time = end_time - start_time - logger.info(f"Total running time: {running_time} seconds") + logger.debug(f"Total running time: {running_time} seconds")