From 45f22e906f51e4cfe65a3d9d4ecc8c41fdc33fc4 Mon Sep 17 00:00:00 2001 From: Azadeh Gholoubi <51101867+azadeh-gh@users.noreply.github.com> Date: Thu, 31 Oct 2024 12:45:59 -0400 Subject: [PATCH 01/22] Add ABI bufr2IODA python converter and JSON file. (#1356) [#115](https://github.com/NOAA-EMC/JEDI-T2O/issues/115) **New files include:** - `parm/ioda/bufr2ioda/bufr2ioda_gsrcsr.json` : JSON containing data format, sensor/bufr2ioda_gsrcsr.json, and satellite information - `ush/ioda/bufr2ioda/bufr2ioda_gsrcsr.py` : bufr2ioda python code for converting ABI GOES-16, GOES-17, and GOES-18 from BUFR to IODA - Also added _abi_ to the observation list in `jcb-prototype_3dvar.yaml.j2` Co-authored-by: Azadeh Gholoubi --- parm/atm/jcb-prototype_3dvar.yaml.j2 | 1 + parm/ioda/bufr2ioda/bufr2ioda_gsrcsr.json | 16 + ush/ioda/bufr2ioda/bufr2ioda_gsrcsr.py | 555 ++++++++++++++++++++++ 3 files changed, 572 insertions(+) create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_gsrcsr.json create mode 100755 ush/ioda/bufr2ioda/bufr2ioda_gsrcsr.py diff --git a/parm/atm/jcb-prototype_3dvar.yaml.j2 b/parm/atm/jcb-prototype_3dvar.yaml.j2 index 7b6c80011..152683110 100644 --- a/parm/atm/jcb-prototype_3dvar.yaml.j2 +++ b/parm/atm/jcb-prototype_3dvar.yaml.j2 @@ -29,3 +29,4 @@ observations: # - satwnd.viirs_npp # - scatwind_ascat_metop-a # - snowcvr + - abi_g16 diff --git a/parm/ioda/bufr2ioda/bufr2ioda_gsrcsr.json b/parm/ioda/bufr2ioda/bufr2ioda_gsrcsr.json new file mode 100644 index 000000000..036f7bd4b --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_gsrcsr.json @@ -0,0 +1,16 @@ +{ + "data_format" : "bufr_d", + "data_type" : "gsrcsr", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "subsets" : ['NC021046'], + "data_description" : "NC021046 ABI, GOES-16; NC021046 ABI, GOES-17, NC021046 ABI, GOES-18", + "data_provider" : "U.S. NOAA/NESDIS", + "sensor_info" : { "sensor_name": "ABI", "sensor_full_name": "Advanced Baseline Imager", "sensor_id": 617 }, + "satellite_info" : [ + { "satellite_name": "g16", "satellite_full_name": "GOES-16", "satellite_id": 270, "launch time": "20171119" }, + { "satellite_name": "g17", "satellite_full_name": "GOES-17", "satellite_id": 271, "launch time": "20180301" }, + { "satellite_name": "g18", "satellite_full_name": "GOES-18", "satellite_id": 272, "launch time": "20220301" } ] +} diff --git a/ush/ioda/bufr2ioda/bufr2ioda_gsrcsr.py b/ush/ioda/bufr2ioda/bufr2ioda_gsrcsr.py new file mode 100755 index 000000000..dc4fc20b0 --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_gsrcsr.py @@ -0,0 +1,555 @@ +#!/usr/bin/env python3 +import argparse +import calendar +import datetime +import json +import math +import os +import time +from datetime import datetime + +import numpy as np +import numpy.ma as ma +from wxflow import Logger + +from pyioda import ioda_obs_space as ioda_ospace +from pyiodaconv import bufr + +# Define and initialize global variables +global float32_fill_value +global int32_fill_value +global int64_fill_value + +float32_fill_value = np.float32(0) +int32_fill_value = np.int32(0) +int64_fill_value = np.int64(0) + + +def bufr_to_ioda(config, logger): + subsets = config["subsets"] + logger.debug(f"Checking subsets = {subsets}") + + # Get parameters from configuration + subsets = config["subsets"] + data_format = config["data_format"] + data_type = config["data_type"] + data_description = config["data_description"] + data_provider = config["data_provider"] + cycle_type = config["cycle_type"] + dump_dir = config["dump_directory"] + ioda_dir = config["ioda_directory"] + cycle = config["cycle_datetime"] + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + + satellite_info_array = config["satellite_info"] + sensor_name = config["sensor_info"]["sensor_name"] + sensor_full_name = config["sensor_info"]["sensor_full_name"] + sensor_id = config["sensor_info"]["sensor_id"] + + # Get derived parameters + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + reference_time = datetime.strptime(cycle, "%Y%m%d%H") + reference_time = reference_time.strftime("%Y-%m-%dT%H:%M:%SZ") + + # General informaton + converter = "BUFR to IODA Converter" + process_level = "Level-2" + platform_description = "NOAA Series of Geostationary Operational Environmental Satellites - 3rd generation since 2016" + sensor_description = "Spinning Enhanced Visible and InfraRed Imager;12 channels, 1 narrow-bandwidth, 1 high-resolution broad-bandwidth VIS" + + logger.info(f"sensor_name = {sensor_name}") + logger.info(f"sensor_full_name = {sensor_full_name}") + logger.info(f"sensor_id = {sensor_id}") + logger.info(f"reference_time = {reference_time}") + + bufrfile = f"{cycle_type}.t{hh}z.{data_type}.tm00.{data_format}" + DATA_PATH = os.path.join( + dump_dir, f"{cycle_type}.{yyyymmdd}", str(hh), "atmos", bufrfile + ) + if not os.path.isfile(DATA_PATH): + logger.info(f"The DATA_PATH is: {DATA_PATH}") + + # ============================================ + # Make the QuerySet for all the data we want + # ============================================ + start_time = time.time() + + logger.info("Making QuerySet") + q = bufr.QuerySet(subsets) + + # MetaData + q.add("latitude", "*/CLATH") + q.add("longitude", "*/CLONH") + q.add("satelliteId", "*/SAID") + q.add("year", "*/YEAR") + q.add("month", "*/MNTH") + q.add("day", "*/DAYS") + q.add("hour", "*/HOUR") + q.add("minute", "*/MINU") + q.add("second", "*/SECO") + q.add("sensorId", "*/SIID[1]") + q.add("sensorZenithAngle", "*/SAZA") + q.add("sensorCentralFrequency", "*/CSRADSEQ/SCCF") + q.add("solarZenithAngle", "*/SOZA") + q.add("cloudFree", "*/CLFRASEQ{2}/NCLDMNT") + q.add("brightnessTemperature", "*/CSRADSEQ/TMBRST") + q.add("ClearSkyStdDev", "*/SDRADSQ/SDTB") + q.add("solarAzimuthAngle", "*/SOLAZI") + q.add("sensorAzimuthAngle", "*/BEARAZ") + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Processing time for making QuerySet : {running_time} seconds") + + # ============================================================== + # Open the BUFR file and execute the QuerySet to get ResultSet + # Use the ResultSet returned to get numpy arrays of the data + # ============================================================== + start_time = time.time() + + logger.info("Executing QuerySet to get ResultSet") + with bufr.File(DATA_PATH) as f: + try: + r = f.execute(q) + except Exception as err: + logger.info(f'Return with {err}') + return + # MetaData + satid = r.get("satelliteId") + instid = r.get("sensorId") + year = r.get("year") + month = r.get("month") + day = r.get("day") + hour = r.get("hour") + minute = r.get("minute") + second = r.get("second") + lat = r.get("latitude") + lon = r.get("longitude") + satzenang = r.get("sensorZenithAngle") + chanfreq = r.get("sensorCentralFrequency", type="float") + BT = r.get("brightnessTemperature") + clrStdDev = r.get("ClearSkyStdDev") + cldFree = r.get("cloudFree", type="float") + solzenang = r.get("solarZenithAngle") + solaziang = r.get("solarAzimuthAngle") + sataziang = r.get("sensorAzimuthAngle") + # DateTime: seconds since Epoch time + # IODA has no support for numpy datetime arrays dtype=datetime64[s] + timestamp = r.get_datetime( + "year", "month", "day", "hour", "minute", "second" + ).astype(np.int64) + + # Global variables declaration + # Set global fill values + float32_fill_value = satzenang.fill_value + int32_fill_value = satid.fill_value + int64_fill_value = timestamp.fill_value.astype(np.int64) + + end_time = time.time() + running_time = end_time - start_time + logger.info( + f"Processing time for executing QuerySet to get ResultSet : {running_time} seconds" + ) + + # ========================= + # Create derived variables + # ========================= + start_time = time.time() + rounded_values = np.where(satzenang % 1 > 0.5, np.ceil(satzenang), np.floor(satzenang)) + # Convert to integer and add 1 + scanpos = rounded_values.astype(np.int32) + 1 + cloudAmount = 100. - cldFree + # Define the conversion factor from degrees to radians + deg2rad = math.pi/180.0 + sataziang = sataziang*deg2rad + viewang = np.full_like(solzenang, float32_fill_value, dtype=np.float32) + # Define Channel dimension for channels 4 to 11 since the other channel values are missing + channel_start = 7 + channel_end = 16 + channum = np.arange(channel_start, channel_end + 1) + # Define wavenumbers for each satellite ID + wavenum_values_dict = { + 270: np.array( + [ + 257037.4, + 162052.9, + 144355.4, + 136322.8, + 118422, + 104089.1, + 96800.1, + 89400.06, + 81529.43, + 75378.98, + ], + dtype=np.float32, + ), + 271: np.array( + [ + 256550.4, + 159490.2, + 136128.6, + 114870.3, + 103420.4, + 92938.72, + 83886.68, + 75122.19, + 83886.68, + 75122.19, + ], + dtype=np.float32, + ), + } + wavenum_fill_value = float32_fill_value + + logger.info("Creating derived variables") + + end_time = time.time() + running_time = end_time - start_time + logger.info( + f"Processing time for creating derived variables : {running_time} seconds" + ) + + # ===================================== + # Split output based on satellite id + # Create IODA ObsSpace + # Write IODA output + # ===================================== + logger.info("Create IODA ObsSpace and Write IODA output based on satellite ID") + + # Find nique satellite identifiers in data to process + unique_satids = np.unique(satid) + logger.info(f"Number of Unique satellite identifiers: {len(unique_satids)}") + logger.info(f"Unique satellite identifiers: {unique_satids}") + logger.debug(f"Loop through unique satellite identifier {unique_satids}") + total_ob_processed = 0 + for sat in unique_satids.tolist(): + start_time = time.time() + + matched = False + for satellite_info in satellite_info_array: + if satellite_info["satellite_id"] == sat: + matched = True + satellite_id = satellite_info["satellite_id"] + satellite_name = satellite_info["satellite_name"] + satinst = sensor_name.lower() + "_" + satellite_name.lower() + logger.debug(f"Split data for {satinst} satid = {sat}") + + if matched: + if satellite_id in wavenum_values_dict: + # Extract the wavenum values for the current satellite ID + Wavenum = wavenum_values_dict[satellite_id] + else: + # If the satellite ID is not in the dictionary + logger.debug(f"satellite ID is not in the dictionary {satellite_id}") + + # Define a boolean mask to subset data from the original data object + satelite_mask = satid == sat + # Define a boolean mask based on the condition 0 < satzenang2 < 80 + satzenang_mask = np.logical_and(0 < satzenang, satzenang < 80) + combined_mask = satzenang_mask & satelite_mask + # MetaData + lon2 = lon[combined_mask] + lat2 = lat[combined_mask] + timestamp2 = timestamp[combined_mask] + satid2 = satid[combined_mask] + instid2 = instid[combined_mask] + satzenang2 = satzenang[combined_mask] + chanfreq2 = chanfreq[6:16] + scanpos2 = scanpos[combined_mask] + solzenang2 = solzenang[combined_mask] + cldFree2 = cldFree[combined_mask] + cloudAmount2 = cloudAmount[combined_mask] + BT2 = BT[combined_mask] + clrStdDev2 = clrStdDev[combined_mask] + viewang2 = viewang.flatten()[combined_mask] + sataziang2 = sataziang.flatten()[combined_mask] + solaziang2 = solaziang.flatten()[combined_mask] + + # Timestamp Range + timestamp2_min = datetime.fromtimestamp(timestamp2.min()) + timestamp2_max = datetime.fromtimestamp(timestamp2.max()) + + # Check unique observation time + unique_timestamp2 = np.unique(timestamp2) + logger.debug(f"Processing output for satid {sat}") + logger.info(f"number of unique_timestamp2 {len(unique_timestamp2)}") + logger.info(f"unique_timestamp2 {unique_timestamp2}") + + # Create the dimensions + dims = { + "Location": np.arange(0, BT2.shape[0]), + "Channel": np.arange(channel_start, channel_end + 1), + } + + # Create IODA ObsSpace + iodafile = f"{cycle_type}.t{hh}z.{satinst}.tm00.nc" + OUTPUT_PATH = os.path.join(ioda_dir, iodafile) + logger.info(f"Create output file : {OUTPUT_PATH}") + obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode="w", dim_dict=dims) + + # Create Global attributes + logger.debug("Write global attributes") + obsspace.write_attr("Converter", converter) + obsspace.write_attr("sourceFiles", bufrfile) + obsspace.write_attr("description", data_description) + obsspace.write_attr("datetimeReference", reference_time) + obsspace.write_attr( + "datetimeRange", [str(timestamp2_min), str(timestamp2_max)] + ) + obsspace.write_attr("sensor", sensor_id) + obsspace.write_attr("platform", satellite_id) + obsspace.write_attr("platformCommonName", satellite_name) + obsspace.write_attr("sensorCommonName", sensor_name) + obsspace.write_attr("processingLevel", process_level) + obsspace.write_attr("platformLongDescription", platform_description) + obsspace.write_attr("sensorLongDescription", sensor_description) + + # Create IODA variables + logger.debug("Write variables: name, type, units, and attributes") + + # Sensor Channel Number + obsspace.create_var( + "MetaData/sensorChannelNumber", + dim_list=["Channel"], + dtype=np.int32, + fillval=int32_fill_value, + ).write_attr("long_name", "Sensor Channel Number").write_data(channum) + + # Sensor Central Frequency + obsspace.create_var( + "MetaData/sensorCentralFrequency", + dim_list=["Channel"], + dtype=chanfreq2.dtype, + fillval=chanfreq2.fill_value, + ).write_attr("units", "Hz").write_attr( + "long_name", "Satellite Channel Center Frequency" + ).write_data( + chanfreq2 + ) + + # Sensor Central Wavenumber + obsspace.create_var( + "MetaData/sensorCentralWavenumber", + dim_list=["Channel"], + dtype=Wavenum.dtype, + fillval=wavenum_fill_value, + ).write_attr("units", "m-1").write_attr( + "long_name", "Sensor Central Wavenumber" + ).write_data( + Wavenum + ) + + if np.any(combined_mask): + # Longitude + obsspace.create_var( + "MetaData/longitude", dtype=lon2.dtype, fillval=lon2.fill_value + ).write_attr("units", "degrees_east").write_attr( + "valid_range", np.array([-180, 180], dtype=np.float32) + ).write_attr( + "long_name", "Longitude" + ).write_data( + lon2 + ) + + # Latitude + obsspace.create_var( + "MetaData/latitude", dtype=lat2.dtype, fillval=lat2.fill_value + ).write_attr("units", "degrees_north").write_attr( + "valid_range", np.array([-90, 90], dtype=np.float32) + ).write_attr( + "long_name", "Latitude" + ).write_data( + lat2 + ) + + # Datetime + obsspace.create_var( + "MetaData/dateTime", dtype=np.int64, fillval=int64_fill_value + ).write_attr("units", "seconds since 1970-01-01T00:00:00Z").write_attr( + "long_name", "Datetime" + ).write_data( + timestamp2 + ) + + # Satellite Identifier + obsspace.create_var( + "MetaData/satelliteIdentifier", + dtype=satid2.dtype, + fillval=satid2.fill_value, + ).write_attr("long_name", "Satellite Identifier").write_data(satid2) + + # Instrument Identifier + obsspace.create_var( + "MetaData/instrumentIdentifier", + dtype=instid2.dtype, + fillval=instid2.fill_value, + ).write_attr("long_name", "Satellite Instrument Identifier").write_data( + instid2 + ) + + # Scan Position (derived variable, need to specified fill value explicitly) + obsspace.create_var( + "MetaData/sensorScanPosition", + dtype=scanpos2.astype(np.int32).dtype, + fillval=int32_fill_value, + ).write_attr("long_name", "Sensor Scan Position").write_data(scanpos2) + + # Sensor Zenith Angle + obsspace.create_var( + "MetaData/sensorZenithAngle", + dtype=satzenang2.dtype, + fillval=satzenang2.fill_value, + ).write_attr("units", "degree").write_attr( + "valid_range", np.array([0, 90], dtype=np.float32) + ).write_attr( + "long_name", "Sensor Zenith Angle" + ).write_data( + satzenang2 + ) + + # Sensor Azimuth Angle + obsspace.create_var( + "MetaData/sensorAzimuthAngle", + dtype=np.float32, + fillval=sataziang2.fill_value, + ).write_attr("units", "degree").write_attr( + "valid_range", np.array([0, 360], dtype=np.float32) + ).write_attr( + "long_name", "Sensor Azimuth Angle" + ).write_data( + sataziang2 + ) + + # Solar Azimuth Angle + obsspace.create_var( + "MetaData/solarAzimuthAngle", + dtype=np.float32, + fillval=solaziang2.fill_value, + ).write_attr("units", "degree").write_attr( + "valid_range", np.array([0, 360], dtype=np.float32) + ).write_attr( + "long_name", "Solar Azimuth Angle" + ).write_data( + solaziang2 + ) + + # Sensor View Angle + obsspace.create_var( + "MetaData/sensorViewAngle", + dtype=np.float32, + fillval=viewang2.fill_value, + ).write_attr("units", "degree").write_attr( + "long_name", "Sensor View Angle" + ).write_data( + viewang2 + ) + + # Solar Zenith Angle + obsspace.create_var( + "MetaData/solarZenithAngle", + dtype=solzenang2.dtype, + fillval=solzenang2.fill_value, + ).write_attr("units", "degree").write_attr( + "valid_range", np.array([0, 180], dtype=np.float32) + ).write_attr( + "long_name", "Solar Zenith Angle" + ).write_data( + solzenang2 + ) + + # Cloud free + obsspace.create_var( + "MetaData/cloudFree", + dtype=cldFree2.dtype, fillval=int32_fill_value + ).write_attr("units", "1").write_attr( + "valid_range", np.array([0, 100], dtype=np.int32) + ).write_attr( + "long_name", "Amount Segment Cloud Free" + ).write_data( + cldFree2 + ) + + # Cloud amount based on computation + obsspace.create_var( + "MetaData/cloudAmount", + dtype=cloudAmount2.dtype, + fillval=cloudAmount2.fill_value, + ).write_attr("units", "1").write_attr( + "valid_range", np.array([0, 100], dtype=np.float32) + ).write_attr( + "long_name", "Amount of cloud coverage in layer" + ).write_data( + cloudAmount2 + ) + + # ObsType based on computation method/spectral band + obsspace.create_var( + "ObsValue/brightnessTemperature", + dim_list=["Location", "Channel"], + dtype=np.float32, + fillval=BT2.fill_value, + ).write_attr("units", "k").write_attr( + "long_name", "Brightness Temperature" + ).write_data( + BT2 + ) + + obsspace.create_var( + "ClearSkyStdDev/brightnessTemperature", + dim_list=["Location", "Channel"], + dtype=np.float32, + fillval=clrStdDev2.fill_value, + ).write_attr( + "long_name", "Standard Deviation Brightness Temperature" + ).write_data( + clrStdDev2 + ) + + else: + logger.debug( + "No valid values (0 Date: Fri, 1 Nov 2024 15:11:38 -0400 Subject: [PATCH 02/22] load esmf/8.5.0 in gaea.intel.lua, use staged CRTM_FIX on all machines (#1327) --- build.sh | 14 ++++++-------- modulefiles/GDAS/gaea.intel.lua | 13 +++++++------ 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/build.sh b/build.sh index 42ec34284..4ebb3666e 100755 --- a/build.sh +++ b/build.sh @@ -103,14 +103,12 @@ WORKFLOW_BUILD=${WORKFLOW_BUILD:-"OFF"} CMAKE_OPTS+=" -DWORKFLOW_TESTS=${WORKFLOW_BUILD}" # JCSDA changed test data things, need to make a dummy CRTM directory -if [[ $BUILD_TARGET == 'hera' ]]; then - if [ -d "$dir_root/bundle/fix/test-data-release/" ]; then rm -rf $dir_root/bundle/fix/test-data-release/; fi - if [ -d "$dir_root/bundle/test-data-release/" ]; then rm -rf $dir_root/bundle/test-data-release/; fi - mkdir -p $dir_root/bundle/fix/test-data-release/ - mkdir -p $dir_root/bundle/test-data-release/ - ln -sf $GDASAPP_TESTDATA/crtm $dir_root/bundle/fix/test-data-release/crtm - ln -sf $GDASAPP_TESTDATA/crtm $dir_root/bundle/test-data-release/crtm -fi +if [ -d "$dir_root/bundle/fix/test-data-release/" ]; then rm -rf $dir_root/bundle/fix/test-data-release/; fi +if [ -d "$dir_root/bundle/test-data-release/" ]; then rm -rf $dir_root/bundle/test-data-release/; fi +mkdir -p $dir_root/bundle/fix/test-data-release/ +mkdir -p $dir_root/bundle/test-data-release/ +ln -sf $GDASAPP_TESTDATA/crtm $dir_root/bundle/fix/test-data-release/crtm +ln -sf $GDASAPP_TESTDATA/crtm $dir_root/bundle/test-data-release/crtm # Configure echo "Configuring ..." diff --git a/modulefiles/GDAS/gaea.intel.lua b/modulefiles/GDAS/gaea.intel.lua index 85d779669..d1aa1df6c 100644 --- a/modulefiles/GDAS/gaea.intel.lua +++ b/modulefiles/GDAS/gaea.intel.lua @@ -10,8 +10,8 @@ prepend_path("MODULEPATH", '/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/u prepend_path("MODULEPATH", '/ncrc/proj/epic/rocoto/modulefiles') -- below two lines get us access to the spack-stack modules -load("stack-intel/2023.1.0") -load("stack-cray-mpich/8.1.25") +load("stack-intel/2023.2.0") +load("stack-cray-mpich/8.1.28") -- JCSDA has 'jedi-fv3-env/unified-dev', but we should load these manually as needed load("cmake/3.23.1") load("gettext/0.20.2") @@ -44,11 +44,12 @@ load("fckit/0.11.0") load("fiat/1.2.0") load("ectrans/1.2.0") load("fms/2023.04") +load("esmf/8.5.0") load("atlas/0.35.1") load("sp/2.5.0") load("gsl-lite/0.37.0") load("libjpeg/2.1.0") -load("krb5/1.16.3") +load("krb5/1.20.1") load("libtirpc/1.3.3") load("hdf/4.2.15") load("jedi-cmake/1.4.0") @@ -84,9 +85,9 @@ local mpinproc = '-n' setenv('MPIEXEC_EXEC', mpiexec) setenv('MPIEXEC_NPROC', mpinproc) -setenv("CRTM_FIX","/gpfs/f5/ufs-ard/world-shared/GDASApp/fix/crtm/2.4.0") -setenv("GDASAPP_TESTDATA","/gpfs/f5/ufs-ard/world-shared/GDASApp/CI/data") -setenv("GDASAPP_UNIT_TEST_DATA_PATH", "/gpfs/f5/ufs-ard/world-shared/GDASApp/CI/data/test") +setenv("CRTM_FIX","/gpfs/f5/nggps_emc/world-shared/GDASApp/fix/crtm/2.4.0") +setenv("GDASAPP_TESTDATA","/gpfs/f5/nggps_emc/world-shared/GDASApp/testdata") +setenv("GDASAPP_UNIT_TEST_DATA_PATH", "/gpfs/f5/nggps_emc/world-shared/GDASApp/unittestdata") whatis("Name: ".. "pkgName") whatis("Version: ".. "pkgVersion") From e9607fc9d27a2d5e9dde9c2f66903d5ec06f4280 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Wed, 6 Nov 2024 08:53:15 -0500 Subject: [PATCH 03/22] Marine B-matrix refactor companion PR (#1346) This PR is a companion to Global Workflow PR [#2992](https://github.com/NOAA-EMC/global-workflow/pull/2992) and JCB-GDAS PR [#36](https://github.com/NOAA-EMC/jcb-gdas/pull/36). It makes a few necessary changes to the marine JCB base YAML required to have JCB working in the B-matrix task in the Global Workflow. --- parm/jcb-gdas | 2 +- parm/soca/berror/soca_diagb.yaml.j2 | 37 ------- parm/soca/berror/soca_ensb.yaml.j2 | 98 ------------------- parm/soca/berror/soca_ensweights.yaml.j2 | 37 ------- .../soca_parameters_diffusion_hz.yaml.j2 | 37 ------- .../soca_parameters_diffusion_vt.yaml.j2 | 33 ------- parm/soca/berror/soca_setcorscales.yaml | 23 ----- parm/soca/berror/soca_vtscales.yaml.j2 | 13 --- parm/soca/gridgen/gridgen.yaml | 5 - parm/soca/marine-jcb-base.yaml | 9 +- parm/soca/soca_fix_stage_500.yaml.j2 | 2 - parm/soca/soca_utils_stage.yaml.j2 | 1 - 12 files changed, 7 insertions(+), 290 deletions(-) delete mode 100644 parm/soca/berror/soca_diagb.yaml.j2 delete mode 100644 parm/soca/berror/soca_ensb.yaml.j2 delete mode 100644 parm/soca/berror/soca_ensweights.yaml.j2 delete mode 100644 parm/soca/berror/soca_parameters_diffusion_hz.yaml.j2 delete mode 100644 parm/soca/berror/soca_parameters_diffusion_vt.yaml.j2 delete mode 100644 parm/soca/berror/soca_setcorscales.yaml delete mode 100644 parm/soca/berror/soca_vtscales.yaml.j2 delete mode 100644 parm/soca/gridgen/gridgen.yaml diff --git a/parm/jcb-gdas b/parm/jcb-gdas index 7717c0e74..b8e995a4c 160000 --- a/parm/jcb-gdas +++ b/parm/jcb-gdas @@ -1 +1 @@ -Subproject commit 7717c0e7401e344a6bce37a4f8ecc11399256936 +Subproject commit b8e995a4cbf01fa4a662c3da3e7d818f8457ec4e diff --git a/parm/soca/berror/soca_diagb.yaml.j2 b/parm/soca/berror/soca_diagb.yaml.j2 deleted file mode 100644 index c0597d358..000000000 --- a/parm/soca/berror/soca_diagb.yaml.j2 +++ /dev/null @@ -1,37 +0,0 @@ -geometry: - mom6_input_nml: mom_input.nml - fields metadata: ./fields_metadata.yaml - -date: '{{ MARINE_WINDOW_END | to_isotime }}' - -background: - date: '{{ MARINE_WINDOW_END | to_isotime }}' - basename: ./bkg/ - ocn_filename: 'ocean.bkg.f009.nc' - ice_filename: 'ice.bkg.f009.nc' - read_from_file: 1 - -background error: - datadir: ./staticb/ - date: '{{ MARINE_WINDOW_MIDDLE | to_isotime }}' - exp: bkgerr_stddev - type: incr - -variables: - name: [tocn, socn, uocn, vocn, hocn, ssh, cicen, hicen, hsnon, mom6_mld] - -rescale: 2.0 # rescales the filtered std. dev. by "rescale" -min sst: 0.0 # Added to sst bkg. err. -max ssh: 0.0 # Limits the amplitude of the unbalanced bkg err -min depth: 500.0 # zero out the bkg. error. at less than min depth -number of halo points: 4 -number of neighbors: 16 - -simple smoothing: - horizontal iterations: 10 - vertical iterations: 1 - -# TODO(G): Too slow for the below scale -#diffusion: -# horizontal: 500.0e3 -# vertical: 3.0 diff --git a/parm/soca/berror/soca_ensb.yaml.j2 b/parm/soca/berror/soca_ensb.yaml.j2 deleted file mode 100644 index 4033f41ba..000000000 --- a/parm/soca/berror/soca_ensb.yaml.j2 +++ /dev/null @@ -1,98 +0,0 @@ -# Configuration for the recentering and re-balancing of the ensemble members -geometry: - mom6_input_nml: mom_input.nml - fields metadata: ./fields_metadata.yaml - -date: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' - -layers variable: [hocn] - -increment variables: [tocn, socn, uocn, vocn, ssh, hocn, cicen, hicen, hsnon] - -set increment variables to zero: [ssh] - -vertical geometry: - date: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' - basename: ./INPUT/ - ocn_filename: MOM.res.nc - read_from_file: 3 - -add recentering increment: false - -soca increments: # Could also be states, but they are read as increments - number of increments: {{ NMEM_ENS }} - pattern: '%mem%' - template: - date: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' - basename: '{{ ENSPERT_RELPATH }}/ens/' - ocn_filename: 'ocean.%mem%.nc' - ice_filename: 'ice.%mem%.nc' - read_from_file: 3 - -steric height: - linear variable changes: - - linear variable change name: BalanceSOCA # Only the steric balance is applied - -ensemble mean output: - datadir: ./staticb/ - date: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' - exp: ens_mean - type: incr - -ssh output: - unbalanced: - datadir: ./staticb/ - date: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' - exp: ssh_unbal_stddev - type: incr - - steric: - datadir: ./staticb/ - date: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' - exp: ssh_steric_stddev - type: incr - - total: - datadir: ./staticb/ - date: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' - exp: ssh_total_stddev - type: incr - - explained variance: - datadir: ./staticb/ - date: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' - exp: steric_explained_variance - type: incr - - recentering error: - datadir: ./staticb/ - date: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' - exp: ssh_recentering_error - type: incr - -background error output: - datadir: ./staticb/ - date: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' - exp: bkgerr_stddev - type: incr - -linear variable change: - linear variable changes: - - linear variable change name: BalanceSOCA - -trajectory: - state variables: [tocn, socn, uocn, vocn, ssh, hocn, layer_depth, mld, cicen, hicen, hsnon] - date: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' - basename: ./INPUT/ - ocn_filename: MOM.res.nc - ice_filename: cice.res.nc - read_from_file: 1 - -output increment: - # TODO: Revert this when fms can take more than 128 charactres file names - datadir: '{{ ENSPERT_RELPATH }}/enspert/' - date: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' - exp: trash - type: incr - output file: 'ocn.pert.steric.%mem%.nc' - pattern: '%mem%' diff --git a/parm/soca/berror/soca_ensweights.yaml.j2 b/parm/soca/berror/soca_ensweights.yaml.j2 deleted file mode 100644 index f677d2a8d..000000000 --- a/parm/soca/berror/soca_ensweights.yaml.j2 +++ /dev/null @@ -1,37 +0,0 @@ -geometry: - mom6_input_nml: mom_input.nml - fields metadata: ./fields_metadata.yaml - -date: '{{ MARINE_WINDOW_MIDDLE | to_isotime }}' - -variables: - ice: [cicen, hicen, hsnon] - ocean: [tocn, socn, uocn, vocn, ssh] - -background: - date: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' - basename: ./INPUT/ - ocn_filename: MOM.res.nc - ice_filename: cice.res.nc - read_from_file: 1 - -weights: - # Need to provide weights^2 when reading from file - ice: 0.0025 # 5% of original variance - ocean: 0.0625 # 25% " " - # Apply localized weights to the ocean ens. B - ocean local weights: - - lon: -172.0 - lat: 11.0 - amplitude: -1.0 - length scale: 700.0 - - lon: -160.0 - lat: 12.0 - amplitude: -1.0 - length scale: 700.0 - -output: - datadir: ./ - date: '{{ MARINE_WINDOW_MIDDLE | to_isotime }}' - exp: ens_weights - type: incr diff --git a/parm/soca/berror/soca_parameters_diffusion_hz.yaml.j2 b/parm/soca/berror/soca_parameters_diffusion_hz.yaml.j2 deleted file mode 100644 index 7d3a78cfb..000000000 --- a/parm/soca/berror/soca_parameters_diffusion_hz.yaml.j2 +++ /dev/null @@ -1,37 +0,0 @@ -geometry: &geom - mom6_input_nml: mom_input.nml - fields metadata: ./fields_metadata.yaml - -background: - read_from_file: 1 - basename: ./INPUT/ - ocn_filename: MOM.res.nc - ice_filename: cice.res.nc - date: '{{ MARINE_WINDOW_END | to_isotime }}' - state variables: [ssh] - -background error: - covariance model: SABER - saber central block: - saber block name: diffusion - geometry: *geom - calibration: - normalization: - method: randomization - iterations: 10000 - - groups: - - horizontal: - model file: - date: '{{ MARINE_WINDOW_END | to_isotime }}' - basename: ./ - ocn_filename: ocn.cor_rh.incr.0001-01-01T00:00:00Z.nc - model variable: ssh - write: - filepath: ./staticb/hz_ocean - - - horizontal: - as gaussian: true - fixed value: 50000.0 - write: - filepath: ./staticb/hz_ice diff --git a/parm/soca/berror/soca_parameters_diffusion_vt.yaml.j2 b/parm/soca/berror/soca_parameters_diffusion_vt.yaml.j2 deleted file mode 100644 index 76ab67e94..000000000 --- a/parm/soca/berror/soca_parameters_diffusion_vt.yaml.j2 +++ /dev/null @@ -1,33 +0,0 @@ -geometry: &geom - mom6_input_nml: mom_input.nml - fields metadata: ./fields_metadata.yaml - -background: - read_from_file: 1 - basename: ./INPUT/ - ocn_filename: MOM.res.nc - ice_filename: cice.res.nc - date: '{{ MARINE_WINDOW_MIDDLE | to_isotime }}' - state variables: [tocn] - -background error: - covariance model: SABER - saber central block: - saber block name: diffusion - geometry: *geom - calibration: - normalization: - # NOTE, not actually used here, since the normalization spec is only used for hz - method: randomization #< other option is "brute force" - iterations: 1000 #< in the real world you'll want to use 1e4 or so - - groups: - - vertical: - as gaussian: true - model file: - date: '{{ MARINE_WINDOW_MIDDLE | to_isotime }}' - basename: ./ - ocn_filename: vt_scales.nc - model variable: tocn - write: - filepath: ./staticb/vt_ocean diff --git a/parm/soca/berror/soca_setcorscales.yaml b/parm/soca/berror/soca_setcorscales.yaml deleted file mode 100644 index 0a91777a1..000000000 --- a/parm/soca/berror/soca_setcorscales.yaml +++ /dev/null @@ -1,23 +0,0 @@ -resolution: - mom6_input_nml: mom_input.nml - fields metadata: ./fields_metadata.yaml - -date: 0001-01-01T00:00:00Z - -corr variables: [ssh] - -scales: - vert layers: 5 # in units of layer - ssh: - rossby mult: 1.00 - min grid mult: 2.0 - -rh output: - datadir: ./ - exp: cor_rh - type: incr - -rv output: - datadir: ./ - exp: cor_rv - type: incr diff --git a/parm/soca/berror/soca_vtscales.yaml.j2 b/parm/soca/berror/soca_vtscales.yaml.j2 deleted file mode 100644 index 8f68b1517..000000000 --- a/parm/soca/berror/soca_vtscales.yaml.j2 +++ /dev/null @@ -1,13 +0,0 @@ -gridspec_filename: soca_gridspec.nc -restart_filename: ./INPUT/MOM.res.nc -mld_filename: './staticb/ocn.bkgerr_stddev.incr.{{ MARINE_WINDOW_END | to_isotime }}.nc' -output_filename: ./vt_scales.nc -output_variable_vt: Temp -output_variable_hz: ave_ssh - -VT_MIN: 5 -VT_MAX: 15 - -HZ_ROSSBY_MULT: 1.0 -HZ_MAX: 200e3 -HZ_MIN_GRID_MULT: 2.0 diff --git a/parm/soca/gridgen/gridgen.yaml b/parm/soca/gridgen/gridgen.yaml deleted file mode 100644 index 34fbdeca6..000000000 --- a/parm/soca/gridgen/gridgen.yaml +++ /dev/null @@ -1,5 +0,0 @@ -geometry: - geom_grid_file: soca_gridspec.nc - mom6_input_nml: mom_input.nml - fields metadata: fields_metadata.yaml - rossby file: rossrad.nc diff --git a/parm/soca/marine-jcb-base.yaml b/parm/soca/marine-jcb-base.yaml index cb5230f1c..d07edcd8c 100644 --- a/parm/soca/marine-jcb-base.yaml +++ b/parm/soca/marine-jcb-base.yaml @@ -17,7 +17,7 @@ final_increment_file: marine_final_increment # Assimilation standard things (not prepended with model choice) # ---------------------------- -window_begin: '{{MARINE_WINDOW_BEGIN}}' +window_begin: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' window_length: '{{MARINE_WINDOW_LENGTH}}' bound_to_include: begin minimizer: RPCG @@ -31,6 +31,10 @@ analysis_variables: [cicen, hicen, hsnon, socn, tocn, uocn, vocn, ssh] # ------------ marine_window_begin: '{{MARINE_WINDOW_BEGIN}}' marine_window_middle: '{{MARINE_WINDOW_MIDDLE}}' +marine_window_begin_iso: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' +marine_window_middle_iso: '{{ MARINE_WINDOW_MIDDLE | to_isotime }}' +marine_window_end_iso: '{{ MARINE_WINDOW_END | to_isotime }}' +enspert_relpath: '{{ ENSPERT_RELPATH }}' # Geometry marine_soca_files_path: . @@ -42,11 +46,10 @@ marine_background_time: '{{MARINE_WINDOW_BEGIN_ISO}}' # Pseudo model marine_forecast_timestep: PT3H -marine_pseudo_model_states: !INC 'bkg_list.yaml' # Background error model background_error_file: '{{berror_model}}' -marine_number_ensemble_members: '{{nmem_ens}}' +marine_number_ensemble_members: '{{NMEM_ENS}}' marine_stddev_time: '{{MARINE_WINDOW_MIDDLE}}' # Observations diff --git a/parm/soca/soca_fix_stage_500.yaml.j2 b/parm/soca/soca_fix_stage_500.yaml.j2 index 0b25073ba..9f030a81f 100644 --- a/parm/soca/soca_fix_stage_500.yaml.j2 +++ b/parm/soca/soca_fix_stage_500.yaml.j2 @@ -9,8 +9,6 @@ copy: - ["{{ SOCA_INPUT_FIX_DIR }}/field_table", "{{ DATA }}/field_table"] - ["{{ SOCA_INPUT_FIX_DIR }}/diag_table", "{{ DATA }}/diag_table"] - ["{{ SOCA_INPUT_FIX_DIR }}/MOM_input", "{{ DATA }}/MOM_input"] -- ["{{ SOCA_INPUT_FIX_DIR }}/fields_metadata.yaml", "{{ DATA }}/fields_metadata.yaml"] -- ["{{ SOCA_INPUT_FIX_DIR }}/obsop_name_map.yaml", "{{ DATA }}/obsop_name_map.yaml"] - ["{{ SOCA_INPUT_FIX_DIR }}/INPUT/grid_spec.nc", "{{ DATA }}/INPUT/grid_spec.nc"] - ["{{ SOCA_INPUT_FIX_DIR }}/INPUT/hycom1_25.nc", "{{ DATA }}/INPUT/hycom1_25.nc"] - ["{{ SOCA_INPUT_FIX_DIR }}/INPUT/layer_coord25.nc", "{{ DATA }}/INPUT/layer_coord25.nc"] diff --git a/parm/soca/soca_utils_stage.yaml.j2 b/parm/soca/soca_utils_stage.yaml.j2 index 26570abe3..462cfccd0 100644 --- a/parm/soca/soca_utils_stage.yaml.j2 +++ b/parm/soca/soca_utils_stage.yaml.j2 @@ -4,4 +4,3 @@ copy: - ["{{ HOMEgfs }}/parm/gdas/soca/fields_metadata.yaml", "{{ DATA }}/fields_metadata.yaml"] - ["{{ HOMEgfs }}/parm/gdas/soca/obsop_name_map.yaml", "{{ DATA }}/obsop_name_map.yaml"] -- ["{{ HOMEgfs }}/parm/gdas/soca/gridgen/gridgen.yaml", "{{ DATA }}/gridgen.yaml"] From 2f1638f126d010ee2f34aa4da5013ce14549d436 Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO <82525974+DavidBurrows-NCO@users.noreply.github.com> Date: Wed, 6 Nov 2024 14:05:14 -0500 Subject: [PATCH 04/22] Update build scripts on Gaea-C5 to conform with ufs-wx-model and global-workflow (#1361) After the recent Gaea-C5 OS upgrade, GDASApp fails to build. This issue corrects Gaea-C5 build and updates the build scripts to conform to ufs-wx-model (following ufs-wx-model https://github.com/ufs-community/ufs-weather-model/pull/2448) and eventual global-workflow updates. Refs NOAA-EMC/global-workflow 3011 https://github.com/NOAA-EMC/global-workflow/issues/3011 Refs NOAA-EMC/global-workflow 3032 https://github.com/NOAA-EMC/global-workflow/pull/3032 Resolves #1360 --- build.sh | 2 +- .../GDAS/{gaea.intel.lua => gaeac5.intel.lua} | 0 ush/detect_machine.sh | 13 ++++++++----- 3 files changed, 9 insertions(+), 6 deletions(-) rename modulefiles/GDAS/{gaea.intel.lua => gaeac5.intel.lua} (100%) diff --git a/build.sh b/build.sh index 4ebb3666e..6445f80cf 100755 --- a/build.sh +++ b/build.sh @@ -71,7 +71,7 @@ while getopts "p:t:c:hvdfa" opt; do done case ${BUILD_TARGET} in - hera | orion | hercules | wcoss2 | noaacloud | gaea | gaeac6 ) + hera | orion | hercules | wcoss2 | noaacloud | gaeac5 | gaeac6 ) echo "Building GDASApp on $BUILD_TARGET" source $dir_root/ush/module-setup.sh module use $dir_root/modulefiles diff --git a/modulefiles/GDAS/gaea.intel.lua b/modulefiles/GDAS/gaeac5.intel.lua similarity index 100% rename from modulefiles/GDAS/gaea.intel.lua rename to modulefiles/GDAS/gaeac5.intel.lua diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index 997c394fa..ab039aebf 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -21,8 +21,8 @@ case $(hostname -f) in dlogin0[1-9].dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dogwood01-9 dlogin10.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dogwood10 - gaea5[1-8]) MACHINE_ID=gaea ;; ### gaea51-58 - gaea5[1-8].ncrc.gov) MACHINE_ID=gaea ;; ### gaea51-58 + gaea5[1-8]) MACHINE_ID=gaeac5 ;; ### gaea51-58 + gaea5[1-8].ncrc.gov) MACHINE_ID=gaeac5 ;; ### gaea51-58 gaea6[1-8]) MACHINE_ID=gaeac6 ;; ### gaea61-68 gaea6[1-8].ncrc.gov) MACHINE_ID=gaeac6 ;; ### gaea61-68 @@ -84,9 +84,12 @@ elif [[ -d /work ]]; then else MACHINE_ID=orion fi -elif [[ -d /gpfs && -d /ncrc ]]; then - # We are on GAEA. - MACHINE_ID=gaea +elif [[ -d /gpfs/f5 ]]; then + # We are on GAEAC5. + MACHINE_ID=gaeac5 +elif [[ -d /gpfs/f6 ]]; then + # We are on GAEAC6. + MACHINE_ID=gaeac6 elif [[ -d /data/prod ]]; then # We are on SSEC's S4 MACHINE_ID=s4 From e5d6563dcc44caa713e75f3626dadac76f3a955a Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Wed, 6 Nov 2024 14:46:55 -0500 Subject: [PATCH 05/22] Use JCB for aerosol DA (#1343) What the title says. Depends on: https://github.com/NOAA-EMC/jcb-gdas/pull/33 https://github.com/NOAA-EMC/jcb-algorithms/pull/5 --- parm/aero/jcb-base.yaml.j2 | 136 ++++++++++++++++++++++++++ parm/aero/jcb-prototype_3dvar.yaml.j2 | 10 ++ parm/jcb-algorithms | 2 +- 3 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 parm/aero/jcb-base.yaml.j2 create mode 100644 parm/aero/jcb-prototype_3dvar.yaml.j2 diff --git a/parm/aero/jcb-base.yaml.j2 b/parm/aero/jcb-base.yaml.j2 new file mode 100644 index 000000000..86988206d --- /dev/null +++ b/parm/aero/jcb-base.yaml.j2 @@ -0,0 +1,136 @@ +# Search path for model and obs for JCB +# ------------------------------------- +algorithm_path: "{{PARMgfs}}/gdas/jcb-algorithms" +app_path_algorithm: "{{PARMgfs}}/gdas/jcb-gdas/algorithm/aero" +app_path_model: "{{PARMgfs}}/gdas/jcb-gdas/model/aero" +app_path_observations: "{{PARMgfs}}/gdas/jcb-gdas/observations/aero" +app_path_observation_chronicle: "{{PARMgfs}}/gdas/jcb-gdas/observation_chronicle/aero" + + +# Places where we deviate from the generic file name of a yaml +# ------------------------------------------------------------ +final_increment_to_latlon_file: aero_final_increment_gaussian +final_increment_file: aero_final_increment_cubed_sphere +model_file: aero_model_pseudo +initial_condition_file: aero_background # Initial conditions for 4D apps is background +background_error_file: "{{BERROR_YAML}}" + +# Assimilation standard things (not prepended with model choice) +# ---------------------------- +window_begin: "{{ AERO_WINDOW_BEGIN | to_isotime }}" +window_length: "{{ AERO_WINDOW_LENGTH }}" +bound_to_include: begin +minimizer: DRPCG +final_diagnostics_departures: anlmob +final_prints_frequency: PT3H +number_of_outer_loops: 2 +analysis_variables: [mass_fraction_of_sulfate_in_air, + mass_fraction_of_hydrophobic_black_carbon_in_air, + mass_fraction_of_hydrophilic_black_carbon_in_air, + mass_fraction_of_hydrophobic_organic_carbon_in_air, + mass_fraction_of_hydrophilic_organic_carbon_in_air, + mass_fraction_of_dust001_in_air, mass_fraction_of_dust002_in_air, + mass_fraction_of_dust003_in_air, mass_fraction_of_dust004_in_air, + mass_fraction_of_dust005_in_air, mass_fraction_of_sea_salt001_in_air, + mass_fraction_of_sea_salt002_in_air, mass_fraction_of_sea_salt003_in_air, + mass_fraction_of_sea_salt004_in_air] + +# Model things +# ------------ +# Geometry +aero_layout_x: {{ layout_x | default(1, true) }} +aero_layout_y: {{ layout_y | default(1, true) }} +aero_npx_ges: {{ npx_ges | default(49, true) }} +aero_npy_ges: {{ npy_ges | default(49, true) }} +aero_npz_ges: {{ npz_ges | default(127, true) }} +aero_npx_anl: {{ npx_anl | default(49, true) }} +aero_npy_anl: {{ npy_anl | default(49, true) }} +aero_npz_anl: {{ npz_anl | default(127, true) }} +aero_npx_clim_b: {{ npx_clim_b | default(49, true) }} +aero_npy_clim_b: {{ npy_clim_b | default(49, true) }} +aero_npz_clim_b: {{ npz_anl | default(127, true) }} + +aero_fv3jedi_files_path: ./fv3jedi # Ideally this would be {{DATA}}/fv3jedi but FMS + +# Background +aero_background_path: ./bkg +aero_background_ensemble_path: ./ens/mem%mem% + +# Default background time is for 3D applications +{% if DOIAU == True %} +aero_background_time_iso: "{{ AERO_WINDOW_BEGIN | to_isotime }}" +{% else %} +aero_background_time_iso: "{{ current_cycle | to_isotime }}" +{% endif %} +aero_cycle_time_iso: "{{ current_cycle | to_isotime }}" +aero_cycle_time_fv3: "{{ current_cycle | to_fv3time }}" + +# time for background error calculation for next cycle +{% set offset_td = "+6H" | to_timedelta %} +{% set background_time = current_cycle | add_to_datetime(offset_td) %} +aero_background_error_time_iso: "{{ background_time | to_isotime }}" +aero_background_error_time_fv3: "{{ background_time | to_fv3time }}" + +# Background error +aero_berror_data_directory: "{{ DATA }}/berror" +aero_berror_diffusion_directory: "{{ DATA }}/diffusion" +aero_standard_devation_path: ./stddev +aero_climatological_b_path: ./clm_stddev +aero_diagb_weight: {{ aero_diagb_weight | default(1.0, true) }} +aero_diagb_static_rescale_factor: {{aero_staticb_rescaling_factor | default(1.0, true) }} +aero_diagb_rescale_factor: {{aero_diagb_rescale | default(1.0, true) }} +aero_diagb_n_halo: {{ aero_diagb_n_halo | default(1, true) }} +aero_diagb_n_neighbors: {{ aero_diagb_n_neighbors | default(1, true) }} +aero_diagb_smooth_horiz_iter: {{ aero_diagb_smooth_horiz_iter | default(1, true) }} +aero_diagb_smooth_vert_iter: {{ aero_diagb_smooth_vert_iter | default(1, true) }} +aero_diffusion_iter: {{ aero_diffusion_iter | default(1, true) }} +aero_diffusion_horiz_len: {{ aero_diffusion_horiz_len | default(1.0, true)}} +aero_diffusion_fixed_val: {{ aero_diffusion_fixed_val | default(1.0, true)}} + +# Forecasting +aero_forecast_timestep: "{{ BKG_TSTEP }}" + +# Observation things +# ------------------ +observations: all_observations + +crtm_coefficient_path: "{{ DATA }}/crtm/" + +# Naming conventions for observational files +aero_obsdataroot_path: "{{COM_OBS}}" + +aero_obsdatain_path: "{{aero_obsdatain_path}}" +aero_obsdatain_prefix: "{{OPREFIX}}" +aero_obsdatain_suffix: ".tm00.nc" + +aero_obsdataout_path: "{{aero_obsdataout_path}}" +aero_obsdataout_prefix: diag_ +aero_obsdataout_suffix: "_{{ current_cycle | to_YMDH }}.nc" + +# Naming conventions for bias correction files +aero_obsbiasroot_path: "{{COM_CHEM_ANALYSIS_PREV}}" + +aero_obsbiasin_path: "{{DATA}}/obs/" +aero_obsbiasin_prefix: "{{GPREFIX}}" +aero_obsbiasin_suffix: ".satbias.nc" +aero_obstlapsein_prefix: "{{GPREFIX}}" +aero_obstlapsein_suffix: ".tlapse.txt" +aero_obsbiascovin_prefix: "{{GPREFIX}}" +aero_obsbiascovin_suffix: ".satbias_cov.nc" + +aero_obsbiasout_path: "{{DATA}}/bc/" +aero_obsbiasout_prefix: "{{APREFIX}}" +aero_obsbiasout_suffix: ".satbias.nc" +aero_obsbiascovout_prefix: "{{APREFIX}}" +aero_obsbiascovout_suffix: ".satbias_cov.nc" + +bias_files: + atms_n20: rad_varbc_params.tar + atms_npp: rad_varbc_params.tar + mtiasi_metop-a: rad_varbc_params.tar + mtiasi_metop-b: rad_varbc_params.tar + amsua_n19: rad_varbc_params.tar + ssmis_f17: rad_varbc_params.tar + ssmis_f18: rad_varbc_params.tar + cris-fsr_n20: rad_varbc_params.tar + cris-fsr_npp: rad_varbc_params.tar diff --git a/parm/aero/jcb-prototype_3dvar.yaml.j2 b/parm/aero/jcb-prototype_3dvar.yaml.j2 new file mode 100644 index 000000000..da4739bca --- /dev/null +++ b/parm/aero/jcb-prototype_3dvar.yaml.j2 @@ -0,0 +1,10 @@ +# Algorithm +# --------- +algorithm: 3dfgat + +# Observation things +# ------------------ +observations: +- viirs_n20_aod +- viirs_npp_aod +# - viirs_n21_aod diff --git a/parm/jcb-algorithms b/parm/jcb-algorithms index a6822d8c1..43d8ff6ba 160000 --- a/parm/jcb-algorithms +++ b/parm/jcb-algorithms @@ -1 +1 @@ -Subproject commit a6822d8c1e72f6b1bf951e378b153cb6df1faee5 +Subproject commit 43d8ff6ba14baf9402ee58d0f3351a143c21211b From 5bb63e7428280e9e876b44e14889c4c08b7b3338 Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Wed, 6 Nov 2024 15:22:16 -0500 Subject: [PATCH 06/22] Rename mtiasi to iasi (#1292) 1. Rename mtiasi to iasi in GDASApp 2. There is a companion [PR #29](https://github.com/NOAA-EMC/jcb-gdas/pull/29) in jcb-gdas - rename mtiasi to iasi - add `reduce obs space` - remove obsolete QC-flag related variables in filters This partly resolves Issue #1291 Co-authored-by: Emily Liu Co-authored-by: Cory Martin --- parm/atm/jcb-prototype_3dvar.yaml.j2 | 4 ++-- parm/atm/jcb-prototype_lgetkf.yaml.j2 | 4 ++-- parm/atm/jcb-prototype_lgetkf_observer.yaml.j2 | 4 ++-- parm/atm/jcb-prototype_lgetkf_solver.yaml.j2 | 4 ++-- .../bufr2ioda/{bufr2ioda_mtiasi.yaml => bufr2ioda_iasi.yaml} | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) rename parm/ioda/bufr2ioda/{bufr2ioda_mtiasi.yaml => bufr2ioda_iasi.yaml} (98%) diff --git a/parm/atm/jcb-prototype_3dvar.yaml.j2 b/parm/atm/jcb-prototype_3dvar.yaml.j2 index 152683110..1975c1f05 100644 --- a/parm/atm/jcb-prototype_3dvar.yaml.j2 +++ b/parm/atm/jcb-prototype_3dvar.yaml.j2 @@ -16,8 +16,8 @@ observations: - conventional_ps - gnssro # - gpsro -# - mtiasi_metop-a -# - mtiasi_metop-b +# - iasi_metop-a +# - iasi_metop-b # - ompsnp_n20 - ompsnp_npp # - ompstc_n20 diff --git a/parm/atm/jcb-prototype_lgetkf.yaml.j2 b/parm/atm/jcb-prototype_lgetkf.yaml.j2 index a1b099bb6..2ed04df3b 100644 --- a/parm/atm/jcb-prototype_lgetkf.yaml.j2 +++ b/parm/atm/jcb-prototype_lgetkf.yaml.j2 @@ -25,8 +25,8 @@ observations: - conventional_ps - gnssro # - gpsro -# - mtiasi_metop-a -# - mtiasi_metop-b +# - iasi_metop-a +# - iasi_metop-b # - ompsnp_n20 - ompsnp_npp # - ompstc_n20 diff --git a/parm/atm/jcb-prototype_lgetkf_observer.yaml.j2 b/parm/atm/jcb-prototype_lgetkf_observer.yaml.j2 index 26654b175..4b800ac8f 100644 --- a/parm/atm/jcb-prototype_lgetkf_observer.yaml.j2 +++ b/parm/atm/jcb-prototype_lgetkf_observer.yaml.j2 @@ -25,8 +25,8 @@ observations: - conventional_ps - gnssro # - gpsro -# - mtiasi_metop-a -# - mtiasi_metop-b +# - iasi_metop-a +# - iasi_metop-b # - ompsnp_n20 - ompsnp_npp # - ompstc_n20 diff --git a/parm/atm/jcb-prototype_lgetkf_solver.yaml.j2 b/parm/atm/jcb-prototype_lgetkf_solver.yaml.j2 index 677934158..b5123dde9 100644 --- a/parm/atm/jcb-prototype_lgetkf_solver.yaml.j2 +++ b/parm/atm/jcb-prototype_lgetkf_solver.yaml.j2 @@ -25,8 +25,8 @@ observations: - conventional_ps - gnssro # - gpsro -# - mtiasi_metop-a -# - mtiasi_metop-b +# - iasi_metop-a +# - iasi_metop-b # - ompsnp_n20 - ompsnp_npp # - ompstc_n20 diff --git a/parm/ioda/bufr2ioda/bufr2ioda_mtiasi.yaml b/parm/ioda/bufr2ioda/bufr2ioda_iasi.yaml similarity index 98% rename from parm/ioda/bufr2ioda/bufr2ioda_mtiasi.yaml rename to parm/ioda/bufr2ioda/bufr2ioda_iasi.yaml index c15df2b62..c11e147c6 100755 --- a/parm/ioda/bufr2ioda/bufr2ioda_mtiasi.yaml +++ b/parm/ioda/bufr2ioda/bufr2ioda_iasi.yaml @@ -94,7 +94,7 @@ observations: ioda: backend: netcdf - obsdataout: "{{ COM_OBS }}/{{ RUN }}.t{{ cyc }}z.mtiasi_$(splitvar).tm00.nc" + obsdataout: "{{ COM_OBS }}/{{ RUN }}.t{{ cyc }}z.iasi_$(splitvar).tm00.nc" dimensions: - name: Channel From 4c9b1d22314845bd8cf388cd5b9603b2cbf9ccfd Mon Sep 17 00:00:00 2001 From: Ed Givelberg Date: Wed, 6 Nov 2024 15:26:03 -0500 Subject: [PATCH 07/22] fixed a bug for empty files (#1351) Added testing for empty files and logging the number of obs. The code exits if the filtering results in no data, so no output file is created. --------- Co-authored-by: Guillaume Vernieres Co-authored-by: RussTreadon-NOAA <26926959+RussTreadon-NOAA@users.noreply.github.com> --- test/marine/CMakeLists.txt | 22 ++++--- .../b2i/b2iconverter/bufr2ioda_converter.py | 14 +++++ .../marine/b2i/b2iconverter/ioda_variables.py | 23 +++++++ .../marine/b2i/b2iconverter/ocean.py | 61 ------------------- 4 files changed, 51 insertions(+), 69 deletions(-) diff --git a/test/marine/CMakeLists.txt b/test/marine/CMakeLists.txt index 92c880a58..c75a5664b 100644 --- a/test/marine/CMakeLists.txt +++ b/test/marine/CMakeLists.txt @@ -28,6 +28,14 @@ install(FILES ${test_input} # bufr to ioda tests: ########################################################################### +set(TEST_WORKING_DIR ${PROJECT_BINARY_DIR}/test/marine) +set(MARINE_BUFR2IODA_DIR ${PROJECT_SOURCE_DIR}/ush/ioda/bufr2ioda/marine) +set(MARINE_BUFR2IODA_DIR ${MARINE_BUFR2IODA_DIR}/b2i) +set(CONFIG_DIR ${PROJECT_SOURCE_DIR}/test/marine/testinput) +set(TESTREF_DIR ${PROJECT_SOURCE_DIR}/test/marine/testref) +set(PYIODACONV_DIR "${PROJECT_SOURCE_DIR}/build/lib/python3.10/") + + # prepare a test.yaml file from test.yaml.in by replacing # placeholder patterns __BUFRINPUTDIR__ and __IODAOUTPUTDIR__ and __OCEANBASIN__ # with actual directory paths @@ -46,14 +54,6 @@ function(CREATE_CONFIG_FILE endfunction() -set(TEST_WORKING_DIR ${PROJECT_BINARY_DIR}/test/marine) - -set(MARINE_BUFR2IODA_DIR ${PROJECT_SOURCE_DIR}/ush/ioda/bufr2ioda/marine) -set(MARINE_BUFR2IODA_DIR ${MARINE_BUFR2IODA_DIR}/b2i) -set(CONFIG_DIR ${PROJECT_SOURCE_DIR}/test/marine/testinput) -set(TESTREF_DIR ${PROJECT_SOURCE_DIR}/test/marine/testref) - - function(CHECK_AND_SET_PATH PATH1 PATH2 RESULT_VAR) # Check if PATH1 exists if(EXISTS ${PATH1}) @@ -157,9 +157,15 @@ function(ADD_INSITU_TEST testname testbufr) COMMAND ${MARINE_BUFR2IODA_DIR}/${TEST}.py -c ${CONFIG_FILE} -t ${TESTREF_DIR}/${TESTREF_FILE} WORKING_DIRECTORY ${TEST_WORKING_DIR} ) + set_property( + TEST test_gdasapp_${TEST} + APPEND PROPERTY + ENVIRONMENT "PYTHONPATH=${PYIODACONV_DIR}:$ENV{PYTHONPATH}" + ) endfunction() + if (GENERATE_BUFR2IODA_TESTS) ADD_INSITU_TEST("profile_argo" "subpfl") ADD_INSITU_TEST("profile_bathy" "bathy") diff --git a/ush/ioda/bufr2ioda/marine/b2i/b2iconverter/bufr2ioda_converter.py b/ush/ioda/bufr2ioda/marine/b2i/b2iconverter/bufr2ioda_converter.py index f65d58523..e104ecce0 100755 --- a/ush/ioda/bufr2ioda/marine/b2i/b2iconverter/bufr2ioda_converter.py +++ b/ush/ioda/bufr2ioda/marine/b2i/b2iconverter/bufr2ioda_converter.py @@ -64,8 +64,22 @@ def run(self): # process query results and set ioda variables self.ioda_vars.set_from_query_result(r) + n_obs = self.ioda_vars.number_of_obs() + self.logger.debug(f"Query result has {n_obs} obs") + if (n_obs == 0): + self.logger.warning(f"No obs! Quitting.") + sys.exit(0) + self.ioda_vars.filter() + n_obs = self.ioda_vars.number_of_obs() + self.logger.debug(f"Filtered result has {n_obs} obs") + if (n_obs == 0): + self.logger.warning(f"No obs! Quitting.") + sys.exit(0) + self.logger.debug(f"Number of temperature obs = {self.ioda_vars.number_of_temp_obs()}") + self.logger.debug(f"Number of salinity obs = {self.ioda_vars.number_of_saln_obs()}") + # set seqNum, PreQC, ObsError, OceanBasin self.ioda_vars.additional_vars.construct() diff --git a/ush/ioda/bufr2ioda/marine/b2i/b2iconverter/ioda_variables.py b/ush/ioda/bufr2ioda/marine/b2i/b2iconverter/ioda_variables.py index 789fe060a..5319e0ff0 100644 --- a/ush/ioda/bufr2ioda/marine/b2i/b2iconverter/ioda_variables.py +++ b/ush/ioda/bufr2ioda/marine/b2i/b2iconverter/ioda_variables.py @@ -44,6 +44,29 @@ def set_salinity_range(self, smin, smax): self.S_min = smin self.S_max = smax + def number_of_temp_obs(self): + try: + if isinstance(self.temp, np.ma.MaskedArray): + return self.temp.count() + else: + return 0 + # except NameError: + except AttributeError: + return 0 + + def number_of_saln_obs(self): + try: + if isinstance(self.saln, np.ma.MaskedArray): + return self.saln.count() + else: + return 0 + # except NameError: + except AttributeError: + return 0 + + def number_of_obs(self): + return max(self.number_of_temp_obs(), self.number_of_saln_obs()) + def build_query(self): q = bufr.QuerySet() q.add('year', '*/YEAR') diff --git a/ush/ioda/bufr2ioda/marine/b2i/b2iconverter/ocean.py b/ush/ioda/bufr2ioda/marine/b2i/b2iconverter/ocean.py index 9e848c5f5..d2063ddcd 100755 --- a/ush/ioda/bufr2ioda/marine/b2i/b2iconverter/ocean.py +++ b/ush/ioda/bufr2ioda/marine/b2i/b2iconverter/ocean.py @@ -5,9 +5,6 @@ import numpy as np import numpy.ma as ma import math -import matplotlib.pyplot as plt -import cartopy.crs as ccrs -import cartopy.feature as cfeature import netCDF4 as nc import xarray as xr @@ -20,8 +17,6 @@ # the main method is get_station_basin which returns the ocean basin # for a list of station coordinates -# there are methods for plotting and printing the ocean basin data -# as well as printing and plotting station basin data class OceanBasin: @@ -54,32 +49,6 @@ def read_nc_file(self): print(f"An IOError occurred: {e}") sys.exit(1) - def print_basin(self): - for i in range(n1): - for j in range(n2): - print(i, j, self.__basin_array[i][j]) - - def plot_basin(self): - # Create a figure and axes with Cartopy projection - fig = plt.figure(figsize=(10, 6)) - ax = fig.add_subplot(1, 1, 1, projection=ccrs.PlateCarree()) - - # Plot the ocean basins using a colormap with 6 colors - # cmap = plt.cm.get_cmap('rainbow', 6) # Choose a colormap with 6 colors - cmap = plt.get_cmap('viridis', 6) # Create a colormap with 6 discrete colors - im = ax.pcolormesh(self.__longitudes, self.__latitudes, self.__basin_array, cmap='viridis', shading='auto', transform=ccrs.PlateCarree()) - - # Add colorbar - cbar = fig.colorbar(im, ax=ax, orientation='vertical', pad=0.05, ticks=np.arange(0, 6)) - cbar.set_label('Ocean Basin', fontsize=12) - # Add title and gridlines - ax.set_title('Ocean Basin Map', fontsize=16) - ax.coastlines() - ax.gridlines(draw_labels=True) - # Show the plot - plt.show() - plt.savefig('ocean_basin.png', dpi=300) - # input: 2 vectors of station coordinates # output: a vector of station ocean basin values def get_station_basin(self, lat, lon): @@ -99,33 +68,3 @@ def get_station_basin(self, lat, lon): i2 = round((lon[i] - lon0) / dlon) ocean_basin.append(self.__basin_array[i1][i2]) return ocean_basin - - def print_station_basin(self, lon, lat, file_path): - ocean_basin = self.get_station_basin(lat, lon) - with open(file_path, 'w') as file: - # Iterate over lon, lat, and ocean_basin arrays simultaneously - for lat_val, lon_val, basin_val in zip(lat, lon, ocean_basin): - file.write(f"{lat_val} {lon_val} {basin_val}\n") - - def plot_stations(self, lon, lat, png_file): - ocean_basin = self.get_station_basin(lon, lat) - - # Initialize the plot - plt.figure(figsize=(12, 8)) - # Create a Cartopy map with PlateCarree projection (latitude/longitude) - ax = plt.axes(projection=ccrs.PlateCarree()) - # Add coastlines and borders - ax.coastlines() - ax.add_feature(cartopy.feature.BORDERS, linestyle=':', linewidth=0.5) - - # Scatter plot with colored dots for each basin type - colors = ['blue', 'green', 'red', 'cyan', 'magenta', 'yellow'] - for basin_type in range(6): - indices = np.where(ocean_basin == basin_type)[0] - ax.scatter(lon[indices], lat[indices], color=colors[basin_type], label=f'Basin {basin_type}', alpha=0.7) - - # Add a legend - plt.legend(loc='lower left') - # Add title and show plot - plt.title('Ocean Basins Plot using Cartopy') - plt.savefig(png_file, dpi=300) From 1df1ad2c8770bbd2018daddede9da8ffebd043b9 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Thu, 7 Nov 2024 10:50:44 -0500 Subject: [PATCH 08/22] Quick indentation bugfix (#1363) Title says it all --- parm/atm/jcb-prototype_3dvar.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/atm/jcb-prototype_3dvar.yaml.j2 b/parm/atm/jcb-prototype_3dvar.yaml.j2 index 1975c1f05..4330a87bd 100644 --- a/parm/atm/jcb-prototype_3dvar.yaml.j2 +++ b/parm/atm/jcb-prototype_3dvar.yaml.j2 @@ -29,4 +29,4 @@ observations: # - satwnd.viirs_npp # - scatwind_ascat_metop-a # - snowcvr - - abi_g16 +# - abi_g16 From dfc871f38fa47c525953af891eafbd237c3b33a8 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA <58948505+AndrewEichmann-NOAA@users.noreply.github.com> Date: Thu, 7 Nov 2024 10:55:44 -0500 Subject: [PATCH 09/22] Completes LETKF task, adds WCDA CI test (#1287) Completes LETKF task, adds WCDA CI test Mutually dependent on https://github.com/NOAA-EMC/global-workflow/pull/2944 --------- Co-authored-by: Cory Martin Co-authored-by: Guillaume Vernieres --- mains/gdas.cc | 2 +- parm/soca/ensda/stage_ens_mem.yaml.j2 | 7 ++- parm/soca/letkf/letkf.yaml.j2 | 29 ++++++------ parm/soca/letkf/letkf_save.yaml.j2 | 20 +++++++++ parm/soca/letkf/letkf_stage.yaml.j2 | 29 +++++++++--- parm/soca/obs/config/adt_rads_all.yaml | 7 +++ parm/soca/obs/config/icec_amsr2_north.yaml | 7 +++ parm/soca/obs/config/icec_amsr2_south.yaml | 7 +++ parm/soca/obs/config/insitu_profile_argo.yaml | 7 +++ parm/soca/obs/config/sst_abi_g16_l3c.yaml | 7 +++ parm/soca/obs/config/sst_abi_g17_l3c.yaml | 7 +++ parm/soca/obs/config/sst_ahi_h08_l3c.yaml | 7 +++ parm/soca/obs/config/sst_avhrr_ma_l3u.yaml | 7 +++ parm/soca/obs/config/sst_avhrr_mb_l3u.yaml | 7 +++ parm/soca/obs/config/sst_avhrr_mc_l3u.yaml | 7 +++ parm/soca/obs/config/sst_viirs_n20_l3u.yaml | 7 +++ parm/soca/obs/config/sst_viirs_npp_l3u.yaml | 7 +++ parm/soca/soca_ens_bkg_stage.yaml.j2 | 6 +-- parm/soca/soca_fix_stage_025.yaml.j2 | 1 - parm/soca/soca_fix_stage_100.yaml.j2 | 1 - parm/soca/soca_fix_stage_500.yaml.j2 | 2 +- test/gw-ci/CMakeLists.txt | 45 +++++++++++++++++-- ush/soca/marine_recenter.py | 2 - ush/soca/prep_ocean_obs.py | 4 +- 24 files changed, 195 insertions(+), 37 deletions(-) create mode 100644 parm/soca/letkf/letkf_save.yaml.j2 diff --git a/mains/gdas.cc b/mains/gdas.cc index f47b82802..05afd244c 100755 --- a/mains/gdas.cc +++ b/mains/gdas.cc @@ -67,7 +67,7 @@ int runApp(int argc, char** argv, const std::string traits, const std::string ap return std::make_unique>(); }; apps["localensembleda"] = []() { - return std::make_unique>(); + return std::make_unique>(); }; apps["variational"] = []() { return std::make_unique>(); diff --git a/parm/soca/ensda/stage_ens_mem.yaml.j2 b/parm/soca/ensda/stage_ens_mem.yaml.j2 index e30a337e7..d0dca6e18 100644 --- a/parm/soca/ensda/stage_ens_mem.yaml.j2 +++ b/parm/soca/ensda/stage_ens_mem.yaml.j2 @@ -9,8 +9,7 @@ # create working directories ###################################### mkdir: -- "{{ DATAenspert }}/ens" - + - "{{ ENSPERT_RELPATH }}/ens" ###################################### # copy ensemble background files ###################################### @@ -22,6 +21,6 @@ copy: '${YMD}':gPDY, '${HH}':gcyc, '${MEMDIR}':"mem" + '%03d' % mem} %} - - ["{{ COM_OCEAN_HISTORY_TMPL | replace_tmpl(tmpl_dict) }}/{{ GDUMP_ENS }}.ocean.t{{ gcyc }}z.inst.f006.nc", "{{ DATAenspert }}/ens/ocean.{{ mem }}.nc"] - - ["{{ COM_ICE_HISTORY_TMPL | replace_tmpl(tmpl_dict) }}/{{ GDUMP_ENS }}.ice.t{{ gcyc }}z.inst.f006.nc", "{{ DATAenspert }}/ens/ice.{{ mem }}.nc"] + - ["{{ COM_OCEAN_HISTORY_TMPL | replace_tmpl(tmpl_dict) }}/{{ GDUMP_ENS }}.ocean.t{{ gcyc }}z.inst.f006.nc", "{{ ENSPERT_RELPATH }}/ens/ocean.{{ mem }}.nc"] + - ["{{ COM_ICE_HISTORY_TMPL | replace_tmpl(tmpl_dict) }}/{{ GDUMP_ENS }}.ice.t{{ gcyc }}z.inst.f006.nc", "{{ ENSPERT_RELPATH }}/ens/ice.{{ mem }}.nc"] {% endfor %} diff --git a/parm/soca/letkf/letkf.yaml.j2 b/parm/soca/letkf/letkf.yaml.j2 index f4c4d4875..1d5e93d7f 100644 --- a/parm/soca/letkf/letkf.yaml.j2 +++ b/parm/soca/letkf/letkf.yaml.j2 @@ -1,3 +1,5 @@ +{% set gcyc = previous_cycle | strftime("%H") %} + geometry: geom_grid_file: soca_gridspec.nc mom6_input_nml: mom_input.nml @@ -11,11 +13,11 @@ background: members from template: template: date: '{{ WINDOW_MIDDLE | to_isotime }}' - ocn_filename: "{{ RUN }}.ocean.t{{ gcyc }}z.inst.f006.nc" - ice_filename: "{{ RUN }}.ice.t{{ gcyc }}z.inst.f006.nc" + ocn_filename: "ocean.%mem%.nc" + ice_filename: "ice.%mem%.nc" read_from_file: 1 - basename: ./ens/mem%mem% - state variables: [socn, tocn, ssh, uocn, vocn, cicen] + basename: {{ ENSPERT_RELPATH }}/ens/ + state variables: [socn, tocn, ssh, hocn, uocn, vocn, cicen] pattern: '%mem%' nmembers: {{ NMEM_ENS }} @@ -38,32 +40,31 @@ local ensemble DA: mult: 1.1 output: - datadir: data_output/ + datadir: letkf_output/ date: *date exp: letkf type: ens output mean prior: - datadir: data_output/ + datadir: letkf_output/ date: *date - exp: letkf + exp: letkf.mean_prior type: fc output variance prior: - datadir: data_output/ + datadir: letkf_output/ date: *date - exp: letkf + exp: letkf.var_prior type: fc output variance posterior: - datadir: data_output/ + datadir: letkf_output/ date: *date - exp: letkf + exp: letkf.var_post type: an output increment: - datadir: data_output/ + datadir: letkf_output/ date: *date exp: letkf.inc - type: an - + type: ens diff --git a/parm/soca/letkf/letkf_save.yaml.j2 b/parm/soca/letkf/letkf_save.yaml.j2 new file mode 100644 index 000000000..194cf222e --- /dev/null +++ b/parm/soca/letkf/letkf_save.yaml.j2 @@ -0,0 +1,20 @@ +{% set PDY = current_cycle | to_YMD %} +{% set cyc = current_cycle | strftime("%H") %} +{% set timestr = WINDOW_BEGIN | to_isotime %} +###################################### +# save letkf analysis to comout +###################################### + +copy: +{% for mem in range(1, NMEM_ENS + 1) %} + {% set tmpl_dict = {'${ROTDIR}':ROTDIR, + '${RUN}': GDUMP_ENS, + '${YMD}':PDY, + '${HH}':cyc, + '${MEMDIR}':"mem" + '%03d' % mem} %} + {% set COMOUT_OCEAN_LETKF_MEM = COM_OCEAN_LETKF_TMPL | replace_tmpl(tmpl_dict) %} + {% set COMOUT_ICE_LETKF_MEM = COM_ICE_LETKF_TMPL | replace_tmpl(tmpl_dict) %} + + - ["{{ DATA }}/letkf_output/ocn.letkf.ens.{{ mem }}.{{ timestr }}.PT3H.nc", "{{ COMOUT_OCEAN_LETKF_MEM }}/{{ GDUMP_ENS }}.ocean.t{{ cyc }}z.analysis.nc"] + - ["{{ DATA }}/letkf_output/ice.letkf.ens.{{ mem }}.{{ timestr }}.PT3H.nc", "{{ COMOUT_ICE_LETKF_MEM }}/{{ GDUMP_ENS }}.ice.t{{ cyc }}z.analysis.nc"] +{% endfor %} diff --git a/parm/soca/letkf/letkf_stage.yaml.j2 b/parm/soca/letkf/letkf_stage.yaml.j2 index 019e1ba37..233e45eb9 100644 --- a/parm/soca/letkf/letkf_stage.yaml.j2 +++ b/parm/soca/letkf/letkf_stage.yaml.j2 @@ -1,11 +1,30 @@ -###################################### -# create working directories +{% set PDY = current_cycle | to_YMD %} +{% set cyc = current_cycle | strftime("%H") %} +{% set gcyc = previous_cycle | strftime("%H") %} ###################################### mkdir: -- "{{ DATA }}/Data" +- "{{ DATA }}/letkf_output" - "{{ DATA }}/obs" -copy: +- "{{ DATA }}/diags" +- "{{ COMOUT_OCEAN_LETKF }}" +- "{{ COMOUT_ICE_LETKF }}" +###################################### +# make comout directories ###################################### -# copy mom input template +{% for mem in range(1, NMEM_ENS + 1) %} + {% set tmpl_dict = {'${ROTDIR}':ROTDIR, + '${RUN}':GDUMP_ENS, + '${YMD}':PDY, + '${HH}':cyc, + '${MEMDIR}':"mem" + '%03d' % mem} %} +- "{{ COM_OCEAN_LETKF_TMPL | replace_tmpl(tmpl_dict) }}" +- "{{ COM_ICE_LETKF_TMPL | replace_tmpl(tmpl_dict) }}" +{% endfor %} ###################################### +# copy mom input template and det bkg +###################################### +copy: - ["{{ PARMgfs }}/gdas/soca/fms/input.nml", "{{ DATA }}/mom_input.nml.tmpl"] +- ["{{ PARMgfs }}/gdas/soca/fields_metadata.yaml", "{{ DATA }}/fields_metadata.yaml"] +- ["{{ COMIN_OCEAN_HISTORY_PREV }}/gdas.ocean.t{{ gcyc }}z.inst.f009.nc", "{{ DATA }}/INPUT/MOM.res.nc"] +- ["{{ COMIN_ICE_HISTORY_PREV }}/gdas.ice.t{{ gcyc }}z.inst.f009.nc", "{{ DATA }}/INPUT/cice.res.nc"] diff --git a/parm/soca/obs/config/adt_rads_all.yaml b/parm/soca/obs/config/adt_rads_all.yaml index 177e58a8f..e5b6dee26 100644 --- a/parm/soca/obs/config/adt_rads_all.yaml +++ b/parm/soca/obs/config/adt_rads_all.yaml @@ -17,6 +17,13 @@ obs operator: name: ADT obs error: covariance model: diagonal +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 obs filters: - filter: Domain Check where: diff --git a/parm/soca/obs/config/icec_amsr2_north.yaml b/parm/soca/obs/config/icec_amsr2_north.yaml index 1b1509671..bd454ae88 100644 --- a/parm/soca/obs/config/icec_amsr2_north.yaml +++ b/parm/soca/obs/config/icec_amsr2_north.yaml @@ -43,3 +43,10 @@ obs filters: where: - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/icec_amsr2_south.yaml b/parm/soca/obs/config/icec_amsr2_south.yaml index 5373b1bee..eedf6800d 100644 --- a/parm/soca/obs/config/icec_amsr2_south.yaml +++ b/parm/soca/obs/config/icec_amsr2_south.yaml @@ -43,3 +43,10 @@ obs filters: where: - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/insitu_profile_argo.yaml b/parm/soca/obs/config/insitu_profile_argo.yaml index b2abaab2c..e533966b8 100644 --- a/parm/soca/obs/config/insitu_profile_argo.yaml +++ b/parm/soca/obs/config/insitu_profile_argo.yaml @@ -625,3 +625,10 @@ obs filters: - ObsError/salinity coefs: - 1000.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/sst_abi_g16_l3c.yaml b/parm/soca/obs/config/sst_abi_g16_l3c.yaml index d96135409..e4bef888d 100644 --- a/parm/soca/obs/config/sst_abi_g16_l3c.yaml +++ b/parm/soca/obs/config/sst_abi_g16_l3c.yaml @@ -54,3 +54,10 @@ obs filters: - ObsError/seaSurfaceTemperature coefs: - 1.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/sst_abi_g17_l3c.yaml b/parm/soca/obs/config/sst_abi_g17_l3c.yaml index 8843da412..c34f5f777 100644 --- a/parm/soca/obs/config/sst_abi_g17_l3c.yaml +++ b/parm/soca/obs/config/sst_abi_g17_l3c.yaml @@ -54,3 +54,10 @@ obs filters: - ObsError/seaSurfaceTemperature coefs: - 1.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/sst_ahi_h08_l3c.yaml b/parm/soca/obs/config/sst_ahi_h08_l3c.yaml index d1842320c..0bf07bab1 100644 --- a/parm/soca/obs/config/sst_ahi_h08_l3c.yaml +++ b/parm/soca/obs/config/sst_ahi_h08_l3c.yaml @@ -54,3 +54,10 @@ obs filters: - ObsError/seaSurfaceTemperature coefs: - 1.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/sst_avhrr_ma_l3u.yaml b/parm/soca/obs/config/sst_avhrr_ma_l3u.yaml index 71f5947a7..1223c1f7f 100644 --- a/parm/soca/obs/config/sst_avhrr_ma_l3u.yaml +++ b/parm/soca/obs/config/sst_avhrr_ma_l3u.yaml @@ -54,3 +54,10 @@ obs filters: - ObsError/seaSurfaceTemperature coefs: - 1.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/sst_avhrr_mb_l3u.yaml b/parm/soca/obs/config/sst_avhrr_mb_l3u.yaml index 090b47ae6..92cde48fe 100644 --- a/parm/soca/obs/config/sst_avhrr_mb_l3u.yaml +++ b/parm/soca/obs/config/sst_avhrr_mb_l3u.yaml @@ -54,3 +54,10 @@ obs filters: - ObsError/seaSurfaceTemperature coefs: - 1.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/sst_avhrr_mc_l3u.yaml b/parm/soca/obs/config/sst_avhrr_mc_l3u.yaml index ab03d548f..fa706616c 100644 --- a/parm/soca/obs/config/sst_avhrr_mc_l3u.yaml +++ b/parm/soca/obs/config/sst_avhrr_mc_l3u.yaml @@ -54,3 +54,10 @@ obs filters: - ObsError/seaSurfaceTemperature coefs: - 1.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/sst_viirs_n20_l3u.yaml b/parm/soca/obs/config/sst_viirs_n20_l3u.yaml index 5941d746c..e78f0f77a 100644 --- a/parm/soca/obs/config/sst_viirs_n20_l3u.yaml +++ b/parm/soca/obs/config/sst_viirs_n20_l3u.yaml @@ -54,3 +54,10 @@ obs filters: - ObsError/seaSurfaceTemperature coefs: - 1.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/sst_viirs_npp_l3u.yaml b/parm/soca/obs/config/sst_viirs_npp_l3u.yaml index 1d0e447ed..6fd0e47e3 100644 --- a/parm/soca/obs/config/sst_viirs_npp_l3u.yaml +++ b/parm/soca/obs/config/sst_viirs_npp_l3u.yaml @@ -54,3 +54,10 @@ obs filters: - ObsError/seaSurfaceTemperature coefs: - 1.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/soca_ens_bkg_stage.yaml.j2 b/parm/soca/soca_ens_bkg_stage.yaml.j2 index a74853b3b..b6f1416fb 100644 --- a/parm/soca/soca_ens_bkg_stage.yaml.j2 +++ b/parm/soca/soca_ens_bkg_stage.yaml.j2 @@ -18,7 +18,7 @@ copy: # define variables # Declare a dict of search and replace terms to GDUMP on each template {% set tmpl_dict = {'ROTDIR':ROTDIR, - 'RUN': RUN, + 'RUN': GDUMP_ENS, 'YMD':gPDY, 'HH':gcyc, 'MEMDIR':"mem" + '%03d' % mem} %} @@ -35,6 +35,6 @@ copy: {% set com_prev_ocn.COMIN_OCEAN_HISTORY_MEM = com_prev_ocn.COMIN_OCEAN_HISTORY_MEM.replace(search_term, replace_term) %} {% set com_prev_ice.COMIN_ICE_HISTORY_MEM = com_prev_ice.COMIN_ICE_HISTORY_MEM.replace(search_term, replace_term) %} {% endfor %} - - ["{{ com_prev_ocn.COMIN_OCEAN_HISTORY_MEM }}/{{ RUN }}.ocean.t{{ gcyc }}z.inst.f006.nc", "{{ DATAens }}/ens/mem{{ '%03d' % mem }}/{{ RUN }}.ocean.t{{ gcyc }}z.inst.f006.nc"] - - ["{{ com_prev_ice.COMIN_ICE_HISTORY_MEM }}/{{ RUN }}.ice.t{{ gcyc }}z.inst.f006.nc", "{{ DATAens }}/ens/mem{{ '%03d' % mem }}/{{ RUN }}.ice.t{{ gcyc }}z.inst.f006.nc"] + - ["{{ com_prev_ocn.COMIN_OCEAN_HISTORY_MEM }}/{{ GDUMP_ENS }}.ocean.t{{ gcyc }}z.inst.f006.nc", "{{ DATAens }}/ens/mem{{ '%03d' % mem }}/{{ GDUMP_ENS }}.ocean.t{{ gcyc }}z.inst.f006.nc"] + - ["{{ com_prev_ice.COMIN_ICE_HISTORY_MEM }}/{{ GDUMP_ENS }}.ice.t{{ gcyc }}z.inst.f006.nc", "{{ DATAens }}/ens/mem{{ '%03d' % mem }}/{{ GDUMP_ENS }}.ice.t{{ gcyc }}z.inst.f006.nc"] {% endfor %} diff --git a/parm/soca/soca_fix_stage_025.yaml.j2 b/parm/soca/soca_fix_stage_025.yaml.j2 index 2c41dcad4..f7b334e7d 100644 --- a/parm/soca/soca_fix_stage_025.yaml.j2 +++ b/parm/soca/soca_fix_stage_025.yaml.j2 @@ -1,4 +1,3 @@ -# TODO(AFE): make resolution dependent mkdir: - "{{ DATA }}/INPUT" ###################################### diff --git a/parm/soca/soca_fix_stage_100.yaml.j2 b/parm/soca/soca_fix_stage_100.yaml.j2 index 0869f7d34..e2f4137a2 100644 --- a/parm/soca/soca_fix_stage_100.yaml.j2 +++ b/parm/soca/soca_fix_stage_100.yaml.j2 @@ -1,4 +1,3 @@ -# TODO(AFE): make resolution dependent mkdir: - "{{ DATA }}/INPUT" ###################################### diff --git a/parm/soca/soca_fix_stage_500.yaml.j2 b/parm/soca/soca_fix_stage_500.yaml.j2 index 9f030a81f..6d6930e0b 100644 --- a/parm/soca/soca_fix_stage_500.yaml.j2 +++ b/parm/soca/soca_fix_stage_500.yaml.j2 @@ -1,4 +1,3 @@ -# TODO(AFE): make resolution dependent mkdir: - "{{ DATA }}/INPUT" ###################################### @@ -9,6 +8,7 @@ copy: - ["{{ SOCA_INPUT_FIX_DIR }}/field_table", "{{ DATA }}/field_table"] - ["{{ SOCA_INPUT_FIX_DIR }}/diag_table", "{{ DATA }}/diag_table"] - ["{{ SOCA_INPUT_FIX_DIR }}/MOM_input", "{{ DATA }}/MOM_input"] +- ["{{ SOCA_INPUT_FIX_DIR }}/obsop_name_map.yaml", "{{ DATA }}/obsop_name_map.yaml"] - ["{{ SOCA_INPUT_FIX_DIR }}/INPUT/grid_spec.nc", "{{ DATA }}/INPUT/grid_spec.nc"] - ["{{ SOCA_INPUT_FIX_DIR }}/INPUT/hycom1_25.nc", "{{ DATA }}/INPUT/hycom1_25.nc"] - ["{{ SOCA_INPUT_FIX_DIR }}/INPUT/layer_coord25.nc", "{{ DATA }}/INPUT/layer_coord25.nc"] diff --git a/test/gw-ci/CMakeLists.txt b/test/gw-ci/CMakeLists.txt index 6c9faedf1..4f1a969e3 100644 --- a/test/gw-ci/CMakeLists.txt +++ b/test/gw-ci/CMakeLists.txt @@ -19,18 +19,39 @@ function(add_cycling_tests pslot YAML_PATH HOMEgfs RUNTESTS PROJECT_SOURCE_DIR T list(GET DATES_LIST 1 FULL_CYCLE) # stage IC's - message(STATUS "staging the 1/2 cycle IC's for ${test_name} ctest") + message(STATUS "staging the 1/2 cycle IC's for ${pslot} ctest") add_test(NAME ${test_name}_gdas_stage_ic_${HALF_CYCLE} COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/run_exp.sh ${pslot} gdas_stage_ic ${HALF_CYCLE}" WORKING_DIRECTORY ${RUNTESTS}) set_tests_properties(${test_name}_gdas_stage_ic_${HALF_CYCLE} PROPERTIES LABELS "manual") + # stage ensemble ics + if (letkf) + message(STATUS "preparing enkfgdas_stage_ic for ${pslot} ctest") + add_test(NAME ${test_name}_enkfgdas_stage_ic_${HALF_CYCLE} + COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/run_exp.sh ${pslot} enkfgdas_stage_ic ${HALF_CYCLE}" + WORKING_DIRECTORY ${RUNTESTS}) + set_tests_properties(${test_name}_enkfgdas_stage_ic_${HALF_CYCLE} PROPERTIES LABELS "manual") + endif() + # 1/2 cycle gdas_fcst message(STATUS "preparing 1/2 cycle gdas_fcst for ${pslot} ctest") - add_test(NAME ${test_name}_gdas_fcst_${HALF_CYCLE} + add_test(NAME ${test_name}_gdas_fcst_seg0_${HALF_CYCLE} COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/run_exp.sh ${pslot} gdas_fcst_seg0 ${HALF_CYCLE}" WORKING_DIRECTORY ${RUNTESTS}) - set_tests_properties(${test_name}_gdas_fcst_${HALF_CYCLE} PROPERTIES LABELS "manual") + set_tests_properties(${test_name}_gdas_fcst_seg0_${HALF_CYCLE} PROPERTIES LABELS "manual") + + # 1/2 cycle enkfgdas_fcst + if (letkf) + set(ENS_MEMS "mem001" "mem002" "mem003") + foreach(ENS_MEM ${ENS_MEMS}) + message(STATUS "preparing 1/2 cycle enkfgdas_fcst_${ENS_MEM} for ${pslot} ctest") + add_test(NAME ${test_name}_enkfgdas_fcst_${ENS_MEM}_${HALF_CYCLE} + COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/run_exp.sh ${pslot} enkfgdas_fcst_${ENS_MEM} ${HALF_CYCLE}" + WORKING_DIRECTORY ${RUNTESTS}) + set_tests_properties(${test_name}_enkfgdas_fcst_${ENS_MEM}_${HALF_CYCLE} PROPERTIES LABELS "manual") + endforeach() + endif() # Select the list of tasks to run for the full cycle message(STATUS "Tasks ${TASK_LIST}") @@ -62,6 +83,24 @@ if (WORKFLOW_TESTS) "gdas_marineanlfinal" ) add_cycling_tests(${pslot} ${YAML_PATH} ${HOMEgfs} ${RUNTESTS} ${PROJECT_SOURCE_DIR} "${TASK_LIST}") + + # WCDA, low-res, ensemble da + # ------------- + set(pslot "WCDA-hyb-C48mx500") + set(letkf TRUE) + set(YAML_PATH ${HOMEgfs}/ci/cases/pr/C48mx500_hybAOWCDA.yaml) + set(TASK_LIST + "gdas_prepoceanobs" + "gdas_marineanlletkf" + # TODO(AFE) waiting until these are working for hybrid + # "gdas_marinebmat" + # "gdas_marineanlinit" + # "gdas_marineanlvar" + # "gdas_marineanlchkpt" + # "gdas_marineanlfinal" + ) + add_cycling_tests(${pslot} ${YAML_PATH} ${HOMEgfs} ${RUNTESTS} ${PROJECT_SOURCE_DIR} "${TASK_LIST}") + set(letkf FALSE) endif() option(RUN_GW_CI "Enable the global-workflow CI tests" OFF) diff --git a/ush/soca/marine_recenter.py b/ush/soca/marine_recenter.py index 5572e2c76..a3814db85 100644 --- a/ush/soca/marine_recenter.py +++ b/ush/soca/marine_recenter.py @@ -68,8 +68,6 @@ def __init__(self, config: Dict) -> None: 'ATM_WINDOW_MIDDLE': window_middle_iso, 'DATA': DATA, 'dump': self.task_config.RUN, - 'fv3jedi_stage_files': self.task_config.FV3JEDI_STAGE_YAML, - 'fv3jedi_stage': self.task_config.FV3JEDI_STAGE_YAML, 'stage_dir': DATA, 'soca_input_fix_dir': self.task_config.SOCA_INPUT_FIX_DIR, 'NMEM_ENS': self.task_config.NMEM_ENS, diff --git a/ush/soca/prep_ocean_obs.py b/ush/soca/prep_ocean_obs.py index 1d6b55e8e..be80bfcc5 100644 --- a/ush/soca/prep_ocean_obs.py +++ b/ush/soca/prep_ocean_obs.py @@ -252,8 +252,8 @@ def finalize(self): for obsspace_to_save in obsspaces_to_save['observations']: - output_file = obsspace_to_save['output file'] - conv_config_file = obsspace_to_save['conversion config file'] + output_file = os.path.basename(obsspace_to_save['output file']) + conv_config_file = os.path.basename(obsspace_to_save['conversion config file']) output_file_dest = os.path.join(COMOUT_OBS, output_file) conv_config_file_dest = os.path.join(COMOUT_OBS, conv_config_file) From 6bc27606f8490cca5faec88413aecf2fd8f8603b Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA <58948505+AndrewEichmann-NOAA@users.noreply.github.com> Date: Tue, 12 Nov 2024 19:17:54 -0500 Subject: [PATCH 10/22] Changes prepoceanobs to using newly refactored BUFR converters (#1352) Just like the title. This mainly changes the converters used to the new ones that use yaml files, the templates for which are included, along with a little script that does most of the work of converting the earlier JSON templates to yaml. Also removes from `ocean.py` some methods for plotting, but can't go into global-workflow because of lack of the necessary python modules in that environment. --- .../bufr2ioda_insitu_profile_argo.yaml | 11 ++++++ .../bufr2ioda_insitu_profile_bathy.yaml | 11 ++++++ .../bufr2ioda_insitu_profile_glider.yaml | 11 ++++++ .../bufr2ioda_insitu_profile_tesac.yaml | 11 ++++++ .../bufr2ioda_insitu_profile_xbtctd.yaml | 11 ++++++ .../bufr2ioda_insitu_surface_trkob.yaml | 11 ++++++ parm/ioda/bufr2ioda/j2y.py | 24 ++++++++++++ parm/soca/obs/obs_list.yaml | 10 ++--- .../marine/b2i/b2iconverter/ioda_variables.py | 1 - ush/soca/prep_ocean_obs.py | 38 ++++++++++++------- ush/soca/prep_ocean_obs_utils.py | 7 ++-- 11 files changed, 122 insertions(+), 24 deletions(-) create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_argo.yaml create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_bathy.yaml create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_glider.yaml create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_tesac.yaml create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_xbtctd.yaml create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.yaml create mode 100644 parm/ioda/bufr2ioda/j2y.py diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_argo.yaml b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_argo.yaml new file mode 100644 index 000000000..7dd28f21c --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_argo.yaml @@ -0,0 +1,11 @@ +cycle_datetime: '{{ current_cycle | to_YMDH }}' +cycle_type: '{{ RUN }}' +data_description: 6-hrly in situ ARGO profiles +data_format: subpfl +data_provider: U.S. NOAA +data_type: argo +dump_directory: '{{ DMPDIR }}' +ioda_directory: '{{ COM_OBS }}' +source: NCEP data tank +subsets: SUBPFL +ocean_basin: '{{ OCEAN_BASIN_FILE }}' diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_bathy.yaml b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_bathy.yaml new file mode 100644 index 000000000..f26d341ea --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_bathy.yaml @@ -0,0 +1,11 @@ +cycle_datetime: '{{ current_cycle | to_YMDH }}' +cycle_type: '{{ RUN }}' +data_description: 6-hrly in situ Bathythermal profiles +data_format: bathy +data_provider: U.S. NOAA +data_type: bathy +dump_directory: '{{ DMPDIR }}' +ioda_directory: '{{ COM_OBS }}' +source: NCEP data tank +subsets: BATHY +ocean_basin: '{{ OCEAN_BASIN_FILE }}' diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_glider.yaml b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_glider.yaml new file mode 100644 index 000000000..cc732df94 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_glider.yaml @@ -0,0 +1,11 @@ +cycle_datetime: '{{ current_cycle | to_YMDH }}' +cycle_type: '{{ RUN }}' +data_description: 6-hrly in situ GLIDER profiles +data_format: subpfl +data_provider: U.S. NOAA +data_type: glider +dump_directory: '{{ DMPDIR }}' +ioda_directory: '{{ COM_OBS }}' +source: NCEP data tank +subsets: SUBPFL +ocean_basin: '{{ OCEAN_BASIN_FILE }}' diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_tesac.yaml b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_tesac.yaml new file mode 100644 index 000000000..01dc441ca --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_tesac.yaml @@ -0,0 +1,11 @@ +cycle_datetime: '{{ current_cycle | to_YMDH }}' +cycle_type: '{{ RUN }}' +data_description: 6-hrly in situ TESAC profiles +data_format: tesac +data_provider: U.S. NOAA +data_type: tesac +dump_directory: '{{ DMPDIR }}' +ioda_directory: '{{ COM_OBS }}' +source: NCEP data tank +subsets: TESAC +ocean_basin: '{{ OCEAN_BASIN_FILE }}' diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_xbtctd.yaml b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_xbtctd.yaml new file mode 100644 index 000000000..49d7f13d2 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_xbtctd.yaml @@ -0,0 +1,11 @@ +cycle_datetime: '{{ current_cycle | to_YMDH }}' +cycle_type: '{{ RUN }}' +data_description: 6-hrly in situ XBT/XCTD profiles +data_format: xbtctd +data_provider: U.S. NOAA +data_type: xbtctd +dump_directory: '{{ DMPDIR }}' +ioda_directory: '{{ COM_OBS }}' +source: NCEP data tank +subsets: XBTCTD +ocean_basin: '{{ OCEAN_BASIN_FILE }}' diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.yaml b/parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.yaml new file mode 100644 index 000000000..1385920c1 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.yaml @@ -0,0 +1,11 @@ +cycle_datetime: '{{ current_cycle | to_YMDH }}' +cycle_type: '{{ RUN }}' +data_description: 6-hrly in situ TRACKOB surface +data_format: trkob +data_provider: U.S. NOAA +data_type: trackob +dump_directory: '{{ DMPDIR }}' +ioda_directory: '{{ COM_OBS }}' +source: NCEP data tank +subsets: TRACKOB +ocean_basin: '{{ OCEAN_BASIN_FILE }}' diff --git a/parm/ioda/bufr2ioda/j2y.py b/parm/ioda/bufr2ioda/j2y.py new file mode 100644 index 000000000..c8b158517 --- /dev/null +++ b/parm/ioda/bufr2ioda/j2y.py @@ -0,0 +1,24 @@ +import json +import yaml +import argparse + +def convert_json_to_yaml(input_file, output_file): + # Load the JSON data from the input file + with open(input_file, 'r') as json_file: + json_data = json.load(json_file) + + # Convert and save as YAML in the output file + with open(output_file, 'w') as yaml_file: + yaml.dump(json_data, yaml_file, default_flow_style=False) + +if __name__ == '__main__': + # Set up argument parser + parser = argparse.ArgumentParser(description='Convert JSON to YAML.') + parser.add_argument('input_file', help='Path to the input JSON file') + parser.add_argument('output_file', help='Path to the output YAML file') + + args = parser.parse_args() + + # Perform the conversion + convert_json_to_yaml(args.input_file, args.output_file) + diff --git a/parm/soca/obs/obs_list.yaml b/parm/soca/obs/obs_list.yaml index c11dc1ace..0ac8ab5af 100644 --- a/parm/soca/obs/obs_list.yaml +++ b/parm/soca/obs/obs_list.yaml @@ -25,15 +25,15 @@ observers: #- !INC ${MARINE_OBS_YAML_DIR}/icec_ssmis_f17_l2.yaml # in situ: monthly -#- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_bathy.yaml +- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_bathy.yaml - !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_argo.yaml -#- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_glider.yaml -#- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_tesac.yaml +- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_glider.yaml +- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_tesac.yaml #- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_tesac_salinity.yaml #- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_marinemammal.yaml -#- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_xbtctd.yaml +- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_xbtctd.yaml #- !INC ${MARINE_OBS_YAML_DIR}/insitu_surface_altkob.yaml -#- !INC ${MARINE_OBS_YAML_DIR}/insitu_surface_trkob.yaml +- !INC ${MARINE_OBS_YAML_DIR}/insitu_surface_trkob.yaml #- !INC ${MARINE_OBS_YAML_DIR}/insitu_surface_trkob_salinity.yaml # in situ: daily diff --git a/ush/ioda/bufr2ioda/marine/b2i/b2iconverter/ioda_variables.py b/ush/ioda/bufr2ioda/marine/b2i/b2iconverter/ioda_variables.py index 5319e0ff0..83a07771e 100644 --- a/ush/ioda/bufr2ioda/marine/b2i/b2iconverter/ioda_variables.py +++ b/ush/ioda/bufr2ioda/marine/b2i/b2iconverter/ioda_variables.py @@ -1,6 +1,5 @@ import numpy as np from pyiodaconv import bufr -from .ocean import OceanBasin from .util import * from .ioda_metadata import IODAMetadata from .ioda_addl_vars import IODAAdditionalVariables diff --git a/ush/soca/prep_ocean_obs.py b/ush/soca/prep_ocean_obs.py index be80bfcc5..da7b2da6a 100644 --- a/ush/soca/prep_ocean_obs.py +++ b/ush/soca/prep_ocean_obs.py @@ -10,6 +10,7 @@ from wxflow import (chdir, FileHandler, logit, + parse_j2yaml, save_as_yaml, Task, YAMLFile) @@ -74,6 +75,7 @@ def initialize(self): SOCA_INPUT_FIX_DIR = self.task_config['SOCA_INPUT_FIX_DIR'] ocean_mask_src = os.path.join(SOCA_INPUT_FIX_DIR, 'RECCAP2_region_masks_all_v20221025.nc') ocean_mask_dest = os.path.join(self.task_config.DATA, 'RECCAP2_region_masks_all_v20221025.nc') + self.task_config['OCEAN_BASIN_FILE'] = ocean_mask_dest try: FileHandler({'copy': [[ocean_mask_src, ocean_mask_dest]]}).sync() @@ -90,11 +92,15 @@ def initialize(self): logger.critical(f"OBSPREP_YAML file {OBSPREP_YAML} does not exist") raise FileNotFoundError - JSON_TMPL_DIR = self.task_config.JSON_TMPL_DIR - BUFR2IODA_PY_DIR = self.task_config.BUFR2IODA_PY_DIR + # TODO (AFE): this should be in the task config file in g-w + BUFR2IODA_TMPL_DIR = os.path.join(self.task_config.HOMEgfs, 'parm/gdas/ioda/bufr2ioda') + # TODO (AFE): this should be in the task config file in g-w, and reaches into GDASApp + # in order to avoid touching the g-w until we know this will remain a task + BUFR2IODA_PY_DIR = os.path.join(self.task_config.HOMEgfs, 'sorc/gdas.cd/ush/ioda/bufr2ioda/marine/b2i') COMIN_OBS = self.task_config.COMIN_OBS COMOUT_OBS = self.task_config['COMOUT_OBS'] + OCEAN_BASIN_FILE = self.task_config['OCEAN_BASIN_FILE'] if not os.path.exists(COMOUT_OBS): os.makedirs(COMOUT_OBS) @@ -146,32 +152,34 @@ def initialize(self): obsprep_space['window end'] = self.window_end ioda_filename = f"{RUN}.t{cyc:02d}z.{obs_space_name}.{cdatestr}.nc4" obsprep_space['output file'] = ioda_filename + ioda_config_file = obtype + '2ioda.yaml' # set up the config file for conversion to IODA for bufr and # netcdf files respectively if obsprep_space['type'] == 'bufr': - gen_bufr_json_config = {'RUN': RUN, - 'current_cycle': cdate, - 'DMPDIR': COMIN_OBS, - 'COM_OBS': COMIN_OBS} - json_config_file = os.path.join(COMIN_OBS, - f"{obtype}_{cdatestr}.json") - obsprep_space['conversion config file'] = json_config_file + bufrconv_config = { + 'RUN': RUN, + 'current_cycle': cdate, + 'DMPDIR': COMIN_OBS, + 'COM_OBS': COMIN_OBS, + 'OCEAN_BASIN_FILE': OCEAN_BASIN_FILE} + obsprep_space['conversion config file'] = ioda_config_file bufr2iodapy = BUFR2IODA_PY_DIR + '/bufr2ioda_' + obtype + '.py' obsprep_space['bufr2ioda converter'] = bufr2iodapy - tmpl_filename = 'bufr2ioda_' + obtype + '.json' - template = os.path.join(JSON_TMPL_DIR, tmpl_filename) + tmpl_filename = 'bufr2ioda_' + obtype + '.yaml' + bufrconv_template = os.path.join(BUFR2IODA_TMPL_DIR, tmpl_filename) + try: - gen_bufr_json(gen_bufr_json_config, template, json_config_file) + bufrconv = parse_j2yaml(bufrconv_template, bufrconv_config) + bufrconv.save(ioda_config_file) except Exception as e: - logger.warning(f"An exeception {e} occured while trying to run gen_bufr_json") + logger.warning(f"An exeception {e} occured while trying to create BUFR2IODA config") logger.warning(f"obtype {obtype} will be skipped") break # go to next observer in OBS_YAML obsspaces_to_convert.append({"obs space": obsprep_space}) elif obsprep_space['type'] == 'nc': - ioda_config_file = obtype + '2ioda.yaml' obsprep_space['conversion config file'] = ioda_config_file save_as_yaml(obsprep_space, ioda_config_file) @@ -260,6 +268,8 @@ def finalize(self): try: FileHandler({'copy': [[output_file, output_file_dest]]}).sync() FileHandler({'copy': [[conv_config_file, conv_config_file_dest]]}).sync() + except Exception as e: + logger.warning(f"An exeception {e} occured while trying to run gen_bufr_json") except OSError: logger.warning(f"Obs file not found, possible IODA converter failure)") continue diff --git a/ush/soca/prep_ocean_obs_utils.py b/ush/soca/prep_ocean_obs_utils.py index 11b18fd37..9ecb06464 100755 --- a/ush/soca/prep_ocean_obs_utils.py +++ b/ush/soca/prep_ocean_obs_utils.py @@ -68,13 +68,12 @@ def run_netcdf_to_ioda(obsspace_to_convert, OCNOBS2IODAEXEC): def run_bufr_to_ioda(obsspace_to_convert): logger.info(f"running run_bufr_to_ioda on {obsspace_to_convert['name']}") - json_output_file = obsspace_to_convert['conversion config file'] + bufrconv_yaml = obsspace_to_convert['conversion config file'] bufr2iodapy = obsspace_to_convert['bufr2ioda converter'] try: - subprocess.run(['python', bufr2iodapy, '-c', json_output_file, '-v'], check=True) - logger.info(f"ran ioda converter on obs space {obsspace_to_convert['name']} successfully") + subprocess.run(['python', bufr2iodapy, '-c', bufrconv_yaml], check=True) return 0 except subprocess.CalledProcessError as e: - logger.warning(f"bufr2ioda converter failed with error {e}, \ + logger.warning(f"bufr2ioda converter failed with error >{e}<, \ return code {e.returncode}") return e.returncode From 1c5cc2e66a3294babddb0ac974d4420ad60e9add Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Wed, 13 Nov 2024 13:02:30 -0500 Subject: [PATCH 11/22] Fix for ensmean option in gdas.x (#1369) When I added `ensmean` as an option, I didn't do it correctly... this PR should allow it to actually work. --- mains/gdas.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mains/gdas.cc b/mains/gdas.cc index 05afd244c..10d826bd9 100755 --- a/mains/gdas.cc +++ b/mains/gdas.cc @@ -56,7 +56,7 @@ int runApp(int argc, char** argv, const std::string traits, const std::string ap apps["converttostructuredgrid"] = []() { return std::make_unique>(); - }; + }; apps["convertstate"] = []() { return std::make_unique>(); }; @@ -106,6 +106,7 @@ int main(int argc, char ** argv) { const std::set validApps = { "converttostructuredgrid", "convertstate", + "ensmean", "hofx4d", "localensembleda", "variational" From 5fb52d368aeda52b205e8323b4d2db7ff3082e70 Mon Sep 17 00:00:00 2001 From: Ed Givelberg Date: Thu, 14 Nov 2024 07:09:24 -0500 Subject: [PATCH 12/22] fixed python version in cmake (#1374) cmake now detects the python version; previously hard-coded. This came up in #1362 --- test/marine/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/marine/CMakeLists.txt b/test/marine/CMakeLists.txt index c75a5664b..46f26cb86 100644 --- a/test/marine/CMakeLists.txt +++ b/test/marine/CMakeLists.txt @@ -28,12 +28,16 @@ install(FILES ${test_input} # bufr to ioda tests: ########################################################################### +find_package(Python REQUIRED) +# Extract the major and minor version (e.g., "3.10" from "3.10.13") +string(REGEX REPLACE "^([0-9]+\\.[0-9]+).*" "\\1" PYTHON_MAJOR_MINOR ${Python_VERSION}) +set(PYIODACONV_DIR "${PROJECT_SOURCE_DIR}/build/lib/python${PYTHON_MAJOR_MINOR}/") + set(TEST_WORKING_DIR ${PROJECT_BINARY_DIR}/test/marine) set(MARINE_BUFR2IODA_DIR ${PROJECT_SOURCE_DIR}/ush/ioda/bufr2ioda/marine) set(MARINE_BUFR2IODA_DIR ${MARINE_BUFR2IODA_DIR}/b2i) set(CONFIG_DIR ${PROJECT_SOURCE_DIR}/test/marine/testinput) set(TESTREF_DIR ${PROJECT_SOURCE_DIR}/test/marine/testref) -set(PYIODACONV_DIR "${PROJECT_SOURCE_DIR}/build/lib/python3.10/") # prepare a test.yaml file from test.yaml.in by replacing From e514b926561bfc8fa3de741876505aff74255c95 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA <26926959+RussTreadon-NOAA@users.noreply.github.com> Date: Fri, 15 Nov 2024 14:56:27 -0500 Subject: [PATCH 13/22] Update JEDI hashes to include Model Variable Renaming Sprint (#1355) --- ci/driver.sh | 139 ++++++---- ci/gw_driver.sh | 144 ++++++---- ci/run_ci.sh | 4 +- ci/run_gw_ci.sh | 4 +- parm/io/fv3jedi_fieldmetadata_fv3inc.yaml | 2 +- parm/io/fv3jedi_fieldmetadata_history.yaml | 12 +- parm/io/fv3jedi_fieldmetadata_restart.yaml | 2 +- parm/jcb-gdas | 2 +- parm/soca/fields_metadata.yaml | 256 +++++++++++++----- parm/soca/letkf/letkf.yaml.j2 | 2 +- parm/soca/marine-jcb-base.yaml | 2 +- .../soca/obs/config/insitu_profile_bathy.yaml | 7 + .../soca/obs/config/insitu_profile_tesac.yaml | 7 + .../soca/obs/config/insitu_surface_trkob.yaml | 7 + parm/soca/obs/obs_list.yaml | 10 +- sorc/fv3-jedi | 2 +- sorc/ioda | 2 +- sorc/iodaconv | 2 +- sorc/oops | 2 +- sorc/saber | 2 +- sorc/soca | 2 +- sorc/ufo | 2 +- sorc/vader | 2 +- .../testinput/gdasapp_fv3jedi_fv3inc.yaml | 2 +- utils/soca/gdas_soca_diagb.h | 21 +- 25 files changed, 432 insertions(+), 207 deletions(-) diff --git a/ci/driver.sh b/ci/driver.sh index ce6372c59..933b9223b 100755 --- a/ci/driver.sh +++ b/ci/driver.sh @@ -1,6 +1,9 @@ #!/bin/bash --login +echo "Start at $(date)" + my_dir="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )" +echo "Set my_dir ${my_dir}" # ============================================================================== usage() { @@ -51,6 +54,14 @@ esac cd $GDAS_CI_ROOT/repo CI_LABEL="${GDAS_CI_HOST}-RT" gh pr list --label "$CI_LABEL" --state "open" | awk '{print $1;}' > $GDAS_CI_ROOT/open_pr_list + +open_pr=`cat $GDAS_CI_ROOT/open_pr_list | wc -l` +if (( $open_pr == 0 )); then + echo "No open PRs with ${CI_LABEL}, exit." + echo "Finish at $(date)" + exit +fi + open_pr_list=$(cat $GDAS_CI_ROOT/open_pr_list) # ============================================================================== @@ -58,72 +69,86 @@ open_pr_list=$(cat $GDAS_CI_ROOT/open_pr_list) repo_url="https://github.com/NOAA-EMC/GDASApp.git" # loop through all open PRs for pr in $open_pr_list; do - gh pr edit $pr --remove-label $CI_LABEL --add-label ${CI_LABEL}-Running - echo "Processing Pull Request #${pr}" + echo " " + echo "Start processing Pull Request #${pr} at $(date)" # get the branch name used for the PR gdasapp_branch=$(gh pr view $pr --json headRefName -q ".headRefName") - - # get the fork information - pr_details=$(gh pr view $pr --repo ${repo_url} --json headRepository,headRepositoryOwner,headRefName) - # extract the necessary info - fork_owner=$(gh pr view $pr --repo ${repo_url} --json headRepositoryOwner --jq '.headRepositoryOwner.login') - fork_name=$(gh pr view $pr --repo ${repo_url} --json headRepository --jq '.headRepository.name') - - # construct the fork URL - gdasapp_url="https://github.com/$fork_owner/${fork_name}.git" + # get additional branch information + branch_owner=$(gh pr view $pr --repo ${repo_url} --json headRepositoryOwner --jq '.headRepositoryOwner.login') + branch_name=$(gh pr view $pr --repo ${repo_url} --json headRepository --jq '.headRepository.name') + pr_assignees=$(gh pr view $pr --repo ${repo_url} --json assignees --jq '.assignees[].login') + + # check if any assignee is authorized to run CI + authorized_by="" + for str in ${pr_assignees[@]}; do + grep $str /scratch1/NCEPDEV/da/role.jedipara/CI/GDASApp/authorized_users + rc=$? + if (( rc == 0 )); then + authorized_by=${str} + echo "FOUND MATCH $str, rc $rc" + break + fi + done + + # Authorized to run CI + if (( rc == 0 )); then + echo "Run CI" + + # update PR label + gh pr edit $pr --remove-label $CI_LABEL --add-label ${CI_LABEL}-Running + + # construct the fork URL + gdasapp_url="https://github.com/$branch_owner/${branch_name}.git" - echo "Fork URL: $gdasapp_url" - echo "Branch Name: $gdasapp_branch" - - # create PR specific directory - if [ -d $GDAS_CI_ROOT/PR/$pr ]; then - rm -rf $GDAS_CI_ROOT/PR/$pr - fi - mkdir -p $GDAS_CI_ROOT/PR/$pr - cd $GDAS_CI_ROOT/PR/$pr - - # clone copy of repo - git clone --recursive --jobs 8 --branch $gdasapp_branch $gdasapp_url - cd GDASApp - - # checkout pull request - git pull - gh pr checkout $pr - git submodule update --init --recursive - - # get commit hash - commit=$(git log --pretty=format:'%h' -n 1) - echo "$commit" > $GDAS_CI_ROOT/PR/$pr/commit - - # load modules - case ${TARGET} in - hera | orion) - echo "Loading modules on $TARGET" - module purge - module use $GDAS_CI_ROOT/PR/$pr/GDASApp/modulefiles - module load GDAS/$TARGET - module list - ;; - *) - echo "Unsupported platform. Exiting with error." - exit 1 - ;; - esac - - # run build and testing command - $my_dir/run_ci.sh -d $GDAS_CI_ROOT/PR/$pr/GDASApp -o $GDAS_CI_ROOT/PR/$pr/output_${commit} - ci_status=$? - gh pr comment $pr --repo ${repo_url} --body-file $GDAS_CI_ROOT/PR/$pr/output_${commit} - if [ $ci_status -eq 0 ]; then - gh pr edit $pr --repo ${repo_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Passed + echo "GDASApp URL: $gdasapp_url" + echo "GDASApp branch Name: $gdasapp_branch" + echo "CI authorized by $authorized_by at $(date)" + + # create PR specific directory + if [ -d $GDAS_CI_ROOT/PR/$pr ]; then + rm -rf $GDAS_CI_ROOT/PR/$pr + fi + mkdir -p $GDAS_CI_ROOT/PR/$pr + cd $GDAS_CI_ROOT/PR/$pr + pwd + + # clone copy of repo + git clone --recursive --jobs 8 --branch $gdasapp_branch $gdasapp_url + cd GDASApp + pwd + + # checkout GDASApp pull request + git pull + gh pr checkout $pr + git submodule update --init --recursive + + # get commit hash + commit=$(git log --pretty=format:'%h' -n 1) + echo "$commit" > $GDAS_CI_ROOT/PR/$pr/commit + + # run build and testing command + echo "Execute $my_dir/run_ci.sh for $GDAS_CI_ROOT/PR/$pr/GDASApp at $(date)" + $my_dir/run_ci.sh -d $GDAS_CI_ROOT/PR/$pr/GDASApp -o $GDAS_CI_ROOT/PR/$pr/output_${commit} + ci_status=$? + echo "After run_ci.sh with ci_status ${ci_status} at $(date)" + gh pr comment $pr --repo ${repo_url} --body-file $GDAS_CI_ROOT/PR/$pr/output_${commit} + if [ $ci_status -eq 0 ]; then + gh pr edit $pr --repo ${repo_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Passed + else + gh pr edit $pr --repo ${repo_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Failed + fi + + # Not authorized to run CI else - gh pr edit $pr --repo ${repo_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Failed + echo "Do NOT run CI" fi + + echo "Finish processing Pull Request #{pr} at $(date)" done # ============================================================================== # scrub working directory for older files find $GDAS_CI_ROOT/PR/* -maxdepth 1 -mtime +3 -exec rm -rf {} \; - +echo "Finish at $(date)" diff --git a/ci/gw_driver.sh b/ci/gw_driver.sh index e85684f84..c40ff4026 100755 --- a/ci/gw_driver.sh +++ b/ci/gw_driver.sh @@ -1,6 +1,9 @@ #!/bin/bash --login +echo "Start at $(date)" + my_dir="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )" +echo "Set my_dir ${my_dir}" # ============================================================================== usage() { @@ -32,7 +35,7 @@ done case ${TARGET} in hera | orion) - echo "Running automated testing with workflow on $TARGET" + echo "Running Automated GW Testing on $TARGET" source $MODULESHOME/init/sh source $my_dir/${TARGET}.sh module purge @@ -51,69 +54,116 @@ esac cd $GDAS_CI_ROOT/repo CI_LABEL="${GDAS_CI_HOST}-GW-RT" gh pr list --label "$CI_LABEL" --state "open" | awk '{print $1;}' > $GDAS_CI_ROOT/open_pr_list_gw + +open_pr=`cat $GDAS_CI_ROOT/open_pr_list_gw | wc -l` +if (( $open_pr == 0 )); then + echo "No open PRs with ${CI_LABEL}, exit." + echo "Finish at $(date)" + exit +fi + open_pr_list=$(cat $GDAS_CI_ROOT/open_pr_list_gw) # ============================================================================== # clone, checkout, build, test, etc. repo_url="https://github.com/NOAA-EMC/GDASApp.git" workflow_url="https://github.com/NOAA-EMC/global-workflow.git" +workflow_branch="develop" # loop through all open PRs for pr in $open_pr_list; do - gh pr edit $pr --remove-label $CI_LABEL --add-label ${CI_LABEL}-Running - echo "Processing Pull Request #${pr}" + echo " " + echo "Start processing Pull Request #${pr} at $(date)" # get the branch name used for the PR gdasapp_branch=$(gh pr view $pr --json headRefName -q ".headRefName") - # check for a companion PR in the global-workflow - companion_pr_exists=$(gh pr list --repo ${workflow_url} --head ${gdasapp_branch} --state open) - if [ -n "$companion_pr_exists" ]; then - # get the PR number - companion_pr=$(echo "$companion_pr_exists" | awk '{print $1;}') - - # extract the necessary info - fork_owner=$(gh pr view $companion_pr --repo $workflow_url --json headRepositoryOwner --jq '.headRepositoryOwner.login') - fork_name=$(gh pr view $companion_pr --repo $workflow_url --json headRepository --jq '.headRepository.name') - - # Construct the fork URL - workflow_url="https://github.com/$fork_owner/$fork_name.git" - - echo "Fork URL: $workflow_url" - echo "Branch Name: $gdasapp_branch" - fi - - # create PR specific directory - if [ -d $GDAS_CI_ROOT/workflow/PR/$pr ]; then - rm -rf $GDAS_CI_ROOT/workflow/PR/$pr - fi - mkdir -p $GDAS_CI_ROOT/workflow/PR/$pr - cd $GDAS_CI_ROOT/workflow/PR/$pr + # get additional branch information + branch_owner=$(gh pr view $pr --repo ${repo_url} --json headRepositoryOwner --jq '.headRepositoryOwner.login') + branch_name=$(gh pr view $pr --repo ${repo_url} --json headRepository --jq '.headRepository.name') + pr_assignees=$(gh pr view $pr --repo ${repo_url} --json assignees --jq '.assignees[].login') + + # check if any assignee is authorized to run CI + authorized_by="" + for str in ${pr_assignees[@]}; do + grep $str /scratch1/NCEPDEV/da/role.jedipara/CI/GDASApp/authorized_users + rc=$? + if (( rc == 0 )); then + authorized_by=${str} + echo "FOUND MATCH $str, rc $rc" + break + fi + done + + # Authorized to run CI + if (( rc == 0 )); then + echo "Run CI" + + # update PR label + gh pr edit $pr --remove-label $CI_LABEL --add-label ${CI_LABEL}-Running + + # check for a companion PR in the global-workflow + companion_pr_exists=$(gh pr list --repo ${workflow_url} --head ${gdasapp_branch} --state open) + if [ -n "$companion_pr_exists" ]; then + # get the PR number + companion_pr=$(echo "$companion_pr_exists" | awk '{print $1;}') + + # extract the necessary info + branch_owner=$(gh pr view $companion_pr --repo $workflow_url --json headRepositoryOwner --jq '.headRepositoryOwner.login') + branch_name=$(gh pr view $companion_pr --repo $workflow_url --json headRepository --jq '.headRepository.name') + + # Construct fork URL. Update workflow branch name + workflow_url="https://github.com/$branch_owner/$branch_name.git" + workflow_branch=$gdasapp_branch + + fi + + echo "Workflow URL: $workflow_url" + echo "Workflow branch name: $workflow_branch" + echo "GDASApp branch name: $gdasapp_branch" + echo "CI authorized by $authorized_by at $(date)" + + # create PR specific directory + if [ -d $GDAS_CI_ROOT/workflow/PR/$pr ]; then + rm -rf $GDAS_CI_ROOT/workflow/PR/$pr + fi + mkdir -p $GDAS_CI_ROOT/workflow/PR/$pr + cd $GDAS_CI_ROOT/workflow/PR/$pr + pwd - # clone global workflow develop branch - git clone --recursive --jobs 8 --branch dev/gdasapp $workflow_url - - # checkout pull request - cd $GDAS_CI_ROOT/workflow/PR/$pr/global-workflow/sorc/gdas.cd - git checkout develop - git pull - gh pr checkout $pr - git submodule update --init --recursive - - # get commit hash - commit=$(git log --pretty=format:'%h' -n 1) - echo "$commit" > $GDAS_CI_ROOT/workflow/PR/$pr/commit - - $my_dir/run_gw_ci.sh -d $GDAS_CI_ROOT/workflow/PR/$pr/global-workflow -o $GDAS_CI_ROOT/workflow/PR/$pr/output_${commit} - ci_status=$? - gh pr comment $pr --body-file $GDAS_CI_ROOT/workflow/PR/$pr/output_${commit} - if [ $ci_status -eq 0 ]; then - gh pr edit $pr --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Passed + # clone global workflow develop branch + git clone --recursive --jobs 8 --branch $workflow_branch $workflow_url + + # checkout GDASApp pull request + cd $GDAS_CI_ROOT/workflow/PR/$pr/global-workflow/sorc/gdas.cd + git pull + gh pr checkout $pr + git submodule update --init --recursive + + # get commit hash + commit=$(git log --pretty=format:'%h' -n 1) + echo "$commit" > $GDAS_CI_ROOT/workflow/PR/$pr/commit + + # run build and testing command + echo "Execute $my_dir/run_gw_ci.sh for $GDAS_CI_ROOT/PR/workflow/PR/$pr/global-workflow at $(date)" + $my_dir/run_gw_ci.sh -d $GDAS_CI_ROOT/workflow/PR/$pr/global-workflow -o $GDAS_CI_ROOT/workflow/PR/$pr/output_${commit} + ci_status=$? + echo "After run_gw_ci.sh with ci_status ${ci_status} at $(date)" + gh pr comment $pr --body-file $GDAS_CI_ROOT/workflow/PR/$pr/output_${commit} + if [ $ci_status -eq 0 ]; then + gh pr edit $pr --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Passed + else + gh pr edit $pr --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Failed + fi + + # Not authorized to run CI else - gh pr edit $pr --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Failed + echo "Do NOT run CI" fi + + echo "Finish processing Pull Request #{pr} at $(date)" done # ============================================================================== # scrub working directory for older files find $GDAS_CI_ROOT/workflow/PR/* -maxdepth 1 -mtime +3 -exec rm -rf {} \; - +echo "Finish at $(date)" diff --git a/ci/run_ci.sh b/ci/run_ci.sh index b62f78a88..1b5f27231 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -1,5 +1,5 @@ #!/bin/bash -#set -eu +set -u # ============================================================================== usage() { @@ -61,7 +61,7 @@ module use $GDAS_MODULE_USE module load GDAS/$TARGET echo "---------------------------------------------------" >> $outfile rm -rf log.ctest -ctest -E "manual" -R gdasapp --output-on-failure &>> log.ctest +ctest -R gdasapp --output-on-failure &>> log.ctest ctest_status=$? npassed=$(cat log.ctest | grep "tests passed") if [ $ctest_status -eq 0 ]; then diff --git a/ci/run_gw_ci.sh b/ci/run_gw_ci.sh index 59758e37f..ba1874107 100755 --- a/ci/run_gw_ci.sh +++ b/ci/run_gw_ci.sh @@ -1,5 +1,5 @@ #!/bin/bash -#set -eu +set -u # ============================================================================== usage() { @@ -31,7 +31,7 @@ done # ============================================================================== # start output file -echo "Automated Global-Workflow GDASApp Testing Results:" > $outfile +echo "Automated GW GDASApp Testing Results:" > $outfile echo "Machine: ${TARGET}" >> $outfile echo '```' >> $outfile echo "Start: $(date) on $(hostname)" >> $outfile diff --git a/parm/io/fv3jedi_fieldmetadata_fv3inc.yaml b/parm/io/fv3jedi_fieldmetadata_fv3inc.yaml index 2f8acb839..4750967b1 100644 --- a/parm/io/fv3jedi_fieldmetadata_fv3inc.yaml +++ b/parm/io/fv3jedi_fieldmetadata_fv3inc.yaml @@ -9,7 +9,7 @@ field metadata: - long name: air_temperature io name: T_inc -- long name: specific_humidity +- long name: water_vapor_mixing_ratio_wrt_moist_air io name: sphum_inc - long name: cloud_liquid_water diff --git a/parm/io/fv3jedi_fieldmetadata_history.yaml b/parm/io/fv3jedi_fieldmetadata_history.yaml index a8532c32c..2e59dccb1 100644 --- a/parm/io/fv3jedi_fieldmetadata_history.yaml +++ b/parm/io/fv3jedi_fieldmetadata_history.yaml @@ -6,7 +6,7 @@ field metadata: - long name: northward_wind io name: vgrd -- long name: specific_humidity +- long name: water_vapor_mixing_ratio_wrt_moist_air io name: spfh - long name: air_temperature @@ -45,11 +45,11 @@ field metadata: - long name: upward_air_velocity io name: dzdt -- long name: surface_pressure +- long name: air_pressure_at_surface io name: pressfc io file: atmosphere -- long name: surface_geopotential_height +- long name: geopotential_height_at_surface io name: hgtsfc - long name: u_component_of_native_D_grid_wind @@ -74,7 +74,7 @@ field metadata: - long name: sheleg io name: weasd -- long name: sea_surface_temperature +- long name: skin_temperature_at_surface io name: tmpsfc - long name: stype @@ -86,8 +86,8 @@ field metadata: - long name: totalSnowDepthMeters io name: snod -- long name: surface_eastward_wind +- long name: eastward_wind_at_surface io name: ugrd_hyblev1 -- long name: surface_northward_wind +- long name: northward_wind_at_surface io name: vgrd_hyblev1 diff --git a/parm/io/fv3jedi_fieldmetadata_restart.yaml b/parm/io/fv3jedi_fieldmetadata_restart.yaml index d4a4a3a09..ccba447dc 100644 --- a/parm/io/fv3jedi_fieldmetadata_restart.yaml +++ b/parm/io/fv3jedi_fieldmetadata_restart.yaml @@ -10,7 +10,7 @@ field metadata: io name: T - long name: air_pressure_thickness - io name: DELP + io name: delp - long name: layer_thickness io name: DZ diff --git a/parm/jcb-gdas b/parm/jcb-gdas index b8e995a4c..c41e7d7ac 160000 --- a/parm/jcb-gdas +++ b/parm/jcb-gdas @@ -1 +1 @@ -Subproject commit b8e995a4cbf01fa4a662c3da3e7d818f8457ec4e +Subproject commit c41e7d7aca3f4053b177709adaa66488f3643980 diff --git a/parm/soca/fields_metadata.yaml b/parm/soca/fields_metadata.yaml index 586d8557e..444bd8418 100644 --- a/parm/soca/fields_metadata.yaml +++ b/parm/soca/fields_metadata.yaml @@ -1,103 +1,150 @@ # -------------------------------------------------------------------------------------------------- # Field metadata for SOCA. Each field can contain the following information: # -# name: Internal name used by soca code and config files +# name: name used by soca and by the rest of JEDI +# name surface: JEDI variable name for 2D surface of a 3D field (Default: ) # grid: "h", "u", or "v" (Default: h) # masked: use land mask if true (Default: true) # levels: "1" or "full_ocn" (Default: 1) -# getval_name: variable name expected by GetValues (Default: ) -# getval_name_surface: GetValues variable name for 2D surface of a 3D field (Default: ) -# io_file: The restart file domain "ocn", "sfc", or "ice" (Default: ) -# io_name: The variable name used in the restart IO (Default: ) +# io file: The restart file domain "ocn", "sfc", or "ice" (Default: ) +# io name: The variable name used in the restart IO (Default: ) +# constant value: Used for "dummy" fields. Sets the entire field to the given constant globally +# This parameter cannot be used with io_file/io_name +# fill value: If the field is masked, this value will be used for the masked areas. +# (Default: 0.0) +# categories: Number of categories for a field with a category dimension (Default: -1) +# If > 0, then the fields "name", and "io name" can use the +# placeholder "" which will be replaced with the category number # -------------------------------------------------------------------------------------------------- # -------------------------------------------------------------------------------------------------- # Ocean state variables # -------------------------------------------------------------------------------------------------- -- name: tocn +- name: sea_water_potential_temperature + name surface: sea_surface_temperature levels: full_ocn - getval name: sea_water_potential_temperature - getval name surface: sea_surface_temperature io file: ocn io name: Temp - fill value: 0.0 -- name: socn +- name: sea_water_salinity + name surface: sea_surface_salinity levels: full_ocn - getval name: sea_water_salinity - getval name surface: sea_surface_salinity io file: ocn io name: Salt property: positive_definite - fill value: 0.0 -- name: uocn +- name: eastward_sea_water_velocity + name surface: surface_eastward_sea_water_velocity grid: u levels: full_ocn - getval name: eastward_sea_water_velocity - getval name surface: surface_eastward_sea_water_velocity io file: ocn io name: u - fill value: 0.0 -- name: vocn +- name: northward_sea_water_velocity + name surface: surface_northward_sea_water_velocity grid: v levels: full_ocn - getval name: northward_sea_water_velocity - getval name surface: surface_northward_sea_water_velocity io file: ocn io name: v - fill value: 0.0 -- name: hocn +- name: sea_water_cell_thickness levels: full_ocn - getval name: sea_water_cell_thickness io file: ocn io name: h - fill value: 0.001 vert interp: false -- name: ssh - getval name: sea_surface_height_above_geoid +- name: sea_surface_height_above_geoid io file: ocn io name: ave_ssh - fill value: 0.0 - name: mom6_mld io file: ocn io name: MLD fill value: 0.0 + +# -------------------------------------------------------------------------------------------------- +# ice state variables with no categories +# -------------------------------------------------------------------------------------------------- +- name: sea_ice_thickness + io file: ice + io name: hi_h #note, was hicen + property: positive_definite + +- name: sea_ice_area_fraction + io file: ice + io name: aice_h #note, was aicen + +- name: sea_ice_snow_thickness + io file: ice + io name: hs_h #note, was hsnon + property: positive_definite + +- name: snow_ice_surface_temperature + io file: ice + io name: Tsfc_h + +- name: air_temperature + io file: ice + io name: Tair_h + +- name: bulk_ice_salinity + io file: ice + io name: sice_h # -------------------------------------------------------------------------------------------------- -# ice state variables +# ice state variables with category dimension # -------------------------------------------------------------------------------------------------- -- name: hicen - getval name: sea_ice_category_thickness +- name: sea_ice_category_area_fraction + categories: 5 io file: ice - io name: hi_h + io sup name: aicen_h + io name: aice_h property: positive_definite - fill value: 0.0 -- name: cicen - getval name: sea_ice_category_area_fraction - getval name surface: sea_ice_area_fraction # note: not accurate, should be "sum" not "surface" +- name: sea_ice_category_volume + categories: 5 io file: ice - io name: aice_h - fill value: 0.0 + io sup name: vicen_h + io name: vice_h + property: positive_definite -- name: hsnon - getval name: sea_ice_category_snow_thickness +- name: sea_ice_snow_category_volume + categories: 5 io file: ice - io name: hs_h + io sup name: vsnon_h + io name: vsno_h property: positive_definite - fill value: 0.0 + +# -------------------------------------------------------------------------------------------------- +# Thermodynamic ice state variables with category and level dimension +# -------------------------------------------------------------------------------------------------- +- name: sea_ice_category_temperature + categories: 5 + levels: 7 + io file: ice + io sup name: Tinz_h + io name: tiz_h + +- name: sea_ice_category_salinity + categories: 5 + levels: 7 + io file: ice + io sup name: Sinz_h + io name: siz_h + +- name: sea_ice_snow_category_temperature + categories: 5 + levels: 1 + io file: ice + io sup name: Tsnz_h + io name: tsz_h + # -------------------------------------------------------------------------------------------------- # wave state variables # -------------------------------------------------------------------------------------------------- -- name: swh - getval name: sea_surface_wave_significant_height +- name: sea_surface_wave_significant_height io file: wav io name: hs property: positive_definite @@ -105,33 +152,28 @@ # -------------------------------------------------------------------------------------------------- # sea surface variables # -------------------------------------------------------------------------------------------------- -- name: sw +- name: net_downwelling_shortwave_radiation masked: false - getval name: net_downwelling_shortwave_radiation io file: sfc io name: sw_rad -- name: lw +- name: net_downwelling_longwave_radiation masked: false - getval name: net_downwelling_longwave_radiation io file: sfc io name: lw_rad -- name: lhf +- name: upward_latent_heat_flux_in_air masked: false - getval name: upward_latent_heat_flux_in_air io file: sfc io name: latent_heat -- name: shf +- name: upward_sensible_heat_flux_in_air masked: false - getval name: upward_sensible_heat_flux_in_air io file: sfc io name: sens_heat -- name: us +- name: friction_velocity_over_water masked: false - getval name: friction_velocity_over_water io file: sfc io name: fric_vel @@ -139,18 +181,16 @@ # -------------------------------------------------------------------------------------------------- # BGC # -------------------------------------------------------------------------------------------------- -- name: chl +- name: mass_concentration_of_chlorophyll_in_sea_water + name surface: sea_surface_chlorophyll levels: full_ocn - getval name: mass_concentration_of_chlorophyll_in_sea_water - getval name surface: sea_surface_chlorophyll io file: ocn io name: chl property: positive_definite -- name: biop +- name: molar_concentration_of_biomass_in_sea_water_in_p_units + name surface: sea_surface_biomass_in_p_units levels: full_ocn - getval name: molar_concentration_of_biomass_in_sea_water_in_p_units - getval name surface: sea_surface_biomass_in_p_units io file: ocn io name: biomass_p property: positive_definite @@ -161,20 +201,106 @@ - name: distance_from_coast masked: false -- name: layer_depth - levels: full_ocn - vert interp: false - - name: mesoscale_representation_error -- name: mld +- name: ocean_mixed_layer_thickness - name: sea_floor_depth_below_sea_surface - name: sea_area_fraction masked: false -- name: surface_temperature_where_sea +- name: skin_temperature_at_surface_where_sea - name: sea_water_depth levels: full_ocn + +- name: latitude +- name: longitude + +# -------------------------------------------------------------------------------------------------- +# variables that VADER should be responsible for +# -------------------------------------------------------------------------------------------------- +- name: sea_water_temperature + levels: full_ocn + +# -------------------------------------------------------------------------------------------------- +- name: dummy_atm1 + constant value: 5.0 + +- name: ozone_thickness + constant value: 275 #The average amount of ozone in the atm. is 300 Dobson Units + +- name: water_vapor #g/cm^2 + constant value: 1.2 + +- name: wind_speed_at_surface + constant value: 6 + +- name: air_pressure_at_surface + constant value: 999 + +- name: relative_humidity + constant value: 89 + +- name: cloud_liquid_water_path + constant value: 163 + +- name: cloud_area_fraction_in_atmosphere_layer + constant value: 80 + +- name: aerosol_optical_thickness + constant value: 0.16 + +- name: single_scattering_albedo + constant value: 0.71 + +- name: asymmetry_parameter + constant value: 0.97 + +#---------------------ocean bio +- name: Carbon_nitrogen_detritus_concentration + levels: full_ocn + io file: bio + io name: CDET + property: positive_definite + +- name: Particulate_inorganic_carbon + levels: full_ocn + io file: bio + io name: PIC + +- name: colored_dissolved_organic_carbon + levels: full_ocn + io file: bio + io name: CDC + +- name: diatom_concentration + levels: full_ocn + io file: bio + io name: DIATOM + +- name: chlorophyte_concentration + levels: full_ocn + io file: bio + io name: CHLORO + +- name: cyano-bacteria_concentration + levels: full_ocn + io file: bio + io name: CYANO + +- name: coccolithophore_concentration + levels: full_ocn + io file: bio + io name: COCCO + +- name: dinoflagellate_concentration + levels: full_ocn + io file: bio + io name: DINO + +- name: phaeocystis_concentration + levels: full_ocn + io file: bio + io name: PHAEO diff --git a/parm/soca/letkf/letkf.yaml.j2 b/parm/soca/letkf/letkf.yaml.j2 index 1d5e93d7f..ef60c6ec9 100644 --- a/parm/soca/letkf/letkf.yaml.j2 +++ b/parm/soca/letkf/letkf.yaml.j2 @@ -17,7 +17,7 @@ background: ice_filename: "ice.%mem%.nc" read_from_file: 1 basename: {{ ENSPERT_RELPATH }}/ens/ - state variables: [socn, tocn, ssh, hocn, uocn, vocn, cicen] + state variables: [sea_water_salinity, sea_water_potential_temperature, sea_surface_height_above_geoid, sea_water_cell_thickness, eastward_sea_water_velocity, northward_sea_water_velocity, sea_ice_area_fraction] pattern: '%mem%' nmembers: {{ NMEM_ENS }} diff --git a/parm/soca/marine-jcb-base.yaml b/parm/soca/marine-jcb-base.yaml index d07edcd8c..3a9d40223 100644 --- a/parm/soca/marine-jcb-base.yaml +++ b/parm/soca/marine-jcb-base.yaml @@ -24,7 +24,7 @@ minimizer: RPCG final_diagnostics_departures: oman final_prints_frequency: PT3H number_of_outer_loops: 1 -analysis_variables: [cicen, hicen, hsnon, socn, tocn, uocn, vocn, ssh] +analysis_variables: [sea_ice_area_fraction, sea_ice_thickness, sea_ice_snow_thickness, sea_water_salinity, sea_water_potential_temperature, eastward_sea_water_velocity, northward_sea_water_velocity, sea_surface_height_above_geoid] # Model things diff --git a/parm/soca/obs/config/insitu_profile_bathy.yaml b/parm/soca/obs/config/insitu_profile_bathy.yaml index 0dc2db0aa..d78cacdb6 100644 --- a/parm/soca/obs/config/insitu_profile_bathy.yaml +++ b/parm/soca/obs/config/insitu_profile_bathy.yaml @@ -26,3 +26,10 @@ obs filters: - ObsError/waterTemperature coefs: - 1000.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/insitu_profile_tesac.yaml b/parm/soca/obs/config/insitu_profile_tesac.yaml index 5c966f88a..b2cf1769d 100644 --- a/parm/soca/obs/config/insitu_profile_tesac.yaml +++ b/parm/soca/obs/config/insitu_profile_tesac.yaml @@ -29,3 +29,10 @@ obs filters: - ObsError/waterTemperature coefs: - 1000.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/insitu_surface_trkob.yaml b/parm/soca/obs/config/insitu_surface_trkob.yaml index 3b058e527..5846b83d6 100644 --- a/parm/soca/obs/config/insitu_surface_trkob.yaml +++ b/parm/soca/obs/config/insitu_surface_trkob.yaml @@ -27,3 +27,10 @@ obs filters: - ObsError/seaSurfaceTemperature coefs: - 1000.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/obs_list.yaml b/parm/soca/obs/obs_list.yaml index 0ac8ab5af..c11dc1ace 100644 --- a/parm/soca/obs/obs_list.yaml +++ b/parm/soca/obs/obs_list.yaml @@ -25,15 +25,15 @@ observers: #- !INC ${MARINE_OBS_YAML_DIR}/icec_ssmis_f17_l2.yaml # in situ: monthly -- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_bathy.yaml +#- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_bathy.yaml - !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_argo.yaml -- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_glider.yaml -- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_tesac.yaml +#- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_glider.yaml +#- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_tesac.yaml #- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_tesac_salinity.yaml #- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_marinemammal.yaml -- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_xbtctd.yaml +#- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_xbtctd.yaml #- !INC ${MARINE_OBS_YAML_DIR}/insitu_surface_altkob.yaml -- !INC ${MARINE_OBS_YAML_DIR}/insitu_surface_trkob.yaml +#- !INC ${MARINE_OBS_YAML_DIR}/insitu_surface_trkob.yaml #- !INC ${MARINE_OBS_YAML_DIR}/insitu_surface_trkob_salinity.yaml # in situ: daily diff --git a/sorc/fv3-jedi b/sorc/fv3-jedi index 88279a632..136dfb9c2 160000 --- a/sorc/fv3-jedi +++ b/sorc/fv3-jedi @@ -1 +1 @@ -Subproject commit 88279a63280c23d6b8974991a8c89380afaf5db7 +Subproject commit 136dfb9c2f8541584e62fa74c616d686602bcdad diff --git a/sorc/ioda b/sorc/ioda index 3fa4a997e..22cd20eae 160000 --- a/sorc/ioda +++ b/sorc/ioda @@ -1 +1 @@ -Subproject commit 3fa4a997e25b3bd018d30e308a26b3e98af0fe6f +Subproject commit 22cd20eae0685914a5b967e13f95779b57bb448c diff --git a/sorc/iodaconv b/sorc/iodaconv index 23e58ed76..6f87a0f27 160000 --- a/sorc/iodaconv +++ b/sorc/iodaconv @@ -1 +1 @@ -Subproject commit 23e58ed76da3628cbd508bd4ac40f8a01c789d7d +Subproject commit 6f87a0f279e836fd604e5b313a25bd1e54bff80e diff --git a/sorc/oops b/sorc/oops index 0d2c235d7..1ba321ff9 160000 --- a/sorc/oops +++ b/sorc/oops @@ -1 +1 @@ -Subproject commit 0d2c235d791e1ba0023ce300103174dddf71aed7 +Subproject commit 1ba321ff912c6338d7362667eff37ddbf569cb18 diff --git a/sorc/saber b/sorc/saber index 1f23a3665..de5015c83 160000 --- a/sorc/saber +++ b/sorc/saber @@ -1 +1 @@ -Subproject commit 1f23a36657f6d10b770348de0f5454e01d377105 +Subproject commit de5015c8328f5b3d64acc99739fbaa64ef571172 diff --git a/sorc/soca b/sorc/soca index 4d7ef21e7..7f2ddb61b 160000 --- a/sorc/soca +++ b/sorc/soca @@ -1 +1 @@ -Subproject commit 4d7ef21e74d78a065156c942a72806ef2e2eb08e +Subproject commit 7f2ddb61bc86796c83dfcd4801c91bffd829ffb4 diff --git a/sorc/ufo b/sorc/ufo index b0cd94558..85ef98cb9 160000 --- a/sorc/ufo +++ b/sorc/ufo @@ -1 +1 @@ -Subproject commit b0cd94558643380ccceea864abac2c34fa291677 +Subproject commit 85ef98cb99f3eae1ed15e39f0cb05046af36fef1 diff --git a/sorc/vader b/sorc/vader index 05eb007e2..3049658d1 160000 --- a/sorc/vader +++ b/sorc/vader @@ -1 +1 @@ -Subproject commit 05eb007e242af3fdc4969c7146a480e12663e452 +Subproject commit 3049658d185c8095caf0e506795d0e5995fa92cf diff --git a/test/fv3jedi/testinput/gdasapp_fv3jedi_fv3inc.yaml b/test/fv3jedi/testinput/gdasapp_fv3jedi_fv3inc.yaml index f59603d92..7ee403ccc 100644 --- a/test/fv3jedi/testinput/gdasapp_fv3jedi_fv3inc.yaml +++ b/test/fv3jedi/testinput/gdasapp_fv3jedi_fv3inc.yaml @@ -1,6 +1,6 @@ variable change: variable change name: Model2GeoVaLs - input variables: &bkgvars [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr,surface_geopotential_height] + input variables: &bkgvars [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr,hgtsfc] output variables: &fv3incrvars [ua,va,t,sphum,ice_wat,liq_wat,o3mr,delp,delz] jedi increment variables: [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr] fv3 increment variables: *fv3incrvars diff --git a/utils/soca/gdas_soca_diagb.h b/utils/soca/gdas_soca_diagb.h index f45ac8313..563d303b9 100644 --- a/utils/soca/gdas_soca_diagb.h +++ b/utils/soca/gdas_soca_diagb.h @@ -261,7 +261,7 @@ namespace gdasapp { // Get the layer thicknesses and convert to layer depth oops::Log::info() << "====================== calculate layer depth" << std::endl; - auto viewHocn = atlas::array::make_view(xbFs["hocn"]); + auto viewHocn = atlas::array::make_view(xbFs["sea_water_cell_thickness"]); atlas::array::ArrayT depth(viewHocn.shape(0), viewHocn.shape(1)); auto viewDepth = atlas::array::make_view(depth); for (atlas::idx_t jnode = 0; jnode < depth.shape(0); ++jnode) { @@ -299,7 +299,7 @@ namespace gdasapp { } // Update the layer thickness halo - nodeColumns.haloExchange(xbFs["hocn"]); + nodeColumns.haloExchange(xbFs["sea_water_cell_thickness"]); // Loop through variables for (auto & var : configD.socaVars.variables()) { @@ -307,7 +307,7 @@ namespace gdasapp { nodeColumns.haloExchange(xbFs[var]); // Skip the layer thickness variable - if (var == "hocn") { + if (var == "sea_water_cell_thickness") { continue; } oops::Log::info() << "====================== std dev for " << var << std::endl; @@ -330,7 +330,7 @@ namespace gdasapp { stdDevFilt(jnode, 0, 0, configD.depthMin, neighbors, 0, viewHocn, bkg, viewBathy, stdDevBkg, false, 4); - if (var == "ssh") { + if (var == "sea_surface_height_above_geoid") { // TODO(G): Extract the unbalanced ssh variance, in the mean time, do this: stdDevBkg(jnode, 0) = std::min(configD.sshMax, stdDevBkg(jnode, 0)); } @@ -353,7 +353,7 @@ namespace gdasapp { if (configD.simpleSmoothing) { for (auto & var : configD.socaVars.variables()) { // Skip the layer thickness variable - if (var == "hocn") { + if (var == "sea_water_cell_thickness") { continue; } @@ -365,7 +365,8 @@ namespace gdasapp { // Loops through nodes and levels for (atlas::idx_t level = 0; level < xbFs[var].shape(1); ++level) { - for (atlas::idx_t jnode = 0; jnode < xbFs["tocn"].shape(0); ++jnode) { + for (atlas::idx_t jnode = 0; + jnode < xbFs["sea_water_potential_temperature"].shape(0); ++jnode) { // Early exit if on a ghost cell if (ghostView(jnode) > 0) { continue; @@ -403,7 +404,8 @@ namespace gdasapp { auto stdDevBkg = atlas::array::make_view(bkgErrFs[var]); auto tmpArray(stdDevBkg); for (int iter = 0; iter < configD.niterVert; ++iter) { - for (atlas::idx_t jnode = 0; jnode < xbFs["tocn"].shape(0); ++jnode) { + for (atlas::idx_t jnode = 0; + jnode < xbFs["sea_water_potential_temperature"].shape(0); ++jnode) { for (atlas::idx_t level = 1; level < xbFs[var].shape(1)-1; ++level) { stdDevBkg(jnode, level) = (tmpArray(jnode, level-1) + tmpArray(jnode, level) + @@ -424,14 +426,15 @@ namespace gdasapp { << std::endl; // Create the diffusion object oops::GeometryData geometryData(geom.functionSpace(), - bkgErrFs["tocn"], true, this->getComm()); + bkgErrFs["sea_water_potential_temperature"], + true, this->getComm()); oops::Diffusion diffuse(geometryData); diffuse.calculateDerivedGeom(geometryData); // Lambda function to construct a field with a constant filtering value auto assignScale = [&](double scale, const std::string& fieldName) { atlas::Field field; - auto levels = xbFs["tocn"].shape(1); + auto levels = xbFs["sea_water_potential_temperature"].shape(1); field = geom.functionSpace().createField(atlas::option::levels(levels) | atlas::option::name(fieldName)); auto viewField = atlas::array::make_view(field); From 2bcedf26cea991fc61f640ccb5be39409b1bd931 Mon Sep 17 00:00:00 2001 From: Mindo Choi <141867620+apchoiCMD@users.noreply.github.com> Date: Fri, 15 Nov 2024 15:45:14 -0500 Subject: [PATCH 14/22] Enable the manual operation of the marine verification tool (#1373) #### This PR enables the Marine Verification Tool to run outside of the g-w CI workflow by submitting an `sbatch` job manually on Hera Includes, - Vrfy task run by a simple driver in the offline #1345 - Improve cosmetic issues we found #1349 - Bug fixes and more #1314 - ~~Move `exgdas_global_marine_analysis_vrfy.py` to `scripts/old` directory~~ Most up-to-date plots can be found at ``` /scratch1/NCEPDEV/da/Mindo.Choi/sandbox/marine_vrfy/gdas.20210827/00/analysis/ocean/vrfy_final_PR ``` The wall time is as follows: ``` [Mindo.Choi@hfe02 vrfy]$ sacct -j 2477688 --format=JobID,JobName,State,ExitCode,Elapsed JobID JobName State ExitCode Elapsed ------------ ---------- ---------- -------- ---------- 2477688 marine_vr+ COMPLETED 0:0 00:11:54 2477688.bat+ batch COMPLETED 0:0 00:11:54 2477688.ext+ extern COMPLETED 0:0 00:11:54 ``` Additional plotting work will be added by consolidating vrfy task as follows: - SST/SSH time series - Omb time series - Spatial SSH/SST/OHC - HTML (?) Close #1314 , Close #1345 , Close #1349 --------- Co-authored-by: Guillaume Vernieres --- scripts/exgdas_global_marine_analysis_vrfy.py | 0 ush/eva/marine_eva_post.py | 4 +- ush/eva/marine_gdas_plots.yaml | 6 +- ush/soca/soca_vrfy.py | 50 +++-- ...gdas_global_marine_analysis_vrfy_manual.py | 210 ++++++++++++++++++ .../run_marine_analysis_vrfy_manual.job | 45 ++++ 6 files changed, 297 insertions(+), 18 deletions(-) mode change 100755 => 100644 scripts/exgdas_global_marine_analysis_vrfy.py create mode 100644 utils/soca/fig_gallery/exgdas_global_marine_analysis_vrfy_manual.py create mode 100644 utils/soca/fig_gallery/run_marine_analysis_vrfy_manual.job diff --git a/scripts/exgdas_global_marine_analysis_vrfy.py b/scripts/exgdas_global_marine_analysis_vrfy.py old mode 100755 new mode 100644 diff --git a/ush/eva/marine_eva_post.py b/ush/eva/marine_eva_post.py index a355621a1..b537ddb3a 100755 --- a/ush/eva/marine_eva_post.py +++ b/ush/eva/marine_eva_post.py @@ -12,7 +12,9 @@ vminmax = {'seaSurfaceTemperature': {'vmin': -2.0, 'vmax': 2.0}, 'seaIceFraction': {'vmin': -0.2, 'vmax': 0.2}, 'seaSurfaceSalinity': {'vmin': -0.2, 'vmax': 0.2}, # TODO: this should be changed - 'absoluteDynamicTopography': {'vmin': -0.2, 'vmax': 0.2}} + 'absoluteDynamicTopography': {'vmin': -0.2, 'vmax': 0.2}, + 'waterTemperature': {'vmin': -2.0, 'vmax': 2.0}, + 'salinity': {'vmin': -0.2, 'vmax': 0.2}} def marine_eva_post(inputyaml, outputdir, diagdir): diff --git a/ush/eva/marine_gdas_plots.yaml b/ush/eva/marine_gdas_plots.yaml index 5bedd1f69..0a903d0c4 100644 --- a/ush/eva/marine_gdas_plots.yaml +++ b/ush/eva/marine_gdas_plots.yaml @@ -73,7 +73,7 @@ graphics: data variable: experiment::OmBQC::${variable} figure: layout: [1,1] - figure size: [11,5] + figure size: [20,10] title: 'OmB post QC | @NAME@ @CYCLE@ | ${variable_title}' output name: map_plots/@NAME@/${variable}/@CHANNELVAR@/@NAME@_${variable}@CHANNELVAR@OmBQC.png tight_layout: true @@ -94,11 +94,11 @@ graphics: data: variable: experiment::OmBQC::${variable} @CHANNELKEY@ - markersize: 1 + markersize: 0.01 label: '$(variable)' colorbar: true # below may need to be edited/removed - cmap: ${dynamic_cmap} + cmap: 'seismic' vmin: ${dynamic_vmin} vmax: ${dynamic_vmax} diff --git a/ush/soca/soca_vrfy.py b/ush/soca/soca_vrfy.py index 854d7ab69..a4060fecd 100755 --- a/ush/soca/soca_vrfy.py +++ b/ush/soca/soca_vrfy.py @@ -38,6 +38,18 @@ def plotConfig(grid_file=[], proj='set me', projs=['Global']): + # Map variable names to their units + variable_units = { + 'ave_ssh': 'meter', + 'Temp': 'deg C', + 'Salt': 'psu', + 'aice_h': 'meter', + 'hi_h': 'meter', + 'hs_h': 'meter', + 'u': 'm/s', + 'v': 'm/s' + } + """ Prepares the configuration for the plotting functions below """ @@ -64,6 +76,9 @@ def plotConfig(grid_file=[], config['variable'] = variable # the variable currently plotted config['projs'] = projs # all the projections etc. config['proj'] = proj + + # Add units to the config for each variable + config['variable_units'] = variable_units return config @@ -78,6 +93,7 @@ def plotHorizontalSlice(config): os.makedirs(dirname, exist_ok=True) variable = config['variable'] + unit = config['variable_units'].get(config['variable'], 'unknown') exp = config['exp'] PDY = config['PDY'] cyc = config['cyc'] @@ -85,12 +101,12 @@ def plotHorizontalSlice(config): if variable in ['Temp', 'Salt', 'u', 'v']: level = config['levels'][0] slice_data = np.squeeze(data[variable])[level, :, :] - label_colorbar = variable + ' Level ' + str(level) + label_colorbar = f"{variable} ({unit}) Level {level}" figname = os.path.join(dirname, variable + '_Level_' + str(level)) title = f"{exp} {PDY} {cyc} {variable} Level {level}" else: slice_data = np.squeeze(data[variable]) - label_colorbar = variable + label_colorbar = f"{variable} ({unit})" figname = os.path.join(dirname, variable + '_' + config['proj']) title = f"{exp} {PDY} {cyc} {variable}" @@ -99,17 +115,17 @@ def plotHorizontalSlice(config): fig, ax = plt.subplots(figsize=(8, 5), subplot_kw={'projection': projs[config['proj']]}) - # Plot the filled contours - contourf_plot = ax.contourf(np.squeeze(grid.lon), + # Use pcolor to plot the data + pcolor_plot = ax.pcolormesh(np.squeeze(grid.lon), np.squeeze(grid.lat), slice_data, - levels=100, vmin=bounds[0], vmax=bounds[1], transform=ccrs.PlateCarree(), - cmap=config['colormap']) + cmap=config['colormap'], + zorder=0) # Add colorbar for filled contours - cbar = fig.colorbar(contourf_plot, ax=ax, shrink=0.75, orientation='horizontal') + cbar = fig.colorbar(pcolor_plot, ax=ax, shrink=0.75, orientation='horizontal') cbar.set_label(label_colorbar) # Add contour lines with specified linewidths @@ -120,16 +136,20 @@ def plotHorizontalSlice(config): levels=contour_levels, colors='black', linewidths=0.1, - transform=ccrs.PlateCarree()) + transform=ccrs.PlateCarree(), + zorder=2) - ax.coastlines() # TODO: make this work on hpc + try: + ax.coastlines() # TODO: make this work on hpc + except Exception as e: + print(f"Warning: could not add coastlines. {e}") ax.set_title(title) if config['proj'] == 'South': ax.set_extent([-180, 180, -90, -50], ccrs.PlateCarree()) if config['proj'] == 'North': ax.set_extent([-180, 180, 50, 90], ccrs.PlateCarree()) # ax.add_feature(cartopy.feature.LAND) # TODO: make this work on hpc - plt.savefig(figname, bbox_inches='tight', dpi=600) + plt.savefig(figname, bbox_inches='tight', dpi=300) plt.close(fig) @@ -138,6 +158,7 @@ def plotZonalSlice(config): Contourf of a zonal slice of an ocean field """ variable = config['variable'] + unit = config['variable_units'].get(config['variable'], 'unknown') exp = config['exp'] PDY = config['PDY'] cyc = config['cyc'] @@ -171,7 +192,7 @@ def plotZonalSlice(config): # Add colorbar for filled contours cbar = fig.colorbar(contourf_plot, ax=ax, shrink=0.5, orientation='horizontal') - cbar.set_label(variable + ' Lat ' + str(lat)) + cbar.set_label(f"{config['variable']} ({unit}) Lat {lat}") # Set the colorbar ticks cbar.set_ticks(contour_levels) @@ -184,7 +205,7 @@ def plotZonalSlice(config): os.makedirs(dirname, exist_ok=True) figname = os.path.join(dirname, config['variable'] + 'zonal_lat_' + str(int(lat)) + '_' + str(int(config['max depth'])) + 'm') - plt.savefig(figname, bbox_inches='tight', dpi=600) + plt.savefig(figname, bbox_inches='tight', dpi=300) plt.close(fig) @@ -193,6 +214,7 @@ def plotMeridionalSlice(config): Contourf of a Meridional slice of an ocean field """ variable = config['variable'] + unit = config['variable_units'].get(config['variable'], 'unknown') exp = config['exp'] PDY = config['PDY'] cyc = config['cyc'] @@ -226,7 +248,7 @@ def plotMeridionalSlice(config): # Add colorbar for filled contours cbar = fig.colorbar(contourf_plot, ax=ax, shrink=0.5, orientation='horizontal') - cbar.set_label(variable + ' Lon ' + str(lon)) + cbar.set_label(f"{config['variable']} ({unit}) Lon {lon}") # Set the colorbar ticks cbar.set_ticks(contour_levels) @@ -239,7 +261,7 @@ def plotMeridionalSlice(config): os.makedirs(dirname, exist_ok=True) figname = os.path.join(dirname, config['variable'] + 'meridional_lon_' + str(int(lon)) + '_' + str(int(config['max depth'])) + 'm') - plt.savefig(figname, bbox_inches='tight', dpi=600) + plt.savefig(figname, bbox_inches='tight', dpi=300) plt.close(fig) diff --git a/utils/soca/fig_gallery/exgdas_global_marine_analysis_vrfy_manual.py b/utils/soca/fig_gallery/exgdas_global_marine_analysis_vrfy_manual.py new file mode 100644 index 000000000..7c8efd0a6 --- /dev/null +++ b/utils/soca/fig_gallery/exgdas_global_marine_analysis_vrfy_manual.py @@ -0,0 +1,210 @@ +import os +import numpy as np +import gen_eva_obs_yaml +import marine_eva_post +import diag_statistics +from multiprocessing import Process +from soca_vrfy import statePlotter, plotConfig +import subprocess + +comout = os.getenv('COM_OCEAN_ANALYSIS') +com_ice_history = os.getenv('COM_ICE_HISTORY_PREV') +com_ocean_history = os.getenv('COM_OCEAN_HISTORY_PREV') +cyc = os.getenv('cyc') +RUN = os.getenv('RUN') + +bcyc = str((int(cyc) - 3) % 24).zfill(2) +gcyc = str((int(cyc) - 6) % 24).zfill(2) +grid_file = os.path.join(comout, f'{RUN}.t'+bcyc+'z.ocngrid.nc') +layer_file = os.path.join(comout, f'{RUN}.t'+cyc+'z.ocninc.nc') + +# for eva +diagdir = os.path.join(comout, 'diags') +HOMEgfs = os.getenv('HOMEgfs') + +# Get flags from environment variables (set in the bash driver) +run_ensemble_analysis = os.getenv('RUN_ENSENBLE_ANALYSIS', 'OFF').upper() == 'ON' +run_bkgerr_analysis = os.getenv('RUN_BACKGROUND_ERROR_ANALYSIS', 'OFF').upper() == 'ON' +run_bkg_analysis = os.getenv('RUN_BACKGROUND_ANALYSIS', 'OFF').upper() == 'ON' +run_increment_analysis = os.getenv('RUN_INCREMENT_ANLYSIS', 'OFF').upper() == 'ON' + +# Initialize an empty list for the main config +configs = [plotConfig(grid_file=grid_file, + data_file=os.path.join(comout, f'{RUN}.t'+cyc+'z.ocnana.nc'), + variables_horiz={'ave_ssh': [-1.8, 1.3], + 'Temp': [-1.8, 34.0], + 'Salt': [32, 40]}, + colormap='nipy_spectral', + comout=os.path.join(comout, 'vrfy', 'ana')), # ocean surface analysis + plotConfig(grid_file=grid_file, + data_file=os.path.join(comout, f'{RUN}.t'+cyc+'z.iceana.nc'), + variables_horiz={'aice_h': [0.0, 1.0], + 'hi_h': [0.0, 4.0], + 'hs_h': [0.0, 0.5]}, + colormap='jet', + projs=['North', 'South', 'Global'], + comout=os.path.join(comout, 'vrfy', 'ana'))] # sea ice analysis + +# Define each config and add to main_config if its flag is True +if run_ensemble_analysis: + config_ens = [plotConfig(grid_file=grid_file, + data_file=os.path.join(comout, f'{RUN}.t{cyc}z.ocn.recentering_error.nc'), + variables_horiz={'ave_ssh': [-1, 1]}, + colormap='seismic', + comout=os.path.join(comout, 'vrfy', 'recentering_error')), # recentering error + plotConfig(grid_file=grid_file, + data_file=os.path.join(comout, f'{RUN}.t{cyc}z.ocn.ssh_steric_stddev.nc'), + variables_horiz={'ave_ssh': [0, 0.8]}, + colormap='gist_ncar', + comout=os.path.join(comout, 'vrfy', 'bkgerr', 'ssh_steric_stddev')), # ssh steric stddev + plotConfig(grid_file=grid_file, + data_file=os.path.join(comout, f'{RUN}.t{cyc}z.ocn.ssh_unbal_stddev.nc'), + variables_horiz={'ave_ssh': [0, 0.8]}, + colormap='gist_ncar', + comout=os.path.join(comout, 'vrfy', 'bkgerr', 'ssh_unbal_stddev')), # ssh unbal stddev + plotConfig(grid_file=grid_file, + data_file=os.path.join(comout, f'{RUN}.t{cyc}z.ocn.ssh_total_stddev.nc'), + variables_horiz={'ave_ssh': [0, 0.8]}, + colormap='gist_ncar', + comout=os.path.join(comout, 'vrfy', 'bkgerr', 'ssh_total_stddev')), # ssh total stddev + plotConfig(grid_file=grid_file, + data_file=os.path.join(comout, f'{RUN}.t{cyc}z.ocn.steric_explained_variance.nc'), + variables_horiz={'ave_ssh': [0, 1]}, + colormap='seismic', + comout=os.path.join(comout, 'vrfy', 'bkgerr', 'steric_explained_variance'))] # steric explained variance + configs.extend(config_ens) + +if run_bkgerr_analysis: + config_bkgerr = [plotConfig(grid_file=grid_file, + layer_file=layer_file, + data_file=os.path.join(comout, os.path.pardir, os.path.pardir, + 'bmatrix', 'ocean', f'{RUN}.t'+cyc+'z.ocean.bkgerr_stddev.nc'), + lats=np.arange(-60, 60, 10), + lons=np.arange(-280, 80, 30), + variables_zonal={'Temp': [0, 2], + 'Salt': [0, 0.2], + 'u': [0, 0.2], + 'v': [0, 0.2]}, + variables_meridional={'Temp': [0, 2], + 'Salt': [0, 0.2], + 'u': [0, 0.2], + 'v': [0, 0.2]}, + variables_horiz={'Temp': [0, 2], + 'Salt': [0, 0.2], + 'u': [0, 0.2], + 'v': [0, 0.2], + 'ave_ssh': [0, 0.1]}, + colormap='jet', + comout=os.path.join(comout, 'vrfy', 'bkgerr'))] # ocn bkgerr stddev + configs.extend(config_bkgerr) + +if run_bkg_analysis: + config_bkg = [plotConfig(grid_file=grid_file, + data_file=os.path.join(com_ice_history, f'{RUN}.ice.t{gcyc}z.inst.f006.nc'), + variables_horiz={'aice_h': [0.0, 1.0], + 'hi_h': [0.0, 4.0], + 'hs_h': [0.0, 0.5]}, + colormap='jet', + projs=['North', 'South', 'Global'], + comout=os.path.join(comout, 'vrfy', 'bkg')), # sea ice background + plotConfig(grid_file=grid_file, + layer_file=layer_file, + data_file=os.path.join(com_ocean_history, f'{RUN}.ocean.t{gcyc}z.inst.f006.nc'), + lats=np.arange(-60, 60, 10), + lons=np.arange(-280, 80, 30), + variables_zonal={'Temp': [-1.8, 34.0], + 'Salt': [32, 40]}, + variables_meridional={'Temp': [-1.8, 34.0], + 'Salt': [32, 40]}, + variables_horiz={'ave_ssh': [-1.8, 1.3], + 'Temp': [-1.8, 34.0], + 'Salt': [32, 40]}, + colormap='nipy_spectral', + comout=os.path.join(comout, 'vrfy', 'bkg'))] + configs.extend(config_bkg) + +if run_increment_analysis: + config_incr = [plotConfig(grid_file=grid_file, + layer_file=layer_file, + data_file=os.path.join(comout, f'{RUN}.t'+cyc+'z.ocninc.nc'), + lats=np.arange(-60, 60, 10), + lons=np.arange(-280, 80, 30), + variables_zonal={'Temp': [-0.5, 0.5], + 'Salt': [-0.1, 0.1]}, + variables_horiz={'Temp': [-0.5, 0.5], + 'Salt': [-0.1, 0.1], + 'ave_ssh': [-0.1, 0.1]}, + variables_meridional={'Temp': [-0.5, 0.5], + 'Salt': [-0.1, 0.1]}, + colormap='seismic', + comout=os.path.join(comout, 'vrfy', 'incr')), # ocean increment + plotConfig(grid_file=grid_file, + data_file=os.path.join(comout, f'{RUN}.t'+cyc+'z.ice.incr.nc'), + lats=np.arange(-60, 60, 10), + variables_horiz={'aice_h': [-0.2, 0.2], + 'hi_h': [-0.5, 0.5], + 'hs_h': [-0.1, 0.1]}, + colormap='seismic', + projs=['North', 'South'], + comout=os.path.join(comout, 'vrfy', 'incr'))] # sea ice increment + configs.extend(config_incr) + + +# plot marine analysis vrfy + +def plot_marine_vrfy(config): + ocnvrfyPlotter = statePlotter(config) + ocnvrfyPlotter.plot() + + +# Number of processes +num_processes = len(configs) + +# Create a list to store the processes +processes = [] + +# Iterate over configs +for config in configs[:num_processes]: + process = Process(target=plot_marine_vrfy, args=(config,)) + process.start() + processes.append(process) + +# Wait for all processes to finish +for process in processes: + process.join() + +####################################### +# eva plots +####################################### + +evadir = os.path.join(HOMEgfs, 'sorc', f'{RUN}.cd', 'ush', 'eva') +marinetemplate = os.path.join(evadir, 'marine_gdas_plots.yaml') +varyaml = os.path.join(comout, 'yaml', 'var_original.yaml') + +# it would be better to refrence the dirs explicitly with the comout path +# but eva doesn't allow for specifying output directories +os.chdir(os.path.join(comout, 'vrfy')) +if not os.path.exists('preevayamls'): + os.makedirs('preevayamls') +if not os.path.exists('evayamls'): + os.makedirs('evayamls') + +gen_eva_obs_yaml.gen_eva_obs_yaml(varyaml, marinetemplate, 'preevayamls') + +files = os.listdir('preevayamls') +for file in files: + infile = os.path.join('preevayamls', file) + marine_eva_post.marine_eva_post(infile, 'evayamls', diagdir) + +files = os.listdir('evayamls') +for file in files: + infile = os.path.join('evayamls', file) + print('running eva on', infile) + subprocess.run(['eva', infile], check=True) + +####################################### +# calculate diag statistics +####################################### + +# As of 11/12/2024 not working +# diag_statistics.get_diag_stats() diff --git a/utils/soca/fig_gallery/run_marine_analysis_vrfy_manual.job b/utils/soca/fig_gallery/run_marine_analysis_vrfy_manual.job new file mode 100644 index 000000000..38ce48ffc --- /dev/null +++ b/utils/soca/fig_gallery/run_marine_analysis_vrfy_manual.job @@ -0,0 +1,45 @@ +#!/bin/bash +#SBATCH --job-name=marine_vrfy # Assign a name to the job (customize as needed) +#SBATCH --account=da-cpu +#SBATCH --qos=debug +#SBATCH -A da-cpu +#SBATCH --output=run_marine_vrfy_analysis.out +#SBATCH --nodes=1 # Request 1 node +#SBATCH --ntasks=40 # Request 40 total tasks (processors across nodes) +#SBATCH --partition=hera # Specify the partition (cluster) named "hera" +#SBATCH --cpus-per-task=1 # Set 1 CPU per task (equivalent to ppn=40 and tpp=1) +#SBATCH --mem=24GB # Request 24GB of memory +#SBATCH --time=00:30:00 # Set the walltime limit to 30 minutes + +# Define HOMEgfs +export HOMEgfs="/scratch1/NCEPDEV/da/Mindo.Choi/workflow_11122024/global-workflow/" + +# Load EVA module +module use ${HOMEgfs}sorc/gdas.cd/modulefiles +module load EVA/hera + +# Set PYTHONPATH using HOMEgfs +export PYTHONPATH="${HOMEgfs}sorc/gdas.cd/ush/:\ +${HOMEgfs}sorc/gdas.cd/ush/eva/:\ +${HOMEgfs}sorc/gdas.cd/ush/soca/:\ +$PYTHONPATH" + +# Set flags to control plotConfig in the Python script +export RUN_ENSENBLE_ANALYSIS=OFF # Check if ensemble run is ON +export RUN_BACKGROUND_ERROR_ANALYSIS=ON +export RUN_BACKGROUND_ANALYSIS=ON +export RUN_INCREMENT_ANLYSIS=ON + +# Define and export the environment variables +export cyc="00" +export RUN="gdas" +export PSLOT="gdas_test" +export PDY="20210827" + +# Define and export environment variables with paths +export COM_OCEAN_ANALYSIS="/scratch1/NCEPDEV/da/Mindo.Choi/sandbox/marine_vrfy/gdas.20210827/00/analysis/ocean" +export COM_ICE_HISTORY_PREV="/scratch1/NCEPDEV/da/Mindo.Choi/sandbox/marine_vrfy/gdas.20210826/18/model/ice/history" +export COM_OCEAN_HISTORY_PREV="/scratch1/NCEPDEV/da/Mindo.Choi/sandbox/marine_vrfy/gdas.20210826/18/model/ocean/history" + +# Excute Marine Verify Analysis +python3 ${HOMEgfs}sorc/gdas.cd/utils/soca/fig_gallery/exgdas_global_marine_analysis_vrfy_manual.py From 71aef363515e3e21ac6e2588387966be7e6afcc9 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA <58948505+AndrewEichmann-NOAA@users.noreply.github.com> Date: Wed, 20 Nov 2024 11:15:14 -0500 Subject: [PATCH 15/22] Fixes gridgen problem and localization for LETKF (#1372) Adds the `gridgen.yaml ` back after being removed to` jcb-gdas`, to allow LETKF to work with a fix within GDASApp. This is a temporary fix until the LETKF task can be converted to using the jcb yamls. Also adds `obs localization` to remaining obs config files - this will have to likewise be carried over to whatever is being used with the obs list. --- parm/soca/gridgen/gridgen.yaml | 5 +++++ parm/soca/obs/config/icec_amsu_mb_l2.yaml | 7 +++++++ parm/soca/obs/config/icec_atms_n20_l2.yaml | 7 +++++++ parm/soca/obs/config/icec_atms_n21_l2.yaml | 7 +++++++ parm/soca/obs/config/icec_atms_npp_l2.yaml | 7 +++++++ parm/soca/obs/config/icec_gmi_gpm_l2.yaml | 7 +++++++ parm/soca/obs/config/icec_ssmis_f17_l2.yaml | 7 +++++++ parm/soca/obs/config/icec_viirs_n20_l2_north.yaml | 7 +++++++ parm/soca/obs/config/icec_viirs_n20_l2_south.yaml | 7 +++++++ parm/soca/obs/config/insitu_profile_dbuoy.yaml | 7 +++++++ parm/soca/obs/config/insitu_profile_dbuoyb.yaml | 7 +++++++ parm/soca/obs/config/insitu_profile_glider.yaml | 7 +++++++ parm/soca/obs/config/insitu_profile_marinemammal.yaml | 7 +++++++ parm/soca/obs/config/insitu_profile_mbuoy.yaml | 7 +++++++ parm/soca/obs/config/insitu_profile_mbuoyb.yaml | 7 +++++++ parm/soca/obs/config/insitu_profile_tesac_salinity.yaml | 7 +++++++ parm/soca/obs/config/insitu_profile_xbtctd.yaml | 7 +++++++ parm/soca/obs/config/insitu_surface_altkob.yaml | 7 +++++++ parm/soca/obs/config/insitu_surface_trkob_salinity.yaml | 7 +++++++ 19 files changed, 131 insertions(+) create mode 100644 parm/soca/gridgen/gridgen.yaml diff --git a/parm/soca/gridgen/gridgen.yaml b/parm/soca/gridgen/gridgen.yaml new file mode 100644 index 000000000..34fbdeca6 --- /dev/null +++ b/parm/soca/gridgen/gridgen.yaml @@ -0,0 +1,5 @@ +geometry: + geom_grid_file: soca_gridspec.nc + mom6_input_nml: mom_input.nml + fields metadata: fields_metadata.yaml + rossby file: rossrad.nc diff --git a/parm/soca/obs/config/icec_amsu_mb_l2.yaml b/parm/soca/obs/config/icec_amsu_mb_l2.yaml index 9ab19e569..cc66e80f4 100644 --- a/parm/soca/obs/config/icec_amsu_mb_l2.yaml +++ b/parm/soca/obs/config/icec_amsu_mb_l2.yaml @@ -43,3 +43,10 @@ obs filters: where: - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/icec_atms_n20_l2.yaml b/parm/soca/obs/config/icec_atms_n20_l2.yaml index d9efd3b35..86edc632b 100644 --- a/parm/soca/obs/config/icec_atms_n20_l2.yaml +++ b/parm/soca/obs/config/icec_atms_n20_l2.yaml @@ -43,3 +43,10 @@ obs filters: where: - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/icec_atms_n21_l2.yaml b/parm/soca/obs/config/icec_atms_n21_l2.yaml index b3e204dd6..b74a70693 100644 --- a/parm/soca/obs/config/icec_atms_n21_l2.yaml +++ b/parm/soca/obs/config/icec_atms_n21_l2.yaml @@ -43,3 +43,10 @@ obs filters: where: - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/icec_atms_npp_l2.yaml b/parm/soca/obs/config/icec_atms_npp_l2.yaml index ede7e0bb1..2c1fb1a73 100644 --- a/parm/soca/obs/config/icec_atms_npp_l2.yaml +++ b/parm/soca/obs/config/icec_atms_npp_l2.yaml @@ -43,3 +43,10 @@ obs filters: where: - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/icec_gmi_gpm_l2.yaml b/parm/soca/obs/config/icec_gmi_gpm_l2.yaml index ba6d3d42a..0bba5b355 100644 --- a/parm/soca/obs/config/icec_gmi_gpm_l2.yaml +++ b/parm/soca/obs/config/icec_gmi_gpm_l2.yaml @@ -43,3 +43,10 @@ obs filters: where: - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/icec_ssmis_f17_l2.yaml b/parm/soca/obs/config/icec_ssmis_f17_l2.yaml index c8e5e40e1..5cec721da 100644 --- a/parm/soca/obs/config/icec_ssmis_f17_l2.yaml +++ b/parm/soca/obs/config/icec_ssmis_f17_l2.yaml @@ -43,3 +43,10 @@ obs filters: where: - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/icec_viirs_n20_l2_north.yaml b/parm/soca/obs/config/icec_viirs_n20_l2_north.yaml index f6a24d7f8..a305ee09a 100644 --- a/parm/soca/obs/config/icec_viirs_n20_l2_north.yaml +++ b/parm/soca/obs/config/icec_viirs_n20_l2_north.yaml @@ -43,3 +43,10 @@ obs filters: where: - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/icec_viirs_n20_l2_south.yaml b/parm/soca/obs/config/icec_viirs_n20_l2_south.yaml index 030b1091f..a295d1a8d 100644 --- a/parm/soca/obs/config/icec_viirs_n20_l2_south.yaml +++ b/parm/soca/obs/config/icec_viirs_n20_l2_south.yaml @@ -43,3 +43,10 @@ obs filters: where: - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/insitu_profile_dbuoy.yaml b/parm/soca/obs/config/insitu_profile_dbuoy.yaml index 2f0b2a873..4a0ae885b 100644 --- a/parm/soca/obs/config/insitu_profile_dbuoy.yaml +++ b/parm/soca/obs/config/insitu_profile_dbuoy.yaml @@ -32,3 +32,10 @@ obs filters: coefs: - 1000.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/insitu_profile_dbuoyb.yaml b/parm/soca/obs/config/insitu_profile_dbuoyb.yaml index d238ada6a..ac70f2b09 100644 --- a/parm/soca/obs/config/insitu_profile_dbuoyb.yaml +++ b/parm/soca/obs/config/insitu_profile_dbuoyb.yaml @@ -31,3 +31,10 @@ obs filters: - ObsError/waterTemperature coefs: - 1000.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/insitu_profile_glider.yaml b/parm/soca/obs/config/insitu_profile_glider.yaml index 366462323..1b9625010 100644 --- a/parm/soca/obs/config/insitu_profile_glider.yaml +++ b/parm/soca/obs/config/insitu_profile_glider.yaml @@ -32,3 +32,10 @@ obs filters: coefs: - 1000.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/insitu_profile_marinemammal.yaml b/parm/soca/obs/config/insitu_profile_marinemammal.yaml index 308ef34e5..a1eb9c44a 100644 --- a/parm/soca/obs/config/insitu_profile_marinemammal.yaml +++ b/parm/soca/obs/config/insitu_profile_marinemammal.yaml @@ -31,3 +31,10 @@ obs filters: - ObsError/waterTemperature coefs: - 1000.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/insitu_profile_mbuoy.yaml b/parm/soca/obs/config/insitu_profile_mbuoy.yaml index 0006a8793..02dfaea52 100644 --- a/parm/soca/obs/config/insitu_profile_mbuoy.yaml +++ b/parm/soca/obs/config/insitu_profile_mbuoy.yaml @@ -31,3 +31,10 @@ obs filters: - ObsError/waterTemperature coefs: - 1000.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/insitu_profile_mbuoyb.yaml b/parm/soca/obs/config/insitu_profile_mbuoyb.yaml index 94ef5a501..873409ccb 100644 --- a/parm/soca/obs/config/insitu_profile_mbuoyb.yaml +++ b/parm/soca/obs/config/insitu_profile_mbuoyb.yaml @@ -31,3 +31,10 @@ obs filters: - ObsError/waterTemperature coefs: - 1000.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/insitu_profile_tesac_salinity.yaml b/parm/soca/obs/config/insitu_profile_tesac_salinity.yaml index 074c29f19..7e7edab59 100644 --- a/parm/soca/obs/config/insitu_profile_tesac_salinity.yaml +++ b/parm/soca/obs/config/insitu_profile_tesac_salinity.yaml @@ -31,3 +31,10 @@ obs filters: - ObsError/salinity coefs: - 1000.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/insitu_profile_xbtctd.yaml b/parm/soca/obs/config/insitu_profile_xbtctd.yaml index d3176f786..101e3739f 100644 --- a/parm/soca/obs/config/insitu_profile_xbtctd.yaml +++ b/parm/soca/obs/config/insitu_profile_xbtctd.yaml @@ -31,3 +31,10 @@ obs filters: - ObsError/waterTemperature coefs: - 1000.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/insitu_surface_altkob.yaml b/parm/soca/obs/config/insitu_surface_altkob.yaml index dd9aa72ad..8ab5a57f8 100644 --- a/parm/soca/obs/config/insitu_surface_altkob.yaml +++ b/parm/soca/obs/config/insitu_surface_altkob.yaml @@ -27,3 +27,10 @@ obs filters: - ObsError/seaSurfaceTemperature coefs: - 1000.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/insitu_surface_trkob_salinity.yaml b/parm/soca/obs/config/insitu_surface_trkob_salinity.yaml index be980a294..bd18b4077 100644 --- a/parm/soca/obs/config/insitu_surface_trkob_salinity.yaml +++ b/parm/soca/obs/config/insitu_surface_trkob_salinity.yaml @@ -27,3 +27,10 @@ obs filters: - ObsError/seaSurfaceSalinity coefs: - 1000.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 From 15c29b1f3ea547eb618819d0f7f8412ed60d308f Mon Sep 17 00:00:00 2001 From: Andy Tangborn <71304161+andytangborn@users.noreply.github.com> Date: Wed, 20 Nov 2024 13:25:31 -0500 Subject: [PATCH 16/22] Add variable rescaling for aerosol species (#1339) This is a draft PR for additions to chem_diagb.h that could allow for a variable rescaling (rs) for each aerosol species. It does not yet include a way to read the values in from a yaml or config file. The it compiles and runs, but I have not yet tested it using 3dvar. And it still has lots of print statements used for testing. It runs in about 40 seconds at C96, and is probably slowed down by the print statements. Any comments will be helpful. Also, note the if statement. I was unsure of the dimensions of an array, so I put in a test. I'll remove it along with the print statements. --- parm/aero/berror/aero_diagb.yaml.j2 | 21 ++++++++++++++++++++- utils/chem/chem_diagb.h | 25 ++++++++++++++++++++----- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/parm/aero/berror/aero_diagb.yaml.j2 b/parm/aero/berror/aero_diagb.yaml.j2 index 31cf00b34..a2b0ea95f 100644 --- a/parm/aero/berror/aero_diagb.yaml.j2 +++ b/parm/aero/berror/aero_diagb.yaml.j2 @@ -76,7 +76,26 @@ variables: - mass_fraction_of_sea_salt003_in_air - mass_fraction_of_sea_salt004_in_air -rescale: {{ aero_diagb_rescale }} # rescales the filtered std. dev. by "rescale" +global rescale: + geometry: + fms initialization: + namelist filename: ./fv3jedi/fmsmpp.nml + field table filename: ./fv3jedi/field_table + akbk: ./fv3jedi/akbk.nc4 + layout: + - {{ layout_x }} + - {{ layout_y }} + nxp: {{ npx_rescale }} + npy: {{ npy_rescale }} + npz: {{ npz_ges }} + field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml + rescale stddev: + filetype: fms restart + skip coupler file: true + dateapath: ./stddev + filename_trcr: rescale.fv_tracer.res.nc + filename_cplr: rescale.coupler.res + number of halo points: {{ aero_diagb_n_halo }} number of neighbors: {{ aero_diagb_n_neighbors }} simple smoothing: diff --git a/utils/chem/chem_diagb.h b/utils/chem/chem_diagb.h index 703d4841f..10fc6b5fe 100644 --- a/utils/chem/chem_diagb.h +++ b/utils/chem/chem_diagb.h @@ -230,13 +230,27 @@ namespace gdasapp { } } - // Rescale - if (fullConfig.has("rescale")) { - double rescale; - fullConfig.get("rescale", rescale); - util::multiplyFieldSet(bkgErrFs, rescale); + // Rescale + if (fullConfig.has("global rescale")) { + const eckit::LocalConfiguration GlobalRescaleConfig(fullConfig, "global rescale"); + const eckit::LocalConfiguration GlobalRescaleGeomConfig(GlobalRescaleConfig, "geometry"); + const fv3jedi::Geometry GlobalRescaleGeom(GlobalRescaleGeomConfig, this-> getComm()); + fv3jedi::Increment global_rescale(GlobalRescaleGeom, chemVars, cycleDate); + global_rescale.zero(); + const eckit::LocalConfiguration GlobalRescaleStdConfig(GlobalRescaleConfig, + "rescale stddev"); + global_rescale.read(GlobalRescaleStdConfig); + // interpolate to background resolution + fv3jedi::Increment global_rescale_interp(geom, global_rescale); + atlas::FieldSet xrsFs; + global_rescale_interp.toFieldSet(xrsFs); + oops::Log::info() << "global rescaling coefficients:" << std::endl; + oops::Log::info() << xrsFs << std::endl; + util::multiplyFieldSets(bkgErrFs, xrsFs); } + + bkgErr.fromFieldSet(bkgErrFs); // Hybrid B option @@ -274,6 +288,7 @@ namespace gdasapp { double rescale_staticb; ClimBConfig.get("staticb rescaling factor", rescale_staticb); + // Combine diagb and climatological background errors fv3jedi::Increment stddev_hybrid(geom, chemVars, cycleDate); stddev_hybrid.zero(); From 0218abf2ae20ada7a73a49c9a86833276024c466 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA <26926959+RussTreadon-NOAA@users.noreply.github.com> Date: Thu, 21 Nov 2024 17:29:51 -0500 Subject: [PATCH 17/22] update jcb-gdas and soca hashes (#1380) --- parm/jcb-gdas | 2 +- sorc/soca | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/jcb-gdas b/parm/jcb-gdas index c41e7d7ac..77fdc329d 160000 --- a/parm/jcb-gdas +++ b/parm/jcb-gdas @@ -1 +1 @@ -Subproject commit c41e7d7aca3f4053b177709adaa66488f3643980 +Subproject commit 77fdc329d45b54529499f4033851e50e1dee0697 diff --git a/sorc/soca b/sorc/soca index 7f2ddb61b..44c4d6e53 160000 --- a/sorc/soca +++ b/sorc/soca @@ -1 +1 @@ -Subproject commit 7f2ddb61bc86796c83dfcd4801c91bffd829ffb4 +Subproject commit 44c4d6e53efa774bbaa038b0ecc4c34e3a33c9df From 9ab7994a0caf6b201613dd7e7ceae482ffa600e0 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Fri, 22 Nov 2024 08:38:54 -0500 Subject: [PATCH 18/22] Comment out rescale section for aero B (#1382) Closes #1381 This PR comments out the species dependent rescaling factor YAML section until we have the corresponding workflow changes added --- parm/aero/berror/aero_diagb.yaml.j2 | 38 ++++++++++++++--------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/parm/aero/berror/aero_diagb.yaml.j2 b/parm/aero/berror/aero_diagb.yaml.j2 index a2b0ea95f..c62947f1e 100644 --- a/parm/aero/berror/aero_diagb.yaml.j2 +++ b/parm/aero/berror/aero_diagb.yaml.j2 @@ -76,25 +76,25 @@ variables: - mass_fraction_of_sea_salt003_in_air - mass_fraction_of_sea_salt004_in_air -global rescale: - geometry: - fms initialization: - namelist filename: ./fv3jedi/fmsmpp.nml - field table filename: ./fv3jedi/field_table - akbk: ./fv3jedi/akbk.nc4 - layout: - - {{ layout_x }} - - {{ layout_y }} - nxp: {{ npx_rescale }} - npy: {{ npy_rescale }} - npz: {{ npz_ges }} - field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml - rescale stddev: - filetype: fms restart - skip coupler file: true - dateapath: ./stddev - filename_trcr: rescale.fv_tracer.res.nc - filename_cplr: rescale.coupler.res +#global rescale: +# geometry: +# fms initialization: +# namelist filename: ./fv3jedi/fmsmpp.nml +# field table filename: ./fv3jedi/field_table +# akbk: ./fv3jedi/akbk.nc4 +# layout: +# - {{ layout_x }} +# - {{ layout_y }} +# npx: {{ npx_rescale }} +# npy: {{ npy_rescale }} +# npz: {{ npz_ges }} +# field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml +# rescale stddev: +# filetype: fms restart +# skip coupler file: true +# dateapath: ./stddev +# filename_trcr: rescale.fv_tracer.res.nc +# filename_cplr: rescale.coupler.res number of halo points: {{ aero_diagb_n_halo }} number of neighbors: {{ aero_diagb_n_neighbors }} From 40ed1a75ea2561c0b08f468640d466d903fea5cf Mon Sep 17 00:00:00 2001 From: KatherineLukens-NOAA <59662895+KatherineLukens-NOAA@users.noreply.github.com> Date: Fri, 22 Nov 2024 15:06:10 -0500 Subject: [PATCH 19/22] Add marine verification figure display capability (HTML) (#1383) This PR adds the capability to display marine verification figures via an internal HTML document. The `README` file lists the steps one should follow to utilize the `*.html` file. The files that should be reviewed are: ``` index_vrfy_marine.html README ``` --- .../fig_gallery/marine_vrfy_display/README | 29 + .../marine_vrfy_display/default.css | 724 +++++++++++++++ .../fig_gallery/marine_vrfy_display/fonts.css | 422 +++++++++ .../index_vrfy_marine.html | 862 ++++++++++++++++++ 4 files changed, 2037 insertions(+) create mode 100644 utils/soca/fig_gallery/marine_vrfy_display/README create mode 100644 utils/soca/fig_gallery/marine_vrfy_display/default.css create mode 100644 utils/soca/fig_gallery/marine_vrfy_display/fonts.css create mode 100644 utils/soca/fig_gallery/marine_vrfy_display/index_vrfy_marine.html diff --git a/utils/soca/fig_gallery/marine_vrfy_display/README b/utils/soca/fig_gallery/marine_vrfy_display/README new file mode 100644 index 000000000..dde23647d --- /dev/null +++ b/utils/soca/fig_gallery/marine_vrfy_display/README @@ -0,0 +1,29 @@ +################################################ +# README for index_vrfy_marine.html +################################################ + +To use, follow these steps: + +---------------------------- + +1. Login to Hera via X2Go and type on the command line: + + firefox index_vrfy_marine.html + +2. In the webpage, + a. Enter the path to your main verification directory. See 2b for info on what to include in this path. + b. Are your figures in your COMROOT or from HPSS? + If COMROOT, the main path should include everything BEFORE /gdas.YearMonthDay subdirectories. + If HPSS, the main path should include everything BEFORE /YearMonthDayHour subdirectories. + Enter COMROOT or HPSS. + c. Enter the date and cycle time you wish to display. + +3. Refresh the webpage tab by clicking the refresh button. You should still see your entered path and date/time. + +4. Navigate the dropdown menu on the left and click to display figures. + +---------------------------- +NOTES: + N1: If you wish to look at figures from a different experiment or for a different date/time, repeat Steps 2-4. + +################################################ diff --git a/utils/soca/fig_gallery/marine_vrfy_display/default.css b/utils/soca/fig_gallery/marine_vrfy_display/default.css new file mode 100644 index 000000000..9eee9a476 --- /dev/null +++ b/utils/soca/fig_gallery/marine_vrfy_display/default.css @@ -0,0 +1,724 @@ + html, body + { + height: 100%; + } + + body + { + margin: 0px; + padding: 0px; + background: #202020; + font-family: 'Source Sans Pro', sans-serif; + font-size: 12pt; + font-weight: 300; + color: #444444; + } + + + h1, h2, h3 + { + margin: 0; + padding: 0; + font-weight: 600; + color: #333333; + } + + p, ol, ul + { + margin-top: 0; + } + + ol, ul + { + padding: 0; + list-style: none; + } + + p + { + line-height: 180%; + } + + strong + { + } + + a + { + color: #2980b9; + } + + a:hover + { + text-decoration: none; + } + + .container + { + overflow: hidden; + margin: 0em auto; + width: 1350px; + } + +/*********************************************************************************/ +/* Image Style */ +/*********************************************************************************/ + + .image + { + display: inline-block; + } + + .image img + { + display: block; + width: 100%; + } + + .image-full + { + display: block; + width: 100%; + margin: 0 0 2em 0; + } + + .image-left + { + float: left; + margin: 0 2em 2em 0; + } + + .image-centered + { + display: block; + margin: 0 0 2em 0; + } + + .image-centered img + { + margin: 0 auto; + width: auto; + } + +/*********************************************************************************/ +/* List Styles */ +/*********************************************************************************/ + + ul.style1 + { + margin: 0 auto; + padding: 0; + width: 80%; + overflow: hidden; + list-style: none; + text-align: left; + color: #6c6c6c + } + + ul.style1 li + { + padding: 1.6em 0em 0em 0em; + margin: 0 0 2.5em 0; + border-top: 1px solid rgba(0,0,0,.1); + } + + ul.style1 li:first-child + { + border-top: none; + } + + ul.style1 p + { + margin: 0; + } + + ul.style1 a + { + display: block; + text-decoration: none; + color: #2D2D2D; + } + + ul.style1 a:hover + { + text-decoration: underline; + } + + ul.style1 h3 + { + padding: 1em 0em 5px 0em; + text-transform: uppercase; + font-size: 1em; + font-weight: 400; + } + + ul.style1 .first + { + padding-top: 0; + background: none; + } + + ul.style1 .date + { + float: left; + position: relative; + width: 80px; + height: 70px; + margin: 0.5em 1.5em 0em 0.5em; + padding: 1.5em 0em 1.5em 0em; + box-shadow: 0px 0px 0px 5px rgba(255,255,255,0.1); + line-height: normal; + text-align: center; + text-transform: uppercase; + text-shadow: 0px 1px 0px rgba(0,0,0,.2); + font-size: 1em; + font-weight: 400; + border-right: 1px solid rgba(0,0,0,.1); + } + + ul.style1 .date:after + { + content: ''; + display: block; + position: absolute; + left: 0; + top: 0; + width: 100%; + height: 100%; + border-radius: 6px; + } + + ul.style1 .date b + { + margin: 0; + padding: 0; + display: block; + margin-top: -5px; + font-size: 1.8em; + font-weight: 700; + } + + ul.style1 .date a + { + } + + ul.style1 .date a:hover + { + text-decoration: none; + } + + +/*********************************************************************************/ +/* Social Icon Styles */ +/*********************************************************************************/ + + ul.contact + { + margin: 0; + padding: 2em 0em 0em 0em; + list-style: none; + } + + ul.contact li + { + display: inline-block; + padding: 0em 0.30em; + font-size: 1em; + } + + ul.contact li span + { + display: none; + margin: 0; + padding: 0; + } + + ul.contact li a + { + color: #FFF; + } + + ul.contact li a:before + { + display: inline-block; + background: #3f3f3f; + width: 40px; + height: 40px; + line-height: 40px; + border-radius: 20px; + text-align: center; + color: #FFFFFF; + } + + ul.contact li a.icon-twitter:before { background: #2DAAE4; } + ul.contact li a.icon-facebook:before { background: #39599F; } + ul.contact li a.icon-dribbble:before { background: #C4376B; } + ul.contact li a.icon-tumblr:before { background: #31516A; } + ul.contact li a.icon-rss:before { background: #F2600B; } + +/*********************************************************************************/ +/* Button Style */ +/*********************************************************************************/ + + .button + { + display: inline-block; + padding: 1em 3em 1em 2em; + letter-spacing: 0.20em; + text-decoration: none; + text-transform: uppercase; + font-weight: 400; + font-size: 0.90em; + color: #FFF; + } + + .button:before + { + display: inline-block; + background: #FFC31F; + margin-right: 1em; + width: 40px; + height: 40px; + line-height: 40px; + border-radius: 20px; + text-align: center; + color: #272925; + } + +/*********************************************************************************/ +/* 4-column */ +/*********************************************************************************/ + + .box1, + .box2, + .box3, + .box4 + { + width: 235px; + } + + .box1, + .box2, + .box3, + { + float: left; + margin-right: 20px; + } + + .box4 + { + float: right; + } + +/*********************************************************************************/ +/* 3-column */ +/*********************************************************************************/ + + .boxA, + .boxB, + .boxC + { + width: 320px; + } + + .boxA, + .boxB, + { + float: left; + margin-right: 20px; + } + + .boxC + { + float: right; + } + +/*********************************************************************************/ +/* 2-column */ +/*********************************************************************************/ + + .tbox1, + .tbox2 + { + width: 575px; + } + + .tbox1 + { + float: left; + } + + .tbox2 + { + float: right; + } + +/*********************************************************************************/ +/* Heading Titles */ +/*********************************************************************************/ + + .title + { + margin-bottom: 3em; + } + + .title h2 + { + text-transform: lowercase; + font-size: 2.8em; + } + + .title .byline + { + font-size: 1.3em; + color: #6F6F6F; + } + +/*********************************************************************************/ +/* Header */ +/*********************************************************************************/ + + #header + { + position: relative; + float: left; + width: 225px; + padding: 3em 0em; + } + +/*********************************************************************************/ +/* Logo */ +/*********************************************************************************/ + + #logo + { + text-align: center; + margin-bottom: 4em; + } + + #logo h1 + { + display: block; + } + + #logo a + { + text-decoration: none; + color: #FFF; + } + + #logo span + { + padding-right: 0.5em; + text-transform: uppercase; + font-size: 0.90em; + color: rgba(255,255,255,0.3); + } + + #logo span a + { + color: rgba(255,255,255,0.5); + } + + #logo img + { + display: inline-block; + margin-bottom: 1em; + border-radius: 50%; + } + +/*********************************************************************************/ +/* Menu */ +/*********************************************************************************/ + + #menu + { + } + + #menu ul + { + } + + #menu li + { + border-top: 1px solid rgba(255,255,255,0.4); + } + + #menu li a, #menu li span + { + display: block; + padding: 1em 1em; + text-align: center; + text-decoration: none; + text-transform: uppercase; + font-weight: 700; + color: rgba(255,255,255,1); + } + + #menu ul li a:hover { + background-color: #2980b9; + border-left: solid 10px #ffffff; + } + + #menu li:hover a, #menu li.active a, #menu li.active span + { + } + + #menu .current_page_item a + { + background: #2980b9; + color: rgba(255,255,255,1); + } + + #menu .icon + { + } + +/*********************************************************************************/ +/* Dropdown */ +/*********************************************************************************/ + .dropbtn { + position: relative; + display: block; + padding: 1em 1em; + width: 100%; + text-align: center; + text-decoration: none; + text-transform: uppercase; + font-family: 'Source Sans Pro', sans-serif; + font-size: 12pt; + font-weight: 700; + color: rgba(255,255,255,1); + background-color: #112e51; + border-top: 1px solid rgba(255,255,255,0.4); + border-left: none; + border-bottom: none; + border-right: none; + cursor: pointer; + } + + #active { + background-color: #2980b9; + color: rgba(255,255,255,1); + } + + .dropbtn:hover { + background-color: #2980b9; + border-left: solid 10px #ffffff; + } + + .dropbtn:focus { + background-color: #2980b9; + border-left: solid 10px #ffffff; + + } + + .dropdown { + position: relative; + display: block; + width: 100%; + } + + /* NESTED DROPDOWNS */ + /* The dropdown content (submenus) */ + .dropdown-content { + display: none; + left: 100%; + top: -50%; + position: absolute; + background-color: #112e51; + min-width: 160px; + z-index: 1; + } + + /*.dropdown-content a { + display: block; + color: rgba(255,255,255,1); + padding: 1em 1em; + border: 1px solid rgba(255,255,255,0.4); + text-align: center; + text-decoration: none; + } + + /* Show the dropdown menu on hover */ + .dropdown:hover .dropdown-content { + display: block; + } + + /* Style for submenu items */ + .dropdown-content div { + padding: 10px; + color: white; + cursor: pointer; + } + + .dropdown-content div:hover { + background-color: #575757; + } + + /* Style for deeper nested dropdown */ + .dropdown-content .dropdown-content { + top: -75%; + left: 100%; + display: none; + position: absolute; + } + + /* Show deeper nested dropdown when hovered over */ + .dropdown-content div:hover .dropdown-content { + display: block; + } + + /* Display dropdown menu when hovering over a parent menu */ + div:hover > .dropdown-menu { + display: block; + } + + /* Style for nested dropdowns */ + /* Ensure only one level of dropdown is visible at a time */ + div > .dropdown-menu > div { + position: relative; + } + + /* Hide the submenus by default */ + .dropdown-menu > .dropdown-menu { + display: none; + left: 100%; /* Position submenu to the right */ + top: 5; + position: absolute; + } + + /* Show only the submenu when hovering over the parent */ + div > .dropdown-menu > div:hover > .dropdown-menu { + display: block; + } + + /* END NESTED DROPDOWNS */ + + .show {display: block;} + +/*********************************************************************************/ +/* Banner */ +/*********************************************************************************/ + + #banner + { + margin-bottom: 0em; + } + +/*********************************************************************************/ +/* Page */ +/*********************************************************************************/ + + #page + { + background: #112e51; +/* background: #2a2a2a; */ + } + +/*********************************************************************************/ +/* Main */ +/*********************************************************************************/ + + #main + { + overflow: hidden; + float: right; + width: 1125px; + padding: 1em 0px 5em 0px; + background: #FFF; + border-top: 6px solid #2980b9; + text-align: center; + } + +/*********************************************************************************/ +/* Featured */ +/*********************************************************************************/ + + #featured + { + overflow: hidden; + margin-bottom: 3em; + padding-top: 5em; + border-top: 1px solid rgba(0,0,0,0.08); + } + +/*********************************************************************************/ +/* Sidebar */ +/*********************************************************************************/ + + #sidebar + { + } + +/*********************************************************************************/ +/* Footer */ +/*********************************************************************************/ + + #footer + { + overflow: hidden; + padding: 0em 0em; + border-top: 1px solid rgba(0,0,0,0.08); + font-weight: bolder; + } + +/*********************************************************************************/ +/* Welcome */ +/*********************************************************************************/ + + #welcome + { + overflow: hidden; + padding: 0em 1em; + border-top: 1px solid rgba(0,0,0,0.08); + font-weight: bolder; + } + +/*********************************************************************************/ +/* Contact */ +/*********************************************************************************/ + + #contact + { + overflow: hidden; + padding: 0em 0em; + border-top: 1px solid rgba(0,0,0,0.08); + font-weight: bolder; + } + +/*********************************************************************************/ +/* Copyright */ +/*********************************************************************************/ + + #copyright + { + overflow: hidden; + padding: 1em 0em; + border-top: 1px solid rgba(0,0,0,0.08); + } + + #copyright span + { + display: block; + letter-spacing: 0.20em; + line-height: 2.5em; + text-align: center; + text-transform: uppercase; + font-size: 0.8em; + font-weight: normal; + color: rgba(0,0,0,0.7); + } + + #copyright a + { + text-decoration: none; + color: rgba(0,0,0,0.9); + } + + .fa + { + display: block; + color: #000; + background: red; + } diff --git a/utils/soca/fig_gallery/marine_vrfy_display/fonts.css b/utils/soca/fig_gallery/marine_vrfy_display/fonts.css new file mode 100644 index 000000000..f90cc4360 --- /dev/null +++ b/utils/soca/fig_gallery/marine_vrfy_display/fonts.css @@ -0,0 +1,422 @@ +@charset 'UTF-8'; + +@font-face{font-family:'FontAwesome';src:url('font/fontawesome-webfont.eot?v=4.0.1');src:url('font/fontawesome-webfont.eot?#iefix&v=4.0.1') format('embedded-opentype'),url('font/fontawesome-webfont.woff?v=4.0.1') format('woff'),url('font/fontawesome-webfont.ttf?v=4.0.1') format('truetype'),url('font/fontawesome-webfont.svg?v=4.0.1#fontawesomeregular') format('svg');font-weight:normal;font-style:normal} + +/*********************************************************************************/ +/* Icons */ +/* Powered by Font Awesome by Dave Gandy | http://fontawesome.io */ +/* Licensed under the SIL OFL 1.1 (font), MIT (CSS) */ +/*********************************************************************************/ + + .fa + { + text-decoration: none; + } + + .fa:before + { + display:inline-block; + font-family: FontAwesome; + font-size: 1.25em; + text-decoration: none; + font-style: normal; + font-weight: normal; + line-height: 1; + -webkit-font-smoothing:antialiased; + -moz-osx-font-smoothing:grayscale; + } + + .fa-lg{font-size:1.3333333333333333em;line-height:.75em;vertical-align:-15%} + .fa-2x{font-size:2em} + .fa-3x{font-size:3em} + .fa-4x{font-size:4em} + .fa-5x{font-size:5em} + .fa-fw{width:1.2857142857142858em;text-align:center} + .fa-ul{padding-left:0;margin-left:2.142857142857143em;list-style-type:none}.fa-ul>li{position:relative} + .fa-li{position:absolute;left:-2.142857142857143em;width:2.142857142857143em;top:.14285714285714285em;text-align:center}.fa-li.fa-lg{left:-1.8571428571428572em} + .fa-border{padding:.2em .25em .15em;border:solid .08em #eee;border-radius:.1em} + .pull-right{float:right} + .pull-left{float:left} + .fa.pull-left{margin-right:.3em} + .fa.pull-right{margin-left:.3em} + .fa-spin{-webkit-animation:spin 2s infinite linear;-moz-animation:spin 2s infinite linear;-o-animation:spin 2s infinite linear;animation:spin 2s infinite linear} + @-moz-keyframes spin{0%{-moz-transform:rotate(0deg)} 100%{-moz-transform:rotate(359deg)}}@-webkit-keyframes spin{0%{-webkit-transform:rotate(0deg)} 100%{-webkit-transform:rotate(359deg)}}@-o-keyframes spin{0%{-o-transform:rotate(0deg)} 100%{-o-transform:rotate(359deg)}}@-ms-keyframes spin{0%{-ms-transform:rotate(0deg)} 100%{-ms-transform:rotate(359deg)}}@keyframes spin{0%{transform:rotate(0deg)} 100%{transform:rotate(359deg)}}.fa-rotate-90{filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=1);-webkit-transform:rotate(90deg);-moz-transform:rotate(90deg);-ms-transform:rotate(90deg);-o-transform:rotate(90deg);transform:rotate(90deg)} + .fa-rotate-180{filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=2);-webkit-transform:rotate(180deg);-moz-transform:rotate(180deg);-ms-transform:rotate(180deg);-o-transform:rotate(180deg);transform:rotate(180deg)} + .fa-rotate-270{filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=3);-webkit-transform:rotate(270deg);-moz-transform:rotate(270deg);-ms-transform:rotate(270deg);-o-transform:rotate(270deg);transform:rotate(270deg)} + .fa-flip-horizontal{filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=0, mirror=1);-webkit-transform:scale(-1, 1);-moz-transform:scale(-1, 1);-ms-transform:scale(-1, 1);-o-transform:scale(-1, 1);transform:scale(-1, 1)} + .fa-flip-vertical{filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=2, mirror=1);-webkit-transform:scale(1, -1);-moz-transform:scale(1, -1);-ms-transform:scale(1, -1);-o-transform:scale(1, -1);transform:scale(1, -1)} + .fa-stack{position:relative;display:inline-block;width:2em;height:2em;line-height:2em;vertical-align:middle} + .fa-stack-1x,.fa-stack-2x{position:absolute;left:0;width:100%;text-align:center} + .fa-stack-1x{line-height:inherit} + .fa-stack-2x{font-size:2em} + .fa-inverse{color:#fff} + .fa-glass:before{content:"\f000"} + .fa-music:before{content:"\f001"} + .fa-search:before{content:"\f002"} + .fa-envelope-o:before{content:"\f003"} + .fa-heart:before{content:"\f004"} + .fa-star:before{content:"\f005"} + .fa-star-o:before{content:"\f006"} + .fa-user:before{content:"\f007"} + .fa-film:before{content:"\f008"} + .fa-th-large:before{content:"\f009"} + .fa-th:before{content:"\f00a"} + .fa-th-list:before{content:"\f00b"} + .fa-check:before{content:"\f00c"} + .fa-times:before{content:"\f00d"} + .fa-search-plus:before{content:"\f00e"} + .fa-search-minus:before{content:"\f010"} + .fa-power-off:before{content:"\f011"} + .fa-signal:before{content:"\f012"} + .fa-gear:before,.fa-cog:before{content:"\f013"} + .fa-trash-o:before{content:"\f014"} + .fa-home:before{content:"\f015"} + .fa-file-o:before{content:"\f016"} + .fa-clock-o:before{content:"\f017"} + .fa-road:before{content:"\f018"} + .fa-download:before{content:"\f019"} + .fa-arrow-circle-o-down:before{content:"\f01a"} + .fa-arrow-circle-o-up:before{content:"\f01b"} + .fa-inbox:before{content:"\f01c"} + .fa-play-circle-o:before{content:"\f01d"} + .fa-rotate-right:before,.fa-repeat:before{content:"\f01e"} + .fa-refresh:before{content:"\f021"} + .fa-list-alt:before{content:"\f022"} + .fa-lock:before{content:"\f023"} + .fa-flag:before{content:"\f024"} + .fa-headphones:before{content:"\f025"} + .fa-volume-off:before{content:"\f026"} + .fa-volume-down:before{content:"\f027"} + .fa-volume-up:before{content:"\f028"} + .fa-qrcode:before{content:"\f029"} + .fa-barcode:before{content:"\f02a"} + .fa-tag:before{content:"\f02b"} + .fa-tags:before{content:"\f02c"} + .fa-book:before{content:"\f02d"} + .fa-bookmark:before{content:"\f02e"} + .fa-print:before{content:"\f02f"} + .fa-camera:before{content:"\f030"} + .fa-font:before{content:"\f031"} + .fa-bold:before{content:"\f032"} + .fa-italic:before{content:"\f033"} + .fa-text-height:before{content:"\f034"} + .fa-text-width:before{content:"\f035"} + .fa-align-left:before{content:"\f036"} + .fa-align-center:before{content:"\f037"} + .fa-align-right:before{content:"\f038"} + .fa-align-justify:before{content:"\f039"} + .fa-list:before{content:"\f03a"} + .fa-dedent:before,.fa-outdent:before{content:"\f03b"} + .fa-indent:before{content:"\f03c"} + .fa-video-camera:before{content:"\f03d"} + .fa-picture-o:before{content:"\f03e"} + .fa-pencil:before{content:"\f040"} + .fa-map-marker:before{content:"\f041"} + .fa-adjust:before{content:"\f042"} + .fa-tint:before{content:"\f043"} + .fa-edit:before,.fa-pencil-square-o:before{content:"\f044"} + .fa-share-square-o:before{content:"\f045"} + .fa-check-square-o:before{content:"\f046"} + .fa-move:before{content:"\f047"} + .fa-step-backward:before{content:"\f048"} + .fa-fast-backward:before{content:"\f049"} + .fa-backward:before{content:"\f04a"} + .fa-play:before{content:"\f04b"} + .fa-pause:before{content:"\f04c"} + .fa-stop:before{content:"\f04d"} + .fa-forward:before{content:"\f04e"} + .fa-fast-forward:before{content:"\f050"} + .fa-step-forward:before{content:"\f051"} + .fa-eject:before{content:"\f052"} + .fa-chevron-left:before{content:"\f053"} + .fa-chevron-right:before{content:"\f054"} + .fa-plus-circle:before{content:"\f055"} + .fa-minus-circle:before{content:"\f056"} + .fa-times-circle:before{content:"\f057"} + .fa-check-circle:before{content:"\f058"} + .fa-question-circle:before{content:"\f059"} + .fa-info-circle:before{content:"\f05a"} + .fa-crosshairs:before{content:"\f05b"} + .fa-times-circle-o:before{content:"\f05c"} + .fa-check-circle-o:before{content:"\f05d"} + .fa-ban:before{content:"\f05e"} + .fa-arrow-left:before{content:"\f060"} + .fa-arrow-right:before{content:"\f061"} + .fa-arrow-up:before{content:"\f062"} + .fa-arrow-down:before{content:"\f063"} + .fa-mail-forward:before,.fa-share:before{content:"\f064"} + .fa-resize-full:before{content:"\f065"} + .fa-resize-small:before{content:"\f066"} + .fa-plus:before{content:"\f067"} + .fa-minus:before{content:"\f068"} + .fa-asterisk:before{content:"\f069"} + .fa-exclamation-circle:before{content:"\f06a"} + .fa-gift:before{content:"\f06b"} + .fa-leaf:before{content:"\f06c"} + .fa-fire:before{content:"\f06d"} + .fa-eye:before{content:"\f06e"} + .fa-eye-slash:before{content:"\f070"} + .fa-warning:before,.fa-exclamation-triangle:before{content:"\f071"} + .fa-plane:before{content:"\f072"} + .fa-calendar:before{content:"\f073"} + .fa-random:before{content:"\f074"} + .fa-comment:before{content:"\f075"} + .fa-magnet:before{content:"\f076"} + .fa-chevron-up:before{content:"\f077"} + .fa-chevron-down:before{content:"\f078"} + .fa-retweet:before{content:"\f079"} + .fa-shopping-cart:before{content:"\f07a"} + .fa-folder:before{content:"\f07b"} + .fa-folder-open:before{content:"\f07c"} + .fa-resize-vertical:before{content:"\f07d"} + .fa-resize-horizontal:before{content:"\f07e"} + .fa-bar-chart-o:before{content:"\f080"} + .fa-twitter-square:before{content:"\f081"} + .fa-facebook-square:before{content:"\f082"} + .fa-camera-retro:before{content:"\f083"} + .fa-key:before{content:"\f084"} + .fa-gears:before,.fa-cogs:before{content:"\f085"} + .fa-comments:before{content:"\f086"} + .fa-thumbs-o-up:before{content:"\f087"} + .fa-thumbs-o-down:before{content:"\f088"} + .fa-star-half:before{content:"\f089"} + .fa-heart-o:before{content:"\f08a"} + .fa-sign-out:before{content:"\f08b"} + .fa-linkedin-square:before{content:"\f08c"} + .fa-thumb-tack:before{content:"\f08d"} + .fa-external-link:before{content:"\f08e"} + .fa-sign-in:before{content:"\f090"} + .fa-trophy:before{content:"\f091"} + .fa-github-square:before{content:"\f092"} + .fa-upload:before{content:"\f093"} + .fa-lemon-o:before{content:"\f094"} + .fa-phone:before{content:"\f095"} + .fa-square-o:before{content:"\f096"} + .fa-bookmark-o:before{content:"\f097"} + .fa-phone-square:before{content:"\f098"} + .fa-twitter:before{content:"\f099"} + .fa-facebook:before{content:"\f09a"} + .fa-github:before{content:"\f09b"} + .fa-unlock:before{content:"\f09c"} + .fa-credit-card:before{content:"\f09d"} + .fa-rss:before{content:"\f09e"} + .fa-hdd-o:before{content:"\f0a0"} + .fa-bullhorn:before{content:"\f0a1"} + .fa-bell:before{content:"\f0f3"} + .fa-certificate:before{content:"\f0a3"} + .fa-hand-o-right:before{content:"\f0a4"} + .fa-hand-o-left:before{content:"\f0a5"} + .fa-hand-o-up:before{content:"\f0a6"} + .fa-hand-o-down:before{content:"\f0a7"} + .fa-arrow-circle-left:before{content:"\f0a8"} + .fa-arrow-circle-right:before{content:"\f0a9"} + .fa-arrow-circle-up:before{content:"\f0aa"} + .fa-arrow-circle-down:before{content:"\f0ab"} + .fa-globe:before{content:"\f0ac"} + .fa-wrench:before{content:"\f0ad"} + .fa-tasks:before{content:"\f0ae"} + .fa-filter:before{content:"\f0b0"} + .fa-briefcase:before{content:"\f0b1"} + .fa-fullscreen:before{content:"\f0b2"} + .fa-group:before{content:"\f0c0"} + .fa-chain:before,.fa-link:before{content:"\f0c1"} + .fa-cloud:before{content:"\f0c2"} + .fa-flask:before{content:"\f0c3"} + .fa-cut:before,.fa-scissors:before{content:"\f0c4"} + .fa-copy:before,.fa-files-o:before{content:"\f0c5"} + .fa-paperclip:before{content:"\f0c6"} + .fa-save:before,.fa-floppy-o:before{content:"\f0c7"} + .fa-square:before{content:"\f0c8"} + .fa-reorder:before{content:"\f0c9"} + .fa-list-ul:before{content:"\f0ca"} + .fa-list-ol:before{content:"\f0cb"} + .fa-strikethrough:before{content:"\f0cc"} + .fa-underline:before{content:"\f0cd"} + .fa-table:before{content:"\f0ce"} + .fa-magic:before{content:"\f0d0"} + .fa-truck:before{content:"\f0d1"} + .fa-pinterest:before{content:"\f0d2"} + .fa-pinterest-square:before{content:"\f0d3"} + .fa-google-plus-square:before{content:"\f0d4"} + .fa-google-plus:before{content:"\f0d5"} + .fa-money:before{content:"\f0d6"} + .fa-caret-down:before{content:"\f0d7"} + .fa-caret-up:before{content:"\f0d8"} + .fa-caret-left:before{content:"\f0d9"} + .fa-caret-right:before{content:"\f0da"} + .fa-columns:before{content:"\f0db"} + .fa-unsorted:before,.fa-sort:before{content:"\f0dc"} + .fa-sort-down:before,.fa-sort-asc:before{content:"\f0dd"} + .fa-sort-up:before,.fa-sort-desc:before{content:"\f0de"} + .fa-envelope:before{content:"\f0e0"} + .fa-linkedin:before{content:"\f0e1"} + .fa-rotate-left:before,.fa-undo:before{content:"\f0e2"} + .fa-legal:before,.fa-gavel:before{content:"\f0e3"} + .fa-dashboard:before,.fa-tachometer:before{content:"\f0e4"} + .fa-comment-o:before{content:"\f0e5"} + .fa-comments-o:before{content:"\f0e6"} + .fa-flash:before,.fa-bolt:before{content:"\f0e7"} + .fa-sitemap:before{content:"\f0e8"} + .fa-umbrella:before{content:"\f0e9"} + .fa-paste:before,.fa-clipboard:before{content:"\f0ea"} + .fa-lightbulb-o:before{content:"\f0eb"} + .fa-exchange:before{content:"\f0ec"} + .fa-cloud-download:before{content:"\f0ed"} + .fa-cloud-upload:before{content:"\f0ee"} + .fa-user-md:before{content:"\f0f0"} + .fa-stethoscope:before{content:"\f0f1"} + .fa-suitcase:before{content:"\f0f2"} + .fa-bell-o:before{content:"\f0a2"} + .fa-coffee:before{content:"\f0f4"} + .fa-cutlery:before{content:"\f0f5"} + .fa-file-text-o:before{content:"\f0f6"} + .fa-building:before{content:"\f0f7"} + .fa-hospital:before{content:"\f0f8"} + .fa-ambulance:before{content:"\f0f9"} + .fa-medkit:before{content:"\f0fa"} + .fa-fighter-jet:before{content:"\f0fb"} + .fa-beer:before{content:"\f0fc"} + .fa-h-square:before{content:"\f0fd"} + .fa-plus-square:before{content:"\f0fe"} + .fa-angle-double-left:before{content:"\f100"} + .fa-angle-double-right:before{content:"\f101"} + .fa-angle-double-up:before{content:"\f102"} + .fa-angle-double-down:before{content:"\f103"} + .fa-angle-left:before{content:"\f104"} + .fa-angle-right:before{content:"\f105"} + .fa-angle-up:before{content:"\f106"} + .fa-angle-down:before{content:"\f107"} + .fa-desktop:before{content:"\f108"} + .fa-laptop:before{content:"\f109"} + .fa-tablet:before{content:"\f10a"} + .fa-mobile-phone:before,.fa-mobile:before{content:"\f10b"} + .fa-circle-o:before{content:"\f10c"} + .fa-quote-left:before{content:"\f10d"} + .fa-quote-right:before{content:"\f10e"} + .fa-spinner:before{content:"\f110"} + .fa-circle:before{content:"\f111"} + .fa-mail-reply:before,.fa-reply:before{content:"\f112"} + .fa-github-alt:before{content:"\f113"} + .fa-folder-o:before{content:"\f114"} + .fa-folder-open-o:before{content:"\f115"} + .fa-expand-o:before{content:"\f116"} + .fa-collapse-o:before{content:"\f117"} + .fa-smile-o:before{content:"\f118"} + .fa-frown-o:before{content:"\f119"} + .fa-meh-o:before{content:"\f11a"} + .fa-gamepad:before{content:"\f11b"} + .fa-keyboard-o:before{content:"\f11c"} + .fa-flag-o:before{content:"\f11d"} + .fa-flag-checkered:before{content:"\f11e"} + .fa-terminal:before{content:"\f120"} + .fa-code:before{content:"\f121"} + .fa-reply-all:before{content:"\f122"} + .fa-mail-reply-all:before{content:"\f122"} + .fa-star-half-empty:before,.fa-star-half-full:before,.fa-star-half-o:before{content:"\f123"} + .fa-location-arrow:before{content:"\f124"} + .fa-crop:before{content:"\f125"} + .fa-code-fork:before{content:"\f126"} + .fa-unlink:before,.fa-chain-broken:before{content:"\f127"} + .fa-question:before{content:"\f128"} + .fa-info:before{content:"\f129"} + .fa-exclamation:before{content:"\f12a"} + .fa-superscript:before{content:"\f12b"} + .fa-subscript:before{content:"\f12c"} + .fa-eraser:before{content:"\f12d"} + .fa-puzzle-piece:before{content:"\f12e"} + .fa-microphone:before{content:"\f130"} + .fa-microphone-slash:before{content:"\f131"} + .fa-shield:before{content:"\f132"} + .fa-calendar-o:before{content:"\f133"} + .fa-fire-extinguisher:before{content:"\f134"} + .fa-rocket:before{content:"\f135"} + .fa-maxcdn:before{content:"\f136"} + .fa-chevron-circle-left:before{content:"\f137"} + .fa-chevron-circle-right:before{content:"\f138"} + .fa-chevron-circle-up:before{content:"\f139"} + .fa-chevron-circle-down:before{content:"\f13a"} + .fa-html5:before{content:"\f13b"} + .fa-css3:before{content:"\f13c"} + .fa-anchor:before{content:"\f13d"} + .fa-unlock-o:before{content:"\f13e"} + .fa-bullseye:before{content:"\f140"} + .fa-ellipsis-horizontal:before{content:"\f141"} + .fa-ellipsis-vertical:before{content:"\f142"} + .fa-rss-square:before{content:"\f143"} + .fa-play-circle:before{content:"\f144"} + .fa-ticket:before{content:"\f145"} + .fa-minus-square:before{content:"\f146"} + .fa-minus-square-o:before{content:"\f147"} + .fa-level-up:before{content:"\f148"} + .fa-level-down:before{content:"\f149"} + .fa-check-square:before{content:"\f14a"} + .fa-pencil-square:before{content:"\f14b"} + .fa-external-link-square:before{content:"\f14c"} + .fa-share-square:before{content:"\f14d"} + .fa-compass:before{content:"\f14e"} + .fa-toggle-down:before,.fa-caret-square-o-down:before{content:"\f150"} + .fa-toggle-up:before,.fa-caret-square-o-up:before{content:"\f151"} + .fa-toggle-right:before,.fa-caret-square-o-right:before{content:"\f152"} + .fa-euro:before,.fa-eur:before{content:"\f153"} + .fa-gbp:before{content:"\f154"} + .fa-dollar:before,.fa-usd:before{content:"\f155"} + .fa-rupee:before,.fa-inr:before{content:"\f156"} + .fa-cny:before,.fa-rmb:before,.fa-yen:before,.fa-jpy:before{content:"\f157"} + .fa-ruble:before,.fa-rouble:before,.fa-rub:before{content:"\f158"} + .fa-won:before,.fa-krw:before{content:"\f159"} + .fa-bitcoin:before,.fa-btc:before{content:"\f15a"} + .fa-file:before{content:"\f15b"} + .fa-file-text:before{content:"\f15c"} + .fa-sort-alpha-asc:before{content:"\f15d"} + .fa-sort-alpha-desc:before{content:"\f15e"} + .fa-sort-amount-asc:before{content:"\f160"} + .fa-sort-amount-desc:before{content:"\f161"} + .fa-sort-numeric-asc:before{content:"\f162"} + .fa-sort-numeric-desc:before{content:"\f163"} + .fa-thumbs-up:before{content:"\f164"} + .fa-thumbs-down:before{content:"\f165"} + .fa-youtube-square:before{content:"\f166"} + .fa-youtube:before{content:"\f167"} + .fa-xing:before{content:"\f168"} + .fa-xing-square:before{content:"\f169"} + .fa-youtube-play:before{content:"\f16a"} + .fa-dropbox:before{content:"\f16b"} + .fa-stack-overflow:before{content:"\f16c"} + .fa-instagram:before{content:"\f16d"} + .fa-flickr:before{content:"\f16e"} + .fa-adn:before{content:"\f170"} + .fa-bitbucket:before{content:"\f171"} + .fa-bitbucket-square:before{content:"\f172"} + .fa-tumblr:before{content:"\f173"} + .fa-tumblr-square:before{content:"\f174"} + .fa-long-arrow-down:before{content:"\f175"} + .fa-long-arrow-up:before{content:"\f176"} + .fa-long-arrow-left:before{content:"\f177"} + .fa-long-arrow-right:before{content:"\f178"} + .fa-apple:before{content:"\f179"} + .fa-windows:before{content:"\f17a"} + .fa-android:before{content:"\f17b"} + .fa-linux:before{content:"\f17c"} + .fa-dribbble:before{content:"\f17d"} + .fa-skype:before{content:"\f17e"} + .fa-foursquare:before{content:"\f180"} + .fa-trello:before{content:"\f181"} + .fa-female:before{content:"\f182"} + .fa-male:before{content:"\f183"} + .fa-gittip:before{content:"\f184"} + .fa-sun-o:before{content:"\f185"} + .fa-moon-o:before{content:"\f186"} + .fa-archive:before{content:"\f187"} + .fa-bug:before{content:"\f188"} + .fa-vk:before{content:"\f189"} + .fa-weibo:before{content:"\f18a"} + .fa-renren:before{content:"\f18b"} + .fa-pagelines:before{content:"\f18c"} + .fa-stack-exchange:before{content:"\f18d"} + .fa-arrow-circle-o-right:before{content:"\f18e"} + .fa-arrow-circle-o-left:before{content:"\f190"} + .fa-toggle-left:before,.fa-caret-square-o-left:before{content:"\f191"} + .fa-dot-circle-o:before{content:"\f192"} + .fa-wheelchair:before{content:"\f193"} + .fa-vimeo-square:before{content:"\f194"} + .fa-turkish-lira:before,.fa-try:before{content:"\f195"} diff --git a/utils/soca/fig_gallery/marine_vrfy_display/index_vrfy_marine.html b/utils/soca/fig_gallery/marine_vrfy_display/index_vrfy_marine.html new file mode 100644 index 000000000..31372866c --- /dev/null +++ b/utils/soca/fig_gallery/marine_vrfy_display/index_vrfy_marine.html @@ -0,0 +1,862 @@ + + + + + +GFS Experiment Verification + + + + + + + + + + + + +
+ +
+

Marine Figures

+

Add path to main verification directory:

+
+ + + +
+

Are the figures in your COMROOT or from HPSS? For example:

+ If COMROOT, the MAIN path should include everything before /gdas.YearMonthDay subdirectories. +
+ If HPSS, the MAIN path should include everything before /YearMonthDayHour subdirectories. +
+ + + +
+ +

Choose a date and cycle time:

+ + +
+ + + +
+
+ + + +
+
+ + + +
+
+ + + +
+ + +
+ + +
+ + + +
+

This page displays verification for the Global Forecast System (GFS) developer experiments.

+

DISCLAIMER: This web page is not "operational" and therefore not subject to 24-hr monitoring by NCEP's Central Operations staff.

+
+ +
+


CONTACT INFORMATION +

+ + + + + + + + +
+ +
+ + From 7a01c5eb323fd14187ebe002f026f0e0115f5973 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Mon, 25 Nov 2024 18:35:13 -0500 Subject: [PATCH 20/22] Add more CI to GW-CI (#1365) This PR is a companion to GW PR [#3120](https://github.com/NOAA-EMC/global-workflow/pull/3120). It does a couple things: 1. 5 GW CI tests are added/extended as CTests in GDASApp, running through to the fcst jobs in the first full-cycle. These CI tests are: ```C96C48_hybatmDA```, ```C96C48_ufs_hybatmDA```, ```C96C48_hybatmaerosnowDA```, ```C48mx500_3DVarAOWCDA```, and ```C48mx500_hybAOWCDA```. 2. Test references are added for ```C96C48_ufs_hybatmDA``` and ```C48mx500_3DVarAOWCDA```, so that we're actually testing the output. 3. These CTests are turned on by default in a workflow build, rather than having to mess with the ```CMakeCache.txt``` file and re-running make. This will allow us to use these tests in nightly testing. 4. ```test/gw-ci/CMakeLists.txt``` is refactored quite a bit. 5. There are 89 CTests, but for 5 CI tests, but I added task dependencies, so they can be run in parallel. The primary motivation for this PR is that we can run CI for our nightly testing of GDASApp. Also, anyone with a PR can easily do CI testing through CTests. --------- Co-authored-by: Russ-Treadon-NOAA --- ci/hera.sh | 1 + ci/orion.sh | 1 + ci/run_ci.sh | 2 +- ci/run_gw_ci.sh | 2 +- parm/atm/jcb-base.yaml.j2 | 5 + parm/atm/jcb-prototype_3dvar-fv3inc.yaml.j2 | 10 + parm/atm/jcb-prototype_3dvar.yaml.j2 | 7 + parm/atm/jcb-prototype_lgetkf-fv3inc.yaml.j2 | 10 + parm/atm/jcb-prototype_lgetkf.yaml.j2 | 7 + .../atm/jcb-prototype_lgetkf_observer.yaml.j2 | 7 + parm/atm/jcb-prototype_lgetkf_solver.yaml.j2 | 7 + parm/jcb-algorithms | 2 +- parm/soca/marine-jcb-3dfgat.yaml.j2 | 7 + parm/soca/marine-jcb-base.yaml | 4 + test/CMakeLists.txt | 7 - test/atm/global-workflow/CMakeLists.txt | 139 +++--- test/atm/global-workflow/config.yaml | 3 + .../jcb-prototype_3dvar-fv3inc.yaml.j2 | 10 + .../jcb-prototype_3dvar.yaml.j2 | 4 +- .../jcb-prototype_lgetkf-fv3inc.yaml.j2 | 10 + .../jcb-prototype_lgetkf.yaml.j2 | 4 +- .../jcb-prototype_lgetkf_observer.yaml.j2 | 4 +- .../jcb-prototype_lgetkf_solver.yaml.j2 | 4 +- test/fv3jedi/CMakeLists.txt | 23 - .../testinput/gdasapp_fv3jedi_fv3inc.yaml | 65 --- .../testoutput/gdasapp_fv3jedi_fv3inc.ref | 41 -- test/gw-ci/CMakeLists.txt | 453 ++++++++++++++---- test/gw-ci/run_exp.sh | 54 ++- .../C48mx500_3DVarAOWCDA_3dfgat.ref | 54 +++ .../C96C48_ufs_hybatmDA_3dvar-fv3inc.ref | 41 ++ .../C96C48_ufs_hybatmDA_3dvar.ref | 124 +++++ .../C96C48_ufs_hybatmDA_lgetkf-fv3inc.ref | 82 ++++ .../C96C48_ufs_hybatmDA_lgetkf_observer.ref | 126 +++++ .../C96C48_ufs_hybatmDA_lgetkf_solver.ref | 180 +++++++ test/testreference/atm_jjob_3dvar-fv3inc.ref | 41 ++ .../atm_jjob_3dvar.ref} | 0 test/testreference/atm_jjob_lgetkf-fv3inc.ref | 123 +++++ .../atm_jjob_lgetkf.ref} | 0 .../atm_jjob_lgetkf_observer.ref} | 0 .../atm_jjob_lgetkf_solver.ref} | 0 ush/CMakeLists.txt | 1 - ush/jediinc2fv3.py | 187 -------- utils/fv3jedi/fv3jedi_fv3inc.h | 22 +- 43 files changed, 1369 insertions(+), 505 deletions(-) create mode 100644 parm/atm/jcb-prototype_3dvar-fv3inc.yaml.j2 create mode 100644 parm/atm/jcb-prototype_lgetkf-fv3inc.yaml.j2 create mode 100644 test/atm/global-workflow/jcb-prototype_3dvar-fv3inc.yaml.j2 create mode 100644 test/atm/global-workflow/jcb-prototype_lgetkf-fv3inc.yaml.j2 delete mode 100644 test/fv3jedi/CMakeLists.txt delete mode 100644 test/fv3jedi/testinput/gdasapp_fv3jedi_fv3inc.yaml delete mode 100644 test/fv3jedi/testoutput/gdasapp_fv3jedi_fv3inc.ref create mode 100644 test/testreference/C48mx500_3DVarAOWCDA_3dfgat.ref create mode 100644 test/testreference/C96C48_ufs_hybatmDA_3dvar-fv3inc.ref create mode 100644 test/testreference/C96C48_ufs_hybatmDA_3dvar.ref create mode 100644 test/testreference/C96C48_ufs_hybatmDA_lgetkf-fv3inc.ref create mode 100644 test/testreference/C96C48_ufs_hybatmDA_lgetkf_observer.ref create mode 100644 test/testreference/C96C48_ufs_hybatmDA_lgetkf_solver.ref create mode 100644 test/testreference/atm_jjob_3dvar-fv3inc.ref rename test/{atm/global-workflow/3dvar.ref => testreference/atm_jjob_3dvar.ref} (100%) create mode 100644 test/testreference/atm_jjob_lgetkf-fv3inc.ref rename test/{atm/global-workflow/lgetkf.ref => testreference/atm_jjob_lgetkf.ref} (100%) rename test/{atm/global-workflow/lgetkf_observer.ref => testreference/atm_jjob_lgetkf_observer.ref} (100%) rename test/{atm/global-workflow/lgetkf_solver.ref => testreference/atm_jjob_lgetkf_solver.ref} (100%) delete mode 100755 ush/jediinc2fv3.py diff --git a/ci/hera.sh b/ci/hera.sh index ec3858370..8a9518f8b 100644 --- a/ci/hera.sh +++ b/ci/hera.sh @@ -6,3 +6,4 @@ export SALLOC_ACCOUNT=$SLURM_ACCOUNT export SBATCH_ACCOUNT=$SLURM_ACCOUNT export SLURM_QOS=debug export PATH=$PATH:/home/role.jedipara/bin +export NTASKS_TESTS=12 diff --git a/ci/orion.sh b/ci/orion.sh index 1232d2828..8c2cf9a5f 100644 --- a/ci/orion.sh +++ b/ci/orion.sh @@ -8,3 +8,4 @@ export SLURM_QOS=debug export SLURM_EXCLUSIVE=user export OMP_NUM_THREADS=1 ulimit -s unlimited +export NTASKS_TESTS=12 diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 1b5f27231..917f70e90 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -61,7 +61,7 @@ module use $GDAS_MODULE_USE module load GDAS/$TARGET echo "---------------------------------------------------" >> $outfile rm -rf log.ctest -ctest -R gdasapp --output-on-failure &>> log.ctest +ctest -j${NTASKS_TESTS} -R gdasapp --output-on-failure &>> log.ctest ctest_status=$? npassed=$(cat log.ctest | grep "tests passed") if [ $ctest_status -eq 0 ]; then diff --git a/ci/run_gw_ci.sh b/ci/run_gw_ci.sh index ba1874107..d3b7de9d0 100755 --- a/ci/run_gw_ci.sh +++ b/ci/run_gw_ci.sh @@ -62,7 +62,7 @@ module use $repodir/sorc/gdas.cd/modulefiles module load GDAS/$TARGET echo "---------------------------------------------------" >> $outfile rm -rf log.ctest -ctest -R gdasapp --output-on-failure &>> log.ctest +ctest -j${NTASKS_TESTS} -R gdasapp --output-on-failure &>> log.ctest ctest_status=$? npassed=$(cat log.ctest | grep "tests passed") if [ $ctest_status -eq 0 ]; then diff --git a/parm/atm/jcb-base.yaml.j2 b/parm/atm/jcb-base.yaml.j2 index d7e1e5907..39c626a84 100644 --- a/parm/atm/jcb-base.yaml.j2 +++ b/parm/atm/jcb-base.yaml.j2 @@ -27,6 +27,11 @@ final_prints_frequency: PT3H number_of_outer_loops: 2 analysis_variables: [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr] +# Testing +# ------- + +do_testing: {{ DO_TEST_MODE | default(false, true) }} + # Model things # ------------ # Geometry diff --git a/parm/atm/jcb-prototype_3dvar-fv3inc.yaml.j2 b/parm/atm/jcb-prototype_3dvar-fv3inc.yaml.j2 new file mode 100644 index 000000000..f7f6db57a --- /dev/null +++ b/parm/atm/jcb-prototype_3dvar-fv3inc.yaml.j2 @@ -0,0 +1,10 @@ +# Algorithm +# --------- +algorithm: fv3jedi_fv3inc_variational + +# Testing things +# -------------- +test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/testreference/C96C48_ufs_hybatmDA_3dvar-fv3inc.ref +test_output_filename: {{ HOMEgfs }}/sorc/gdas.cd/build/gdas/test/testoutput/C96C48_ufs_hybatmDA_3dvar-fv3inc.test.out +test_float_relative_tolerance: 1.0e-3 +test_float_absolute_tolerance: 1.0e-5 diff --git a/parm/atm/jcb-prototype_3dvar.yaml.j2 b/parm/atm/jcb-prototype_3dvar.yaml.j2 index 4330a87bd..b89b65105 100644 --- a/parm/atm/jcb-prototype_3dvar.yaml.j2 +++ b/parm/atm/jcb-prototype_3dvar.yaml.j2 @@ -30,3 +30,10 @@ observations: # - scatwind_ascat_metop-a # - snowcvr # - abi_g16 + +# Testing things +# -------------- +test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/testreference/C96C48_ufs_hybatmDA_3dvar.ref +test_output_filename: {{ HOMEgfs }}/sorc/gdas.cd/build/gdas/test/testoutput/C96C48_ufs_hybatmDA_3dvar.test.out +test_float_relative_tolerance: 1.0e-3 +test_float_absolute_tolerance: 1.0e-5 diff --git a/parm/atm/jcb-prototype_lgetkf-fv3inc.yaml.j2 b/parm/atm/jcb-prototype_lgetkf-fv3inc.yaml.j2 new file mode 100644 index 000000000..9845103df --- /dev/null +++ b/parm/atm/jcb-prototype_lgetkf-fv3inc.yaml.j2 @@ -0,0 +1,10 @@ +# Algorithm +# --------- +algorithm: fv3jedi_fv3inc_lgetkf + +# Testing things +# -------------- +test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/testreference/C96C48_ufs_hybatmDA_lgetkf-fv3inc.ref +test_output_filename: {{ HOMEgfs }}/sorc/gdas.cd/build/gdas/test/testoutput/C96C48_ufs_hybatmDA_lgetkf-fv3inc.test.out +test_float_relative_tolerance: 1.0e-3 +test_float_absolute_tolerance: 1.0e-5 diff --git a/parm/atm/jcb-prototype_lgetkf.yaml.j2 b/parm/atm/jcb-prototype_lgetkf.yaml.j2 index 2ed04df3b..04f9faafd 100644 --- a/parm/atm/jcb-prototype_lgetkf.yaml.j2 +++ b/parm/atm/jcb-prototype_lgetkf.yaml.j2 @@ -38,3 +38,10 @@ observations: - satwnd.viirs_npp # - scatwind_ascat_metop-a # - snowcvr + +# Testing things +# -------------- +test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/testreference/C96C48_ufs_hybatmDA_lgetkf.ref +test_output_filename: {{ HOMEgfs }}/sorc/gdas.cd/build/gdas/test/testoutput/C96C48_ufs_hybatmDA_lgetkf.test.out +test_float_relative_tolerance: 1.0e-3 +test_float_absolute_tolerance: 1.0e-5 diff --git a/parm/atm/jcb-prototype_lgetkf_observer.yaml.j2 b/parm/atm/jcb-prototype_lgetkf_observer.yaml.j2 index 4b800ac8f..39e682415 100644 --- a/parm/atm/jcb-prototype_lgetkf_observer.yaml.j2 +++ b/parm/atm/jcb-prototype_lgetkf_observer.yaml.j2 @@ -44,3 +44,10 @@ atmosphere_obsdataout_prefix: diag_lobs_ # Distribution type distribution_type: RoundRobin + +# Testing things +# -------------- +test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/testreference/C96C48_ufs_hybatmDA_lgetkf_observer.ref +test_output_filename: {{ HOMEgfs }}/sorc/gdas.cd/build/gdas/test/testoutput/C96C48_ufs_hybatmDA_lgetkf_observer.test.out +test_float_relative_tolerance: 1.0e-3 +test_float_absolute_tolerance: 1.0e-5 diff --git a/parm/atm/jcb-prototype_lgetkf_solver.yaml.j2 b/parm/atm/jcb-prototype_lgetkf_solver.yaml.j2 index b5123dde9..7abccdd6e 100644 --- a/parm/atm/jcb-prototype_lgetkf_solver.yaml.j2 +++ b/parm/atm/jcb-prototype_lgetkf_solver.yaml.j2 @@ -47,3 +47,10 @@ atmosphere_obsdataout_prefix: diag_solv_ # Distribution type distribution_type: Halo + +# Testing things +# -------------- +test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/testreference/C96C48_ufs_hybatmDA_lgetkf_solver.ref +test_output_filename: {{ HOMEgfs }}/sorc/gdas.cd/build/gdas/test/testoutput/C96C48_ufs_hybatmDA_lgetkf_solver.test.out +test_float_relative_tolerance: 1.0e-3 +test_float_absolute_tolerance: 1.0e-5 diff --git a/parm/jcb-algorithms b/parm/jcb-algorithms index 43d8ff6ba..0f4423943 160000 --- a/parm/jcb-algorithms +++ b/parm/jcb-algorithms @@ -1 +1 @@ -Subproject commit 43d8ff6ba14baf9402ee58d0f3351a143c21211b +Subproject commit 0f4423943e1c2bdd4b9d0a4218e30df83c5e2c66 diff --git a/parm/soca/marine-jcb-3dfgat.yaml.j2 b/parm/soca/marine-jcb-3dfgat.yaml.j2 index acdfa002d..59d0b9ff5 100644 --- a/parm/soca/marine-jcb-3dfgat.yaml.j2 +++ b/parm/soca/marine-jcb-3dfgat.yaml.j2 @@ -3,3 +3,10 @@ algorithm: 3dfgat # Observations observations: !INC ${OBS_LIST_SHORT} + +# Testing things +# -------------- +test_reference_filename: '{{HOMEgfs}}/sorc/gdas.cd/test/testreference/C48mx500_3DVarAOWCDA_3dfgat.ref' +test_output_filename: '{{HOMEgfs}}/sorc/gdas.cd/build/gdas/test/testoutput/C48mx500_3DVarAOWCDA_3dfgat.test.out' +test_float_relative_tolerance: 1.0e-3 +test_float_absolute_tolerance: 1.0e-5 diff --git a/parm/soca/marine-jcb-base.yaml b/parm/soca/marine-jcb-base.yaml index 3a9d40223..1b55dc7d2 100644 --- a/parm/soca/marine-jcb-base.yaml +++ b/parm/soca/marine-jcb-base.yaml @@ -26,6 +26,10 @@ final_prints_frequency: PT3H number_of_outer_loops: 1 analysis_variables: [sea_ice_area_fraction, sea_ice_thickness, sea_ice_snow_thickness, sea_water_salinity, sea_water_potential_temperature, eastward_sea_water_velocity, northward_sea_water_velocity, sea_surface_height_above_geoid] +# Testing +# ------- + +do_testing: '{{ DO_TEST_MODE | default(false, true) }}' # Model things # ------------ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index cae0d55ec..a31f5443e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -24,7 +24,6 @@ file(ARCHIVE_EXTRACT INPUT ${CMAKE_CURRENT_BINARY_DIR}/${TAR} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) # list of test binary/data files to install list(APPEND test_data - ${CMAKE_CURRENT_BINARY_DIR}/testdata/atminc_compress.nc4 ${CMAKE_CURRENT_BINARY_DIR}/testdata/adt.nc ${CMAKE_CURRENT_BINARY_DIR}/testdata/sst.nc ${CMAKE_CURRENT_BINARY_DIR}/testdata/sss.nc @@ -120,11 +119,6 @@ add_test(NAME test_gdasapp_check_yaml_keys # WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) # commented out above test because it does not really do as advertised with all these templates which are not valid yaml -# test for ush/jediinc2fv3.py -add_test(NAME test_gdasapp_jedi_increment_to_fv3 - COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/ush/jediinc2fv3.py ${PROJECT_BINARY_DIR}/test/testdata/atmges_compress.nc4 ${PROJECT_BINARY_DIR}/test/testdata/atminc_compress.nc4 ${PROJECT_BINARY_DIR}/test/testoutput/fv_increment.nc - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) - # high level tests that require the global-workflow # TODO(AFE) see GDASApp issue #1213 #if (WORKFLOW_TESTS) @@ -136,7 +130,6 @@ add_test(NAME test_gdasapp_jedi_increment_to_fv3 #endif() if (${BUILD_GDASBUNDLE}) - add_subdirectory(fv3jedi) # fv3jedi tests add_subdirectory(snow) # snow da tests add_subdirectory(gw-ci) # subset of the global-workflow ci tests endif() diff --git a/test/atm/global-workflow/CMakeLists.txt b/test/atm/global-workflow/CMakeLists.txt index 925a73677..daa5b8edf 100644 --- a/test/atm/global-workflow/CMakeLists.txt +++ b/test/atm/global-workflow/CMakeLists.txt @@ -1,63 +1,86 @@ # test for creating an experiment directory within the global-workflow file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) -add_test(NAME test_gdasapp_setup_atm_cycled_exp - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/setup_workflow_exp.sh - ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) - -add_test(NAME test_gdasapp_atm_jjob_var_init - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_var_init.sh - ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) - -add_test(NAME test_gdasapp_atm_jjob_var_run - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_var_run.sh - ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) - -add_test(NAME test_gdasapp_atm_jjob_var_inc - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_var_inc.sh - ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) - -add_test(NAME test_gdasapp_atm_jjob_var_final - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_var_final.sh - ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) - -add_test(NAME test_gdasapp_atm_jjob_ens_init - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_init.sh - ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) - -add_test(NAME test_gdasapp_atm_jjob_ens_letkf - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_letkf.sh - ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) - -add_test(NAME test_gdasapp_atm_jjob_ens_init_split - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_init_split.sh - ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) - -add_test(NAME test_gdasapp_atm_jjob_ens_obs - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_obs.sh - ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) - -add_test(NAME test_gdasapp_atm_jjob_ens_sol - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_sol.sh - ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) +ecbuild_add_test(TARGET test_gdasapp_setup_atm_cycled_exp + TYPE SCRIPT + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/setup_workflow_exp.sh + ARGS ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) + +ecbuild_add_test(TARGET test_gdasapp_atm_jjob_var_init + TYPE SCRIPT + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_var_init.sh + ARGS ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun + TEST_DEPENDS test_gdasapp_setup_atm_cycled_exp) + +ecbuild_add_test(TARGET test_gdasapp_atm_jjob_var_run + TYPE SCRIPT + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_var_run.sh + ARGS ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun + TEST_DEPENDS test_gdasapp_atm_jjob_var_init) + +ecbuild_add_test(TARGET test_gdasapp_atm_jjob_var_inc + TYPE SCRIPT + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_var_inc.sh + ARGS ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun + TEST_DEPENDS test_gdasapp_atm_jjob_var_run) + +ecbuild_add_test(TARGET test_gdasapp_atm_jjob_var_final + TYPE SCRIPT + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_var_final.sh + ARGS ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun + TEST_DEPENDS test_gdasapp_atm_jjob_var_inc) + +ecbuild_add_test(TARGET test_gdasapp_atm_jjob_ens_init + TYPE SCRIPT + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_init.sh + ARGS ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun + TEST_DEPENDS test_gdasapp_setup_atm_cycled_exp) + +ecbuild_add_test(TARGET test_gdasapp_atm_jjob_ens_letkf + TYPE SCRIPT + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_letkf.sh + ARGS ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun + TEST_DEPENDS test_gdasapp_atm_jjob_ens_init) + +ecbuild_add_test(TARGET test_gdasapp_atm_jjob_ens_init_split + TYPE SCRIPT + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_init_split.sh + ARGS ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun + TEST_DEPENDS test_gdasapp_atm_jjob_ens_letkf) + +ecbuild_add_test(TARGET test_gdasapp_atm_jjob_ens_obs + TYPE SCRIPT + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_obs.sh + ARGS ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun + TEST_DEPENDS test_gdasapp_atm_jjob_ens_init_split) + +ecbuild_add_test(TARGET test_gdasapp_atm_jjob_ens_sol + TYPE SCRIPT + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_sol.sh + ARGS ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun + TEST_DEPENDS test_gdasapp_atm_jjob_ens_obs) -add_test(NAME test_gdasapp_atm_jjob_ens_inc - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_inc.sh - ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) - -add_test(NAME test_gdasapp_atm_jjob_ens_final - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_final.sh - ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) +ecbuild_add_test(TARGET test_gdasapp_atm_jjob_ens_inc + TYPE SCRIPT + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_inc.sh + ARGS ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun + TEST_DEPENDS test_gdasapp_atm_jjob_ens_sol) + +ecbuild_add_test(TARGET test_gdasapp_atm_jjob_ens_final + TYPE SCRIPT + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_final.sh + ARGS ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun + TEST_DEPENDS test_gdasapp_atm_jjob_ens_inc) diff --git a/test/atm/global-workflow/config.yaml b/test/atm/global-workflow/config.yaml index d8bf2e4c2..d6d4b0343 100644 --- a/test/atm/global-workflow/config.yaml +++ b/test/atm/global-workflow/config.yaml @@ -8,9 +8,11 @@ base: DUMPDIR: "@dumpdir@" STMP: "@bindir@/test/atm/global-workflow/testrun" PTMP: "@bindir@/test/atm/global-workflow/testrun" + DO_TEST_MODE: "YES" atmanl: JCB_ALGO_YAML_VAR: "@srcdir@/test/atm/global-workflow/jcb-prototype_3dvar.yaml.j2" + JCB_ALGO_YAML_FV3INC: "@srcdir@/test/atm/global-workflow/jcb-prototype_3dvar-fv3inc.yaml.j2" STATICB_TYPE: "identity" ATMRES_ANL: "C48" LAYOUT_X_ATMANL: 1 @@ -20,5 +22,6 @@ atmensanl: JCB_ALGO_YAML_LETKF: "@srcdir@/test/atm/global-workflow/jcb-prototype_lgetkf.yaml.j2" JCB_ALGO_YAML_OBS: "@srcdir@/test/atm/global-workflow/jcb-prototype_lgetkf_observer.yaml.j2" JCB_ALGO_YAML_SOL: "@srcdir@/test/atm/global-workflow/jcb-prototype_lgetkf_solver.yaml.j2" + JCB_ALGO_YAML_FV3INC: "@srcdir@/test/atm/global-workflow/jcb-prototype_lgetkf-fv3inc.yaml.j2" LAYOUT_X_ATMENSANL: 1 LAYOUT_Y_ATMENSANL: 1 diff --git a/test/atm/global-workflow/jcb-prototype_3dvar-fv3inc.yaml.j2 b/test/atm/global-workflow/jcb-prototype_3dvar-fv3inc.yaml.j2 new file mode 100644 index 000000000..9686fbbce --- /dev/null +++ b/test/atm/global-workflow/jcb-prototype_3dvar-fv3inc.yaml.j2 @@ -0,0 +1,10 @@ +# Algorithm +# --------- +algorithm: fv3jedi_fv3inc_variational + +# Testing things +# -------------- +test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/testreference/atm_jjob_3dvar-fv3inc.ref +test_output_filename: {{ HOMEgfs }}/sorc/gdas.cd/build/gdas/test/testoutput/atm_jjob_3dvar-fv3inc.test.out +test_float_relative_tolerance: 1.0e-3 +test_float_absolute_tolerance: 1.0e-5 diff --git a/test/atm/global-workflow/jcb-prototype_3dvar.yaml.j2 b/test/atm/global-workflow/jcb-prototype_3dvar.yaml.j2 index e16f86ecc..7465a4274 100644 --- a/test/atm/global-workflow/jcb-prototype_3dvar.yaml.j2 +++ b/test/atm/global-workflow/jcb-prototype_3dvar.yaml.j2 @@ -17,7 +17,7 @@ atmosphere_obsdatain_suffix: ".{{ current_cycle | to_YMDH }}.nc" # Testing things # -------------- -test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/atm/global-workflow/3dvar.ref -test_output_filename: ./3dvar.out +test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/testreference/atm_jjob_3dvar.ref +test_output_filename: {{ HOMEgfs }}/sorc/gdas.cd/build/gdas/test/testoutput/atm_jjob_3dvar.test.out test_float_relative_tolerance: 1.0e-3 test_float_absolute_tolerance: 1.0e-5 diff --git a/test/atm/global-workflow/jcb-prototype_lgetkf-fv3inc.yaml.j2 b/test/atm/global-workflow/jcb-prototype_lgetkf-fv3inc.yaml.j2 new file mode 100644 index 000000000..cd11e771d --- /dev/null +++ b/test/atm/global-workflow/jcb-prototype_lgetkf-fv3inc.yaml.j2 @@ -0,0 +1,10 @@ +# Algorithm +# --------- +algorithm: fv3jedi_fv3inc_lgetkf + +# Testing things +# -------------- +test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/testreference/atm_jjob_lgetkf-fv3inc.ref +test_output_filename: {{ HOMEgfs }}/sorc/gdas.cd/build/gdas/test/testoutput/atm_jjob_lgetkf-fv3inc.test.out +test_float_relative_tolerance: 1.0e-3 +test_float_absolute_tolerance: 1.0e-5 diff --git a/test/atm/global-workflow/jcb-prototype_lgetkf.yaml.j2 b/test/atm/global-workflow/jcb-prototype_lgetkf.yaml.j2 index b83b7d695..e6b1f9593 100644 --- a/test/atm/global-workflow/jcb-prototype_lgetkf.yaml.j2 +++ b/test/atm/global-workflow/jcb-prototype_lgetkf.yaml.j2 @@ -22,7 +22,7 @@ atmosphere_obsdatain_suffix: ".{{ current_cycle | to_YMDH }}.nc" # Testing things # -------------- -test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/atm/global-workflow/lgetkf.ref -test_output_filename: ./lgetkf.out +test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/testreference/atm_jjob_lgetkf.ref +test_output_filename: {{ HOMEgfs }}/sorc/gdas.cd/build/gdas/test/testoutput/atm_jjob_lgetkf.test.out test_float_relative_tolerance: 1.0e-3 test_float_absolute_tolerance: 1.0e-5 diff --git a/test/atm/global-workflow/jcb-prototype_lgetkf_observer.yaml.j2 b/test/atm/global-workflow/jcb-prototype_lgetkf_observer.yaml.j2 index ec1182019..ac0fcd7dd 100644 --- a/test/atm/global-workflow/jcb-prototype_lgetkf_observer.yaml.j2 +++ b/test/atm/global-workflow/jcb-prototype_lgetkf_observer.yaml.j2 @@ -28,7 +28,7 @@ distribution_type: RoundRobin # Testing things # -------------- -test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/atm/global-workflow/lgetkf_observer.ref -test_output_filename: ./lgetkf_observer.out +test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/testreference/atm_jjob_lgetkf_observer.ref +test_output_filename: {{ HOMEgfs }}/sorc/gdas.cd/build/gdas/test/testoutput/atm_jjob_lgetkf_observer.test.out test_float_relative_tolerance: 1.0e-3 test_float_absolute_tolerance: 1.0e-5 diff --git a/test/atm/global-workflow/jcb-prototype_lgetkf_solver.yaml.j2 b/test/atm/global-workflow/jcb-prototype_lgetkf_solver.yaml.j2 index a6ad34751..2c3bd2aae 100644 --- a/test/atm/global-workflow/jcb-prototype_lgetkf_solver.yaml.j2 +++ b/test/atm/global-workflow/jcb-prototype_lgetkf_solver.yaml.j2 @@ -28,7 +28,7 @@ distribution_type: Halo # Testing things # -------------- -test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/atm/global-workflow/lgetkf_solver.ref -test_output_filename: ./lgetkf_solver.out +test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/testreference/atm_jjob_lgetkf_solver.ref +test_output_filename: {{ HOMEgfs }}/sorc/gdas.cd/build/gdas/test/testoutput/atm_jjob_lgetkf_solver.test.out test_float_relative_tolerance: 1.0e-3 test_float_absolute_tolerance: 1.0e-5 diff --git a/test/fv3jedi/CMakeLists.txt b/test/fv3jedi/CMakeLists.txt deleted file mode 100644 index 9b75510b9..000000000 --- a/test/fv3jedi/CMakeLists.txt +++ /dev/null @@ -1,23 +0,0 @@ -# Subdirectories -file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/fv3jedi/testinput) -file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/fv3jedi/testoutput) -file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/fv3jedi/fv3jedi) - -# Test YAMLs -file(CREATE_LINK ${PROJECT_SOURCE_DIR}/test/fv3jedi/testinput/gdasapp_fv3jedi_fv3inc.yaml ${PROJECT_BINARY_DIR}/test/fv3jedi/testinput/gdasapp_fv3jedi_fv3inc.yaml SYMBOLIC) - -# Test reference files -file(CREATE_LINK ${PROJECT_SOURCE_DIR}/test/fv3jedi/testoutput/gdasapp_fv3jedi_fv3inc.ref ${PROJECT_BINARY_DIR}/test/fv3jedi/testoutput/gdasapp_fv3jedi_fv3inc.ref SYMBOLIC) - -# FV3 files -file(CREATE_LINK ${PROJECT_SOURCE_DIR}/parm/io/fv3jedi_fieldmetadata_fv3inc.yaml ${PROJECT_BINARY_DIR}/test/fv3jedi/fv3jedi/fv3jedi_fieldmetadata_fv3inc.yaml SYMBOLIC) -file(CREATE_LINK ${PROJECT_SOURCE_DIR}/parm/io/fv3jedi_fieldmetadata_history.yaml ${PROJECT_BINARY_DIR}/test/fv3jedi/fv3jedi/fv3jedi_fieldmetadata_history.yaml SYMBOLIC) -file(CREATE_LINK ${PROJECT_SOURCE_DIR}/sorc/fv3-jedi/test/Data/fv3files/akbk127.nc4 ${PROJECT_BINARY_DIR}/test/fv3jedi/fv3jedi/akbk127.nc4 SYMBOLIC) -file(CREATE_LINK ${PROJECT_SOURCE_DIR}/sorc/fv3-jedi/test/Data/fv3files/fmsmpp.nml ${PROJECT_BINARY_DIR}/test/fv3jedi/fv3jedi/fmsmpp.nml SYMBOLIC) -file(CREATE_LINK ${PROJECT_SOURCE_DIR}/sorc/fv3-jedi/test/Data/fv3files/field_table_gfdl ${PROJECT_BINARY_DIR}/test/fv3jedi/fv3jedi/field_table_gfdl SYMBOLIC) - -# Tests -add_test(NAME test_gdasapp_fv3jedi_fv3inc - COMMAND srun -n6 ${CMAKE_BINARY_DIR}/bin/fv3jedi_fv3inc.x ${PROJECT_BINARY_DIR}/test/fv3jedi/testinput/gdasapp_fv3jedi_fv3inc.yaml - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/fv3jedi) - diff --git a/test/fv3jedi/testinput/gdasapp_fv3jedi_fv3inc.yaml b/test/fv3jedi/testinput/gdasapp_fv3jedi_fv3inc.yaml deleted file mode 100644 index 7ee403ccc..000000000 --- a/test/fv3jedi/testinput/gdasapp_fv3jedi_fv3inc.yaml +++ /dev/null @@ -1,65 +0,0 @@ -variable change: - variable change name: Model2GeoVaLs - input variables: &bkgvars [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr,hgtsfc] - output variables: &fv3incrvars [ua,va,t,sphum,ice_wat,liq_wat,o3mr,delp,delz] -jedi increment variables: [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr] -fv3 increment variables: *fv3incrvars -background geometry: - fms initialization: - namelist filename: ./fv3jedi/fmsmpp.nml - field table filename: ./fv3jedi/field_table_gfdl - akbk: ./fv3jedi/akbk127.nc4 - layout: - - '1' - - '1' - npx: '13' - npy: '13' - npz: '127' - field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_history.yaml -jedi increment geometry: - fms initialization: - namelist filename: ./fv3jedi/fmsmpp.nml - field table filename: ./fv3jedi/field_table_gfdl - akbk: ./fv3jedi/akbk127.nc4 - layout: - - '1' - - '1' - npx: '13' - npy: '13' - npz: '127' - field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_history.yaml -fv3 increment geometry: - fms initialization: - namelist filename: ./fv3jedi/fmsmpp.nml - field table filename: ./fv3jedi/field_table_gfdl - akbk: ./fv3jedi/akbk127.nc4 - layout: - - '1' - - '1' - npx: '13' - npy: '13' - npz: '127' - field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_fv3inc.yaml -members: -- background input: - filetype: cube sphere history - datapath: ../testdata/ - provider: ufs - datetime: '2021-07-31T12:00:00Z' - filename: gdas.t06z.atmf006.nc - state variables: *bkgvars - jedi increment input: - filetype: cube sphere history - datapath: ../testdata/ - filename: atminc.20210731.120000.nc4 - provider: ufs - fv3 increment output: - filetype: cube sphere history - filename: atminc.20210731_120000.nc4 - provider: ufs - -test: - reference filename: testoutput/gdasapp_fv3jedi_fv3inc.ref - test output filename: testoutput/gdasapp_fv3jedi_fv3inc.test.out - float relative tolerance: 1.0e-3 - diff --git a/test/fv3jedi/testoutput/gdasapp_fv3jedi_fv3inc.ref b/test/fv3jedi/testoutput/gdasapp_fv3jedi_fv3inc.ref deleted file mode 100644 index a4be55d39..000000000 --- a/test/fv3jedi/testoutput/gdasapp_fv3jedi_fv3inc.ref +++ /dev/null @@ -1,41 +0,0 @@ -Background State: - ----------------------------------------------------------------------------------------------------- -State print | number of fields = 9 | cube sphere face size: C12 -eastward_wind | Min:-7.6655158996582031e+01 Max:+1.3484048461914062e+02 RMS:+2.4474819956360815e+01 -northward_wind | Min:-5.7591831207275391e+01 Max:+6.1521602630615234e+01 RMS:+9.0604680778982232e+00 -air_temperature | Min:+1.6467715454101562e+02 Max:+3.1937854003906250e+02 RMS:+2.5118590413162758e+02 -surface_pressure | Min:+5.6141089843750000e+04 Max:+1.0346660156250000e+05 RMS:+9.8890740204548158e+04 -specific_humidity | Min:+1.2364032020428795e-09 Max:+2.1250871941447258e-02 RMS:+5.1335857648077696e-03 -cloud_liquid_ice | Min:+0.0000000000000000e+00 Max:+5.9844314819201827e-04 RMS:+1.2911033014274613e-05 -cloud_liquid_water | Min:+0.0000000000000000e+00 Max:+1.1054204078391194e-03 RMS:+3.7434027002184771e-05 -ozone_mass_mixing_ratio | Min:+8.6089233519714981e-09 Max:+1.6528036212548614e-05 RMS:+4.4014999048943556e-06 -sfc_geopotential_height_times_grav | Min:-3.9759671688079834e+00 Max:+4.8742519531250000e+03 RMS:+6.2112179267749525e+02 ----------------------------------------------------------------------------------------------------- -JEDI Increment: - ----------------------------------------------------------------------------------------------------- -Increment print | number of fields = 8 | cube sphere face size: C12 -eastward_wind | Min:-4.3422836419308410e+00 Max:+1.2320940067737499e+01 RMS:+3.0957235443709130e-01 -northward_wind | Min:-4.1090470888107049e+00 Max:+5.4552721209750796e+00 RMS:+3.1062842460180656e-01 -air_temperature | Min:-5.2980343087781989e-01 Max:+5.1811022097894011e-01 RMS:+3.5920751813835923e-02 -surface_pressure | Min:-1.3399192319669237e+02 Max:+6.8315024915413233e+01 RMS:+1.5538727344814935e+01 -specific_humidity | Min:-2.8092260972819617e-04 Max:+2.9434075393080551e-04 RMS:+1.6532405760382759e-05 -cloud_liquid_ice | Min:+0.0000000000000000e+00 Max:+0.0000000000000000e+00 RMS:+0.0000000000000000e+00 -cloud_liquid_water | Min:+0.0000000000000000e+00 Max:+0.0000000000000000e+00 RMS:+0.0000000000000000e+00 -ozone_mass_mixing_ratio | Min:+0.0000000000000000e+00 Max:+0.0000000000000000e+00 RMS:+0.0000000000000000e+00 ----------------------------------------------------------------------------------------------------- -FV3 Increment: - ----------------------------------------------------------------------------------------------------- -Increment print | number of fields = 9 | cube sphere face size: C12 -eastward_wind | Min:-4.3422836419308410e+00 Max:+1.2320940067737499e+01 RMS:+3.0957235443709130e-01 -northward_wind | Min:-4.1090470888107049e+00 Max:+5.4552721209750796e+00 RMS:+3.1062842460180656e-01 -air_temperature | Min:-5.2980343087781989e-01 Max:+5.1811022097894011e-01 RMS:+3.5920751813835923e-02 -specific_humidity | Min:-2.8092260972819617e-04 Max:+2.9434075393080551e-04 RMS:+1.6532405760382759e-05 -cloud_liquid_ice | Min:+0.0000000000000000e+00 Max:+0.0000000000000000e+00 RMS:+0.0000000000000000e+00 -cloud_liquid_water | Min:+0.0000000000000000e+00 Max:+0.0000000000000000e+00 RMS:+0.0000000000000000e+00 -ozone_mass_mixing_ratio | Min:+0.0000000000000000e+00 Max:+0.0000000000000000e+00 RMS:+0.0000000000000000e+00 -air_pressure_thickness | Min:-2.9992886080290191e+00 Max:+1.5291703492039233e+00 RMS:+1.7535872214547940e-01 -layer_thickness | Min:-4.6699236754648155e-01 Max:+7.4693987323735200e-01 RMS:+3.1162055487823255e-02 ----------------------------------------------------------------------------------------------------- diff --git a/test/gw-ci/CMakeLists.txt b/test/gw-ci/CMakeLists.txt index 4f1a969e3..5a07090d0 100644 --- a/test/gw-ci/CMakeLists.txt +++ b/test/gw-ci/CMakeLists.txt @@ -1,11 +1,220 @@ +# Function that adds a test for a given GW task +function(add_task task_name test_prefix is_full_cycle HALF_CYCLE FULL_CYCLE pslot WORKING_DIRECTORY PROJECT_SOURCE_DIR) + + # Get subtask names if task is a metatask + if("${task_name}" STREQUAL "gdas_fcst") + set(subtask_names_list + "gdas_fcst_seg0") + elseif("${task_name}" STREQUAL "enkfgdas_fcst") + if("${pslot}" STREQUAL "C48mx500_hybAOWCDA") + set(subtask_names_list + "enkfgdas_fcst_mem001" + "enkfgdas_fcst_mem002" + "enkfgdas_fcst_mem003") + else() + set(subtask_names_list + "enkfgdas_fcst_mem001" + "enkfgdas_fcst_mem002") + endif() + elseif("${task_name}" STREQUAL "gdas_atmos_prod") + set(subtask_names_list + "gdas_atmos_prod_f000" + "gdas_atmos_prod_f001" + "gdas_atmos_prod_f002" + "gdas_atmos_prod_f003" + "gdas_atmos_prod_f004" + "gdas_atmos_prod_f005" + "gdas_atmos_prod_f006" + "gdas_atmos_prod_f007" + "gdas_atmos_prod_f008" + "gdas_atmos_prod_f009") + elseif("${task_name}" STREQUAL "enkfgdas_epmn") + set(subtask_names_list + "enkfgdas_epos001" + "enkfgdas_epos002" + "enkfgdas_epos003" + "enkfgdas_epos004" + "enkfgdas_epos005" + "enkfgdas_epos006") + elseif("${task_name}" STREQUAL "enkfgdas_ecmn") + if ("${pslot}" STREQUAL "C96C48_ufs_hybatmDA") + set(subtask_names_list + "enkfgdas_ecen000") + else() + set(subtask_names_list + "enkfgdas_ecen000" + "enkfgdas_ecen001" + "enkfgdas_ecen002") + endif() + else() + set(subtask_names_list ${task_name}) + endif() + + # Convert list to comma-delimited string + string(JOIN "," subtask_names ${subtask_names_list}) + + # Get task dependencies + set(TEST_DEPENDS) + if(NOT ${is_full_cycle}) + if("${task_name}" STREQUAL "gdas_fcst") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_stage_ic_${HALF_CYCLE}") + elseif("${task_name}" STREQUAL "gdas_atmos_prod") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_fcst_${HALF_CYCLE}") + elseif("${task_name}" STREQUAL "enkfgdas_fcst") + list(APPEND TEST_DEPENDS "${test_prefix}_enkfgdas_stage_ic_${HALF_CYCLE}") + elseif("${task_name}" STREQUAL "enkfgdas_echgres") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_fcst_${HALF_CYCLE}") + list(APPEND TEST_DEPENDS "${test_prefix}_enkfgdas_fcst_${HALF_CYCLE}") + elseif("${task_name}" STREQUAL "enkfgdas_epmn") + list(APPEND TEST_DEPENDS "${test_prefix}_enkfgdas_fcst_${HALF_CYCLE}") + elseif("${task_name}" STREQUAL "gdas_aeroanlgenb") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_fcst_${HALF_CYCLE}") + else() + list(APPEND TEST_DEPENDS "${test_prefix}") + endif() + else() + if("${task_name}" STREQUAL "gdas_prep") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_fcst_${HALF_CYCLE}") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_atmos_prod_${HALF_CYCLE}") + elseif("${task_name}" STREQUAL "gdas_anal") + list(APPEND TEST_DEPENDS "${test_prefix}_enkfgdas_epmn_${HALF_CYCLE}") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_prep_${FULL_CYCLE}") + elseif("${task_name}" STREQUAL "gdas_prepatmiodaobs") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_prep_${FULL_CYCLE}") + elseif("${task_name}" STREQUAL "gdas_atmanlinit") + list(APPEND TEST_DEPENDS "${test_prefix}_enkfgdas_epmn_${HALF_CYCLE}") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_prepatmiodaobs_${FULL_CYCLE}") + elseif("${task_name}" STREQUAL "gdas_atmanlvar") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_atmanlinit_${FULL_CYCLE}") + elseif("${task_name}" STREQUAL "gdas_atmanlfv3inc") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_atmanlvar_${FULL_CYCLE}") + elseif("${task_name}" STREQUAL "gdas_atmanlfinal") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_atmanlfv3inc_${FULL_CYCLE}") + elseif("${task_name}" STREQUAL "gdas_aeroanlinit") + list(APPEND TEST_DEPENDS "${test_prefix}_aeroanlgenb_${HALF_CYCLE}") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_prep_${FULL_CYCLE}") + elseif("${task_name}" STREQUAL "gdas_aeroanlvar") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_aeroanlinit_${FULL_CYCLE}") + elseif("${task_name}" STREQUAL "gdas_aeroanlfinal") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_aeroanlvar_${FULL_CYCLE}") + elseif("${task_name}" STREQUAL "gdas_snowanl") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_prep_${FULL_CYCLE}") + elseif("${task_name}" STREQUAL "gdas_sfcanl") + if("${pslot}" STREQUAL "C96C48_ufs_hybatmDA") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_atmanlfinal_${FULL_CYCLE}") + elseif("${pslot}" STREQUAL "C96C48_hybatmaerosnowDA") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_anal_${FULL_CYCLE}") + list(APPEND TEST_DEPENDS "${test_prefix}_snowanal_${FULL_CYCLE}") + else() + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_anal_${FULL_CYCLE}") + endif() + elseif("${task_name}" STREQUAL "gdas_analcalc") + list(APPEND TEST_DEPENDS "${test_prefix}_enkfgdas_echgres_${HALF_CYCLE}") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_sfcanl_${FULL_CYCLE}") + elseif("${task_name}" STREQUAL "gdas_fcst") + if("${pslot}" STREQUAL "C96C48_hybatmaerosnowDA") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_sfcanl_${FULL_CYCLE}") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_aeroanlfinal_${FULL_CYCLE}") + elseif("${pslot}" STREQUAL "C48mx500_3DVarAOWCDA") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_sfcanl_${FULL_CYCLE}") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_marineanlfinal_${FULL_CYCLE}") + else() + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_sfcanl_${FULL_CYCLE}") + endif() + elseif("${task_name}" STREQUAL "enkfgdas_atmensanlinit") + list(APPEND TEST_DEPENDS "${test_prefix}_enkfgdas_epmn_${HALF_CYCLE}") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_prepatmiodaobs_${FULL_CYCLE}") + elseif("${task_name}" STREQUAL "enkfgdas_atmensanlobs") + list(APPEND TEST_DEPENDS "${test_prefix}_enkfgdas_atmensanlinit_${FULL_CYCLE}") + elseif("${task_name}" STREQUAL "enkfgdas_atmensanlsol") + list(APPEND TEST_DEPENDS "${test_prefix}_enkfgdas_atmensanlobs_${FULL_CYCLE}") + elseif("${task_name}" STREQUAL "enkfgdas_atmensanlfv3inc") + list(APPEND TEST_DEPENDS "${test_prefix}_enkfgdas_atmensanlsol_${FULL_CYCLE}") + elseif("${task_name}" STREQUAL "enkfgdas_atmensanlfinal") + list(APPEND TEST_DEPENDS "${test_prefix}_enkfgdas_atmensanlfv3inc_${FULL_CYCLE}") + elseif("${task_name}" STREQUAL "enkfgdas_eobs") + set(TEST_DEPENDS) + list(APPEND TEST_DEPENDS "${test_prefix}_enkfgdas_epmn_${HALF_CYCLE}") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_prep_${FULL_CYCLE}") + elseif("${task_name}" STREQUAL "enkfgdas_ediag") + list(APPEND TEST_DEPENDS "${test_prefix}_enkfgdas_eobs_${FULL_CYCLE}") + elseif("${task_name}" STREQUAL "enkfgdas_eupd") + list(APPEND TEST_DEPENDS "${test_prefix}_enkfgdas_ediag_${FULL_CYCLE}") + elseif("${task_name}" STREQUAL "enkfgdas_ecmn") + if("${pslot}" STREQUAL "C96C48_ufs_hybatmDA") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_analcalc_${FULL_CYCLE}") + list(APPEND TEST_DEPENDS "${test_prefix}_enkfgdas_atmensanlfinal_${FULL_CYCLE}") + else() + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_analcalc_${FULL_CYCLE}") + list(APPEND TEST_DEPENDS "${test_prefix}_enkfgdas_eupd_${FULL_CYCLE}") + endif() + elseif("${task_name}" STREQUAL "enkfgdas_esnowrecen") + list(APPEND TEST_DEPENDS "${test_prefix}_enkfgdas_epmn_${HALF_CYCLE}") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_snowanl_${FULL_CYCLE}") + elseif("${task_name}" STREQUAL "enkfgdas_esfc") + if("${pslot}" STREQUAL "C96C48_ufs_hybatmDA") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_analcalc_${FULL_CYCLE}") + list(APPEND TEST_DEPENDS "${test_prefix}_enkfgdas_atmensanlfinal_${FULL_CYCLE}") + elseif("${pslot}" STREQUAL "C96C48_hybatmaerosnowDA") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_analcalc_${FULL_CYCLE}") + list(APPEND TEST_DEPENDS "${test_prefix}_enkfgdas_eupd_${FULL_CYCLE}") + list(APPEND TEST_DEPENDS "${test_prefix}_enkfgdas_esnowrecen_${FULL_CYCLE}") + else() + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_analcalc_${FULL_CYCLE}") + list(APPEND TEST_DEPENDS "${test_prefix}_enkfgdas_eupd_${FULL_CYCLE}") + endif() + elseif("${task_name}" STREQUAL "enkfgdas_fcst") + list(APPEND TEST_DEPENDS "${test_prefix}_enkfgdas_ecmn_${FULL_CYCLE}") + list(APPEND TEST_DEPENDS "${test_prefix}_enkfgdas_esfc_${FULL_CYCLE}") + elseif("${task_name}" STREQUAL "gdas_prepoceanobs") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_fcst_${HALF_CYCLE}") + elseif("${task_name}" STREQUAL "gdas_marinebmat") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_fcst_${HALF_CYCLE}") + elseif("${task_name}" STREQUAL "gdas_marineanlinit") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_fcst_${HALF_CYCLE}") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_prepoceanobs_${FULL_CYCLE}") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_marinebmat_${FULL_CYCLE}") + elseif("${task_name}" STREQUAL "gdas_marineanlvar") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_marineanlinit_${FULL_CYCLE}") + elseif("${task_name}" STREQUAL "gdas_marineanlchkpt") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_marineanlvar_${FULL_CYCLE}") + elseif("${task_name}" STREQUAL "gdas_marineanlfinal") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_marineanlchkpt_${FULL_CYCLE}") + elseif("${task_name}" STREQUAL "gdas_marineanlletkf") + list(APPEND TEST_DEPENDS "${test_prefix}_enkfgdas_fcst_${HALF_CYCLE}") + list(APPEND TEST_DEPENDS "${test_prefix}_gdas_prepoceanobs_${FULL_CYCLE}") + else() + list(APPEND TEST_DEPENDS "${test_prefix}") + endif() + endif() + + # Set cycle + if(${is_full_cycle}) + set(cycle ${FULL_CYCLE}) + else() + set(cycle ${HALF_CYCLE}) + endif() + + # Add Task + set(test_name ${test_prefix}_${task_name}_${cycle}) + message(STATUS "preparing ${subtask_names} for ${test_prefix} ctest") + ecbuild_add_test(TARGET ${test_name} + TYPE SCRIPT + COMMAND ${PROJECT_SOURCE_DIR}/test/gw-ci/run_exp.sh + ARGS ${pslot} ${cycle} ${subtask_names} + WORKING_DIRECTORY ${WORKING_DIRECTORY} + TEST_DEPENDS ${TEST_DEPENDS}) +endfunction() + # Function that generates the 1/2 cycle forecast and DA tasks -function(add_cycling_tests pslot YAML_PATH HOMEgfs RUNTESTS PROJECT_SOURCE_DIR TASK_LIST) - set(test_name test_gdasapp_${pslot}) +function(add_cycling_tests pslot YAML_PATH HOMEgfs WORKING_DIRECTORY PROJECT_SOURCE_DIR HALF_CYCLE_TASKS FULL_CYCLE_TASKS) + set(test_prefix test_gdasapp_${pslot}) # Prepare the COMROOT and EXPDIR for the cycling ctests - add_test(NAME ${test_name} - COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/create_exp.sh ${YAML_PATH} ${pslot} ${HOMEgfs} ${RUNTESTS}" - WORKING_DIRECTORY ${RUNTESTS}) - set_tests_properties(${test_name} PROPERTIES LABELS "manual") + ecbuild_add_test(TARGET ${test_prefix} + TYPE SCRIPT + COMMAND ${PROJECT_SOURCE_DIR}/test/gw-ci/create_exp.sh + ARGS ${YAML_PATH} ${pslot} ${HOMEgfs} ${WORKING_DIRECTORY} + WORKING_DIRECTORY ${WORKING_DIRECTORY}) # Get the 1/2 cycle and full cycle's dates execute_process( @@ -18,78 +227,151 @@ function(add_cycling_tests pslot YAML_PATH HOMEgfs RUNTESTS PROJECT_SOURCE_DIR T list(GET DATES_LIST 0 HALF_CYCLE) list(GET DATES_LIST 1 FULL_CYCLE) - # stage IC's - message(STATUS "staging the 1/2 cycle IC's for ${pslot} ctest") - add_test(NAME ${test_name}_gdas_stage_ic_${HALF_CYCLE} - COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/run_exp.sh ${pslot} gdas_stage_ic ${HALF_CYCLE}" - WORKING_DIRECTORY ${RUNTESTS}) - set_tests_properties(${test_name}_gdas_stage_ic_${HALF_CYCLE} PROPERTIES LABELS "manual") - - # stage ensemble ics - if (letkf) - message(STATUS "preparing enkfgdas_stage_ic for ${pslot} ctest") - add_test(NAME ${test_name}_enkfgdas_stage_ic_${HALF_CYCLE} - COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/run_exp.sh ${pslot} enkfgdas_stage_ic ${HALF_CYCLE}" - WORKING_DIRECTORY ${RUNTESTS}) - set_tests_properties(${test_name}_enkfgdas_stage_ic_${HALF_CYCLE} PROPERTIES LABELS "manual") - endif() - - # 1/2 cycle gdas_fcst - message(STATUS "preparing 1/2 cycle gdas_fcst for ${pslot} ctest") - add_test(NAME ${test_name}_gdas_fcst_seg0_${HALF_CYCLE} - COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/run_exp.sh ${pslot} gdas_fcst_seg0 ${HALF_CYCLE}" - WORKING_DIRECTORY ${RUNTESTS}) - set_tests_properties(${test_name}_gdas_fcst_seg0_${HALF_CYCLE} PROPERTIES LABELS "manual") - - # 1/2 cycle enkfgdas_fcst - if (letkf) - set(ENS_MEMS "mem001" "mem002" "mem003") - foreach(ENS_MEM ${ENS_MEMS}) - message(STATUS "preparing 1/2 cycle enkfgdas_fcst_${ENS_MEM} for ${pslot} ctest") - add_test(NAME ${test_name}_enkfgdas_fcst_${ENS_MEM}_${HALF_CYCLE} - COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/run_exp.sh ${pslot} enkfgdas_fcst_${ENS_MEM} ${HALF_CYCLE}" - WORKING_DIRECTORY ${RUNTESTS}) - set_tests_properties(${test_name}_enkfgdas_fcst_${ENS_MEM}_${HALF_CYCLE} PROPERTIES LABELS "manual") - endforeach() - endif() - + # Select the list of tasks to run for the half cycle + message(STATUS "Half-cycle tasks ${HALF_CYCLE_TASKS}") + set(is_full_cycle FALSE) + foreach(task_name ${HALF_CYCLE_TASKS}) + add_task(${task_name} ${test_prefix} ${is_full_cycle} ${HALF_CYCLE} ${FULL_CYCLE} ${pslot} ${WORKING_DIRECTORY} ${PROJECT_SOURCE_DIR}) + endforeach() + # Select the list of tasks to run for the full cycle - message(STATUS "Tasks ${TASK_LIST}") - - foreach(task ${TASK_LIST}) - message(STATUS "preparing the full cycle ${task} for ${pslot} ctest") - add_test(NAME ${test_name}_${task}_${FULL_CYCLE} - COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/run_exp.sh ${pslot} ${task} ${FULL_CYCLE}" - WORKING_DIRECTORY ${RUNTESTS}) - set_tests_properties(${test_name}_${task}_${FULL_CYCLE} PROPERTIES LABELS "manual") + message(STATUS "Full-cycle tasks ${FULL_CYCLE_TASKS}") + set(is_full_cycle TRUE) + foreach(task_name ${FULL_CYCLE_TASKS}) + add_task(${task_name} ${test_prefix} ${is_full_cycle} ${HALF_CYCLE} ${FULL_CYCLE} ${pslot} ${WORKING_DIRECTORY} ${PROJECT_SOURCE_DIR}) endforeach() endfunction() if (WORKFLOW_TESTS) # Setup the environement set(HOMEgfs ${CMAKE_SOURCE_DIR}/../../..) - set(RUNTESTS ${CMAKE_CURRENT_BINARY_DIR}/../../test/gw-ci) + set(WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../test/gw-ci) + + # GSI Atm DA C96/C48 + # ------------------ + set(pslot "C96C48_hybatmDA") + set(YAML_PATH ${HOMEgfs}/ci/cases/pr/${pslot}.yaml) + set(HALF_CYCLE_TASKS + "gdas_stage_ic" + "gdas_fcst" + "gdas_atmos_prod" + "enkfgdas_stage_ic" + "enkfgdas_fcst" + "enkfgdas_echgres" + "enkfgdas_epmn") + set(FULL_CYCLE_TASKS + "gdas_prep" + "gdas_anal" + "gdas_sfcanl" + "gdas_analcalc" + "gdas_fcst" + "enkfgdas_eobs" + "enkfgdas_ediag" + "enkfgdas_eupd" + "enkfgdas_ecmn" + "enkfgdas_esfc" + "enkfgdas_fcst" + ) - # 3DVAR WCDA, low-res + add_cycling_tests(${pslot} ${YAML_PATH} ${HOMEgfs} ${WORKING_DIRECTORY} ${PROJECT_SOURCE_DIR} "${HALF_CYCLE_TASKS}" "${FULL_CYCLE_TASKS}") + # JEDI Atm DA C96/C48 # ------------------- - set(pslot "WCDA-3DVAR-C48mx500") - set(YAML_PATH ${HOMEgfs}/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml) - set(TASK_LIST + set(pslot "C96C48_ufs_hybatmDA") + set(YAML_PATH ${HOMEgfs}/ci/cases/pr/${pslot}.yaml) + set(HALF_CYCLE_TASKS + "gdas_stage_ic" + "gdas_fcst" + "gdas_atmos_prod" + "enkfgdas_stage_ic" + "enkfgdas_fcst" + "enkfgdas_echgres" + "enkfgdas_epmn") + set(FULL_CYCLE_TASKS + "gdas_prep" + "gdas_prepatmiodaobs" + "gdas_atmanlinit" + "gdas_atmanlvar" + "gdas_atmanlfv3inc" + "gdas_atmanlfinal" + "gdas_sfcanl" + "gdas_analcalc" + "gdas_fcst" + "enkfgdas_atmensanlinit" + "enkfgdas_atmensanlobs" + "enkfgdas_atmensanlsol" + "enkfgdas_atmensanlfv3inc" + "enkfgdas_atmensanlfinal" + "enkfgdas_ecmn" + "enkfgdas_esfc" + "enkfgdas_fcst" + ) + add_cycling_tests(${pslot} ${YAML_PATH} ${HOMEgfs} ${WORKING_DIRECTORY} ${PROJECT_SOURCE_DIR} "${HALF_CYCLE_TASKS}" "${FULL_CYCLE_TASKS}") + + # Aero-Land DA C96 + # ---------------- + set(pslot "C96C48_hybatmaerosnowDA") + set(YAML_PATH ${HOMEgfs}/ci/cases/pr/${pslot}.yaml) + set(HALF_CYCLE_TASKS + "gdas_stage_ic" + "gdas_fcst" + "gdas_atmos_prod" + "gdas_aeroanlgenb" + "enkfgdas_stage_ic" + "enkfgdas_fcst" + "enkfgdas_echgres" + "enkfgdas_epmn") + set(FULL_CYCLE_TASKS + "gdas_prep" + "gdas_anal" + "gdas_aeroanlinit" + "gdas_aeroanlvar" + "gdas_aeroanlfinal" + "gdas_snowanl" + "gdas_sfcanl" + "gdas_analcalc" + "gdas_fcst" + "enkfgdas_eobs" + "enkfgdas_ediag" + "enkfgdas_eupd" + "enkfgdas_ecmn" + "enkfgdas_esnowrecen" + "enkfgdas_esfc" + "enkfgdas_fcst" + ) + add_cycling_tests(${pslot} ${YAML_PATH} ${HOMEgfs} ${WORKING_DIRECTORY} ${PROJECT_SOURCE_DIR} "${HALF_CYCLE_TASKS}" "${FULL_CYCLE_TASKS}") + + # GSI Atm DA C48, JEDI Marine DA 500 + # ---------------------------------- + set(pslot "C48mx500_3DVarAOWCDA") + set(YAML_PATH ${HOMEgfs}/ci/cases/pr/${pslot}.yaml) + set(HALF_CYCLE_TASKS + "gdas_stage_ic" + "gdas_fcst") + set(FULL_CYCLE_TASKS "gdas_prepoceanobs" "gdas_marinebmat" "gdas_marineanlinit" "gdas_marineanlvar" "gdas_marineanlchkpt" "gdas_marineanlfinal" + "gdas_prep" + "gdas_anal" + "gdas_sfcanl" + "gdas_fcst" ) - add_cycling_tests(${pslot} ${YAML_PATH} ${HOMEgfs} ${RUNTESTS} ${PROJECT_SOURCE_DIR} "${TASK_LIST}") + add_cycling_tests(${pslot} ${YAML_PATH} ${HOMEgfs} ${WORKING_DIRECTORY} ${PROJECT_SOURCE_DIR} "${HALF_CYCLE_TASKS}" "${FULL_CYCLE_TASKS}") # WCDA, low-res, ensemble da # ------------- - set(pslot "WCDA-hyb-C48mx500") + set(pslot "C48mx500_hybAOWCDA") set(letkf TRUE) - set(YAML_PATH ${HOMEgfs}/ci/cases/pr/C48mx500_hybAOWCDA.yaml) - set(TASK_LIST + set(YAML_PATH ${HOMEgfs}/ci/cases/pr/${pslot}.yaml) + set(HALF_CYCLE_TASKS + "gdas_stage_ic" + "gdas_fcst" + "enkfgdas_stage_ic" + "enkfgdas_fcst") + set(FULL_CYCLE_TASKS "gdas_prepoceanobs" "gdas_marineanlletkf" # TODO(AFE) waiting until these are working for hybrid @@ -99,40 +381,31 @@ if (WORKFLOW_TESTS) # "gdas_marineanlchkpt" # "gdas_marineanlfinal" ) - add_cycling_tests(${pslot} ${YAML_PATH} ${HOMEgfs} ${RUNTESTS} ${PROJECT_SOURCE_DIR} "${TASK_LIST}") + add_cycling_tests(${pslot} ${YAML_PATH} ${HOMEgfs} ${WORKING_DIRECTORY} ${PROJECT_SOURCE_DIR} "${HALF_CYCLE_TASKS}" "${FULL_CYCLE_TASKS}") set(letkf FALSE) -endif() - -option(RUN_GW_CI "Enable the global-workflow CI tests" OFF) -if (RUN_GW_CI) - # Aero-Land DA, C96 - # ----------------- - set(pslot "Aero-Snow-3DVAR-C96") - set(YAML_PATH ${HOMEgfs}/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml) - set(TASK_LIST) # empty list for now - add_cycling_tests(${pslot} ${YAML_PATH} ${HOMEgfs} ${RUNTESTS} ${PROJECT_SOURCE_DIR} "${TASK_LIST}") - - # Atm DA, C96/C48 - # --------------- - set(pslot "Atm-hyb-C96C48") - set(YAML_PATH ${HOMEgfs}/ci/cases/pr/C96C48_ufs_hybatmDA.yaml) - set(TASK_LIST) # empty list for now - add_cycling_tests(${pslot} ${YAML_PATH} ${HOMEgfs} ${RUNTESTS} ${PROJECT_SOURCE_DIR} "${TASK_LIST}") # GFSv17, 3DVAR prototype # ----------------------- - set(pslot "GFSv17-3DVAR-C384mx025") - set(YAML_PATH ${HOMEgfs}/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml) - set(TASK_LIST - "gdas_prepoceanobs" - "gdas_marinebmat" - "gdas_marineanlinit" - "gdas_marineanlvar" - "gdas_marineanlchkpt" - "gdas_marineanlfinal" - "gdas_ocnanalvrfy" - "gdas_prep" - "gdas_anal" - ) - add_cycling_tests(${pslot} ${YAML_PATH} ${HOMEgfs} ${RUNTESTS} ${PROJECT_SOURCE_DIR} "${TASK_LIST}") + option(TEST_GFS17 "Enable the GFSv17 prototype CI tests" OFF) + if(TEST_GFS17) + set(pslot "GFSv17-3DVAR-C384mx025") + set(YAML_PATH ${HOMEgfs}/ci/cases/gfsv17/${pslot}.yaml) + set(HALF_CYCLE_TASKS + "gdas_stage_ic" + "gdas_fcst") + set(FULL_CYCLE_TASKS + "gdas_prepoceanobs" + "gdas_marinebmat" + "gdas_marineanlinit" + "gdas_marineanlvar" + "gdas_marineanlchkpt" + "gdas_marineanlfinal" + "gdas_ocnanalvrfy" + "gdas_prep" + "gdas_anal" + "gdas_sfcanl" + "gdas_fcst" + ) + add_cycling_tests(${pslot} ${YAML_PATH} ${HOMEgfs} ${WORKING_DIRECTORY} ${PROJECT_SOURCE_DIR} "${HALF_CYCLE_TASKS}" "${FULL_CYCLE_TASKS}") + endif() endif() diff --git a/test/gw-ci/run_exp.sh b/test/gw-ci/run_exp.sh index 8040860a0..3c02dbb1a 100755 --- a/test/gw-ci/run_exp.sh +++ b/test/gw-ci/run_exp.sh @@ -1,39 +1,55 @@ #!/bin/bash pslot=$1 -TASK_NAME=$2 -CYCLE=$3 +CYCLE=$2 +shift +shift +task_args=("$@") # Define the workflow XML and database files WORKFLOW_XML=${pslot}/EXPDIR/${pslot}/${pslot}.xml WORKFLOW_DB=${pslot}/EXPDIR/${pslot}/${pslot}.db # Boot the task -echo "booting $TASK_NAME for cycle $CYCLE" +echo "booting ${TASK_ARRAY[@]} for cycle $CYCLE" if [[ ! -e "$WORKFLOW_DB" ]]; then - rocotorun -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$TASK_NAME" -c "$CYCLE" + rocotorun -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$task_args" -c "$CYCLE" fi -rocotoboot -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$TASK_NAME" -c "$CYCLE" +rocotoboot -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$task_args" -c "$CYCLE" +# Loop through tasks +IFS=',' read -r -a TASK_ARRAY <<< "$task_args" +num_tasks=${#TASK_ARRAY[@]} while true; do # Update the status of the task - rocotorun -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$TASK_NAME" -c "$CYCLE" + rocotorun -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$task_args" -c "$CYCLE" - # Check the task status - OUTPUT=$(rocotostat -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$TASK_NAME" -c "$CYCLE") - STATUS=$(echo "$OUTPUT" | awk '$2 == task {print $4}' task="$TASK_NAME") + num_succeeded=0 + for task in "${TASK_ARRAY[@]}"; do + + # Check the task status + OUTPUT=$(rocotostat -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$task" -c "$CYCLE") + STATUS=$(echo "$OUTPUT" | awk '$2 == task {print $4}' task="$task") - if [[ "$STATUS" == "SUCCEEDED" ]]; then - echo "The task succeeded." + if [[ "$STATUS" == "SUCCEEDED" ]]; then + echo "$pslot"_"$task"_"$CYCLE"" succeeded." + num_succeeded=$((num_succeeded + 1)) + elif [[ "$STATUS" == "FAILED" ]]; then + echo "$pslot"_"$task"_"$CYCLE"" failed." + exit 1 + elif [[ "$STATUS" == "DEAD" ]]; then + echo "$pslot"_"$task"_"$CYCLE"" is dead." + exit 1 + elif [[ "$STATUS" == "SUBMITTING" ]] || [[ "$STATUS" == "QUEUED" ]] || [[ "$STATUS" == "RUNNING" ]]; then + echo "$pslot"_"$task"_"$CYCLE"" is in state: $STATUS" + else + echo "$pslot"_"$task"_"$CYCLE"" is in unrecognized state: $STATUS. Rewinding..." + rocotorewind -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$task_args" -c "$CYCLE" + rocotoboot -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$task_args" -c "$CYCLE" + fi + done + if [[ "$num_succeeded" == "$num_tasks" ]]; then exit 0 - elif [[ "$STATUS" == "FAILED" ]]; then - echo "The task failed." - exit 1 - elif [[ "$STATUS" == "DEAD" ]]; then - echo "The task is dead." - exit 1 - else - echo "The task is in state: $STATUS" fi sleep 10 done diff --git a/test/testreference/C48mx500_3DVarAOWCDA_3dfgat.ref b/test/testreference/C48mx500_3DVarAOWCDA_3dfgat.ref new file mode 100644 index 000000000..b28df9aec --- /dev/null +++ b/test/testreference/C48mx500_3DVarAOWCDA_3dfgat.ref @@ -0,0 +1,54 @@ +Norm of input parameter StdDev: 4.1916150456516459e+02 +CostJb : Nonlinear Jb = 0.0000000000000000e+00 +CostJo : Nonlinear Jo(adt_rads_all) = 3.6538107293180053e+02, nobs = 154056, Jo/n = 2.3717419180804417e-03, err = 3.5902523361909755e+00 +CostJo : Nonlinear Jo(sst_avhrr_ma_l3u) = 2.2674500789797807e+05, nobs = 80795, Jo/n = 2.8064237625840467e+00, err = 3.1933209964416054e-01 +CostJo : Nonlinear Jo(sst_avhrr_mb_l3u) = 2.4093341994206450e+05, nobs = 87590, Jo/n = 2.7506955125249970e+00, err = 3.1303879303248278e-01 +CostJo : Nonlinear Jo(sst_viirs_npp_l3u) = 3.5095100144345453e+05, nobs = 86464, Jo/n = 4.0589262750214488e+00, err = 2.7352638780435701e-01 +CostJo : Nonlinear Jo(sst_viirs_n20_l3u) = 3.4617882174260722e+05, nobs = 82328, Jo/n = 4.2048734542635220e+00, err = 2.7610924066693887e-01 +CostJo : Nonlinear Jo(sst_abi_g16_l3c) = 0.0000000000000000e+00 --- No Observations +CostJo : Nonlinear Jo(sst_abi_g17_l3c) = 0.0000000000000000e+00 --- No Observations +CostJo : Nonlinear Jo(sst_ahi_h08_l3c) = 0.0000000000000000e+00 --- No Observations +CostJo : Nonlinear Jo(icec_amsr2_north) = 1.3776003730013903e+05, nobs = 238447, Jo/n = 5.7773860564460455e-01, err = 1.1973958453579239e-01 +CostJo : Nonlinear Jo(icec_amsr2_south) = 1.3525377877554770e+06, nobs = 294824, Jo/n = 4.5876108720981907e+00, err = 1.6104199583000647e-01 +CostJo : Nonlinear Jo(insitu_profile_argo) = 1.2371155464032217e+03, nobs = 74525, Jo/n = 1.6600007331811092e-02, err = 1.0000000000000000e+01 +CostFunction: Nonlinear J = 2.6567085727010556e+06 +RPCGMinimizer: reduction in residual norm = 3.3320242450517824e-02 +CostFunction::addIncrement: Analysis: + Valid time: 2021-03-24T18:00:00Z + sea_ice_area_fraction min=-0.3191478642087981 max=1.9094064942717854 mean=0.1380060365165569 + sea_ice_thickness min=0.0000000000000000 max=4.7237067222595215 mean=0.1716933516882944 + sea_ice_snow_thickness min=0.0000000000000000 max=0.5992891192436218 mean=0.0227558953028715 + sea_water_salinity min=-0.0004127016278705 max=40.0432853698730469 mean=33.6826135810349925 +sea_water_potential_temperature min=-13.5424639498294965 max=31.5094358260619565 mean=11.4876284245572435 + eastward_sea_water_velocity min=-0.9187383651733398 max=0.6746135354042053 mean=-0.0008460743757450 + northward_sea_water_velocity min=-0.6511899232864380 max=0.9435366392135620 mean=0.0030693165578485 + sea_surface_height_above_geoid min=-1.9814533174212232 max=0.8682830617264085 mean=-0.3311748439341126 + sea_water_cell_thickness min=0.0000000000000000 max=5416.6992187500000000 mean=128.6267671163319903 + ocean_mixed_layer_thickness min=2.3304977416992188 max=2417.2331242561340332 mean=148.3113220148310063 + sea_water_depth min=2.3304977416992188 max=2949.9484505653381348 mean=150.4446913535238650 + + + + + + + + + + + + + +CostJb : Nonlinear Jb = 319.2500686210223648 +CostJo : Nonlinear Jo(adt_rads_all) = 269.8785811470986573, nobs = 154056, Jo/n = 0.0017518212932122, err = 3.5902523361909755 +CostJo : Nonlinear Jo(sst_avhrr_ma_l3u) = 97733.3440574856213061, nobs = 80795, Jo/n = 1.2096459441485936, err = 0.3193320996441605 +CostJo : Nonlinear Jo(sst_avhrr_mb_l3u) = 102482.6442049633478746, nobs = 87590, Jo/n = 1.1700267633858128, err = 0.3130387930324828 +CostJo : Nonlinear Jo(sst_viirs_npp_l3u) = 179681.9646639756683726, nobs = 86464, Jo/n = 2.0781130258139302, err = 0.2735263878043570 +CostJo : Nonlinear Jo(sst_viirs_n20_l3u) = 165441.5019567124545574, nobs = 82328, Jo/n = 2.0095411276444519, err = 0.2761092406669389 +CostJo : Nonlinear Jo(sst_abi_g16_l3c) = 0.0000000000000000 --- No Observations +CostJo : Nonlinear Jo(sst_abi_g17_l3c) = 0.0000000000000000 --- No Observations +CostJo : Nonlinear Jo(sst_ahi_h08_l3c) = 0.0000000000000000 --- No Observations +CostJo : Nonlinear Jo(icec_amsr2_north) = 96049.0640872453514021, nobs = 238447, Jo/n = 0.4028109562596525, err = 0.1197395845357924 +CostJo : Nonlinear Jo(icec_amsr2_south) = 507683.7047332451329567, nobs = 294824, Jo/n = 1.7219890671493676, err = 0.1610419958300065 +CostJo : Nonlinear Jo(insitu_profile_argo) = 608.7904107136805578, nobs = 74525, Jo/n = 0.0081689421095428, err = 10.0000000000000000 +CostFunction: Nonlinear J = 1150270.1427641094196588 diff --git a/test/testreference/C96C48_ufs_hybatmDA_3dvar-fv3inc.ref b/test/testreference/C96C48_ufs_hybatmDA_3dvar-fv3inc.ref new file mode 100644 index 000000000..6f3adb18d --- /dev/null +++ b/test/testreference/C96C48_ufs_hybatmDA_3dvar-fv3inc.ref @@ -0,0 +1,41 @@ +Background State: + +---------------------------------------------------------------------------------------------------- +State print | number of fields = 9 | cube sphere face size: C96 +eastward_wind | Min:-8.7593299865722656e+01 Max:+1.1421906280517578e+02 RMS:+1.7446423933066384e+01 +northward_wind | Min:-9.0978759765625000e+01 Max:+8.9302459716796875e+01 RMS:+9.8048456519148388e+00 +air_temperature | Min:+1.7290483093261719e+02 Max:+3.1017132568359375e+02 RMS:+2.5022377762999352e+02 +air_pressure_at_surface | Min:+5.1309082031250000e+04 Max:+1.0433011718750000e+05 RMS:+9.8914626690611054e+04 +water_vapor_mixing_ratio_wrt_moist_air | Min:+9.9999986069576607e-10 Max:+2.1381899714469910e-02 RMS:+5.0230919069545812e-03 +cloud_liquid_ice | Min:+0.0000000000000000e+00 Max:+5.7223520707339048e-04 RMS:+9.4296924830407741e-06 +cloud_liquid_water | Min:+0.0000000000000000e+00 Max:+1.6279185656458139e-03 RMS:+3.7809568286336720e-05 +ozone_mass_mixing_ratio | Min:+3.6120087476554374e-10 Max:+1.7313132048002444e-05 RMS:+4.4655432291945023e-06 +geopotential_height_at_surface | Min:-2.9894643783569336e+01 Max:+5.3870561523437500e+03 RMS:+6.4442219321892003e+02 +---------------------------------------------------------------------------------------------------- +JEDI Increment: + +---------------------------------------------------------------------------------------------------- +Increment print | number of fields = 8 | cube sphere face size: C48 +eastward_wind | Min:-2.1115621251510674e-07 Max:+1.1444664416160322e-07 RMS:+1.8401316033983295e-09 +northward_wind | Min:-1.0219621415785696e-07 Max:+2.6249988138715707e-07 RMS:+1.8295902505163720e-09 +air_temperature | Min:-5.2000757477799198e-08 Max:+6.6504355800134363e-08 RMS:+7.1626194815483433e-10 +air_pressure_at_surface | Min:-2.0445149857550859e-06 Max:+9.8957389127463102e-07 RMS:+4.2160438885855514e-08 +water_vapor_mixing_ratio_wrt_moist_air | Min:-3.5212104499554586e-11 Max:+2.4915642032191654e-11 RMS:+2.7352479311235747e-13 +cloud_liquid_ice | Min:-5.2622388732733935e-13 Max:+3.8117236092690345e-13 RMS:+3.4198803820164734e-15 +cloud_liquid_water | Min:-1.1565164516162685e-11 Max:+3.1649457890922744e-12 RMS:+1.5995494806702646e-14 +ozone_mass_mixing_ratio | Min:-1.0346794247831136e-06 Max:+1.3451623930433847e-06 RMS:+3.6858146749007146e-08 +---------------------------------------------------------------------------------------------------- +FV3 Increment: + +---------------------------------------------------------------------------------------------------- +Increment print | number of fields = 9 | cube sphere face size: C48 +eastward_wind | Min:-2.1115621251510674e-07 Max:+1.1444664416160322e-07 RMS:+1.8401316033983295e-09 +northward_wind | Min:-1.0219621415785696e-07 Max:+2.6249988138715707e-07 RMS:+1.8295902505163720e-09 +air_temperature | Min:-5.2000757477799198e-08 Max:+6.6504355800134363e-08 RMS:+7.1626194815483433e-10 +water_vapor_mixing_ratio_wrt_moist_air | Min:-3.5212104499554586e-11 Max:+2.4915642032191654e-11 RMS:+2.7352479311235747e-13 +cloud_liquid_ice | Min:-5.2622388732733935e-13 Max:+3.8117236092690345e-13 RMS:+3.4198803820164734e-15 +cloud_liquid_water | Min:-1.1565164516162685e-11 Max:+3.1649457890922744e-12 RMS:+1.5995494806702646e-14 +ozone_mass_mixing_ratio | Min:-1.0346794247831136e-06 Max:+1.3451623930433847e-06 RMS:+3.6858146749007146e-08 +air_pressure_thickness | Min:-3.8906364352442324e-08 Max:+1.7884758563013747e-08 RMS:+4.2747856122164087e-10 +layer_thickness | Min:-1.9763365344260819e-07 Max:+1.8311084204469807e-07 RMS:+3.6402384455212584e-09 +---------------------------------------------------------------------------------------------------- diff --git a/test/testreference/C96C48_ufs_hybatmDA_3dvar.ref b/test/testreference/C96C48_ufs_hybatmDA_3dvar.ref new file mode 100644 index 000000000..711f08fb2 --- /dev/null +++ b/test/testreference/C96C48_ufs_hybatmDA_3dvar.ref @@ -0,0 +1,124 @@ +CostJb : Nonlinear Jb = 0.0000000000000000e+00 +CostJo : Nonlinear Jo(Aircraft) = 6.0590797038184456e+05, nobs = 504555, Jo/n = 1.2008759607611550e+00, err = 2.2437239435917040e+00 +CostJo : Nonlinear Jo(ascatw_ascat_metop-b) = 1.5949936485170507e+04, nobs = 26314, Jo/n = 6.0613880387514274e-01, err = 1.8277692518568445e+00 +CostJo : Nonlinear Jo(ATMS N20) = 4.8387730216799086e+04, nobs = 156588, Jo/n = 3.0901301643037199e-01, err = 5.2594000470964719e+00 +CostJo : Nonlinear Jo(surface_ps) = 2.6308196193705146e+04, nobs = 107322, Jo/n = 2.4513330159431565e-01, err = 5.9305643428180190e+02 +CostJo : Nonlinear Jo(gnssrobndnbam) = 6.1501893265653157e+05, nobs = 473864, Jo/n = 1.2978806844506685e+00, err = 3.2933330370258057e-04 +CostJo : Nonlinear Jo(ompsnp_npp) = 2.1269336098572853e+03, nobs = 3087, Jo/n = 6.8899695816562534e-01, err = 2.4001367423170068e+00 +CostJo : Nonlinear Jo(ompstc_npp) = 3.3784707883660294e+03, nobs = 3696, Jo/n = 9.1408841676570063e-01, err = 6.0000000000000000e+00 +CostJo : Nonlinear Jo(satwind_goes-16) = 6.9383960384868606e+03, nobs = 193154, Jo/n = 3.5921575729660586e-02, err = 1.1892541771722902e+01 +CostFunction: Nonlinear J = 1.3240165663707610e+06 +DRPCGMinimizer: reduction in residual norm = 2.8257552285466486e-01 +CostFunction::addIncrement: Analysis: +---------------------------------------------------------------------------------------------------- +State print | number of fields = 22 | cube sphere face size: C96 +eastward_wind | Min:-8.7593299865709369e+01 Max:+1.1421906280506562e+02 RMS:+1.7446423933063677e+01 +northward_wind | Min:-9.0978759765616502e+01 Max:+8.9302459716826505e+01 RMS:+9.8048456519093889e+00 +air_temperature | Min:+1.7290483093261747e+02 Max:+3.1017132568363934e+02 RMS:+2.5022377762999326e+02 +air_pressure_thickness | Min:+6.0599988698959351e-01 Max:+1.7837935791015625e+03 RMS:+9.5699700087025974e+02 +air_pressure_at_surface | Min:+5.1309082031249884e+04 Max:+1.0433011718749903e+05 RMS:+9.8914626690610530e+04 +water_vapor_mixing_ratio_wrt_moist_air | Min:+9.9999986088507587e-10 Max:+2.1381899714470930e-02 RMS:+5.0230919069543158e-03 +cloud_liquid_ice | Min:+0.0000000000000000e+00 Max:+5.7223520707321528e-04 RMS:+9.4296924830162644e-06 +cloud_liquid_water | Min:+0.0000000000000000e+00 Max:+1.6279185656660684e-03 RMS:+3.7809568286304757e-05 +ozone_mass_mixing_ratio | Min:+3.8859406171368113e-10 Max:+1.7313133576612057e-05 RMS:+4.4663053755671636e-06 +geopotential_height_at_surface | Min:-2.9894643783569336e+01 Max:+5.3870561523437500e+03 RMS:+6.4442219321892003e+02 +slmsk | Min:+0.0000000000000000e+00 Max:+2.0000000000000000e+00 RMS:+6.5312859958249614e-01 +sheleg | Min:+0.0000000000000000e+00 Max:+2.6371386718750000e+02 RMS:+2.2054824502787593e+01 +skin_temperature_at_surface | Min:+2.1746235656738281e+02 Max:+3.2932260131835938e+02 RMS:+2.8737006087270038e+02 +vtype | Min:+0.0000000000000000e+00 Max:+2.0000000000000000e+01 RMS:+5.8667091445155686e+00 +stype | Min:+0.0000000000000000e+00 Max:+1.6000000000000000e+01 RMS:+4.3549806522511076e+00 +vfrac | Min:+0.0000000000000000e+00 Max:+9.8026573181152344e+01 RMS:+1.9275201636346665e+01 +stc | Min:+2.0000000000000000e+02 Max:+3.2045886230468750e+02 RMS:+2.8806776004459306e+02 +soilMoistureVolumetric | Min:+2.1923782303929329e-02 Max:+1.0000000000000000e+00 RMS:+8.6145451167852249e-01 +totalSnowDepthMeters | Min:+0.0000000000000000e+00 Max:+1.2327475547790527e+00 RMS:+1.5878111306662487e-01 +eastward_wind_at_surface | Min:-1.8106302261352539e+01 Max:+2.0922132492065430e+01 RMS:+5.2142386237867280e+00 +northward_wind_at_surface | Min:-2.0834318161010742e+01 Max:+1.9025196075439453e+01 RMS:+4.8322666159970842e+00 +f10m | Min:+9.3901658058166504e-01 Max:+1.0698120594024658e+00 RMS:+9.9592865511053685e-01 +---------------------------------------------------------------------------------------------------- + + + +Obs bias coefficients: +--------------------------------------------------------------- + constant: Min= -0.8381630182271099, Max= 0.8200770020481498, Norm= 2.2810923110282308 + lapseRate_order_2: Min= -0.2969009876260876, Max= 0.3889850080012900, Norm= 0.7014865852485521 + lapseRate: Min= -0.5954660177230082, Max= 1.6232719421370738, Norm= 2.1316868531033393 + emissivityJacobian: Min= -0.0097089996561411, Max= 8.3582897185072884, Norm= 10.2580018725806656 +sensorScanAngle_order_4: Min= -3.4036920070662289, Max= 0.8113600015621025, Norm= 4.8886281019448123 +sensorScanAngle_order_3: Min= -0.7697319984440882, Max= 0.3303030133246191, Norm= 0.9762309707883376 +sensorScanAngle_order_2: Min= -0.5401099920283099, Max= 2.2389800548543319, Norm= 2.4920946301939773 + sensorScanAngle: Min= -0.1069049984217430, Max= 0.4472059905526856, Norm= 0.8480380113023768 +--------------------------------------------------------------- + + + + + + +CostJb : Nonlinear Jb = 0.0000000000001377 +CostJo : Nonlinear Jo(Aircraft) = 605907.9700265740975738, nobs = 504555, Jo/n = 1.2008759600570287, err = 2.2437239435917040 +CostJo : Nonlinear Jo(ascatw_ascat_metop-b) = 15949.9364848986988363, nobs = 26314, Jo/n = 0.6061388038648133, err = 1.8277692518568445 +CostJo : Nonlinear Jo(ATMS N20) = 48387.7302149529932649, nobs = 156588, Jo/n = 0.3090130164185825, err = 5.2594000470964719 +CostJo : Nonlinear Jo(surface_ps) = 26308.1961930343022686, nobs = 107322, Jo/n = 0.2451333015880649, err = 593.0564342818018986 +CostJo : Nonlinear Jo(gnssrobndnbam) = 615018.9326265617273748, nobs = 473864, Jo/n = 1.2978806843874229, err = 0.0003293333037026 +CostJo : Nonlinear Jo(ompsnp_npp) = 1931.1242066182112467, nobs = 3087, Jo/n = 0.6255666364166541, err = 2.4001367423170068 +CostJo : Nonlinear Jo(ompstc_npp) = 1175.2354163386241908, nobs = 3696, Jo/n = 0.3179749503080693, err = 6.0000000000000000 +CostJo : Nonlinear Jo(satwind_goes-16) = 6938.3960383742578415, nobs = 193154, Jo/n = 0.0359215757290776, err = 11.8925417717229021 +CostFunction: Nonlinear J = 1321617.5212073526345193 +DRPCGMinimizer: reduction in residual norm = 0.8446076379600224 +CostFunction::addIncrement: Analysis: +---------------------------------------------------------------------------------------------------- +State print | number of fields = 22 | cube sphere face size: C96 +eastward_wind | Min:-8.7593299865597274e+01 Max:+1.1421906280413639e+02 RMS:+1.7446423933041416e+01 +northward_wind | Min:-9.0978759765544794e+01 Max:+8.9302459717076374e+01 RMS:+9.8048456518642482e+00 +air_temperature | Min:+1.7290483093261994e+02 Max:+3.1017132568402411e+02 RMS:+2.5022377762999653e+02 +air_pressure_thickness | Min:+6.0599988698959351e-01 Max:+1.7837935791015625e+03 RMS:+9.5699700087025974e+02 +air_pressure_at_surface | Min:+5.1309082031248872e+04 Max:+1.0433011718749085e+05 RMS:+9.8914626690605888e+04 +water_vapor_mixing_ratio_wrt_moist_air | Min:+9.9999986293294490e-10 Max:+2.1381899714479527e-02 RMS:+5.0230919069522385e-03 +cloud_liquid_ice | Min:+0.0000000000000000e+00 Max:+5.7223520707173773e-04 RMS:+9.4296924828094189e-06 +cloud_liquid_water | Min:+0.0000000000000000e+00 Max:+1.6279185658369072e-03 RMS:+3.7809568286035265e-05 +ozone_mass_mixing_ratio | Min:+0.0000000000000000e+00 Max:+1.7393152519974760e-05 RMS:+4.4687802775349384e-06 +geopotential_height_at_surface | Min:-2.9894643783569336e+01 Max:+5.3870561523437500e+03 RMS:+6.4442219321892003e+02 +slmsk | Min:+0.0000000000000000e+00 Max:+2.0000000000000000e+00 RMS:+6.5312859958249614e-01 +sheleg | Min:+0.0000000000000000e+00 Max:+2.6371386718750000e+02 RMS:+2.2054824502787593e+01 +skin_temperature_at_surface | Min:+2.1746235656738281e+02 Max:+3.2932260131835938e+02 RMS:+2.8737006087270038e+02 +vtype | Min:+0.0000000000000000e+00 Max:+2.0000000000000000e+01 RMS:+5.8667091445155686e+00 +stype | Min:+0.0000000000000000e+00 Max:+1.6000000000000000e+01 RMS:+4.3549806522511076e+00 +vfrac | Min:+0.0000000000000000e+00 Max:+9.8026573181152344e+01 RMS:+1.9275201636346665e+01 +stc | Min:+2.0000000000000000e+02 Max:+3.2045886230468750e+02 RMS:+2.8806776004459306e+02 +soilMoistureVolumetric | Min:+2.1923782303929329e-02 Max:+1.0000000000000000e+00 RMS:+8.6145451167852249e-01 +totalSnowDepthMeters | Min:+0.0000000000000000e+00 Max:+1.2327475547790527e+00 RMS:+1.5878111306662487e-01 +eastward_wind_at_surface | Min:-1.8106302261352539e+01 Max:+2.0922132492065430e+01 RMS:+5.2142386237867280e+00 +northward_wind_at_surface | Min:-2.0834318161010742e+01 Max:+1.9025196075439453e+01 RMS:+4.8322666159970842e+00 +f10m | Min:+9.3901658058166504e-01 Max:+1.0698120594024658e+00 RMS:+9.9592865511053685e-01 +---------------------------------------------------------------------------------------------------- + + + +Obs bias coefficients: +--------------------------------------------------------------- + constant: Min= -0.8381630182312128, Max= 0.8200770020452602, Norm= 2.2810923110345662 + lapseRate_order_2: Min= -0.2969009876342316, Max= 0.3889850080009739, Norm= 0.7014865852476552 + lapseRate: Min= -0.5954660177223734, Max= 1.6232719421235957, Norm= 2.1316868530892328 + emissivityJacobian: Min= -0.0097089996561434, Max= 8.3582897174897202, Norm= 10.2580018711068170 +sensorScanAngle_order_4: Min= -3.4036920070781176, Max= 0.8113600015458797, Norm= 4.8886281019598847 +sensorScanAngle_order_3: Min= -0.7697319984481772, Max= 0.3303030133236201, Norm= 0.9762309707916220 +sensorScanAngle_order_2: Min= -0.5401099920369742, Max= 2.2389800548457712, Norm= 2.4920946301898259 + sensorScanAngle: Min= -0.1069049984223674, Max= 0.4472059905508586, Norm= 0.8480380112990616 +--------------------------------------------------------------- + + + + + + +CostJb : Nonlinear Jb = 0.0000000000008880 +CostJo : Nonlinear Jo(Aircraft) = 605907.9670297848060727, nobs = 504555, Jo/n = 1.2008759541175587, err = 2.2437239435917040 +CostJo : Nonlinear Jo(ascatw_ascat_metop-b) = 15949.9364826060973428, nobs = 26314, Jo/n = 0.6061388037776886, err = 1.8277692518568445 +CostJo : Nonlinear Jo(ATMS N20) = 48387.7301996692985995, nobs = 156588, Jo/n = 0.3090130163209780, err = 5.2594000470964719 +CostJo : Nonlinear Jo(surface_ps) = 26308.1961873747168283, nobs = 107322, Jo/n = 0.2451333015353303, err = 593.0564342818018986 +CostJo : Nonlinear Jo(gnssrobndnbam) = 615018.9323734971694648, nobs = 473864, Jo/n = 1.2978806838533781, err = 0.0003293333037026 +CostJo : Nonlinear Jo(ompsnp_npp) = 1105.2453930547394521, nobs = 3087, Jo/n = 0.3580321972966438, err = 2.4001367423170068 +CostJo : Nonlinear Jo(ompstc_npp) = 842.3521445536282499, nobs = 3696, Jo/n = 0.2279091300199211, err = 6.0000000000000000 +CostJo : Nonlinear Jo(satwind_goes-16) = 6938.3960374245170897, nobs = 193154, Jo/n = 0.0359215757241606, err = 11.8925417717229021 +CostFunction: Nonlinear J = 1320458.7558479649014771 diff --git a/test/testreference/C96C48_ufs_hybatmDA_lgetkf-fv3inc.ref b/test/testreference/C96C48_ufs_hybatmDA_lgetkf-fv3inc.ref new file mode 100644 index 000000000..103e05260 --- /dev/null +++ b/test/testreference/C96C48_ufs_hybatmDA_lgetkf-fv3inc.ref @@ -0,0 +1,82 @@ +Background State for member 0: + +---------------------------------------------------------------------------------------------------- +State print | number of fields = 9 | cube sphere face size: C48 +eastward_wind | Min:-8.4384330749511719e+01 Max:+1.1146717834472656e+02 RMS:+1.7423407399233511e+01 +northward_wind | Min:-8.6144805908203125e+01 Max:+8.9331993103027344e+01 RMS:+1.0017299694477177e+01 +air_temperature | Min:+1.7350303649902344e+02 Max:+3.0961535644531250e+02 RMS:+2.5015669840056216e+02 +air_pressure_at_surface | Min:+5.2320046875000000e+04 Max:+1.0452482812500000e+05 RMS:+9.8914997456124955e+04 +water_vapor_mixing_ratio_wrt_moist_air | Min:+1.6847316430812498e-08 Max:+2.2395884618163109e-02 RMS:+5.0107896876815721e-03 +cloud_liquid_ice | Min:+0.0000000000000000e+00 Max:+4.1855275048874319e-04 RMS:+1.0354889627583217e-05 +cloud_liquid_water | Min:+0.0000000000000000e+00 Max:+1.3839458115398884e-03 RMS:+3.5588872198565466e-05 +ozone_mass_mixing_ratio | Min:+1.1339260641562987e-09 Max:+1.7277885490329936e-05 RMS:+4.4686682637740296e-06 +geopotential_height_at_surface | Min:-2.7018375396728516e+01 Max:+5.2697558593750000e+03 RMS:+6.3772571434211386e+02 +---------------------------------------------------------------------------------------------------- +JEDI Increment for member 0: + +---------------------------------------------------------------------------------------------------- +Increment print | number of fields = 8 | cube sphere face size: C48 +eastward_wind | Min:-2.6713170463703555e+01 Max:+2.7247719731829164e+01 RMS:+7.7003410130506966e-01 +northward_wind | Min:-2.2266548354929135e+01 Max:+2.4111106836816546e+01 RMS:+7.7874592260732178e-01 +air_temperature | Min:-1.0204740456386372e+01 Max:+1.0612363461439131e+01 RMS:+3.2383456606394428e-01 +air_pressure_at_surface | Min:-1.6632427368676872e-03 Max:+1.3842245934938546e-03 RMS:+8.3556099392947144e-05 +water_vapor_mixing_ratio_wrt_moist_air | Min:-1.0075023053001828e-02 Max:+1.3216991071147419e-02 RMS:+1.3161159034094137e-04 +cloud_liquid_ice | Min:-3.2384493993478425e-04 Max:+2.5782120798697029e-04 RMS:+3.5186992249237140e-06 +cloud_liquid_water | Min:-1.3410840615217380e-03 Max:+1.0793991164780980e-03 RMS:+1.2675198989331570e-05 +ozone_mass_mixing_ratio | Min:-1.7321578649115885e-06 Max:+2.2122365151317159e-06 RMS:+3.4501654041399942e-08 +---------------------------------------------------------------------------------------------------- +FV3 Increment for member 0: + +---------------------------------------------------------------------------------------------------- +Increment print | number of fields = 9 | cube sphere face size: C48 +eastward_wind | Min:-2.6713170463703555e+01 Max:+2.7247719731829164e+01 RMS:+7.7003410130506966e-01 +northward_wind | Min:-2.2266548354929135e+01 Max:+2.4111106836816546e+01 RMS:+7.7874592260732178e-01 +air_temperature | Min:-1.0204740456386372e+01 Max:+1.0612363461439131e+01 RMS:+3.2383456606394428e-01 +water_vapor_mixing_ratio_wrt_moist_air | Min:-1.0075023053001828e-02 Max:+1.3216991071147419e-02 RMS:+1.3161159034094137e-04 +cloud_liquid_ice | Min:-3.2384493993478425e-04 Max:+2.5782120798697029e-04 RMS:+3.5186992249237140e-06 +cloud_liquid_water | Min:-1.3410840615217380e-03 Max:+1.0793991164780980e-03 RMS:+1.2675198989331570e-05 +ozone_mass_mixing_ratio | Min:-1.7321578649115885e-06 Max:+2.2122365151317159e-06 RMS:+3.4501654041399942e-08 +air_pressure_thickness | Min:-3.7230194720905274e-05 Max:+3.0984621844254434e-05 RMS:+9.4295307947072803e-07 +layer_thickness | Min:-5.0314572597610095e+01 Max:+3.1207257614340051e+01 RMS:+6.5903625246599373e-01 +---------------------------------------------------------------------------------------------------- +Background State for member 1: + +---------------------------------------------------------------------------------------------------- +State print | number of fields = 9 | cube sphere face size: C48 +eastward_wind | Min:-8.5966735839843750e+01 Max:+1.1211631774902344e+02 RMS:+1.7425666524535583e+01 +northward_wind | Min:-8.5594192504882812e+01 Max:+8.9450973510742188e+01 RMS:+1.0012910607932993e+01 +air_temperature | Min:+1.7322386169433594e+02 Max:+3.1173611450195312e+02 RMS:+2.5015480177211401e+02 +air_pressure_at_surface | Min:+5.2300289062500000e+04 Max:+1.0443091406250000e+05 RMS:+9.8912612346289476e+04 +water_vapor_mixing_ratio_wrt_moist_air | Min:+1.7588122958045460e-08 Max:+2.1704806014895439e-02 RMS:+4.9960608009716814e-03 +cloud_liquid_ice | Min:+0.0000000000000000e+00 Max:+4.2980920989066362e-04 RMS:+9.7698907168770129e-06 +cloud_liquid_water | Min:+0.0000000000000000e+00 Max:+1.4689513482153416e-03 RMS:+3.5167035611299903e-05 +ozone_mass_mixing_ratio | Min:+2.9807353807420611e-10 Max:+1.7195472537423484e-05 RMS:+4.4692040486361164e-06 +geopotential_height_at_surface | Min:-2.7018375396728516e+01 Max:+5.2697558593750000e+03 RMS:+6.3772571434211386e+02 +---------------------------------------------------------------------------------------------------- +JEDI Increment for member 1: + +---------------------------------------------------------------------------------------------------- +Increment print | number of fields = 8 | cube sphere face size: C48 +eastward_wind | Min:-2.7118603154153071e+01 Max:+2.5039255461317936e+01 RMS:+7.9016696986758683e-01 +northward_wind | Min:-1.9134521911266908e+01 Max:+1.8052220490594621e+01 RMS:+8.1416259656824774e-01 +air_temperature | Min:-1.0334410919697632e+01 Max:+1.0023267385714213e+01 RMS:+3.2993662752227909e-01 +air_pressure_at_surface | Min:-1.7015457730735761e-03 Max:+1.4328894367281464e-03 RMS:+8.3597529649529631e-05 +water_vapor_mixing_ratio_wrt_moist_air | Min:-1.0940498213010061e-02 Max:+1.4176037926216224e-02 RMS:+1.3345901067359821e-04 +cloud_liquid_ice | Min:-2.7333859171390946e-04 Max:+4.8852641654798684e-04 RMS:+3.6400194778040982e-06 +cloud_liquid_water | Min:-1.1354531767787950e-03 Max:+1.0641496243961217e-03 RMS:+1.3080375599165519e-05 +ozone_mass_mixing_ratio | Min:-1.8008616277179968e-06 Max:+2.4391037763833551e-06 RMS:+3.6841977469037158e-08 +---------------------------------------------------------------------------------------------------- +FV3 Increment for member 1: + +---------------------------------------------------------------------------------------------------- +Increment print | number of fields = 9 | cube sphere face size: C48 +eastward_wind | Min:-2.7118603154153071e+01 Max:+2.5039255461317936e+01 RMS:+7.9016696986758683e-01 +northward_wind | Min:-1.9134521911266908e+01 Max:+1.8052220490594621e+01 RMS:+8.1416259656824774e-01 +air_temperature | Min:-1.0334410919697632e+01 Max:+1.0023267385714213e+01 RMS:+3.2993662752227909e-01 +water_vapor_mixing_ratio_wrt_moist_air | Min:-1.0940498213010061e-02 Max:+1.4176037926216224e-02 RMS:+1.3345901067359821e-04 +cloud_liquid_ice | Min:-2.7333859171390946e-04 Max:+4.8852641654798684e-04 RMS:+3.6400194778040982e-06 +cloud_liquid_water | Min:-1.1354531767787950e-03 Max:+1.0641496243961217e-03 RMS:+1.3080375599165519e-05 +ozone_mass_mixing_ratio | Min:-1.8008616277179968e-06 Max:+2.4391037763833551e-06 RMS:+3.6841977469037158e-08 +air_pressure_thickness | Min:-3.8087571738287807e-05 Max:+3.2073941838461906e-05 RMS:+9.4342063209947081e-07 +layer_thickness | Min:-4.7521592877819785e+01 Max:+3.1680899109200254e+01 RMS:+6.5375317024288038e-01 +---------------------------------------------------------------------------------------------------- diff --git a/test/testreference/C96C48_ufs_hybatmDA_lgetkf_observer.ref b/test/testreference/C96C48_ufs_hybatmDA_lgetkf_observer.ref new file mode 100644 index 000000000..e85f63696 --- /dev/null +++ b/test/testreference/C96C48_ufs_hybatmDA_lgetkf_observer.ref @@ -0,0 +1,126 @@ +Initial state for member 1: +---------------------------------------------------------------------------------------------------- +State print | number of fields = 23 | cube sphere face size: C48 +eastward_wind | Min:-8.4384330749511719e+01 Max:+1.1146717834472656e+02 RMS:+1.7423407399233511e+01 +northward_wind | Min:-8.6144805908203125e+01 Max:+8.9331993103027344e+01 RMS:+1.0017299694477179e+01 +air_temperature | Min:+1.7350303649902344e+02 Max:+3.0961535644531250e+02 RMS:+2.5015669840056216e+02 +layer_thickness | Min:-3.0059777832031250e+03 Max:-1.6207923889160156e+01 RMS:+9.8503674614826332e+02 +air_pressure_thickness | Min:+6.0599988698959351e-01 Max:+1.7881016845703125e+03 RMS:+9.5698921151558159e+02 +air_pressure_at_surface | Min:+5.2320046875000000e+04 Max:+1.0452482812500000e+05 RMS:+9.8914997456124955e+04 +water_vapor_mixing_ratio_wrt_moist_air | Min:+1.6847316430812498e-08 Max:+2.2395884618163109e-02 RMS:+5.0107896876815721e-03 +cloud_liquid_ice | Min:+0.0000000000000000e+00 Max:+4.1855275048874319e-04 RMS:+1.0354889627583219e-05 +cloud_liquid_water | Min:+0.0000000000000000e+00 Max:+1.3839458115398884e-03 RMS:+3.5588872198565459e-05 +ozone_mass_mixing_ratio | Min:+1.1339260641562987e-09 Max:+1.7277885490329936e-05 RMS:+4.4686682637740296e-06 +geopotential_height_at_surface | Min:-2.7018375396728516e+01 Max:+5.2697558593750000e+03 RMS:+6.3772571434211386e+02 +slmsk | Min:+0.0000000000000000e+00 Max:+2.0000000000000000e+00 RMS:+6.5462209907135938e-01 +sheleg | Min:+0.0000000000000000e+00 Max:+2.5217851257324219e+02 RMS:+2.2019818126018798e+01 +skin_temperature_at_surface | Min:+2.1759780883789062e+02 Max:+3.2456860351562500e+02 RMS:+2.8741259834843908e+02 +vtype | Min:+0.0000000000000000e+00 Max:+2.0000000000000000e+01 RMS:+6.1246929250310931e+00 +stype | Min:+0.0000000000000000e+00 Max:+1.6000000000000000e+01 RMS:+4.5755536879077185e+00 +vfrac | Min:+0.0000000000000000e+00 Max:+9.8551864624023438e+01 RMS:+2.0170282476619530e+01 +stc | Min:+2.0000000000000000e+02 Max:+3.1568060302734375e+02 RMS:+2.8817196445239324e+02 +soilMoistureVolumetric | Min:+2.1992070600390434e-02 Max:+1.0000000000000000e+00 RMS:+8.4975035958924516e-01 +totalSnowDepthMeters | Min:+0.0000000000000000e+00 Max:+1.0927191972732544e+00 RMS:+1.5092084570439257e-01 +eastward_wind_at_surface | Min:-1.8143459320068359e+01 Max:+2.0627843856811523e+01 RMS:+5.1169707890467366e+00 +northward_wind_at_surface | Min:-2.3821027755737305e+01 Max:+1.9497446060180664e+01 RMS:+4.8067548196805943e+00 +f10m | Min:+9.4103527069091797e-01 Max:+1.0685453414916992e+00 RMS:+9.9573423073115486e-01 +---------------------------------------------------------------------------------------------------- +Initial state for member 2: +---------------------------------------------------------------------------------------------------- +State print | number of fields = 23 | cube sphere face size: C48 +eastward_wind | Min:-8.5966735839843750e+01 Max:+1.1211631774902344e+02 RMS:+1.7425666524535583e+01 +northward_wind | Min:-8.5594192504882812e+01 Max:+8.9450973510742188e+01 RMS:+1.0012910607932993e+01 +air_temperature | Min:+1.7322386169433594e+02 Max:+3.1173611450195312e+02 RMS:+2.5015480177211401e+02 +layer_thickness | Min:-3.0079758300781250e+03 Max:-1.6234376907348633e+01 RMS:+9.8509317417150282e+02 +air_pressure_thickness | Min:+6.0599988698959351e-01 Max:+1.7860076904296875e+03 RMS:+9.5696337286916196e+02 +air_pressure_at_surface | Min:+5.2300289062500000e+04 Max:+1.0443091406250000e+05 RMS:+9.8912612346289476e+04 +water_vapor_mixing_ratio_wrt_moist_air | Min:+1.7588122958045460e-08 Max:+2.1704806014895439e-02 RMS:+4.9960608009716806e-03 +cloud_liquid_ice | Min:+0.0000000000000000e+00 Max:+4.2980920989066362e-04 RMS:+9.7698907168770146e-06 +cloud_liquid_water | Min:+0.0000000000000000e+00 Max:+1.4689513482153416e-03 RMS:+3.5167035611299903e-05 +ozone_mass_mixing_ratio | Min:+2.9807353807420611e-10 Max:+1.7195472537423484e-05 RMS:+4.4692040486361164e-06 +geopotential_height_at_surface | Min:-2.7018375396728516e+01 Max:+5.2697558593750000e+03 RMS:+6.3772571434211386e+02 +slmsk | Min:+0.0000000000000000e+00 Max:+2.0000000000000000e+00 RMS:+6.5462209907135938e-01 +sheleg | Min:+0.0000000000000000e+00 Max:+2.4906845092773438e+02 RMS:+2.2004970461687567e+01 +skin_temperature_at_surface | Min:+2.1673643493652344e+02 Max:+3.2559057617187500e+02 RMS:+2.8745909600453388e+02 +vtype | Min:+0.0000000000000000e+00 Max:+2.0000000000000000e+01 RMS:+6.1246929250310931e+00 +stype | Min:+0.0000000000000000e+00 Max:+1.6000000000000000e+01 RMS:+4.5755536879077185e+00 +vfrac | Min:+0.0000000000000000e+00 Max:+9.8551864624023438e+01 RMS:+2.0170282476619530e+01 +stc | Min:+2.0000000000000000e+02 Max:+3.1603344726562500e+02 RMS:+2.8817511849953127e+02 +soilMoistureVolumetric | Min:+2.1997468546032906e-02 Max:+1.0000000000000000e+00 RMS:+8.4973303187978066e-01 +totalSnowDepthMeters | Min:+0.0000000000000000e+00 Max:+1.0929538011550903e+00 RMS:+1.5094966458993353e-01 +eastward_wind_at_surface | Min:-1.9632795333862305e+01 Max:+2.2031156539916992e+01 RMS:+5.1474973436096771e+00 +northward_wind_at_surface | Min:-2.2859689712524414e+01 Max:+1.9478809356689453e+01 RMS:+4.7531537396512338e+00 +f10m | Min:+9.4079720973968506e-01 Max:+1.0689357519149780e+00 RMS:+9.9569948613238035e-01 +---------------------------------------------------------------------------------------------------- +H(x) for member 1: +Aircraft nobs= 502771 Min=-80.55443499910179, Max=305.5072533751332, RMS=146.5850485230079 + +ascatw_ascat_metop-b nobs= 24988 Min=-14.90743210285059, Max=18.50735802131708, RMS=5.626101623328659 + +ATMS N20 nobs= 17107090 Min=129.7859007557375, Max=294.7571288275636, RMS=237.7675940436646 + +surface_ps nobs= 199096 Min=481.5556154686088, Max=106952.1342381349, RMS=94358.04308199322 + +gnssrobndnbam nobs= 792394 Min=3.54653995010946e-06, Max=0.05689811581521509, RMS=0.00729813001231048 + +ompsnp_npp nobs= 3652 Min=0.02233893266611933, Max=544.9735566336279, RMS=74.39666518228908 + +ompstc_npp nobs= 29925 Min=232.08666141862, Max=578.874275460677, RMS=330.1687050473581 + +satwind_goes-16 nobs= 192150 Min=-46.72026247379588, Max=73.49080322835941, RMS=13.50004896322415 + + +H(x) for member 2: +Aircraft nobs= 502771 Min=-83.52929132190421, Max=303.935382876328, RMS=146.5832501835669 + +ascatw_ascat_metop-b nobs= 24988 Min=-15.95012800292468, Max=19.32845599082289, RMS=5.590280602334444 + +ATMS N20 nobs= 17107090 Min=129.5870694672627, Max=294.7645909817928, RMS=237.7536249801317 + +surface_ps nobs= 199096 Min=475.424805046184, Max=106814.5789947868, RMS=94338.64524856347 + +gnssrobndnbam nobs= 792394 Min=3.535338326285228e-06, Max=0.05521000303350442, RMS=0.007290866963061116 + +ompsnp_npp nobs= 3652 Min=0.02236964099830396, Max=551.3730864849784, RMS=74.49227809990998 + +ompstc_npp nobs= 29925 Min=232.3157492522477, Max=581.8884351421542, RMS=330.2400711699158 + +satwind_goes-16 nobs= 192150 Min=-45.04364111939185, Max=72.45210083322863, RMS=13.31321129007812 + + +H(x) ensemble background mean: +Aircraft nobs= 670256 Min=-82.041863160503, Max=304.2592158533807, RMS=127.3605543707548 + +ascatw_ascat_metop-b nobs= 241220 Min=-15.81413469653508, Max=19.88922395519342, RMS=5.523473979723838 + +ATMS N20 nobs= 17107090 Min=129.6864851115001, Max=294.7608599046782, RMS=237.7603500470373 + +surface_ps nobs= 199096 Min=486.4346696969737, Max=106883.3566164608, RMS=94348.33691646729 + +gnssrobndnbam nobs= 792394 Min=3.540939138197344e-06, Max=0.04945358726186298, RMS=0.007290515506537144 + +ompsnp_npp nobs= 3652 Min=0.02235738516446045, Max=548.1733215593032, RMS=74.44382038546588 + +ompstc_npp nobs= 29925 Min=232.3457460802846, Max=580.3706053589909, RMS=330.2028035413688 + +satwind_goes-16 nobs= 1605788 Min=-48.18299380448379, Max=74.78775267937297, RMS=10.02015189273681 + + +background y - H(x): +Aircraft nobs= 514449 Min=-244.1546176605621, Max=114.0538314416612, RMS=3.825505485330922 + +ascatw_ascat_metop-b nobs= 241220 Min=-15.53489644753131, Max=19.32628866297402, RMS=2.327498313731722 + +ATMS N20 nobs= 17107090 Min=-139.4066954833384, Max=99999997810.8074, RMS=5096987910.765858 + +surface_ps nobs= 116459 Min=-101684.8005703992, Max=75903.15428094343, RMS=2988.11265787499 + +gnssrobndnbam nobs= 791231 Min=-0.0302615994327759, Max=0.01662471709650585, RMS=0.0008693038108764198 + +ompsnp_npp nobs= 3652 Min=-30.64844591762643, Max=60.86103708368728, RMS=3.943997228450417 + +ompstc_npp nobs= 28934 Min=-98.77455818390638, Max=716.7959660681558, RMS=17.05887006514246 + +satwind_goes-16 nobs= 1605788 Min=-51.62964751676423, Max=38.23192217860554, RMS=3.10218655904646 + + diff --git a/test/testreference/C96C48_ufs_hybatmDA_lgetkf_solver.ref b/test/testreference/C96C48_ufs_hybatmDA_lgetkf_solver.ref new file mode 100644 index 000000000..7c07c4dfa --- /dev/null +++ b/test/testreference/C96C48_ufs_hybatmDA_lgetkf_solver.ref @@ -0,0 +1,180 @@ +Initial state for member 1: +---------------------------------------------------------------------------------------------------- +State print | number of fields = 23 | cube sphere face size: C48 +eastward_wind | Min:-8.4384330749511719e+01 Max:+1.1146717834472656e+02 RMS:+1.7423407399233511e+01 +northward_wind | Min:-8.6144805908203125e+01 Max:+8.9331993103027344e+01 RMS:+1.0017299694477179e+01 +air_temperature | Min:+1.7350303649902344e+02 Max:+3.0961535644531250e+02 RMS:+2.5015669840056216e+02 +layer_thickness | Min:-3.0059777832031250e+03 Max:-1.6207923889160156e+01 RMS:+9.8503674614826332e+02 +air_pressure_thickness | Min:+6.0599988698959351e-01 Max:+1.7881016845703125e+03 RMS:+9.5698921151558159e+02 +air_pressure_at_surface | Min:+5.2320046875000000e+04 Max:+1.0452482812500000e+05 RMS:+9.8914997456124955e+04 +water_vapor_mixing_ratio_wrt_moist_air | Min:+1.6847316430812498e-08 Max:+2.2395884618163109e-02 RMS:+5.0107896876815721e-03 +cloud_liquid_ice | Min:+0.0000000000000000e+00 Max:+4.1855275048874319e-04 RMS:+1.0354889627583219e-05 +cloud_liquid_water | Min:+0.0000000000000000e+00 Max:+1.3839458115398884e-03 RMS:+3.5588872198565459e-05 +ozone_mass_mixing_ratio | Min:+1.1339260641562987e-09 Max:+1.7277885490329936e-05 RMS:+4.4686682637740296e-06 +geopotential_height_at_surface | Min:-2.7018375396728516e+01 Max:+5.2697558593750000e+03 RMS:+6.3772571434211386e+02 +slmsk | Min:+0.0000000000000000e+00 Max:+2.0000000000000000e+00 RMS:+6.5462209907135938e-01 +sheleg | Min:+0.0000000000000000e+00 Max:+2.5217851257324219e+02 RMS:+2.2019818126018798e+01 +skin_temperature_at_surface | Min:+2.1759780883789062e+02 Max:+3.2456860351562500e+02 RMS:+2.8741259834843908e+02 +vtype | Min:+0.0000000000000000e+00 Max:+2.0000000000000000e+01 RMS:+6.1246929250310931e+00 +stype | Min:+0.0000000000000000e+00 Max:+1.6000000000000000e+01 RMS:+4.5755536879077185e+00 +vfrac | Min:+0.0000000000000000e+00 Max:+9.8551864624023438e+01 RMS:+2.0170282476619530e+01 +stc | Min:+2.0000000000000000e+02 Max:+3.1568060302734375e+02 RMS:+2.8817196445239324e+02 +soilMoistureVolumetric | Min:+2.1992070600390434e-02 Max:+1.0000000000000000e+00 RMS:+8.4975035958924516e-01 +totalSnowDepthMeters | Min:+0.0000000000000000e+00 Max:+1.0927191972732544e+00 RMS:+1.5092084570439257e-01 +eastward_wind_at_surface | Min:-1.8143459320068359e+01 Max:+2.0627843856811523e+01 RMS:+5.1169707890467366e+00 +northward_wind_at_surface | Min:-2.3821027755737305e+01 Max:+1.9497446060180664e+01 RMS:+4.8067548196805943e+00 +f10m | Min:+9.4103527069091797e-01 Max:+1.0685453414916992e+00 RMS:+9.9573423073115486e-01 +---------------------------------------------------------------------------------------------------- +Initial state for member 2: +---------------------------------------------------------------------------------------------------- +State print | number of fields = 23 | cube sphere face size: C48 +eastward_wind | Min:-8.5966735839843750e+01 Max:+1.1211631774902344e+02 RMS:+1.7425666524535583e+01 +northward_wind | Min:-8.5594192504882812e+01 Max:+8.9450973510742188e+01 RMS:+1.0012910607932993e+01 +air_temperature | Min:+1.7322386169433594e+02 Max:+3.1173611450195312e+02 RMS:+2.5015480177211401e+02 +layer_thickness | Min:-3.0079758300781250e+03 Max:-1.6234376907348633e+01 RMS:+9.8509317417150282e+02 +air_pressure_thickness | Min:+6.0599988698959351e-01 Max:+1.7860076904296875e+03 RMS:+9.5696337286916196e+02 +air_pressure_at_surface | Min:+5.2300289062500000e+04 Max:+1.0443091406250000e+05 RMS:+9.8912612346289476e+04 +water_vapor_mixing_ratio_wrt_moist_air | Min:+1.7588122958045460e-08 Max:+2.1704806014895439e-02 RMS:+4.9960608009716806e-03 +cloud_liquid_ice | Min:+0.0000000000000000e+00 Max:+4.2980920989066362e-04 RMS:+9.7698907168770146e-06 +cloud_liquid_water | Min:+0.0000000000000000e+00 Max:+1.4689513482153416e-03 RMS:+3.5167035611299903e-05 +ozone_mass_mixing_ratio | Min:+2.9807353807420611e-10 Max:+1.7195472537423484e-05 RMS:+4.4692040486361164e-06 +geopotential_height_at_surface | Min:-2.7018375396728516e+01 Max:+5.2697558593750000e+03 RMS:+6.3772571434211386e+02 +slmsk | Min:+0.0000000000000000e+00 Max:+2.0000000000000000e+00 RMS:+6.5462209907135938e-01 +sheleg | Min:+0.0000000000000000e+00 Max:+2.4906845092773438e+02 RMS:+2.2004970461687567e+01 +skin_temperature_at_surface | Min:+2.1673643493652344e+02 Max:+3.2559057617187500e+02 RMS:+2.8745909600453388e+02 +vtype | Min:+0.0000000000000000e+00 Max:+2.0000000000000000e+01 RMS:+6.1246929250310931e+00 +stype | Min:+0.0000000000000000e+00 Max:+1.6000000000000000e+01 RMS:+4.5755536879077185e+00 +vfrac | Min:+0.0000000000000000e+00 Max:+9.8551864624023438e+01 RMS:+2.0170282476619530e+01 +stc | Min:+2.0000000000000000e+02 Max:+3.1603344726562500e+02 RMS:+2.8817511849953127e+02 +soilMoistureVolumetric | Min:+2.1997468546032906e-02 Max:+1.0000000000000000e+00 RMS:+8.4973303187978066e-01 +totalSnowDepthMeters | Min:+0.0000000000000000e+00 Max:+1.0929538011550903e+00 RMS:+1.5094966458993353e-01 +eastward_wind_at_surface | Min:-1.9632795333862305e+01 Max:+2.2031156539916992e+01 RMS:+5.1474973436096771e+00 +northward_wind_at_surface | Min:-2.2859689712524414e+01 Max:+1.9478809356689453e+01 RMS:+4.7531537396512338e+00 +f10m | Min:+9.4079720973968506e-01 Max:+1.0689357519149780e+00 RMS:+9.9569948613238035e-01 +---------------------------------------------------------------------------------------------------- +H(x) for member 1: +Aircraft nobs= 502771 Min=-80.55443572998047, Max=305.5072631835938, RMS=146.5850485206547 + +ascatw_ascat_metop-b nobs= 24988 Min=-14.90743255615234, Max=18.50735855102539, RMS=5.626101621409232 + +ATMS N20 nobs= 17107090 Min=129.7859039306641, Max=294.7571411132812, RMS=237.7675940456118 + +surface_ps nobs= 199096 Min=481.5556030273438, Max=106952.1328125, RMS=94358.04310219502 + +gnssrobndnbam nobs= 792394 Min=3.546540028764866e-06, Max=0.05689811706542969, RMS=0.007298130011564753 + +ompsnp_npp nobs= 3652 Min=0.02233893238008022, Max=544.9735717773438, RMS=74.39666527674214 + +ompstc_npp nobs= 29925 Min=232.0866546630859, Max=578.874267578125, RMS=330.1687049872166 + +satwind_goes-16 nobs= 192150 Min=-46.72026062011719, Max=73.49080657958984, RMS=13.50004896377961 + + +H(x) for member 2: +Aircraft nobs= 502771 Min=-83.52928924560547, Max=303.9353942871094, RMS=146.5832501984888 + +ascatw_ascat_metop-b nobs= 24988 Min=-15.95012760162354, Max=19.32845687866211, RMS=5.590280603593994 + +ATMS N20 nobs= 17107090 Min=129.5870666503906, Max=294.7645874023438, RMS=237.7536249814979 + +surface_ps nobs= 199096 Min=475.4248046875, Max=106814.578125, RMS=94338.64525474468 + +gnssrobndnbam nobs= 792394 Min=3.53533823727048e-06, Max=0.05521000176668167, RMS=0.007290866961668531 + +ompsnp_npp nobs= 3652 Min=0.02236964181065559, Max=551.3731079101562, RMS=74.49227830514708 + +ompstc_npp nobs= 29925 Min=232.3157501220703, Max=581.888427734375, RMS=330.2400711798879 + +satwind_goes-16 nobs= 192150 Min=-45.04364013671875, Max=72.45210266113281, RMS=13.31321128986013 + + +H(x) ensemble background mean: +Aircraft nobs= 502771 Min=-82.04186248779297, Max=304.2592163085938, RMS=146.582395066166 + +ascatw_ascat_metop-b nobs= 24988 Min=-15.35133123397827, Max=18.91790771484375, RMS=5.590053671147163 + +ATMS N20 nobs= 17107090 Min=129.6864852905273, Max=294.7608642578125, RMS=237.7603500486925 + +surface_ps nobs= 199096 Min=486.4346618652344, Max=106883.35546875, RMS=94348.33692965737 + +gnssrobndnbam nobs= 792394 Min=3.540939133017673e-06, Max=0.04945358820259571, RMS=0.007290515505449794 + +ompsnp_npp nobs= 3652 Min=0.02235738560557365, Max=548.17333984375, RMS=74.44382053535134 + +ompstc_npp nobs= 29925 Min=232.3457412719727, Max=580.37060546875, RMS=330.2028035163413 + +satwind_goes-16 nobs= 192150 Min=-45.88195037841797, Max=72.75027465820312, RMS=13.36167441823028 + + +background y - H(x): +Aircraft nobs= 502771 Min=-18.89637756347656, Max=18.92713928222656, RMS=3.217299765305571 + +ascatw_ascat_metop-b nobs= 24988 Min=-4.990096844732761, Max=4.997781611979008, RMS=1.78136290653079 + +ATMS N20 nobs= 17107090 Min=-139.4066925048828, Max=99999997810.8074, RMS=5096987910.765857 + +surface_ps nobs= 116459 Min=-101684.80078125, Max=75903.154296875, RMS=2988.112660654301 + +gnssrobndnbam nobs= 791231 Min=-0.03026160039007664, Max=0.01662471704185009, RMS=0.000869303809760421 + +ompsnp_npp nobs= 3652 Min=-30.64844512939453, Max=60.86103820800781, RMS=3.943997189932038 + +ompstc_npp nobs= 28934 Min=-98.7745361328125, Max=716.7959594726562, RMS=17.05887006085864 + +satwind_goes-16 nobs= 192150 Min=-17.58694839477539, Max=18.68439626693726, RMS=3.267043143712932 + + +Background mean : +---------------------------------------------------------------------------------------------------- +State print | number of fields = 23 | cube sphere face size: C48 +eastward_wind | Min:-8.5175533294677734e+01 Max:+1.1179174804687500e+02 RMS:+1.7400557939433298e+01 +northward_wind | Min:-8.5735511779785156e+01 Max:+8.9157825469970703e+01 RMS:+9.9701645189144461e+00 +air_temperature | Min:+1.7338398742675781e+02 Max:+3.1067573547363281e+02 RMS:+2.5015547362929473e+02 +layer_thickness | Min:-3.0054658203125000e+03 Max:-1.6221150398254395e+01 RMS:+9.8506306362137718e+02 +air_pressure_thickness | Min:+6.0599988698959351e-01 Max:+1.7870546875000000e+03 RMS:+9.5697619937906575e+02 +air_pressure_at_surface | Min:+5.2310167968750000e+04 Max:+1.0447787109375000e+05 RMS:+9.8913797930278190e+04 +water_vapor_mixing_ratio_wrt_moist_air | Min:+2.9691944014587079e-08 Max:+2.1115131676197052e-02 RMS:+4.9994705712860729e-03 +cloud_liquid_ice | Min:+0.0000000000000000e+00 Max:+3.6185300268698484e-04 RMS:+9.3862057121105856e-06 +cloud_liquid_water | Min:+0.0000000000000000e+00 Max:+1.2367462622933090e-03 RMS:+3.1415946634547486e-05 +ozone_mass_mixing_ratio | Min:+7.5237392704874395e-10 Max:+1.7233576727448963e-05 RMS:+4.4688532813447449e-06 +geopotential_height_at_surface | Min:-2.7018375396728516e+01 Max:+5.2697558593750000e+03 RMS:+6.3772571434211386e+02 +slmsk | Min:+0.0000000000000000e+00 Max:+2.0000000000000000e+00 RMS:+6.5462209907135938e-01 +sheleg | Min:+0.0000000000000000e+00 Max:+2.4910437774658203e+02 RMS:+2.2007651817656669e+01 +skin_temperature_at_surface | Min:+2.1716712188720703e+02 Max:+3.2174871826171875e+02 RMS:+2.8743544448926787e+02 +vtype | Min:+0.0000000000000000e+00 Max:+2.0000000000000000e+01 RMS:+6.1246929250310931e+00 +stype | Min:+0.0000000000000000e+00 Max:+1.6000000000000000e+01 RMS:+4.5755536879077185e+00 +vfrac | Min:+0.0000000000000000e+00 Max:+9.8551864624023438e+01 RMS:+2.0170282476619530e+01 +stc | Min:+2.0000000000000000e+02 Max:+3.1577014160156250e+02 RMS:+2.8817349407786276e+02 +soilMoistureVolumetric | Min:+2.1994769573211670e-02 Max:+1.0000000000000000e+00 RMS:+8.4973696890011863e-01 +totalSnowDepthMeters | Min:+0.0000000000000000e+00 Max:+1.0928364992141724e+00 RMS:+1.5091512135815568e-01 +eastward_wind_at_surface | Min:-1.8888127326965332e+01 Max:+2.1329500198364258e+01 RMS:+5.1094029653114390e+00 +northward_wind_at_surface | Min:-2.3228111267089844e+01 Max:+1.8953988075256348e+01 RMS:+4.7537442465404594e+00 +f10m | Min:+9.4170856475830078e-01 Max:+1.0687405467033386e+00 RMS:+9.9571647867703006e-01 +---------------------------------------------------------------------------------------------------- +Analysis mean : +---------------------------------------------------------------------------------------------------- +State print | number of fields = 23 | cube sphere face size: C48 +eastward_wind | Min:-8.4774324321502988e+01 Max:+1.1170619450293722e+02 RMS:+1.7400375509763755e+01 +northward_wind | Min:-8.7214121288366655e+01 Max:+8.9233807495233577e+01 RMS:+9.9938074328966984e+00 +air_temperature | Min:+1.7338398731904834e+02 Max:+3.1021734492129713e+02 RMS:+2.5015182619184384e+02 +layer_thickness | Min:-3.0054658092822870e+03 Max:-1.6234013903099573e+01 RMS:+9.8506487778080520e+02 +air_pressure_thickness | Min:+6.0599988698959351e-01 Max:+1.7869309532732013e+03 RMS:+9.5697423778426378e+02 +air_pressure_at_surface | Min:+5.2310167956090001e+04 Max:+1.0447787101137359e+05 RMS:+9.8913797926518338e+04 +water_vapor_mixing_ratio_wrt_moist_air | Min:+0.0000000000000000e+00 Max:+2.1115131089317944e-02 RMS:+4.9974721017221301e-03 +cloud_liquid_ice | Min:+0.0000000000000000e+00 Max:+4.4276663301760246e-04 RMS:+9.6220406276590353e-06 +cloud_liquid_water | Min:+0.0000000000000000e+00 Max:+1.3226195416814611e-03 RMS:+3.3209346549199534e-05 +ozone_mass_mixing_ratio | Min:+1.0218316416270972e-09 Max:+1.7364107739907706e-05 RMS:+4.4689530099725226e-06 +geopotential_height_at_surface | Min:-2.7018375396728516e+01 Max:+5.2697558593750000e+03 RMS:+6.3772571434211386e+02 +slmsk | Min:+0.0000000000000000e+00 Max:+2.0000000000000000e+00 RMS:+6.5462209907135938e-01 +sheleg | Min:+0.0000000000000000e+00 Max:+2.4910437774658203e+02 RMS:+2.2007651817656669e+01 +skin_temperature_at_surface | Min:+2.1716712188720703e+02 Max:+3.2174871826171875e+02 RMS:+2.8743544448926787e+02 +vtype | Min:+0.0000000000000000e+00 Max:+2.0000000000000000e+01 RMS:+6.1246929250310931e+00 +stype | Min:+0.0000000000000000e+00 Max:+1.6000000000000000e+01 RMS:+4.5755536879077185e+00 +vfrac | Min:+0.0000000000000000e+00 Max:+9.8551864624023438e+01 RMS:+2.0170282476619530e+01 +stc | Min:+2.0000000000000000e+02 Max:+3.1577014160156250e+02 RMS:+2.8817349407786276e+02 +soilMoistureVolumetric | Min:+2.1994769573211670e-02 Max:+1.0000000000000000e+00 RMS:+8.4973696890011863e-01 +totalSnowDepthMeters | Min:+0.0000000000000000e+00 Max:+1.0928364992141724e+00 RMS:+1.5091512135815568e-01 +eastward_wind_at_surface | Min:-1.8888127326965332e+01 Max:+2.1329500198364258e+01 RMS:+5.1094029653114390e+00 +northward_wind_at_surface | Min:-2.3228111267089844e+01 Max:+1.8953988075256348e+01 RMS:+4.7537442465404594e+00 +f10m | Min:+9.4170856475830078e-01 Max:+1.0687405467033386e+00 RMS:+9.9571647867703006e-01 +---------------------------------------------------------------------------------------------------- diff --git a/test/testreference/atm_jjob_3dvar-fv3inc.ref b/test/testreference/atm_jjob_3dvar-fv3inc.ref new file mode 100644 index 000000000..98a94646b --- /dev/null +++ b/test/testreference/atm_jjob_3dvar-fv3inc.ref @@ -0,0 +1,41 @@ +Background State: + +---------------------------------------------------------------------------------------------------- +State print | number of fields = 9 | cube sphere face size: C48 +eastward_wind | Min:-5.5495644220059390e+01 Max:+8.4934651204487366e+01 RMS:+1.6388250288532191e+01 +northward_wind | Min:-7.3712421229093422e+01 Max:+7.6984825502186652e+01 RMS:+8.1424144891567511e+00 +air_temperature | Min:+1.7482158077318496e+02 Max:+3.1467235233685437e+02 RMS:+2.4978424883064508e+02 +surface_pressure | Min:+5.3298485514404747e+04 Max:+1.0398027192902798e+05 RMS:+9.8897229693652524e+04 +specific_humidity | Min:+1.2691624667179020e-08 Max:+2.0041369334954085e-02 RMS:+4.7792638023125686e-03 +cloud_liquid_ice | Min:+0.0000000000000000e+00 Max:+1.2024392024662985e-03 RMS:+1.4396968866569995e-05 +cloud_liquid_water | Min:+0.0000000000000000e+00 Max:+1.5935686618789048e-03 RMS:+4.1931345590543333e-05 +ozone_mass_mixing_ratio | Min:+1.1998327271379204e-08 Max:+1.7808431790670831e-05 RMS:+4.4947917149861064e-06 +surface_geopotential_height | Min:-2.7018394470214844e+01 Max:+5.2697460937500000e+03 RMS:+6.3032575784066080e+02 +---------------------------------------------------------------------------------------------------- +JEDI Increment: + +---------------------------------------------------------------------------------------------------- +Increment print | number of fields = 8 | cube sphere face size: C48 +eastward_wind | Min:-4.9503077015344843e-07 Max:+7.0838511279092131e-07 RMS:+2.4490899978286723e-09 +northward_wind | Min:-7.2703516684757830e-07 Max:+2.8853321110133834e-07 RMS:+2.1376104971902294e-09 +air_temperature | Min:-2.1723477061641461e-07 Max:+6.5020122974601691e-07 RMS:+1.0998492962503341e-08 +surface_pressure | Min:-7.8580342233181000e-10 Max:+3.2014213502407074e-10 RMS:+1.5159255545534699e-11 +specific_humidity | Min:-6.7425976452941262e-04 Max:+7.2974731745455677e-04 RMS:+2.1043490698817701e-06 +cloud_liquid_ice | Min:-2.7736911125384438e-06 Max:+5.0834107240617123e-07 RMS:+4.7607939912271645e-08 +cloud_liquid_water | Min:-3.7332445167104997e-04 Max:+3.4355489504058373e-04 RMS:+3.0033170533500212e-06 +ozone_mass_mixing_ratio | Min:+0.0000000000000000e+00 Max:+0.0000000000000000e+00 RMS:+0.0000000000000000e+00 +---------------------------------------------------------------------------------------------------- +FV3 Increment: + +---------------------------------------------------------------------------------------------------- +Increment print | number of fields = 9 | cube sphere face size: C48 +eastward_wind | Min:-4.9503077015344843e-07 Max:+7.0838511279092131e-07 RMS:+2.4490899978286723e-09 +northward_wind | Min:-7.2703516684757830e-07 Max:+2.8853321110133834e-07 RMS:+2.1376104971902294e-09 +air_temperature | Min:-2.1723477061641461e-07 Max:+6.5020122974601691e-07 RMS:+1.0998492962503341e-08 +specific_humidity | Min:-6.7425976452941262e-04 Max:+7.2974731745455677e-04 RMS:+2.1043490698817701e-06 +cloud_liquid_ice | Min:-2.7736911125384438e-06 Max:+5.0834107240617123e-07 RMS:+4.7607939912271645e-08 +cloud_liquid_water | Min:-3.7332445167104997e-04 Max:+3.4355489504058373e-04 RMS:+3.0033170533500212e-06 +ozone_mass_mixing_ratio | Min:+0.0000000000000000e+00 Max:+0.0000000000000000e+00 RMS:+0.0000000000000000e+00 +air_pressure_thickness | Min:-2.9103830456733704e-11 Max:+2.9103830456733704e-11 RMS:+3.3056019160193798e-13 +layer_thickness | Min:-1.0638235174337751e-01 Max:+1.0329449278924585e-01 RMS:+2.8485086711777381e-04 +---------------------------------------------------------------------------------------------------- diff --git a/test/atm/global-workflow/3dvar.ref b/test/testreference/atm_jjob_3dvar.ref similarity index 100% rename from test/atm/global-workflow/3dvar.ref rename to test/testreference/atm_jjob_3dvar.ref diff --git a/test/testreference/atm_jjob_lgetkf-fv3inc.ref b/test/testreference/atm_jjob_lgetkf-fv3inc.ref new file mode 100644 index 000000000..2c8fed453 --- /dev/null +++ b/test/testreference/atm_jjob_lgetkf-fv3inc.ref @@ -0,0 +1,123 @@ +Background State for member 0: + +---------------------------------------------------------------------------------------------------- +State print | number of fields = 9 | cube sphere face size: C48 +eastward_wind | Min:-5.1619864857418477e+01 Max:+8.6812084442971653e+01 RMS:+1.5983411875766325e+01 +northward_wind | Min:-7.3094846805319690e+01 Max:+7.0353817207582651e+01 RMS:+7.7265473774262041e+00 +air_temperature | Min:+1.7646396818493386e+02 Max:+3.1441817730817547e+02 RMS:+2.4989439192167012e+02 +surface_pressure | Min:+5.3259705655314101e+04 Max:+1.0392378167916714e+05 RMS:+9.8893465602866549e+04 +specific_humidity | Min:+9.5164908108891825e-09 Max:+2.0222136340880194e-02 RMS:+4.8552459174929334e-03 +cloud_liquid_ice | Min:-1.3545705512023003e-20 Max:+6.7304686933261399e-04 RMS:+1.0726563909939355e-05 +cloud_liquid_water | Min:-5.4244251425755909e-20 Max:+1.2879383569881558e-03 RMS:+3.8382491358281577e-05 +ozone_mass_mixing_ratio | Min:+2.9375003505643131e-08 Max:+1.8014885502109894e-05 RMS:+4.4946424525503736e-06 +surface_geopotential_height | Min:-2.7018394470214844e+01 Max:+5.2697460937500000e+03 RMS:+6.3032575784066080e+02 +---------------------------------------------------------------------------------------------------- +JEDI Increment for member 0: + +---------------------------------------------------------------------------------------------------- +Increment print | number of fields = 8 | cube sphere face size: C48 +eastward_wind | Min:-1.7416909136434541e+01 Max:+1.8325867271120192e+01 RMS:+7.2689113073983769e-01 +northward_wind | Min:-1.5335235321425948e+01 Max:+2.1574447615122054e+01 RMS:+7.1328003377466020e-01 +air_temperature | Min:-8.6048230478038406e+00 Max:+8.1497937417599609e+00 RMS:+3.1081395800884187e-01 +surface_pressure | Min:-4.9202937637815580e-04 Max:+8.7924188102306289e-04 RMS:+5.8051418356002832e-05 +specific_humidity | Min:-6.6741214369962698e-03 Max:+9.0731153226882751e-03 RMS:+1.4380776566867527e-04 +cloud_liquid_ice | Min:-4.9931336523754066e-04 Max:+2.6181751401796260e-04 RMS:+3.2747284981508197e-06 +cloud_liquid_water | Min:-1.0141458910771410e-03 Max:+1.1396207487973308e-03 RMS:+1.0447469223790289e-05 +ozone_mass_mixing_ratio | Min:-2.9909193396824413e-06 Max:+3.4967162777667096e-06 RMS:+5.0924831319987802e-08 +---------------------------------------------------------------------------------------------------- +FV3 Increment for member 0: + +---------------------------------------------------------------------------------------------------- +Increment print | number of fields = 9 | cube sphere face size: C48 +eastward_wind | Min:-1.7416909136434541e+01 Max:+1.8325867271120192e+01 RMS:+7.2689113073983769e-01 +northward_wind | Min:-1.5335235321425948e+01 Max:+2.1574447615122054e+01 RMS:+7.1328003377466020e-01 +air_temperature | Min:-8.6048230478038406e+00 Max:+8.1497937417599609e+00 RMS:+3.1081395800884187e-01 +specific_humidity | Min:-6.6741214369962698e-03 Max:+9.0731153226882751e-03 RMS:+1.4380776566867527e-04 +cloud_liquid_ice | Min:-4.9931336523754066e-04 Max:+2.6181751401796260e-04 RMS:+3.2747284981508197e-06 +cloud_liquid_water | Min:-1.0141458910771410e-03 Max:+1.1396207487973308e-03 RMS:+1.0447469223790289e-05 +ozone_mass_mixing_ratio | Min:-2.9909193396824413e-06 Max:+3.4967162777667096e-06 RMS:+5.0924831319987802e-08 +air_pressure_thickness | Min:-1.1013638868462294e-05 Max:+1.9681036064866930e-05 RMS:+6.5512588707706875e-07 +layer_thickness | Min:-2.8045559103640699e+01 Max:+2.1697786412500136e+01 RMS:+6.5544095172004369e-01 +---------------------------------------------------------------------------------------------------- +Background State for member 1: + +---------------------------------------------------------------------------------------------------- +State print | number of fields = 9 | cube sphere face size: C48 +eastward_wind | Min:-5.2853490758012072e+01 Max:+8.7248708527964936e+01 RMS:+1.5955809745480765e+01 +northward_wind | Min:-7.2178651628585953e+01 Max:+7.2568167230757609e+01 RMS:+7.7335061167195889e+00 +air_temperature | Min:+1.7660670217895057e+02 Max:+3.1526174222386965e+02 RMS:+2.4989392943476341e+02 +surface_pressure | Min:+5.3317702550070273e+04 Max:+1.0394629485957995e+05 RMS:+9.8892949759388735e+04 +specific_humidity | Min:+2.1514597275760548e-08 Max:+1.9663168048133908e-02 RMS:+4.8569105861498299e-03 +cloud_liquid_ice | Min:-1.3544569699620215e-20 Max:+7.9557019872326799e-04 RMS:+1.0816147029169270e-05 +cloud_liquid_water | Min:-5.4200711162051571e-20 Max:+1.2298889446687945e-03 RMS:+3.8689917911663205e-05 +ozone_mass_mixing_ratio | Min:+1.6238717942558199e-08 Max:+1.8031521448783218e-05 RMS:+4.4947640660308833e-06 +surface_geopotential_height | Min:-2.7018394470214844e+01 Max:+5.2697460937500000e+03 RMS:+6.3032575784066080e+02 +---------------------------------------------------------------------------------------------------- +JEDI Increment for member 1: + +---------------------------------------------------------------------------------------------------- +Increment print | number of fields = 8 | cube sphere face size: C48 +eastward_wind | Min:-1.7542991285853031e+01 Max:+2.0968845531775742e+01 RMS:+7.2825863145362402e-01 +northward_wind | Min:-1.5283258164430123e+01 Max:+2.0478626874987171e+01 RMS:+7.1562489129210427e-01 +air_temperature | Min:-1.0374519800275246e+01 Max:+9.3297652767893524e+00 RMS:+3.1237829519350230e-01 +surface_pressure | Min:-4.8986046327570421e-04 Max:+9.0817638056250871e-04 RMS:+5.9230402548777353e-05 +specific_humidity | Min:-6.9184909302780918e-03 Max:+8.2127508558306646e-03 RMS:+1.4099077159566875e-04 +cloud_liquid_ice | Min:-5.2537411650465499e-04 Max:+3.1949478810012653e-04 RMS:+3.2097080991341253e-06 +cloud_liquid_water | Min:-1.0684141562485590e-03 Max:+1.1931964279486475e-03 RMS:+1.0416514064617798e-05 +ozone_mass_mixing_ratio | Min:-3.1852115564505394e-06 Max:+3.2851197161513411e-06 RMS:+5.1928484225101225e-08 +---------------------------------------------------------------------------------------------------- +FV3 Increment for member 1: + +---------------------------------------------------------------------------------------------------- +Increment print | number of fields = 9 | cube sphere face size: C48 +eastward_wind | Min:-1.7542991285853031e+01 Max:+2.0968845531775742e+01 RMS:+7.2825863145362402e-01 +northward_wind | Min:-1.5283258164430123e+01 Max:+2.0478626874987171e+01 RMS:+7.1562489129210427e-01 +air_temperature | Min:-1.0374519800275246e+01 Max:+9.3297652767893524e+00 RMS:+3.1237829519350230e-01 +specific_humidity | Min:-6.9184909302780918e-03 Max:+8.2127508558306646e-03 RMS:+1.4099077159566875e-04 +cloud_liquid_ice | Min:-5.2537411650465499e-04 Max:+3.1949478810012653e-04 RMS:+3.2097080991341253e-06 +cloud_liquid_water | Min:-1.0684141562485590e-03 Max:+1.1931964279486475e-03 RMS:+1.0416514064617798e-05 +ozone_mass_mixing_ratio | Min:-3.1852115564505394e-06 Max:+3.2851197161513411e-06 RMS:+5.1928484225101225e-08 +air_pressure_thickness | Min:-1.0965086403302848e-05 Max:+2.0328705431893468e-05 RMS:+6.6843104175422882e-07 +layer_thickness | Min:-3.2106148452359776e+01 Max:+2.5923789087046316e+01 RMS:+6.6540710201306896e-01 +---------------------------------------------------------------------------------------------------- +Background State for member 2: + +---------------------------------------------------------------------------------------------------- +State print | number of fields = 9 | cube sphere face size: C48 +eastward_wind | Min:-5.3776366655868109e+01 Max:+8.6331467223030344e+01 RMS:+1.5960680883840784e+01 +northward_wind | Min:-7.0176400159320124e+01 Max:+6.9531049915936578e+01 RMS:+7.7410680097841684e+00 +air_temperature | Min:+1.7648447482434227e+02 Max:+3.1491330999751426e+02 RMS:+2.4989313307279673e+02 +surface_pressure | Min:+5.3350124363319999e+04 Max:+1.0389818047809265e+05 RMS:+9.8892179510558097e+04 +specific_humidity | Min:+2.1846447498856659e-08 Max:+2.0007929878326374e-02 RMS:+4.8392192734415156e-03 +cloud_liquid_ice | Min:-6.7740196346531239e-21 Max:+6.2473018245821010e-04 RMS:+1.0889666282638220e-05 +cloud_liquid_water | Min:-5.4107584399974539e-20 Max:+1.2132298501853738e-03 RMS:+3.8405804865664137e-05 +ozone_mass_mixing_ratio | Min:+1.2289325508931378e-08 Max:+1.8036413152177077e-05 RMS:+4.4950884931704925e-06 +surface_geopotential_height | Min:-2.7018394470214844e+01 Max:+5.2697460937500000e+03 RMS:+6.3032575784066080e+02 +---------------------------------------------------------------------------------------------------- +JEDI Increment for member 2: + +---------------------------------------------------------------------------------------------------- +Increment print | number of fields = 8 | cube sphere face size: C48 +eastward_wind | Min:-1.8014062913663103e+01 Max:+1.8516732485049666e+01 RMS:+7.3383694849572323e-01 +northward_wind | Min:-1.5579280119728031e+01 Max:+2.2762102719398591e+01 RMS:+7.2377717441299694e-01 +air_temperature | Min:-1.0570848751737598e+01 Max:+8.3564092475959129e+00 RMS:+3.1645038788954982e-01 +surface_pressure | Min:-4.7656900311210393e-04 Max:+9.1220166839889316e-04 RMS:+5.8396146445486219e-05 +specific_humidity | Min:-5.2439197475218757e-03 Max:+8.2607880293262357e-03 RMS:+1.4211303446855155e-04 +cloud_liquid_ice | Min:-5.2671607196952148e-04 Max:+2.7498589550193714e-04 RMS:+3.2193226262458792e-06 +cloud_liquid_water | Min:-1.0851123947058787e-03 Max:+1.2045791096627974e-03 RMS:+1.0548571598067807e-05 +ozone_mass_mixing_ratio | Min:-3.2510682480215228e-06 Max:+3.3849184382668304e-06 RMS:+5.2379967545874409e-08 +---------------------------------------------------------------------------------------------------- +FV3 Increment for member 2: + +---------------------------------------------------------------------------------------------------- +Increment print | number of fields = 9 | cube sphere face size: C48 +eastward_wind | Min:-1.8014062913663103e+01 Max:+1.8516732485049666e+01 RMS:+7.3383694849572323e-01 +northward_wind | Min:-1.5579280119728031e+01 Max:+2.2762102719398591e+01 RMS:+7.2377717441299694e-01 +air_temperature | Min:-1.0570848751737598e+01 Max:+8.3564092475959129e+00 RMS:+3.1645038788954982e-01 +specific_humidity | Min:-5.2439197475218757e-03 Max:+8.2607880293262357e-03 RMS:+1.4211303446855155e-04 +cloud_liquid_ice | Min:-5.2671607196952148e-04 Max:+2.7498589550193714e-04 RMS:+3.2193226262458792e-06 +cloud_liquid_water | Min:-1.0851123947058787e-03 Max:+1.2045791096627974e-03 RMS:+1.0548571598067807e-05 +ozone_mass_mixing_ratio | Min:-3.2510682480215228e-06 Max:+3.3849184382668304e-06 RMS:+5.2379967545874409e-08 +air_pressure_thickness | Min:-1.0667572496458888e-05 Max:+2.0418810890987515e-05 RMS:+6.5901623607235396e-07 +layer_thickness | Min:-2.8377559817174188e+01 Max:+2.5490176089209854e+01 RMS:+6.7263739596740091e-01 +---------------------------------------------------------------------------------------------------- diff --git a/test/atm/global-workflow/lgetkf.ref b/test/testreference/atm_jjob_lgetkf.ref similarity index 100% rename from test/atm/global-workflow/lgetkf.ref rename to test/testreference/atm_jjob_lgetkf.ref diff --git a/test/atm/global-workflow/lgetkf_observer.ref b/test/testreference/atm_jjob_lgetkf_observer.ref similarity index 100% rename from test/atm/global-workflow/lgetkf_observer.ref rename to test/testreference/atm_jjob_lgetkf_observer.ref diff --git a/test/atm/global-workflow/lgetkf_solver.ref b/test/testreference/atm_jjob_lgetkf_solver.ref similarity index 100% rename from test/atm/global-workflow/lgetkf_solver.ref rename to test/testreference/atm_jjob_lgetkf_solver.ref diff --git a/ush/CMakeLists.txt b/ush/CMakeLists.txt index 2bf242db4..65f8bbfe8 100644 --- a/ush/CMakeLists.txt +++ b/ush/CMakeLists.txt @@ -1,6 +1,5 @@ list(APPEND ush_files check_yaml_keys.py - jediinc2fv3.py genYAML ) diff --git a/ush/jediinc2fv3.py b/ush/jediinc2fv3.py deleted file mode 100755 index 3bee852df..000000000 --- a/ush/jediinc2fv3.py +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env python3 -# translate FV3-JEDI increment to FV3 readable format with delp and hydrostatic delz calculation -import argparse -import netCDF4 as nc -import numpy as np -import logging -import os - -vardict = { - 'ua': 'u_inc', - 'va': 'v_inc', - 't': 'T_inc', - 'T': 'T_inc', - 'sphum': 'sphum_inc', - 'liq_wat': 'liq_wat_inc', - 'o3mr': 'o3mr_inc', - 'ice_wat': 'icmr_inc', - 'lat': 'lat', - 'lon': 'lon', -} - - -def jedi_inc_to_fv3(FV3ges, FV3JEDIinc, FV3inc): - logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S') - # Check if required input netcdf files exist. If not present, exit with error message. - try: - with nc.Dataset(FV3ges, 'r'), nc.Dataset(FV3JEDIinc, 'r'): - ncges = nc.Dataset(FV3ges, 'r') - ncin = nc.Dataset(FV3JEDIinc, 'r') - ncout = nc.Dataset(FV3inc, 'w', format='NETCDF4') - create_fv3inc(ncges, ncin, ncout) - - except FileNotFoundError as e: - logging.error(f"Error occurred with message {e}") - raise - - -def create_fv3inc(ncges, ncin, ncout): - - # Copy over dimensions - for name, dimension in ncin.dimensions.items(): - if name == 'time': - continue - elif name == 'edge': - nameout = 'ilev' - else: - nameout = name - ncout.createDimension(nameout, - (len(dimension) if not dimension.isunlimited() else None)) - # Some global attributes - ncout.source = 'jediinc2fv3.py' - ncout.comment = 'Increment produced by FV3-JEDI and modified for use by FV3 read' - - # Create all the dummy vertical coordinate variables - nlevs = len(ncin.dimensions['lev']) - nlats = len(ncin.dimensions['lat']) - nlons = len(ncin.dimensions['lon']) - pfull = range(1, nlevs+1) - phalf = range(1, nlevs+2) - levvar = ncout.createVariable('lev', 'f4', ('lev')) - levvar[:] = pfull - pfullvar = ncout.createVariable('pfull', 'f4', ('lev')) - pfullvar[:] = pfull - ilevvar = ncout.createVariable('ilev', 'f4', ('ilev')) - ilevvar[:] = phalf - hyaivar = ncout.createVariable('hyai', 'f4', ('ilev')) - hyaivar[:] = phalf - hybivar = ncout.createVariable('hybi', 'f4', ('ilev')) - hybivar[:] = phalf - - # Rename and change dimensionality of fields - for name, variable in ncin.variables.items(): - if len(variable.dimensions) in [4]: - dimsout = variable.dimensions[1:] - dimsout_inc = dimsout - else: - dimsout = variable.dimensions - - if name in vardict: - # The temperature increment is needed to compute the delz - # increment. Either T or t may be used in jedi increment - # files. Set tinc to the appropriate case. - if name in ['T', 't']: - tinc = name - - x = ncout.createVariable(vardict[name], 'f4', dimsout) - if len(variable.dimensions) in [4]: - ncout[vardict[name]][:] = ncin[name][0, ...] - else: - ncout[vardict[name]][:] = ncin[name][:] - - # Populate increment and guess fields - # Note: increment and guess fields have same shape - # ps_inc is (time, lat, lon), ps_ges is {time, grid_yt, grid_xt) - # t_inc is (time, lev, lat, lon), t_ges is (time, pfull, grid_yt, grid_xt) - - ps_ges = ncges.variables['pressfc'][:] - t_ges = ncges.variables['tmp'][:] - q_ges = ncges.variables['spfh'][:] - - ps_inc = ncin.variables['ps'][:] - t_inc = ncin.variables[tinc][:] - q_inc = ncin.variables['sphum'][:] - - # Compute analysis ps - ps_anl = np.zeros((nlats, nlons), float) - ps_anl = ps_ges + ps_inc - - # Set constants and compute derived constants - grav = 9.80665 - airmw = 28.965 - h2omw = 18.015 - runiv = 8314.47 - rdry = runiv/airmw - rvap = runiv/h2omw - cpdry = 3.5*rdry - fv = (rvap/rdry)-1 - rdog = rdry/grav - kappa = rdry/cpdry - kap1 = kappa+1.0 - kapr = 1.0/kappa - - # Get ak,bk from guess file - nc_attrs = ncges.ncattrs() - ak = ncges.getncattr('ak') - bk = ncges.getncattr('bk') - - # Compute guess and analysis interface pressures - prsi_ges = np.zeros((nlevs+1, nlats, nlons), float) - prsi_anl = np.zeros((nlevs+1, nlats, nlons), float) - for k in range(0, nlevs+1): - prsi_ges[k, :, :] = ak[k]+bk[k]*ps_ges[:, :] - prsi_anl[k, :, :] = ak[k]+bk[k]*ps_anl[:, :] - - # Compute pressure increment (delp_inc). Compute - # guess and analysis layer pressures using Philips method - delp_inc = np.zeros((nlevs, nlats, nlons), float) - prsl_ges = np.zeros((nlevs, nlats, nlons), float) - prsl_anl = np.zeros((nlevs, nlats, nlons), float) - for k in range(0, nlevs): - dbk = bk[k+1] - bk[k] - delp_inc[k, :, :] = ps_inc[:, :] * dbk - prsl_ges[k, :, :] = ((prsi_ges[k+1, :, :]**kap1 - prsi_ges[k, :, :]**kap1) / (kap1*(prsi_ges[k+1, :, :] - prsi_ges[k, :, :])))**kapr - prsl_anl[k, :, :] = ((prsi_anl[k+1, :, :]**kap1 - prsi_anl[k, :, :]**kap1) / (kap1*(prsi_anl[k+1, :, :] - prsi_anl[k, :, :])))**kapr - - # Write delp increment to output file - x = ncout.createVariable('delp_inc', 'f4', dimsout_inc) - ncout['delp_inc'][:] = delp_inc[:] - - # Compute analysis temperature andl specific humidity - t_anl = np.zeros((nlevs, nlats, nlons), float) - q_anl = np.zeros((nlevs, nlats, nlons), float) - t_anl = t_ges + t_inc - q_anl = q_ges + q_inc - - # Compute guess and analysis virtual temperature - tv_ges = np.zeros((nlevs, nlats, nlons), float) - tv_anl = np.zeros((nlevs, nlats, nlons), float) - tv_ges = t_ges * (1.0 + fv*q_ges) - tv_anl = t_anl * (1.0 + fv*q_anl) - - # Compute guess and analysis delz and delz increment - delz_ges = np.zeros((nlevs, nlats, nlons), float) - delz_anl = np.zeros((nlevs, nlats, nlons), float) - delz_inc = np.zeros((nlevs, nlats, nlons), float) - for k in range(0, nlevs): - if k == 0: - delz_ges[k, :, :] = rdog * tv_ges[:, k, :, :] * np.log(prsl_ges[k, :, :]/prsi_ges[k+1, :, :]) - delz_anl[k, :, :] = rdog * tv_anl[:, k, :, :] * np.log(prsl_anl[k, :, :]/prsi_anl[k+1, :, :]) - else: - delz_ges[k, :, :] = rdog * tv_ges[:, k, :, :] * np.log(prsi_ges[k, :, :]/prsi_ges[k+1, :, :]) - delz_anl[k, :, :] = rdog * tv_anl[:, k, :, :] * np.log(prsi_anl[k, :, :]/prsi_anl[k+1, :, :]) - - delz_inc[k, :, :] = delz_anl[k, :, :] - delz_ges[k, :, :] - - # Write delz increment to output file - x = ncout.createVariable('delz_inc', 'f4', dimsout_inc) - ncout['delz_inc'][:] = delz_inc[:] - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument('FV3background', type=str, help='Input FV3 background file') - parser.add_argument('FV3JEDIincrement', type=str, help='Input FV3-JEDI LatLon Increment File') - parser.add_argument('FV3increment', type=str, help='Output FV3 Increment File') - args = parser.parse_args() - jedi_inc_to_fv3(args.FV3background, args.FV3JEDIincrement, args.FV3increment) diff --git a/utils/fv3jedi/fv3jedi_fv3inc.h b/utils/fv3jedi/fv3jedi_fv3inc.h index 8044692a0..183e426aa 100644 --- a/utils/fv3jedi/fv3jedi_fv3inc.h +++ b/utils/fv3jedi/fv3jedi_fv3inc.h @@ -110,12 +110,21 @@ namespace gdasapp { // Read background state fv3jedi::State xxBkg(stateGeom, stateInputConfig); - oops::Log::test() << "Background State: " << std::endl << xxBkg << std::endl; // Read JEDI increment fv3jedi::Increment dxJEDI(jediIncrGeom, jediIncrVars, xxBkg.validTime()); dxJEDI.read(jediIncrInputConfig); - oops::Log::test() << "JEDI Increment: " << std::endl << dxJEDI << std::endl; + + // Testing output for inputs + if ( nmem == 1 ) { + oops::Log::test() << "Background State:" << std::endl << xxBkg << std::endl; + oops::Log::test() << "JEDI Increment:" << std::endl << dxJEDI << std::endl; + } else { + oops::Log::test() << "Background State for member " << imem << ":" << std::endl \ + << xxBkg << std::endl; + oops::Log::test() << "JEDI Increment for member " << imem << ":" << std::endl \ + << dxJEDI << std::endl; + } // Increment conversion // --------------------------------------------------------------------------------- @@ -156,7 +165,14 @@ namespace gdasapp { } } dxFV3.fromFieldSet(dxFsFV3); - oops::Log::test() << "FV3 Increment: " << std::endl << dxFV3 << std::endl; + + // Test output for outputs + if ( nmem == 1 ) { + oops::Log::test() << "FV3 Increment:" << std::endl << dxFV3 << std::endl; + } else { + oops::Log::test() << "FV3 Increment for member " << imem << ":" << std::endl \ + << dxFV3 << std::endl; + } // Write FV3 increment dxFV3.write(fv3IncrOuputConfig); From 9b78c7d01c017a9f3600bb222396f7251386ed7b Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Wed, 27 Nov 2024 08:16:30 -0500 Subject: [PATCH 21/22] Make sure gfs_* DA jobs are not regression tests (#1390) This PR addresses the bug @RussTreadon-NOAA found that ```gfs_atmanlvar``` was being run as a regression test and using the same test reference as ```gdas_atmanlvar``` in GW PR [#3120](https://github.com/NOAA-EMC/global-workflow/pull/3120). See https://github.com/NOAA-EMC/global-workflow/pull/3120#issuecomment-2501783018 I've moved all activation of testing mode in JCB out of the JCB base YAMLs and into the JCB algorithm YAMLs. I test the ```RUN``` variables to make sure it's not equal to ```gfs```. I re-ran all the regression tests, and they all passed. --- parm/atm/jcb-base.yaml.j2 | 5 ----- parm/atm/jcb-prototype_3dvar-fv3inc.yaml.j2 | 3 +++ parm/atm/jcb-prototype_3dvar.yaml.j2 | 3 +++ parm/atm/jcb-prototype_lgetkf-fv3inc.yaml.j2 | 3 +++ parm/atm/jcb-prototype_lgetkf.yaml.j2 | 3 +++ parm/atm/jcb-prototype_lgetkf_observer.yaml.j2 | 3 +++ parm/atm/jcb-prototype_lgetkf_solver.yaml.j2 | 3 +++ parm/soca/marine-jcb-3dfgat.yaml.j2 | 3 +++ parm/soca/marine-jcb-base.yaml | 5 ----- test/atm/global-workflow/jcb-prototype_3dvar-fv3inc.yaml.j2 | 3 +++ test/atm/global-workflow/jcb-prototype_3dvar.yaml.j2 | 3 +++ test/atm/global-workflow/jcb-prototype_lgetkf-fv3inc.yaml.j2 | 3 +++ test/atm/global-workflow/jcb-prototype_lgetkf.yaml.j2 | 3 +++ .../global-workflow/jcb-prototype_lgetkf_observer.yaml.j2 | 3 +++ test/atm/global-workflow/jcb-prototype_lgetkf_solver.yaml.j2 | 3 +++ 15 files changed, 39 insertions(+), 10 deletions(-) diff --git a/parm/atm/jcb-base.yaml.j2 b/parm/atm/jcb-base.yaml.j2 index 39c626a84..d7e1e5907 100644 --- a/parm/atm/jcb-base.yaml.j2 +++ b/parm/atm/jcb-base.yaml.j2 @@ -27,11 +27,6 @@ final_prints_frequency: PT3H number_of_outer_loops: 2 analysis_variables: [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr] -# Testing -# ------- - -do_testing: {{ DO_TEST_MODE | default(false, true) }} - # Model things # ------------ # Geometry diff --git a/parm/atm/jcb-prototype_3dvar-fv3inc.yaml.j2 b/parm/atm/jcb-prototype_3dvar-fv3inc.yaml.j2 index f7f6db57a..c0aaa3d38 100644 --- a/parm/atm/jcb-prototype_3dvar-fv3inc.yaml.j2 +++ b/parm/atm/jcb-prototype_3dvar-fv3inc.yaml.j2 @@ -4,7 +4,10 @@ algorithm: fv3jedi_fv3inc_variational # Testing things # -------------- +{% if DO_TEST_MODE and RUN == 'gdas' %} +do_testing: true test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/testreference/C96C48_ufs_hybatmDA_3dvar-fv3inc.ref test_output_filename: {{ HOMEgfs }}/sorc/gdas.cd/build/gdas/test/testoutput/C96C48_ufs_hybatmDA_3dvar-fv3inc.test.out test_float_relative_tolerance: 1.0e-3 test_float_absolute_tolerance: 1.0e-5 +{% endif %} diff --git a/parm/atm/jcb-prototype_3dvar.yaml.j2 b/parm/atm/jcb-prototype_3dvar.yaml.j2 index b89b65105..e1e2ff836 100644 --- a/parm/atm/jcb-prototype_3dvar.yaml.j2 +++ b/parm/atm/jcb-prototype_3dvar.yaml.j2 @@ -33,7 +33,10 @@ observations: # Testing things # -------------- +{% if DO_TEST_MODE and RUN == 'gdas' %} +do_testing: true test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/testreference/C96C48_ufs_hybatmDA_3dvar.ref test_output_filename: {{ HOMEgfs }}/sorc/gdas.cd/build/gdas/test/testoutput/C96C48_ufs_hybatmDA_3dvar.test.out test_float_relative_tolerance: 1.0e-3 test_float_absolute_tolerance: 1.0e-5 +{% endif %} diff --git a/parm/atm/jcb-prototype_lgetkf-fv3inc.yaml.j2 b/parm/atm/jcb-prototype_lgetkf-fv3inc.yaml.j2 index 9845103df..f7bf0049b 100644 --- a/parm/atm/jcb-prototype_lgetkf-fv3inc.yaml.j2 +++ b/parm/atm/jcb-prototype_lgetkf-fv3inc.yaml.j2 @@ -4,7 +4,10 @@ algorithm: fv3jedi_fv3inc_lgetkf # Testing things # -------------- +{% if DO_TEST_MODE and RUN == 'enkfgdas' %} +do_testing: true test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/testreference/C96C48_ufs_hybatmDA_lgetkf-fv3inc.ref test_output_filename: {{ HOMEgfs }}/sorc/gdas.cd/build/gdas/test/testoutput/C96C48_ufs_hybatmDA_lgetkf-fv3inc.test.out test_float_relative_tolerance: 1.0e-3 test_float_absolute_tolerance: 1.0e-5 +{% endif %} diff --git a/parm/atm/jcb-prototype_lgetkf.yaml.j2 b/parm/atm/jcb-prototype_lgetkf.yaml.j2 index 04f9faafd..03e364825 100644 --- a/parm/atm/jcb-prototype_lgetkf.yaml.j2 +++ b/parm/atm/jcb-prototype_lgetkf.yaml.j2 @@ -41,7 +41,10 @@ observations: # Testing things # -------------- +{% if DO_TEST_MODE and RUN == 'enkfgdas' %} +do_testing: true test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/testreference/C96C48_ufs_hybatmDA_lgetkf.ref test_output_filename: {{ HOMEgfs }}/sorc/gdas.cd/build/gdas/test/testoutput/C96C48_ufs_hybatmDA_lgetkf.test.out test_float_relative_tolerance: 1.0e-3 test_float_absolute_tolerance: 1.0e-5 +{% endif %} diff --git a/parm/atm/jcb-prototype_lgetkf_observer.yaml.j2 b/parm/atm/jcb-prototype_lgetkf_observer.yaml.j2 index 39e682415..62bc4b813 100644 --- a/parm/atm/jcb-prototype_lgetkf_observer.yaml.j2 +++ b/parm/atm/jcb-prototype_lgetkf_observer.yaml.j2 @@ -47,7 +47,10 @@ distribution_type: RoundRobin # Testing things # -------------- +{% if DO_TEST_MODE and RUN == 'enkfgdas' %} +do_testing: true test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/testreference/C96C48_ufs_hybatmDA_lgetkf_observer.ref test_output_filename: {{ HOMEgfs }}/sorc/gdas.cd/build/gdas/test/testoutput/C96C48_ufs_hybatmDA_lgetkf_observer.test.out test_float_relative_tolerance: 1.0e-3 test_float_absolute_tolerance: 1.0e-5 +{% endif %} diff --git a/parm/atm/jcb-prototype_lgetkf_solver.yaml.j2 b/parm/atm/jcb-prototype_lgetkf_solver.yaml.j2 index 7abccdd6e..ca62821aa 100644 --- a/parm/atm/jcb-prototype_lgetkf_solver.yaml.j2 +++ b/parm/atm/jcb-prototype_lgetkf_solver.yaml.j2 @@ -50,7 +50,10 @@ distribution_type: Halo # Testing things # -------------- +{% if DO_TEST_MODE and RUN == 'enkfgdas' %} +do_testing: true test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/testreference/C96C48_ufs_hybatmDA_lgetkf_solver.ref test_output_filename: {{ HOMEgfs }}/sorc/gdas.cd/build/gdas/test/testoutput/C96C48_ufs_hybatmDA_lgetkf_solver.test.out test_float_relative_tolerance: 1.0e-3 test_float_absolute_tolerance: 1.0e-5 +{% endif %} diff --git a/parm/soca/marine-jcb-3dfgat.yaml.j2 b/parm/soca/marine-jcb-3dfgat.yaml.j2 index 59d0b9ff5..f5301fb32 100644 --- a/parm/soca/marine-jcb-3dfgat.yaml.j2 +++ b/parm/soca/marine-jcb-3dfgat.yaml.j2 @@ -6,7 +6,10 @@ observations: !INC ${OBS_LIST_SHORT} # Testing things # -------------- +{% if DO_TEST_MODE and RUN == 'gdas' %} +do_testing: true test_reference_filename: '{{HOMEgfs}}/sorc/gdas.cd/test/testreference/C48mx500_3DVarAOWCDA_3dfgat.ref' test_output_filename: '{{HOMEgfs}}/sorc/gdas.cd/build/gdas/test/testoutput/C48mx500_3DVarAOWCDA_3dfgat.test.out' test_float_relative_tolerance: 1.0e-3 test_float_absolute_tolerance: 1.0e-5 +{% endif %} diff --git a/parm/soca/marine-jcb-base.yaml b/parm/soca/marine-jcb-base.yaml index 1b55dc7d2..7ca450c7c 100644 --- a/parm/soca/marine-jcb-base.yaml +++ b/parm/soca/marine-jcb-base.yaml @@ -26,11 +26,6 @@ final_prints_frequency: PT3H number_of_outer_loops: 1 analysis_variables: [sea_ice_area_fraction, sea_ice_thickness, sea_ice_snow_thickness, sea_water_salinity, sea_water_potential_temperature, eastward_sea_water_velocity, northward_sea_water_velocity, sea_surface_height_above_geoid] -# Testing -# ------- - -do_testing: '{{ DO_TEST_MODE | default(false, true) }}' - # Model things # ------------ marine_window_begin: '{{MARINE_WINDOW_BEGIN}}' diff --git a/test/atm/global-workflow/jcb-prototype_3dvar-fv3inc.yaml.j2 b/test/atm/global-workflow/jcb-prototype_3dvar-fv3inc.yaml.j2 index 9686fbbce..f30be7ad9 100644 --- a/test/atm/global-workflow/jcb-prototype_3dvar-fv3inc.yaml.j2 +++ b/test/atm/global-workflow/jcb-prototype_3dvar-fv3inc.yaml.j2 @@ -4,7 +4,10 @@ algorithm: fv3jedi_fv3inc_variational # Testing things # -------------- +{% if DO_TEST_MODE and RUN == 'gdas' %} +do_testing: true test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/testreference/atm_jjob_3dvar-fv3inc.ref test_output_filename: {{ HOMEgfs }}/sorc/gdas.cd/build/gdas/test/testoutput/atm_jjob_3dvar-fv3inc.test.out test_float_relative_tolerance: 1.0e-3 test_float_absolute_tolerance: 1.0e-5 +{% endif %} diff --git a/test/atm/global-workflow/jcb-prototype_3dvar.yaml.j2 b/test/atm/global-workflow/jcb-prototype_3dvar.yaml.j2 index 7465a4274..104d8004d 100644 --- a/test/atm/global-workflow/jcb-prototype_3dvar.yaml.j2 +++ b/test/atm/global-workflow/jcb-prototype_3dvar.yaml.j2 @@ -17,7 +17,10 @@ atmosphere_obsdatain_suffix: ".{{ current_cycle | to_YMDH }}.nc" # Testing things # -------------- +{% if DO_TEST_MODE and RUN == 'gdas' %} +do_testing: true test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/testreference/atm_jjob_3dvar.ref test_output_filename: {{ HOMEgfs }}/sorc/gdas.cd/build/gdas/test/testoutput/atm_jjob_3dvar.test.out test_float_relative_tolerance: 1.0e-3 test_float_absolute_tolerance: 1.0e-5 +{% endif %} diff --git a/test/atm/global-workflow/jcb-prototype_lgetkf-fv3inc.yaml.j2 b/test/atm/global-workflow/jcb-prototype_lgetkf-fv3inc.yaml.j2 index cd11e771d..1238593e1 100644 --- a/test/atm/global-workflow/jcb-prototype_lgetkf-fv3inc.yaml.j2 +++ b/test/atm/global-workflow/jcb-prototype_lgetkf-fv3inc.yaml.j2 @@ -4,7 +4,10 @@ algorithm: fv3jedi_fv3inc_lgetkf # Testing things # -------------- +{% if DO_TEST_MODE and RUN == 'enkfgdas' %} +do_testing: true test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/testreference/atm_jjob_lgetkf-fv3inc.ref test_output_filename: {{ HOMEgfs }}/sorc/gdas.cd/build/gdas/test/testoutput/atm_jjob_lgetkf-fv3inc.test.out test_float_relative_tolerance: 1.0e-3 test_float_absolute_tolerance: 1.0e-5 +{% endif %} diff --git a/test/atm/global-workflow/jcb-prototype_lgetkf.yaml.j2 b/test/atm/global-workflow/jcb-prototype_lgetkf.yaml.j2 index e6b1f9593..0b01cdd8c 100644 --- a/test/atm/global-workflow/jcb-prototype_lgetkf.yaml.j2 +++ b/test/atm/global-workflow/jcb-prototype_lgetkf.yaml.j2 @@ -22,7 +22,10 @@ atmosphere_obsdatain_suffix: ".{{ current_cycle | to_YMDH }}.nc" # Testing things # -------------- +{% if DO_TEST_MODE and RUN == 'enkfgdas' %} +do_testing: true test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/testreference/atm_jjob_lgetkf.ref test_output_filename: {{ HOMEgfs }}/sorc/gdas.cd/build/gdas/test/testoutput/atm_jjob_lgetkf.test.out test_float_relative_tolerance: 1.0e-3 test_float_absolute_tolerance: 1.0e-5 +{% endif %} diff --git a/test/atm/global-workflow/jcb-prototype_lgetkf_observer.yaml.j2 b/test/atm/global-workflow/jcb-prototype_lgetkf_observer.yaml.j2 index ac0fcd7dd..f6d76e22d 100644 --- a/test/atm/global-workflow/jcb-prototype_lgetkf_observer.yaml.j2 +++ b/test/atm/global-workflow/jcb-prototype_lgetkf_observer.yaml.j2 @@ -28,7 +28,10 @@ distribution_type: RoundRobin # Testing things # -------------- +{% if DO_TEST_MODE and RUN == 'enkfgdas' %} +do_testing: true test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/testreference/atm_jjob_lgetkf_observer.ref test_output_filename: {{ HOMEgfs }}/sorc/gdas.cd/build/gdas/test/testoutput/atm_jjob_lgetkf_observer.test.out test_float_relative_tolerance: 1.0e-3 test_float_absolute_tolerance: 1.0e-5 +{% endif %} diff --git a/test/atm/global-workflow/jcb-prototype_lgetkf_solver.yaml.j2 b/test/atm/global-workflow/jcb-prototype_lgetkf_solver.yaml.j2 index 2c3bd2aae..78215fefe 100644 --- a/test/atm/global-workflow/jcb-prototype_lgetkf_solver.yaml.j2 +++ b/test/atm/global-workflow/jcb-prototype_lgetkf_solver.yaml.j2 @@ -28,7 +28,10 @@ distribution_type: Halo # Testing things # -------------- +{% if DO_TEST_MODE and RUN == 'enkfgdas' %} +do_testing: true test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/testreference/atm_jjob_lgetkf_solver.ref test_output_filename: {{ HOMEgfs }}/sorc/gdas.cd/build/gdas/test/testoutput/atm_jjob_lgetkf_solver.test.out test_float_relative_tolerance: 1.0e-3 test_float_absolute_tolerance: 1.0e-5 +{% endif %} From 76208eb619bdc086e87e4ed2b812438bea3e32f5 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA <26926959+RussTreadon-NOAA@users.noreply.github.com> Date: Sat, 30 Nov 2024 11:12:41 -0500 Subject: [PATCH 22/22] add prepdata dependency to ioda converter ctests (#1392) --- utils/test/CMakeLists.txt | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/utils/test/CMakeLists.txt b/utils/test/CMakeLists.txt index 00adbad26..0f62cd33c 100644 --- a/utils/test/CMakeLists.txt +++ b/utils/test/CMakeLists.txt @@ -15,10 +15,10 @@ list( APPEND utils_test_input ) set( gdas_utils_test_ref + testref/rads2ioda.test testref/ghrsst2ioda.test testref/rtofstmp.test testref/rtofssal.test - testref/rads2ioda.test testref/smap2ioda.test testref/smos2ioda.test testref/icecabi2ioda.test @@ -67,14 +67,16 @@ ecbuild_add_test( TARGET test_gdasapp_util_rads2ioda COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_obsprovider2ioda.x ARGS "../testinput/gdas_rads2ioda.yaml" LIBS gdas-utils - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc + TEST_DEPENDS test_gdasapp_util_prepdata) # Test the GHRSST to IODA converter ecbuild_add_test( TARGET test_gdasapp_util_ghrsst2ioda COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_obsprovider2ioda.x ARGS "../testinput/gdas_ghrsst2ioda.yaml" LIBS gdas-utils - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc + TEST_DEPENDS test_gdasapp_util_prepdata) # copy rtofs binary input files to the testing area # and generate the tests @@ -129,46 +131,53 @@ ecbuild_add_test( TARGET test_gdasapp_util_smap2ioda COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_obsprovider2ioda.x ARGS "../testinput/gdas_smap2ioda.yaml" LIBS gdas-utils - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc + TEST_DEPENDS test_gdasapp_util_prepdata) # Test the SMOS to IODA converter ecbuild_add_test( TARGET test_gdasapp_util_smos2ioda COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_obsprovider2ioda.x ARGS "../testinput/gdas_smos2ioda.yaml" LIBS gdas-utils - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc + TEST_DEPENDS test_gdasapp_util_prepdata) # Test the VIIRS AOD to IODA converter ecbuild_add_test( TARGET test_gdasapp_util_viirsaod2ioda COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_obsprovider2ioda.x ARGS "../testinput/gdas_viirsaod2ioda.yaml" LIBS gdas-utils - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc + TEST_DEPENDS test_gdasapp_util_prepdata) # Test the ABI to IODA converter ecbuild_add_test( TARGET test_gdasapp_util_icecabi2ioda COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_obsprovider2ioda.x ARGS "../testinput/gdas_icecabi2ioda.yaml" LIBS gdas-utils - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc + TEST_DEPENDS test_gdasapp_util_prepdata) # Test the AMSR2 to IODA converter ecbuild_add_test( TARGET test_gdasapp_util_icecamsr2ioda COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_obsprovider2ioda.x ARGS "../testinput/gdas_icecamsr2ioda.yaml" LIBS gdas-utils - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc + TEST_DEPENDS test_gdasapp_util_prepdata) # Test the MIRS to IODA converter ecbuild_add_test( TARGET test_gdasapp_util_icecmirs2ioda COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_obsprovider2ioda.x ARGS "../testinput/gdas_icecmirs2ioda.yaml" LIBS gdas-utils - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc + TEST_DEPENDS test_gdasapp_util_prepdata) # Test the JPSSRR to IODA converter ecbuild_add_test( TARGET test_gdasapp_util_icecjpssrr2ioda COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_obsprovider2ioda.x ARGS "../testinput/gdas_icecjpssrr2ioda.yaml" LIBS gdas-utils - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc + TEST_DEPENDS test_gdasapp_util_prepdata)