From 146c790039bbbe294a4e124029732520aedab504 Mon Sep 17 00:00:00 2001 From: Mindo Choi <141867620+apchoiCMD@users.noreply.github.com> Date: Tue, 19 Dec 2023 16:50:45 -0500 Subject: [PATCH] Fix test reference mismatch (#786) #### What this PR fixed - Two new ioda converters that work with `mktime` in a C library function that need to ensure consistent behavior across different machines with different time zones (e.g., Orion (CST) and Hera (UTC)) - Set the time zone explicitly using the `tzset` function with UTC time zone - Used ```oops/utils``` to make dateime to Julian date so now code is a lot simpler than before Fixed https://github.com/NOAA-EMC/GDASApp/issues/783 --------- Co-authored-by: Guillaume Vernieres --- utils/obsproc/IcecAmsr2Ioda.h | 37 +++++++++++++++++++-------- utils/obsproc/Smap2Ioda.h | 29 +++++++++++++++------ utils/test/CMakeLists.txt | 22 ++++++++-------- utils/test/testref/icecamsr2ioda.test | 6 ++--- utils/test/testref/rads2ioda.test | 2 +- utils/test/testref/smap2ioda.test | 6 ++--- 6 files changed, 64 insertions(+), 38 deletions(-) diff --git a/utils/obsproc/IcecAmsr2Ioda.h b/utils/obsproc/IcecAmsr2Ioda.h index 61bacfade..e2d0b2e5d 100644 --- a/utils/obsproc/IcecAmsr2Ioda.h +++ b/utils/obsproc/IcecAmsr2Ioda.h @@ -1,9 +1,11 @@ #pragma once #include +#include #include #include #include // NOLINT (using C API) +#include #include #include @@ -13,6 +15,7 @@ #include "ioda/Group.h" #include "ioda/ObsGroup.h" +#include "oops/util/dateFunctions.h" #include "NetCDFToIodaConverter.h" @@ -66,18 +69,30 @@ namespace gdasapp { iodaVars.referenceDate_ = "seconds since 1970-01-01T00:00:00Z"; size_t index = 0; - std::tm timeinfo = {}; for (int i = 0; i < ntimes; i += dimTimeSize) { - timeinfo.tm_year = oneTmpdateTimeVal[i] - 1900; // Year since 1900 - timeinfo.tm_mon = oneTmpdateTimeVal[i + 1] - 1; // 0-based; 8 represents Sep - timeinfo.tm_mday = oneTmpdateTimeVal[i + 2]; - timeinfo.tm_hour = oneTmpdateTimeVal[i + 3]; - timeinfo.tm_min = oneTmpdateTimeVal[i + 4]; - timeinfo.tm_sec = oneTmpdateTimeVal[i + 5]; - - // Calculate and store the seconds since the Unix epoch - time_t epochtime = std::mktime(&timeinfo); - iodaVars.datetime_(index) = static_cast(epochtime); + int year = oneTmpdateTimeVal[i]; + int month = oneTmpdateTimeVal[i+1]; + int day = oneTmpdateTimeVal[i+2]; + int hour = oneTmpdateTimeVal[i+3]; + int minute = oneTmpdateTimeVal[i+4]; + int second = static_cast(oneTmpdateTimeVal[i+5]); + + // Replace Fillvalue -9999 to 0 to avoid crash in dateToJulian + if (year == -9999 || month == -9999 || day == -9999 || + hour == -9999 || minute == -9999 || second == -9999) { + year = month = day = hour = minute = second = 0; + } + + // Convert a date to Julian date + uint64_t julianDate = util::datefunctions::dateToJulian(year, month, day); + + // Subtract Julian day from January 1, 1970 (convert to epoch) + int daysSinceEpoch = julianDate - 2440588; + + // Calculate seconds only from HHMMSS + int secondsOffset = util::datefunctions::hmsToSeconds(hour, minute, second); + + iodaVars.datetime_(index) = static_cast(daysSinceEpoch*86400.0f) + secondsOffset; index++; } diff --git a/utils/obsproc/Smap2Ioda.h b/utils/obsproc/Smap2Ioda.h index be3c2d8e3..cb8fb52fe 100644 --- a/utils/obsproc/Smap2Ioda.h +++ b/utils/obsproc/Smap2Ioda.h @@ -1,7 +1,10 @@ #pragma once +#include +#include #include #include // NOLINT (using C API) +#include #include #include @@ -11,6 +14,7 @@ #include "ioda/Group.h" #include "ioda/ObsGroup.h" +#include "oops/util/dateFunctions.h" #include "oops/util/DateTime.h" #include "NetCDFToIodaConverter.h" @@ -70,14 +74,23 @@ namespace gdasapp { iodaVars.referenceDate_ = "seconds since 1970-01-01T00:00:00Z"; - // calculate the seconds of Jan 1 of startyear since unix epoch - std::tm tm{}; - // defaults are zero, Jan is zero - tm.tm_year = startYear - 1900; - tm.tm_mday = 1; - time_t unixStartYear = mktime(&tm); + int year = startYear; + int month = 1; + int day = 1; - int unixStartDay = unixStartYear + startDay * 86400; + // Replace Fillvalue -9999 to 0 to avoid crash in dateToJulian + if (year == -9999) { + year = 0; + } + + // Convert a date to Julian date + uint64_t julianDate = util::datefunctions::dateToJulian(year, month, day); + + // Subtract Julian day from January 1, 1970 (convert to epoch) + int daysSinceEpoch = julianDate - 2440588; + + // Calculate seconds + int secondsSinceEpoch = (daysSinceEpoch + startDay) * 86400; int loc; for (int i = 0; i < dim0; i++) { @@ -88,7 +101,7 @@ namespace gdasapp { iodaVars.obsVal_(loc) = sss[i][j]; iodaVars.obsError_(loc) = sss_error[i][j]; iodaVars.preQc_(loc) = sss_qc[i][j]; - iodaVars.datetime_(loc) = static_cast(obsTime[j] + unixStartDay); + iodaVars.datetime_(loc) = static_cast(obsTime[j] + secondsSinceEpoch); } } diff --git a/utils/test/CMakeLists.txt b/utils/test/CMakeLists.txt index cf74db8aa..a39bad869 100644 --- a/utils/test/CMakeLists.txt +++ b/utils/test/CMakeLists.txt @@ -64,12 +64,11 @@ ecbuild_add_test( TARGET test_gdasapp_util_ghrsst2ioda WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) # Test the SMAP to IODA converter -# TODO(Mindo): Turn back on when date is fixed -#ecbuild_add_test( TARGET test_gdasapp_util_smap2ioda -# COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_obsprovider2ioda.x -# ARGS "../testinput/gdas_smap2ioda.yaml" -# LIBS gdas-utils -# WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) +ecbuild_add_test( TARGET test_gdasapp_util_smap2ioda + COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_obsprovider2ioda.x + ARGS "../testinput/gdas_smap2ioda.yaml" + LIBS gdas-utils + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) # Test the SMOS to IODA converter ecbuild_add_test( TARGET test_gdasapp_util_smos2ioda @@ -86,9 +85,8 @@ ecbuild_add_test( TARGET test_gdasapp_util_viirsaod2ioda WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) # Test the AMSR2 to IODA converter -# TODO(Mindo): Turn back on when date is fixed -#ecbuild_add_test( TARGET test_gdasapp_util_icecamsr2ioda -# COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_obsprovider2ioda.x -# ARGS "../testinput/gdas_icecamsr2ioda.yaml" -# LIBS gdas-utils -# WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) +ecbuild_add_test( TARGET test_gdasapp_util_icecamsr2ioda + COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_obsprovider2ioda.x + ARGS "../testinput/gdas_icecamsr2ioda.yaml" + LIBS gdas-utils + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) diff --git a/utils/test/testref/icecamsr2ioda.test b/utils/test/testref/icecamsr2ioda.test index a8ef1c959..da4819145 100644 --- a/utils/test/testref/icecamsr2ioda.test +++ b/utils/test/testref/icecamsr2ioda.test @@ -21,6 +21,6 @@ latitude: Max: 86.8975 Sum: 1346.66 datetime: - Min: 1625066557 - Max: 1625126209 - Sum: 27626797699 + Min: 1625048557 + Max: 1625108209 + Sum: 27626462899 diff --git a/utils/test/testref/rads2ioda.test b/utils/test/testref/rads2ioda.test index f6e0283ce..479a672ed 100644 --- a/utils/test/testref/rads2ioda.test +++ b/utils/test/testref/rads2ioda.test @@ -1,4 +1,4 @@ -iReading: [rads_adt_3a_2021181.nc4,rads_adt_3b_2021181.nc4] +Reading: [rads_adt_3a_2021181.nc4,rads_adt_3b_2021181.nc4] seconds since 1858-11-17T00:00:00Z obsVal: Min: 0.1671 diff --git a/utils/test/testref/smap2ioda.test b/utils/test/testref/smap2ioda.test index c644a5324..bc8916f2f 100644 --- a/utils/test/testref/smap2ioda.test +++ b/utils/test/testref/smap2ioda.test @@ -21,6 +21,6 @@ latitude: Max: 40.8618 Sum: -370.853 datetime: - Min: 1625197568 - Max: 1625204736 - Sum: 48756048640 + Min: 1625175936 + Max: 1625183104 + Sum: 48755399680