Fix test reference mismatch (#786)

#### What this PR fixed - Two new ioda converters <del>that work with `mktime` in a C library function</del> that need to ensure consistent behavior across different machines with different time zones (e.g., Orion (CST) and Hera (UTC)) - <del>Set the time zone explicitly using the `tzset` function with UTC time zone</del> - Used ```oops/utils``` to make dateime to Julian date so now code is a lot simpler than before Fixed #783 --------- Co-authored-by: Guillaume Vernieres <[email protected]>
NOAA-EMC · Dec 19, 2023 · 146c790 · 146c790
1 parent 911af66
commit 146c790
Show file tree

Hide file tree

Showing 6 changed files with 64 additions and 38 deletions.
diff --git a/utils/obsproc/IcecAmsr2Ioda.h b/utils/obsproc/IcecAmsr2Ioda.h
@@ -1,9 +1,11 @@
 #pragma once
 
 #include <ctime>
+#include <iomanip>
 #include <iostream>
 #include <map>
 #include <netcdf>    // NOLINT (using C API)
+#include <sstream>
 #include <string>
 #include <vector>
 
@@ -13,6 +15,7 @@
 
 #include "ioda/Group.h"
 #include "ioda/ObsGroup.h"
+#include "oops/util/dateFunctions.h"
 
 #include "NetCDFToIodaConverter.h"
 
@@ -66,18 +69,30 @@ namespace gdasapp {
       iodaVars.referenceDate_ = "seconds since 1970-01-01T00:00:00Z";
 
       size_t index = 0;
-      std::tm timeinfo = {};
       for (int i = 0; i < ntimes; i += dimTimeSize) {
-        timeinfo.tm_year = oneTmpdateTimeVal[i] - 1900;  // Year since 1900
-        timeinfo.tm_mon = oneTmpdateTimeVal[i + 1] - 1;  // 0-based; 8 represents Sep
-        timeinfo.tm_mday = oneTmpdateTimeVal[i + 2];
-        timeinfo.tm_hour = oneTmpdateTimeVal[i + 3];
-        timeinfo.tm_min = oneTmpdateTimeVal[i + 4];
-        timeinfo.tm_sec = oneTmpdateTimeVal[i + 5];
-
-        // Calculate and store the seconds since the Unix epoch
-        time_t epochtime = std::mktime(&timeinfo);
-        iodaVars.datetime_(index) = static_cast<int64_t>(epochtime);
+        int year = oneTmpdateTimeVal[i];
+        int month = oneTmpdateTimeVal[i+1];
+        int day = oneTmpdateTimeVal[i+2];
+        int hour = oneTmpdateTimeVal[i+3];
+        int minute =  oneTmpdateTimeVal[i+4];
+        int second = static_cast<int>(oneTmpdateTimeVal[i+5]);
+
+        // Replace Fillvalue -9999 to 0 to avoid crash in dateToJulian
+        if (year == -9999 || month == -9999 || day == -9999 ||
+          hour == -9999 || minute == -9999 || second == -9999) {
+          year = month = day = hour = minute = second = 0;
+        }
+
+        // Convert a date to Julian date
+        uint64_t julianDate = util::datefunctions::dateToJulian(year, month, day);
+
+        // Subtract Julian day from January 1, 1970 (convert to epoch)
+        int daysSinceEpoch = julianDate - 2440588;
+
+        // Calculate seconds only from HHMMSS
+        int secondsOffset = util::datefunctions::hmsToSeconds(hour, minute, second);
+
+        iodaVars.datetime_(index) = static_cast<int64_t>(daysSinceEpoch*86400.0f) + secondsOffset;
         index++;
       }
 

diff --git a/utils/obsproc/Smap2Ioda.h b/utils/obsproc/Smap2Ioda.h
@@ -1,7 +1,10 @@
 #pragma once
 
+#include <ctime>
+#include <iomanip>
 #include <iostream>
 #include <netcdf>    // NOLINT (using C API)
+#include <sstream>
 #include <string>
 #include <vector>
 
@@ -11,6 +14,7 @@
 
 #include "ioda/Group.h"
 #include "ioda/ObsGroup.h"
+#include "oops/util/dateFunctions.h"
 #include "oops/util/DateTime.h"
 
 #include "NetCDFToIodaConverter.h"
@@ -70,14 +74,23 @@ namespace gdasapp {
 
       iodaVars.referenceDate_ = "seconds since 1970-01-01T00:00:00Z";
 
-      // calculate the seconds of Jan 1 of startyear since unix epoch
-      std::tm tm{};
-      // defaults are zero, Jan is zero
-      tm.tm_year = startYear - 1900;
-      tm.tm_mday = 1;
-      time_t unixStartYear = mktime(&tm);
+      int year = startYear;
+      int month = 1;
+      int day = 1;
 
-      int unixStartDay = unixStartYear + startDay * 86400;
+      // Replace Fillvalue -9999 to 0 to avoid crash in dateToJulian
+      if (year == -9999) {
+        year = 0;
+      }
+
+      // Convert a date to Julian date
+      uint64_t julianDate = util::datefunctions::dateToJulian(year, month, day);
+
+      // Subtract Julian day from January 1, 1970 (convert to epoch)
+      int daysSinceEpoch = julianDate - 2440588;
+
+      // Calculate seconds
+      int secondsSinceEpoch = (daysSinceEpoch + startDay) * 86400;
 
       int loc;
       for (int i = 0; i < dim0; i++) {
@@ -88,7 +101,7 @@ namespace gdasapp {
           iodaVars.obsVal_(loc) = sss[i][j];
           iodaVars.obsError_(loc) = sss_error[i][j];
           iodaVars.preQc_(loc) = sss_qc[i][j];
-          iodaVars.datetime_(loc) =  static_cast<int64_t>(obsTime[j] + unixStartDay);
+          iodaVars.datetime_(loc) =  static_cast<int64_t>(obsTime[j] + secondsSinceEpoch);
         }
       }
 

diff --git a/utils/test/CMakeLists.txt b/utils/test/CMakeLists.txt
@@ -64,12 +64,11 @@ ecbuild_add_test( TARGET  test_gdasapp_util_ghrsst2ioda
                   WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc)
 
 # Test the SMAP to IODA converter
-# TODO(Mindo): Turn back on when date is fixed
-#ecbuild_add_test( TARGET  test_gdasapp_util_smap2ioda
-#                  COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_obsprovider2ioda.x
-#                  ARGS    "../testinput/gdas_smap2ioda.yaml"
-#                  LIBS    gdas-utils
-#                  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc)
+ecbuild_add_test( TARGET  test_gdasapp_util_smap2ioda
+                  COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_obsprovider2ioda.x
+                  ARGS    "../testinput/gdas_smap2ioda.yaml"
+                  LIBS    gdas-utils
+                  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc)
 
 # Test the SMOS to IODA converter
 ecbuild_add_test( TARGET  test_gdasapp_util_smos2ioda
@@ -86,9 +85,8 @@ ecbuild_add_test( TARGET  test_gdasapp_util_viirsaod2ioda
                   WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc)
 
 # Test the AMSR2 to IODA converter
-# TODO(Mindo): Turn back on when date is fixed
-#ecbuild_add_test( TARGET  test_gdasapp_util_icecamsr2ioda
-#                  COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_obsprovider2ioda.x
-#                  ARGS    "../testinput/gdas_icecamsr2ioda.yaml"
-#                  LIBS    gdas-utils
-#                  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc)
+ecbuild_add_test( TARGET  test_gdasapp_util_icecamsr2ioda
+                  COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_obsprovider2ioda.x
+                  ARGS    "../testinput/gdas_icecamsr2ioda.yaml"
+                  LIBS    gdas-utils
+                  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc)
diff --git a/utils/test/testref/icecamsr2ioda.test b/utils/test/testref/icecamsr2ioda.test
@@ -21,6 +21,6 @@ latitude:
     Max: 86.8975
     Sum: 1346.66
 datetime:
-    Min: 1625066557
-    Max: 1625126209
-    Sum: 27626797699
+    Min: 1625048557
+    Max: 1625108209
+    Sum: 27626462899
diff --git a/utils/test/testref/rads2ioda.test b/utils/test/testref/rads2ioda.test
@@ -1,4 +1,4 @@
-iReading: [rads_adt_3a_2021181.nc4,rads_adt_3b_2021181.nc4]
+Reading: [rads_adt_3a_2021181.nc4,rads_adt_3b_2021181.nc4]
 seconds since 1858-11-17T00:00:00Z
 obsVal:
     Min: 0.1671

diff --git a/utils/test/testref/smap2ioda.test b/utils/test/testref/smap2ioda.test
@@ -21,6 +21,6 @@ latitude:
     Max: 40.8618
     Sum: -370.853
 datetime:
-    Min: 1625197568
-    Max: 1625204736
-    Sum: 48756048640
+    Min: 1625175936
+    Max: 1625183104
+    Sum: 48755399680