Add AMSR2 for ICEC to IODA converter (#657)

* fixed conflict file merging from team members * Update preparation of data for Rad obs * unnecessary testdata files deleted * Adding converter for ASMR2_ICEC * Read two Dim and write one Dim like IODA Format * modified code for icec_asmr2 ioda converter * Only work for reading dateTime from multi-dimensions of Asmr2 * Completed from arrays to vectors * follow-up recent develop branch * Modified AMSR2_ICEC code * Modified AMSR2_ICEC code * Modified AMSR2_ICEC code * Modified AMSR2_ICEC code * Modified AMSR2_ICEC code * yaml for amsr2 and rad are edited * yaml for amsr2 and rad are edited * final version, cleanup and addressing PR review * included PR review * May be final PR review
NOAA-EMC · Oct 6, 2023 · f9bddeb · f9bddeb
1 parent ea26739
commit f9bddeb
Show file tree

Hide file tree

Showing 6 changed files with 115 additions and 3 deletions.
diff --git a/utils/obsproc/IcecAmsr2Ioda.h b/utils/obsproc/IcecAmsr2Ioda.h
@@ -0,0 +1,94 @@
+#pragma once
+
+#include <ctime>
+#include <iostream>
+#include <map>
+#include <netcdf>    // NOLINT (using C API)
+#include <string>
+#include <vector>
+
+#include "eckit/config/LocalConfiguration.h"
+
+#include <Eigen/Dense>    // NOLINT
+
+#include "ioda/Group.h"
+#include "ioda/ObsGroup.h"
+
+#include "NetCDFToIodaConverter.h"
+
+namespace gdasapp {
+
+  class IcecAmsr2Ioda : public NetCDFToIodaConverter {
+   public:
+    explicit IcecAmsr2Ioda(const eckit::Configuration & fullConfig, const eckit::mpi::Comm & comm)
+      : NetCDFToIodaConverter(fullConfig, comm) {
+      variable_ = "seaIceFraction";
+    }
+
+    // Read netcdf file and populate iodaVars
+    gdasapp::IodaVars providerToIodaVars(const std::string fileName) final {
+      oops::Log::info() << "Processing files provided by the AMSR2" << std::endl;
+
+      // Open the NetCDF file in read-only mode
+      netCDF::NcFile ncFile(fileName, netCDF::NcFile::read);
+
+      // Get the number of obs in the file
+      int dimxSize = ncFile.getDim("Number_of_X_Dimension").getSize();
+      int dimySize = ncFile.getDim("Number_of_Y_Dimension").getSize();
+      int dimTimeSize = ncFile.getDim("Time_Dimension").getSize();
+      int nobs = dimxSize * dimySize;
+      int ntimes = dimxSize * dimySize * dimTimeSize;
+
+      // Create instance of iodaVars object
+      gdasapp::IodaVars iodaVars(nobs, {}, {});
+
+      oops::Log::debug() << "--- iodaVars.location: " << iodaVars.location << std::endl;
+
+      // Read non-optional metadata: longitude and latitude
+      std::vector<float> oneDimLatVal(iodaVars.location);
+      ncFile.getVar("Latitude").getVar(oneDimLatVal.data());
+
+      std::vector<float> oneDimLonVal(iodaVars.location);
+      ncFile.getVar("Longitude").getVar(oneDimLonVal.data());
+
+      // Read Quality Flags as a preQc
+      std::vector<int64_t> oneDimFlagsVal(iodaVars.location);
+      ncFile.getVar("Flags").getVar(oneDimFlagsVal.data());
+
+      // Get Ice_Concentration obs values
+      std::vector<int> oneDimObsVal(iodaVars.location);
+      ncFile.getVar("NASA_Team_2_Ice_Concentration").getVar(oneDimObsVal.data());
+
+      // Read and process the dateTime
+      std::vector<int> oneTmpdateTimeVal(ntimes);
+      ncFile.getVar("Scan_Time").getVar(oneTmpdateTimeVal.data());
+      iodaVars.referenceDate = "seconds since 1970-01-01T00:00:00Z";
+
+      std::tm timeinfo = {};
+      for (int i = 0; i < ntimes; i += dimTimeSize) {
+        for (int j = 0; j < dimTimeSize && i + j < ntimes; j++) {
+        }
+        timeinfo.tm_year = oneTmpdateTimeVal[i] - 1900;  // Year since 1900
+        timeinfo.tm_mon = oneTmpdateTimeVal[i + 1] - 1;  // 0-based; 8 represents Sep
+        timeinfo.tm_mday = oneTmpdateTimeVal[i + 2];
+        timeinfo.tm_hour = oneTmpdateTimeVal[i + 3];
+        timeinfo.tm_min = oneTmpdateTimeVal[i + 4];
+        timeinfo.tm_sec = oneTmpdateTimeVal[i + 5];
+
+        // Calculate and store the seconds since the Unix epoch
+        time_t epochtime = std::mktime(&timeinfo);
+        iodaVars.datetime(i/6) = static_cast<int64_t>(epochtime);
+      }
+
+      // Update non-optional Eigen arrays
+      for (int i = 0; i < iodaVars.location; i++) {
+        iodaVars.longitude(i) = oneDimLonVal[i];
+        iodaVars.latitude(i) = oneDimLatVal[i];
+        iodaVars.obsVal(i) = static_cast<int64_t>(oneDimObsVal[i]*0.01);
+        iodaVars.obsError(i) = 0.1;  // Do something for obs error
+        iodaVars.preQc(i) = oneDimFlagsVal[i];
+      }
+      return iodaVars;
+    };
+  };  // class IcecAmsr2Ioda
+}  // namespace gdasapp
diff --git a/utils/obsproc/NetCDFToIodaConverter.h b/utils/obsproc/NetCDFToIodaConverter.h
@@ -196,7 +196,7 @@ namespace gdasapp {
         ioda::Variable tmpIntMeta;
         int count = 0;
         for (const std::string& strMeta : iodaVars.intMetadataName) {
-          tmpIntMeta = ogrp.vars.createWithScales<float>("MetaData/"+strMeta,
+          tmpIntMeta = ogrp.vars.createWithScales<int>("MetaData/"+strMeta,
                                                          {ogrp.vars["Location"]}, int_params);
           tmpIntMeta.writeWithEigenRegular(iodaVars.intMetadata.col(count));
           count++;
@@ -207,7 +207,7 @@ namespace gdasapp {
         count = 0;
         for (const std::string& strMeta : iodaVars.floatMetadataName) {
           tmpFloatMeta = ogrp.vars.createWithScales<float>("MetaData/"+strMeta,
-                                                      {ogrp.vars["Location"]}, int_params);
+                                                      {ogrp.vars["Location"]}, float_params);
           tmpFloatMeta.writeWithEigenRegular(iodaVars.floatMetadata.col(count));
           count++;
         }

diff --git a/utils/obsproc/applications/gdas_obsprovider2ioda.h b/utils/obsproc/applications/gdas_obsprovider2ioda.h
@@ -6,6 +6,7 @@
 #include "oops/mpi/mpi.h"
 #include "oops/runs/Application.h"
 
+#include "../IcecAmsr2Ioda.h"
 #include "../Rads2Ioda.h"
 #include "../Smap2Ioda.h"
 #include "../Smos2Ioda.h"
@@ -33,6 +34,9 @@ namespace gdasapp {
       } else if (provider == "SMOS") {
         Smos2Ioda conv2ioda(fullConfig, this->getComm());
         conv2ioda.writeToIoda();
+      } else if (provider == "AMSR2") {
+        IcecAmsr2Ioda conv2ioda(fullConfig, this->getComm());
+        conv2ioda.writeToIoda();
       } else {
         oops::Log::info() << "Provider not implemented" << std::endl;
         return 1;

diff --git a/utils/test/CMakeLists.txt b/utils/test/CMakeLists.txt
@@ -4,6 +4,7 @@ list( APPEND utils_test_input
   testinput/gdas_rads2ioda.yaml
   testinput/gdas_smap2ioda.yaml
   testinput/gdas_smos2ioda.yaml
+  testinput/gdas_icecamsr2ioda.yaml
 )
 
 file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/testinput)
@@ -53,3 +54,10 @@ ecbuild_add_test( TARGET  test_gdasapp_util_smos2ioda
                   ARGS    "../testinput/gdas_smos2ioda.yaml"
                   LIBS    gdas-utils
                   WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc)
+
+# Test the AMSR2 to IODA converter
+ecbuild_add_test( TARGET  test_gdasapp_util_icecamsr2ioda
+                  COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_obsprovider2ioda.x
+                  ARGS    "../testinput/gdas_icecamsr2ioda.yaml"
+                  LIBS    gdas-utils
+                  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc)
diff --git a/utils/test/testinput/gdas_icecamsr2ioda.yaml b/utils/test/testinput/gdas_icecamsr2ioda.yaml
@@ -0,0 +1,6 @@
+provider: AMSR2
+window begin: 2018-04-15T06:00:00Z
+window end: 2018-04-15T12:00:00Z
+output file: icec_amsr2_north_1.ioda.nc
+input files:
+- icec_amsr2_north_1.nc4
diff --git a/utils/test/testinput/gdas_smos2ioda.yaml b/utils/test/testinput/gdas_smos2ioda.yaml
@@ -1,7 +1,7 @@
 provider: SMOS
 window begin: 2018-04-15T06:00:00Z
 window end: 2018-04-15T12:00:00Z
-output file: sss_smos.ioda.nc4
+output file: sss_smos.ioda.nc
 input files:
 - sss_smos_1.nc4
 - sss_smos_2.nc4