Feature list depth option

src/fdb5/CMakeLists.txt

@@ -39,7 +39,8 @@ list( APPEND fdb5_srcs
     api/helpers/FDBToolRequest.cc
     api/helpers/FDBToolRequest.h
     api/helpers/DumpIterator.h
-    api/helpers/ListIterator.cc
+    api/helpers/ListElement.cc
+    api/helpers/ListElement.h
     api/helpers/ListIterator.h
     api/helpers/LockIterator.h
     api/helpers/MoveIterator.h

src/fdb5/api/DistFDB.cc

@@ -179,11 +179,12 @@ auto DistFDB::queryInternal(const FDBToolRequest& request, const QueryFN& fn) ->
 }
 
 
-ListIterator DistFDB::list(const FDBToolRequest& request) {
+ListIterator DistFDB::list(const FDBToolRequest& request, int level) {
     LOG_DEBUG_LIB(LibFdb5) << "DistFDB::list() : " << request << std::endl;
     return queryInternal(request,
-                         [](FDB& fdb, const FDBToolRequest& request) {
-                            return fdb.list(request);
+                         [level](FDB& fdb, const FDBToolRequest& request) {
+                            bool deduplicate = false; // never deduplicate on inner calls
+                            return fdb.list(request, deduplicate, level);
                          });
 }
 

src/fdb5/api/DistFDB.h

@@ -45,7 +45,7 @@ class DistFDB : public FDBBase {
 
     ListIterator inspect(const metkit::mars::MarsRequest& request) override;
 
-    ListIterator list(const FDBToolRequest& request) override;
+    ListIterator list(const FDBToolRequest& request, int level) override;
 
     DumpIterator dump(const FDBToolRequest& request, bool simple) override;
 

src/fdb5/api/FDB.cc

@@ -13,22 +13,30 @@
  * (Project ID: 671951) www.nextgenio.eu
  */
 
+#include <cstddef>
+#include <sstream>
+#include <string>
+#include <vector>
+
 #include "eckit/config/Resource.h"
+#include "eckit/exception/Exceptions.h"
 #include "eckit/io/DataHandle.h"
 #include "eckit/io/MemoryHandle.h"
 #include "eckit/log/Log.h"
 #include "eckit/message/Message.h"
 #include "eckit/message/Reader.h"
 
-#include "eckit/system/Plugin.h"
 #include "eckit/system/LibraryManager.h"
 
+#include "metkit/hypercube/HyperCube.h"
 #include "metkit/hypercube/HyperCubePayloaded.h"
 
 #include "fdb5/LibFdb5.h"
 #include "fdb5/api/FDB.h"
 #include "fdb5/api/FDBFactory.h"
 #include "fdb5/api/helpers/FDBToolRequest.h"
+#include "fdb5/api/helpers/ListIterator.h"
+#include "fdb5/database/FieldLocation.h"
 #include "fdb5/database/Key.h"
 #include "fdb5/io/HandleGatherer.h"
 #include "fdb5/message/MessageDecoder.h"
@@ -149,7 +157,7 @@ bool FDB::sorted(const metkit::mars::MarsRequest &request) {
 class ListElementDeduplicator : public metkit::hypercube::Deduplicator<ListElement> {
 public:
     bool toReplace(const ListElement& existing, const ListElement& replacement) const override {
-        return existing.timestamp() < replacement.timestamp();
+        return existing.attributes().timestamp < replacement.attributes().timestamp;
     }
 };
 
@@ -168,7 +176,7 @@ eckit::DataHandle* FDB::read(const std::vector<eckit::URI>& uris, bool sorted) {
     }
     return result.dataHandle();
 }
-    
+
 
 eckit::DataHandle* FDB::read(ListIterator& it, bool sorted) {
     eckit::Timer timer;
@@ -181,20 +189,18 @@ eckit::DataHandle* FDB::read(ListIterator& it, bool sorted) {
     if (dedup) {
         if (it.next(el)) {
             // build the request representing the tensor-product of all retrieved fields
-            metkit::mars::MarsRequest cubeRequest = el.combinedKey().request();
-            std::vector<ListElement> elements{el};
+            metkit::mars::MarsRequest cubeRequest = el.key().request();
+            std::vector<ListElement>  elements {el};
 
             while (it.next(el)) {
-                cubeRequest.merge(el.combinedKey().request());
+                cubeRequest.merge(el.key().request());
                 elements.push_back(el);
             }
 
             // checking all retrieved fields against the hypercube, to remove duplicates
-            ListElementDeduplicator dedup;
-            metkit::hypercube::HyperCubePayloaded<ListElement> cube(cubeRequest, dedup);
-            for(auto el: elements) {
-                cube.add(el.combinedKey().request(), el);
-            }
+            ListElementDeduplicator deduplicator;
+            metkit::hypercube::HyperCubePayloaded<ListElement> cube(cubeRequest, deduplicator);
+            for (const auto& elem : elements) { cube.add(elem.key().request(), el); }
 
             if (cube.countVacant() > 0) {
                 std::stringstream ss;
@@ -205,7 +211,7 @@ eckit::DataHandle* FDB::read(ListIterator& it, bool sorted) {
                 eckit::Log::warning() << ss.str() << std::endl;
             }
 
-            for (size_t i=0; i< cube.size(); i++) {
+            for (std::size_t i = 0; i < cube.size(); i++) {
                 ListElement element;
                 if (cube.find(i, element)) {
                     result.add(element.location().dataHandle());
@@ -230,8 +236,8 @@ ListIterator FDB::inspect(const metkit::mars::MarsRequest& request) {
     return internal_->inspect(request);
 }
 
-ListIterator FDB::list(const FDBToolRequest& request, bool deduplicate) {
-    return ListIterator(internal_->list(request), deduplicate);
+ListIterator FDB::list(const FDBToolRequest& request, const bool deduplicate, const int level) {
+    return {internal_->list(request, level), deduplicate};
 }
 
 DumpIterator FDB::dump(const FDBToolRequest& request, bool simple) {

src/fdb5/api/FDB.h

@@ -21,6 +21,7 @@
 
 #include <memory>
 #include <iosfwd>
+#include <string>
 
 #include "eckit/distributed/Transport.h"
 
@@ -94,7 +95,7 @@ class FDB {
 
     ListIterator inspect(const metkit::mars::MarsRequest& request);
 
-    ListIterator list(const FDBToolRequest& request, bool deduplicate=false);
+    ListIterator list(const FDBToolRequest& request, bool deduplicate=false, int level=3);
 
     DumpIterator dump(const FDBToolRequest& request, bool simple=false);
 

src/fdb5/api/FDBFactory.h

@@ -71,7 +71,7 @@ class FDBBase : private eckit::NonCopyable {
 
     virtual ListIterator inspect(const metkit::mars::MarsRequest& request) = 0;
 
-    virtual ListIterator list(const FDBToolRequest& request) = 0;
+    virtual ListIterator list(const FDBToolRequest& request, int level) = 0;
 
     virtual DumpIterator dump(const FDBToolRequest& request, bool simple) = 0;
 

src/fdb5/api/LocalFDB.cc

@@ -15,7 +15,6 @@
 
 #include "eckit/container/Queue.h"
 #include "eckit/log/Log.h"
-#include "eckit/message/Message.h"
 
 #include "fdb5/api/helpers/ListIterator.h"
 #include "fdb5/api/helpers/FDBToolRequest.h"
@@ -34,7 +33,6 @@
 #include "fdb5/api/local/DumpVisitor.h"
 #include "fdb5/api/local/ListVisitor.h"
 #include "fdb5/api/local/PurgeVisitor.h"
-#include "fdb5/api/local/QueryVisitor.h"
 #include "fdb5/api/local/StatsVisitor.h"
 #include "fdb5/api/local/StatusVisitor.h"
 #include "fdb5/api/local/WipeVisitor.h"
@@ -83,9 +81,9 @@ APIIterator<typename VisitorType::ValueType> LocalFDB::queryInternal(const FDBTo
     return QueryIterator(new AsyncIterator(async_worker));
 }
 
-ListIterator LocalFDB::list(const FDBToolRequest& request) {
+ListIterator LocalFDB::list(const FDBToolRequest& request, const int level) {
     LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::list() : " << request << std::endl;
-    return queryInternal<ListVisitor>(request);
+    return queryInternal<ListVisitor>(request, level);
 }
 
 DumpIterator LocalFDB::dump(const FDBToolRequest &request, bool simple) {

src/fdb5/api/LocalFDB.h

@@ -41,7 +41,7 @@ class LocalFDB : public FDBBase {
 
     ListIterator inspect(const metkit::mars::MarsRequest& request) override;
 
-    ListIterator list(const FDBToolRequest& request) override;
+    ListIterator list(const FDBToolRequest& request, int level) override;
 
     DumpIterator dump(const FDBToolRequest& request, bool simple) override;
 

src/fdb5/api/RemoteFDB.cc

@@ -13,28 +13,33 @@
  * (Project ID: 671951) www.nextgenio.eu
  */
 
-#include <functional>
 #include <unistd.h>
+#include <cstdint>
+#include <mutex>
+#include <sstream>
+#include <string>
 
 #include "fdb5/api/RemoteFDB.h"
 #include "fdb5/LibFdb5.h"
+#include "fdb5/api/helpers/ListElement.h"
 #include "fdb5/io/HandleGatherer.h"
 #include "fdb5/remote/Messages.h"
 #include "fdb5/remote/RemoteFieldLocation.h"
 #include "fdb5/api/helpers/FDBToolRequest.h"
 #include "fdb5/database/Key.h"
 
+#include "eckit/config/Configuration.h"
 #include "eckit/config/LocalConfiguration.h"
+#include "eckit/config/Resource.h"
+#include "eckit/exception/Exceptions.h"
 #include "eckit/io/Buffer.h"
 #include "eckit/log/Bytes.h"
 #include "eckit/log/Log.h"
 #include "eckit/message/Message.h"
-#include "eckit/distributed/Transport.h"
-#include "eckit/config/Resource.h"
+#include "eckit/os/BackTrace.h"
+#include "eckit/runtime/Main.h"
 #include "eckit/serialisation/MemoryStream.h"
 #include "eckit/utils/Translator.h"
-#include "eckit/runtime/Main.h"
-#include "eckit/os/BackTrace.h"
 
 #include "metkit/mars/MarsRequest.h"
 
@@ -62,7 +67,7 @@ class ConnectionError : public eckit::Exception {
     ConnectionError(const int);
     ConnectionError(const int, const eckit::net::Endpoint&);
 
-    bool retryOnClient() const override { return true; } 
+    bool retryOnClient() const override { return true; }
 };
 
 ConnectionError::ConnectionError(const int retries) {
@@ -549,13 +554,20 @@ struct BaseAPIHelper {
     static ValueType valueFromStream(eckit::Stream& s, RemoteFDB* fdb) { return ValueType(s); }
 };
 
-using ListHelper = BaseAPIHelper<ListElement, fdb5::remote::Message::List>;
+struct ListHelper : public BaseAPIHelper<ListElement, fdb5::remote::Message::List> {
+
+    ListHelper(int level) : level_(level) {}
+    void encodeExtra(eckit::Stream& s) const { s << level_; }
+
+private:
+    int level_;
+};
 
 struct InspectHelper : BaseAPIHelper<ListElement, fdb5::remote::Message::Inspect> {
 
     static ListElement valueFromStream(eckit::Stream& s, RemoteFDB* fdb) {
         ListElement elem(s);
-        return ListElement(elem.key(), RemoteFieldLocation(fdb, elem.location()).make_shared(), elem.timestamp());
+        return {elem.keys(), RemoteFieldLocation(fdb, elem.location()), elem.attributes().timestamp};
     }
 };
 
@@ -625,7 +637,7 @@ struct WipeHelper : BaseAPIHelper<WipeElement, fdb5::remote::Message::Wipe> {
 
 struct MoveHelper : BaseAPIHelper<MoveElement, fdb5::remote::Message::Move> {
 
-    
+
     MoveHelper(const eckit::URI& dest) :
         dest_(dest) {}
 
@@ -715,8 +727,8 @@ auto RemoteFDB::forwardApiCall(const HelperClass& helper, const FDBToolRequest&
            );
 }
 
-ListIterator RemoteFDB::list(const FDBToolRequest& request) {
-    return forwardApiCall(ListHelper(), request);
+ListIterator RemoteFDB::list(const FDBToolRequest& request, const int level) {
+    return forwardApiCall(ListHelper(level), request);
 }
 
 ListIterator RemoteFDB::inspect(const metkit::mars::MarsRequest& request) {

src/fdb5/api/RemoteFDB.h

@@ -62,7 +62,7 @@ class RemoteFDB : public FDBBase {
 
     ListIterator inspect(const metkit::mars::MarsRequest& request) override;
 
-    ListIterator list(const FDBToolRequest& request) override;
+    ListIterator list(const FDBToolRequest& request, int level) override;
 
     DumpIterator dump(const FDBToolRequest& request, bool simple) override;
 

src/fdb5/api/SelectFDB.cc

@@ -137,12 +137,10 @@ auto SelectFDB::queryInternal(const FDBToolRequest& request, const QueryFN& fn)
     return QueryIterator(new APIAggregateIterator<ValueType>(std::move(iterQueue)));
 }
 
-ListIterator SelectFDB::list(const FDBToolRequest& request) {
+ListIterator SelectFDB::list(const FDBToolRequest& request, const int level) {
     LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::list() >> " << request << std::endl;
     return queryInternal(request,
-                         [](FDB& fdb, const FDBToolRequest& request) {
-                            return fdb.list(request);
-                         });
+                         [level](FDB& fdb, const FDBToolRequest& request) { return fdb.list(request, false, level); });
 }
 
 DumpIterator SelectFDB::dump(const FDBToolRequest& request, bool simple) {

src/fdb5/api/SelectFDB.h

@@ -50,7 +50,7 @@ class SelectFDB : public FDBBase {
 
     ListIterator inspect(const metkit::mars::MarsRequest& request) override;
 
-    ListIterator list(const FDBToolRequest& request) override;
+    ListIterator list(const FDBToolRequest& request, int level) override;
 
     DumpIterator dump(const FDBToolRequest& request, bool simple) override;