rule schema key

src/fdb5/CMakeLists.txt

@@ -39,7 +39,8 @@ list( APPEND fdb5_srcs
     api/helpers/FDBToolRequest.cc
     api/helpers/FDBToolRequest.h
     api/helpers/DumpIterator.h
-    api/helpers/ListIterator.cc
+    api/helpers/ListElement.cc
+    api/helpers/ListElement.h
     api/helpers/ListIterator.h
     api/helpers/LockIterator.h
     api/helpers/MoveIterator.h
@@ -78,6 +79,8 @@ list( APPEND fdb5_srcs
     database/AxisRegistry.h
     database/BaseArchiveVisitor.cc
     database/BaseArchiveVisitor.h
+    database/BaseKey.cc
+    database/BaseKey.h
     database/Catalogue.cc
     database/Catalogue.h
     database/DatabaseNotFoundException.cc
@@ -124,6 +127,7 @@ list( APPEND fdb5_srcs
     database/IndexFactory.h
     database/Key.cc
     database/Key.h
+    database/ReadVisitor.cc
     database/ReadVisitor.h
     database/Report.cc
     database/Report.h
@@ -444,6 +448,7 @@ list( APPEND fdb5_tools
     fdb-schema
     fdb-where
     fdb-info
+    fdb-inspect
     fdb-status
     fdb-lock
     fdb-unlock

src/fdb5/api/DistFDB.cc

@@ -179,11 +179,12 @@ auto DistFDB::queryInternal(const FDBToolRequest& request, const QueryFN& fn) ->
 }
 
 
-ListIterator DistFDB::list(const FDBToolRequest& request) {
+ListIterator DistFDB::list(const FDBToolRequest& request, int level) {
     LOG_DEBUG_LIB(LibFdb5) << "DistFDB::list() : " << request << std::endl;
     return queryInternal(request,
-                         [](FDB& fdb, const FDBToolRequest& request) {
-                            return fdb.list(request);
+                         [level](FDB& fdb, const FDBToolRequest& request) {
+                            bool deduplicate = false; // never deduplicate on inner calls
+                            return fdb.list(request, deduplicate, level);
                          });
 }
 

src/fdb5/api/DistFDB.h

@@ -45,7 +45,7 @@ class DistFDB : public FDBBase {
 
     ListIterator inspect(const metkit::mars::MarsRequest& request) override;
 
-    ListIterator list(const FDBToolRequest& request) override;
+    ListIterator list(const FDBToolRequest& request, int level) override;
 
     AxesIterator axesIterator(const FDBToolRequest& request, int level=3) override { NOTIMP; }
 

src/fdb5/api/FDB.cc

@@ -13,10 +13,17 @@
  * (Project ID: 671951) www.nextgenio.eu
  */
 
+#include <cstddef>
+#include <sstream>
+#include <string>
+#include <vector>
+
 #include "eckit/config/Resource.h"
+#include "eckit/exception/Exceptions.h"
 #include "eckit/io/DataHandle.h"
 #include "eckit/io/MemoryHandle.h"
 #include "eckit/log/Log.h"
+#include "eckit/log/Timer.h"
 #include "eckit/message/Message.h"
 #include "eckit/message/Reader.h"
 
@@ -26,6 +33,9 @@
 #include "fdb5/api/FDB.h"
 #include "fdb5/api/FDBFactory.h"
 #include "fdb5/api/helpers/FDBToolRequest.h"
+#include "fdb5/api/helpers/ListElement.h"
+#include "fdb5/api/helpers/ListIterator.h"
+#include "fdb5/database/FieldLocation.h"
 #include "fdb5/database/Key.h"
 #include "fdb5/io/HandleGatherer.h"
 #include "fdb5/message/MessageDecoder.h"
@@ -110,12 +120,11 @@ void FDB::archive(const Key& key, const void* data, size_t length) {
     Key keyInternal(key);
 
     // step in archival requests from the model is just an integer. We need to include the stepunit
-    auto stepunit = keyInternal.find("stepunits");
-    if (stepunit != keyInternal.end()) {
-        if (stepunit->second.size()>0 && static_cast<char>(tolower(stepunit->second[0])) != 'h') {
-            auto step = keyInternal.find("step");
-            if (step != keyInternal.end()) {
-                std::string canonicalStep = config().schema().registry().lookupType("step").toKey(step->second + static_cast<char>(tolower(stepunit->second[0])));
+    if (const auto [stepunit, found] = keyInternal.find("stepunits"); found) {
+        if (stepunit->second.size() > 0 && static_cast<char>(tolower(stepunit->second[0])) != 'h') {
+            if (auto [step, foundStep] = keyInternal.find("step"); foundStep) {
+                std::string canonicalStep = config().schema().registry().lookupType("step").toKey(
+                    step->second + static_cast<char>(tolower(stepunit->second[0])));
                 keyInternal.set("step", canonicalStep);
             }
         }
@@ -180,19 +189,17 @@ eckit::DataHandle* FDB::read(ListIterator& it, bool sorted) {
         if (it.next(el)) {
             // build the request representing the tensor-product of all retrieved fields
             metkit::mars::MarsRequest cubeRequest = el.combinedKey().request();
-            std::vector<ListElement> elements{el};
+            std::vector<ListElement>  elements {el};
 
             while (it.next(el)) {
                 cubeRequest.merge(el.combinedKey().request());
                 elements.push_back(el);
             }
 
             // checking all retrieved fields against the hypercube, to remove duplicates
-            ListElementDeduplicator dedup;
-            metkit::hypercube::HyperCubePayloaded<ListElement> cube(cubeRequest, dedup);
-            for(auto el: elements) {
-                cube.add(el.combinedKey().request(), el);
-            }
+            ListElementDeduplicator deduplicator;
+            metkit::hypercube::HyperCubePayloaded<ListElement> cube(cubeRequest, deduplicator);
+            for (const auto& elem : elements) { cube.add(elem.combinedKey().request(), el); }
 
             if (cube.countVacant() > 0) {
                 std::stringstream ss;
@@ -203,7 +210,7 @@ eckit::DataHandle* FDB::read(ListIterator& it, bool sorted) {
                 eckit::Log::warning() << ss.str() << std::endl;
             }
 
-            for (size_t i=0; i< cube.size(); i++) {
+            for (std::size_t i = 0; i < cube.size(); i++) {
                 ListElement element;
                 if (cube.find(i, element)) {
                     result.add(element.location().dataHandle());
@@ -228,8 +235,8 @@ ListIterator FDB::inspect(const metkit::mars::MarsRequest& request) {
     return internal_->inspect(request);
 }
 
-ListIterator FDB::list(const FDBToolRequest& request, bool deduplicate) {
-    return ListIterator(internal_->list(request), deduplicate);
+ListIterator FDB::list(const FDBToolRequest& request, const bool deduplicate, const int level) {
+    return {internal_->list(request, level), deduplicate};
 }
 
 DumpIterator FDB::dump(const FDBToolRequest& request, bool simple) {

src/fdb5/api/FDB.h

@@ -21,6 +21,7 @@
 
 #include <memory>
 #include <iosfwd>
+#include <string>
 
 #include "eckit/distributed/Transport.h"
 
@@ -95,7 +96,7 @@ class FDB {
 
     ListIterator inspect(const metkit::mars::MarsRequest& request);
 
-    ListIterator list(const FDBToolRequest& request, bool deduplicate=false);
+    ListIterator list(const FDBToolRequest& request, bool deduplicate=false, int level=3);
 
     DumpIterator dump(const FDBToolRequest& request, bool simple=false);
 

src/fdb5/api/FDBFactory.h

@@ -75,7 +75,7 @@ class FDBBase : private eckit::NonCopyable {
 
     virtual ListIterator inspect(const metkit::mars::MarsRequest& request) = 0;
 
-    virtual ListIterator list(const FDBToolRequest& request) = 0;
+    virtual ListIterator list(const FDBToolRequest& request, int level) = 0;
 
     virtual DumpIterator dump(const FDBToolRequest& request, bool simple) = 0;
 

src/fdb5/api/LocalFDB.cc

@@ -15,7 +15,6 @@
 
 #include "eckit/container/Queue.h"
 #include "eckit/log/Log.h"
-#include "eckit/message/Message.h"
 
 #include "fdb5/api/helpers/ListIterator.h"
 #include "fdb5/api/helpers/FDBToolRequest.h"
@@ -34,7 +33,6 @@
 #include "fdb5/api/local/DumpVisitor.h"
 #include "fdb5/api/local/ListVisitor.h"
 #include "fdb5/api/local/PurgeVisitor.h"
-#include "fdb5/api/local/QueryVisitor.h"
 #include "fdb5/api/local/StatsVisitor.h"
 #include "fdb5/api/local/StatusVisitor.h"
 #include "fdb5/api/local/WipeVisitor.h"
@@ -83,9 +81,9 @@ APIIterator<typename VisitorType::ValueType> LocalFDB::queryInternal(const FDBTo
     return QueryIterator(new AsyncIterator(async_worker));
 }
 
-ListIterator LocalFDB::list(const FDBToolRequest& request) {
+ListIterator LocalFDB::list(const FDBToolRequest& request, const int level) {
     LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::list() : " << request << std::endl;
-    return queryInternal<ListVisitor>(request);
+    return queryInternal<ListVisitor>(request, level);
 }
 
 DumpIterator LocalFDB::dump(const FDBToolRequest &request, bool simple) {
@@ -127,7 +125,7 @@ ControlIterator LocalFDB::control(const FDBToolRequest& request,
 
 AxesIterator LocalFDB::axesIterator(const FDBToolRequest& request, int level) {
     LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::axesIterator() : " << request << std::endl;
-    return queryInternal<AxesVisitor>(request, config_, level);
+    return queryInternal<AxesVisitor>(request, level);
 }
 
 void LocalFDB::flush() {

src/fdb5/api/LocalFDB.h

@@ -41,7 +41,7 @@ class LocalFDB : public FDBBase {
 
     ListIterator inspect(const metkit::mars::MarsRequest& request) override;
 
-    ListIterator list(const FDBToolRequest& request) override;
+    ListIterator list(const FDBToolRequest& request, int level) override;
 
     DumpIterator dump(const FDBToolRequest& request, bool simple) override;
 

src/fdb5/api/RemoteFDB.cc

@@ -6,10 +6,11 @@
 #include "eckit/serialisation/MemoryStream.h"
 #include "fdb5/api/helpers/FDBToolRequest.h"
 
+#include "fdb5/LibFdb5.h"
 #include "fdb5/api/RemoteFDB.h"
+#include "fdb5/api/helpers/ListElement.h"
 #include "fdb5/database/Archiver.h"
 #include "fdb5/database/Inspector.h"
-#include "fdb5/LibFdb5.h"
 
 #include "fdb5/remote/client/ClientConnectionRouter.h"
 #include "fdb5/remote/RemoteFieldLocation.h"
@@ -36,6 +37,8 @@ using StatsHelper = BaseAPIHelper<fdb5::StatsElement, fdb5::remote::Message::Sta
 
 struct ListHelper : BaseAPIHelper<fdb5::ListElement, fdb5::remote::Message::List> {
 
+    ListHelper(const int depth) : depth_(depth) { }
+
     static fdb5::ListElement valueFromStream(eckit::Stream& s, fdb5::RemoteFDB* fdb) {
         fdb5::ListElement elem(s);
 
@@ -48,11 +51,16 @@ struct ListHelper : BaseAPIHelper<fdb5::ListElement, fdb5::remote::Message::List
             eckit::net::Endpoint fieldLocationEndpoint{elem.location().uri().host(), elem.location().uri().port()};
 
             std::shared_ptr<const fdb5::FieldLocation> remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(fieldLocationEndpoint), static_cast<const RemoteFieldLocation&>(elem.location())).make_shared();
-            return fdb5::ListElement(elem.key(), remoteLocation, elem.timestamp());
+            return fdb5::ListElement(elem.keys(), remoteLocation, elem.timestamp());
         }
         std::shared_ptr<const fdb5::FieldLocation> remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(), elem.location()).make_shared();
-        return fdb5::ListElement(elem.key(), remoteLocation, elem.timestamp());
+        return fdb5::ListElement(elem.keys(), remoteLocation, elem.timestamp());
     }
+
+    void encodeExtra(eckit::Stream& s) const { s << depth_; }
+
+private:
+    int depth_ {3};
 };
 
 struct AxesHelper : BaseAPIHelper<fdb5::AxesElement, fdb5::remote::Message::Axes> {
@@ -78,10 +86,10 @@ struct InspectHelper : BaseAPIHelper<fdb5::ListElement, fdb5::remote::Message::I
             eckit::net::Endpoint fieldLocationEndpoint{elem.location().uri().host(), elem.location().uri().port()};
 
             std::shared_ptr<const fdb5::FieldLocation> remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(fieldLocationEndpoint), static_cast<const RemoteFieldLocation&>(elem.location())).make_shared();
-            return fdb5::ListElement(elem.key(), remoteLocation, elem.timestamp());
+            return fdb5::ListElement(elem.keys(), remoteLocation, elem.timestamp());
         }
         std::shared_ptr<const fdb5::FieldLocation> remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(), elem.location()).make_shared();
-        return fdb5::ListElement(elem.key(), remoteLocation, elem.timestamp());
+        return fdb5::ListElement(elem.keys(), remoteLocation, elem.timestamp());
     }
 };
 
@@ -124,7 +132,7 @@ RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name
 
     std::vector<std::string> stores;
     std::vector<std::string> fieldLocationEndpoints;
-    
+
     for (size_t i=0; i<numStores; i++) {
         std::string store;
         s >> store;
@@ -225,12 +233,12 @@ auto RemoteFDB::forwardApiCall(const HelperClass& helper, const FDBToolRequest&
         );
 }
 
-ListIterator RemoteFDB::list(const FDBToolRequest& request) {
-    return forwardApiCall(ListHelper(), request);
+ListIterator RemoteFDB::list(const FDBToolRequest& request, const int depth) {
+    return forwardApiCall(ListHelper(depth), request);
 }
 
-AxesIterator RemoteFDB::axesIterator(const FDBToolRequest& request, int level) {
-    return forwardApiCall(AxesHelper(level), request);
+AxesIterator RemoteFDB::axesIterator(const FDBToolRequest& request, const int depth) {
+    return forwardApiCall(AxesHelper(depth), request);
 }
 
 ListIterator RemoteFDB::inspect(const metkit::mars::MarsRequest& request) {
@@ -247,7 +255,7 @@ void RemoteFDB::print(std::ostream& s) const {
 
 // Client
 bool RemoteFDB::handle(remote::Message message, uint32_t requestID) {
-    
+
     switch (message) {
         case fdb5::remote::Message::Complete: {
 

src/fdb5/api/RemoteFDB.h

@@ -44,9 +44,9 @@ class RemoteFDB : public LocalFDB, public remote::Client {
 
     ListIterator inspect(const metkit::mars::MarsRequest& request) override;
 
-    ListIterator list(const FDBToolRequest& request) override;
+    ListIterator list(const FDBToolRequest& request, int depth) override;
 
-    AxesIterator axesIterator(const FDBToolRequest& request, int level=3) override;
+    AxesIterator axesIterator(const FDBToolRequest& request, int depth = 3) override;
 
     DumpIterator dump(const FDBToolRequest& request, bool simple) override { NOTIMP; }
 

src/fdb5/api/SelectFDB.cc

@@ -137,12 +137,10 @@ auto SelectFDB::queryInternal(const FDBToolRequest& request, const QueryFN& fn)
     return QueryIterator(new APIAggregateIterator<ValueType>(std::move(iterQueue)));
 }
 
-ListIterator SelectFDB::list(const FDBToolRequest& request) {
+ListIterator SelectFDB::list(const FDBToolRequest& request, const int level) {
     LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::list() >> " << request << std::endl;
     return queryInternal(request,
-                         [](FDB& fdb, const FDBToolRequest& request) {
-                            return fdb.list(request);
-                         });
+                         [level](FDB& fdb, const FDBToolRequest& request) { return fdb.list(request, false, level); });
 }
 
 DumpIterator SelectFDB::dump(const FDBToolRequest& request, bool simple) {
@@ -217,20 +215,13 @@ void SelectFDB::print(std::ostream &s) const {
 }
 
 bool SelectFDB::matches(const Key& key, const SelectMap &select, bool requireMissing) const {
+    for (const auto& [keyword, regex] : select) {
+        const auto [iter, found] = key.find(keyword);
 
-    for (const auto& kv : select) {
+        if (!found && requireMissing) { return false; }
 
-        const std::string& k(kv.first);
-        const eckit::Regex& re(kv.second);
-
-        eckit::StringDict::const_iterator i = key.find(k);
-        if (i == key.end()) {
-            if (requireMissing) return false;
-        } else if (!re.match(i->second)) {
-            return false;
-        }
+        if (!regex.match(iter->second)) { return false; }
     }
-
     return true;
 }
 

src/fdb5/api/SelectFDB.h

@@ -50,7 +50,7 @@ class SelectFDB : public FDBBase {
 
     ListIterator inspect(const metkit::mars::MarsRequest& request) override;
 
-    ListIterator list(const FDBToolRequest& request) override;
+    ListIterator list(const FDBToolRequest& request, int level) override;
 
     DumpIterator dump(const FDBToolRequest& request, bool simple) override;