From 862778c1f1d187da0a41dd5d9ce013b17ff728db Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 24 Jul 2024 16:01:49 +0200 Subject: [PATCH 01/61] test(toKey): add check step value --- tests/fdb/type/test_toKey.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/fdb/type/test_toKey.cc b/tests/fdb/type/test_toKey.cc index 5819a0416..ee46109d3 100644 --- a/tests/fdb/type/test_toKey.cc +++ b/tests/fdb/type/test_toKey.cc @@ -124,6 +124,8 @@ CASE( "Step & ClimateDaily - expansion" ) { key.set("levelist", "50"); key.set("param", "129.128"); + EXPECT(key.valuesToString() == "20210427:dacl:0000:ei:7799:g:pb:pl:2-12:99:100:50:129.128"); + fdb5::Archiver archiver; fdb5::ArchiveVisitor visitor(archiver, key, data, 4); config.schema().expand(key, visitor); @@ -131,8 +133,7 @@ CASE( "Step & ClimateDaily - expansion" ) { EXPECT(key.canonicalValue("date") == "0427"); EXPECT(key.canonicalValue("time") == "0000"); - - std::cout << key.valuesToString() << std::endl; + EXPECT(key.canonicalValue("step") == "2-12"); EXPECT(key.valuesToString() == "0427:dacl:0000:ei:7799:g:pb:pl:2-12:99:100:50:129.128"); From d60aeb802840a2fe6bbe327bf964ec9a244486f3 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 29 Jul 2024 17:41:14 +0200 Subject: [PATCH 02/61] feat(list): cherry-pick level from Simon branch --- src/fdb5/api/DistFDB.cc | 7 ++++--- src/fdb5/api/DistFDB.h | 2 +- src/fdb5/api/FDB.cc | 6 +++--- src/fdb5/api/FDB.h | 2 +- src/fdb5/api/FDBFactory.h | 2 +- src/fdb5/api/LocalFDB.cc | 4 ++-- src/fdb5/api/LocalFDB.h | 2 +- src/fdb5/api/RemoteFDB.cc | 17 ++++++++++++----- src/fdb5/api/RemoteFDB.h | 2 +- src/fdb5/api/SelectFDB.cc | 6 ++---- src/fdb5/api/SelectFDB.h | 2 +- src/fdb5/api/local/ListVisitor.h | 21 ++++++++++++++++++--- src/fdb5/database/FieldLocation.cc | 26 ++++++++++++++++++++++++++ src/fdb5/database/FieldLocation.h | 2 ++ src/fdb5/database/Manager.cc | 2 +- src/fdb5/tools/fdb-list.cc | 13 ++++++++----- tests/fdb/api/ApiSpy.h | 3 ++- 17 files changed, 86 insertions(+), 33 deletions(-) diff --git a/src/fdb5/api/DistFDB.cc b/src/fdb5/api/DistFDB.cc index fbbe524b5..37b28a247 100644 --- a/src/fdb5/api/DistFDB.cc +++ b/src/fdb5/api/DistFDB.cc @@ -179,11 +179,12 @@ auto DistFDB::queryInternal(const FDBToolRequest& request, const QueryFN& fn) -> } -ListIterator DistFDB::list(const FDBToolRequest& request) { +ListIterator DistFDB::list(const FDBToolRequest& request, int level) { LOG_DEBUG_LIB(LibFdb5) << "DistFDB::list() : " << request << std::endl; return queryInternal(request, - [](FDB& fdb, const FDBToolRequest& request) { - return fdb.list(request); + [level](FDB& fdb, const FDBToolRequest& request) { + bool deduplicate = false; // never deduplicate on inner calls + return fdb.list(request, deduplicate, level); }); } diff --git a/src/fdb5/api/DistFDB.h b/src/fdb5/api/DistFDB.h index 494009e22..cbd8f3048 100644 --- a/src/fdb5/api/DistFDB.h +++ b/src/fdb5/api/DistFDB.h @@ -45,7 +45,7 @@ class DistFDB : public FDBBase { ListIterator inspect(const metkit::mars::MarsRequest& request) override; - ListIterator list(const FDBToolRequest& request) override; + ListIterator list(const FDBToolRequest& request, int level) override; DumpIterator dump(const FDBToolRequest& request, bool simple) override; diff --git a/src/fdb5/api/FDB.cc b/src/fdb5/api/FDB.cc index 218d07a17..29bf72d4c 100644 --- a/src/fdb5/api/FDB.cc +++ b/src/fdb5/api/FDB.cc @@ -168,7 +168,7 @@ eckit::DataHandle* FDB::read(const std::vector& uris, bool sorted) { } return result.dataHandle(); } - + eckit::DataHandle* FDB::read(ListIterator& it, bool sorted) { eckit::Timer timer; @@ -230,8 +230,8 @@ ListIterator FDB::inspect(const metkit::mars::MarsRequest& request) { return internal_->inspect(request); } -ListIterator FDB::list(const FDBToolRequest& request, bool deduplicate) { - return ListIterator(internal_->list(request), deduplicate); +ListIterator FDB::list(const FDBToolRequest& request, const bool deduplicate, const int level) { + return {internal_->list(request, level), deduplicate}; } DumpIterator FDB::dump(const FDBToolRequest& request, bool simple) { diff --git a/src/fdb5/api/FDB.h b/src/fdb5/api/FDB.h index 6be852565..32b1fdf16 100644 --- a/src/fdb5/api/FDB.h +++ b/src/fdb5/api/FDB.h @@ -94,7 +94,7 @@ class FDB { ListIterator inspect(const metkit::mars::MarsRequest& request); - ListIterator list(const FDBToolRequest& request, bool deduplicate=false); + ListIterator list(const FDBToolRequest& request, bool deduplicate=false, int level=3); DumpIterator dump(const FDBToolRequest& request, bool simple=false); diff --git a/src/fdb5/api/FDBFactory.h b/src/fdb5/api/FDBFactory.h index 6cd3c1c37..9d28e35b9 100644 --- a/src/fdb5/api/FDBFactory.h +++ b/src/fdb5/api/FDBFactory.h @@ -71,7 +71,7 @@ class FDBBase : private eckit::NonCopyable { virtual ListIterator inspect(const metkit::mars::MarsRequest& request) = 0; - virtual ListIterator list(const FDBToolRequest& request) = 0; + virtual ListIterator list(const FDBToolRequest& request, int level) = 0; virtual DumpIterator dump(const FDBToolRequest& request, bool simple) = 0; diff --git a/src/fdb5/api/LocalFDB.cc b/src/fdb5/api/LocalFDB.cc index 3d86255e4..11897ea72 100644 --- a/src/fdb5/api/LocalFDB.cc +++ b/src/fdb5/api/LocalFDB.cc @@ -83,9 +83,9 @@ APIIterator LocalFDB::queryInternal(const FDBTo return QueryIterator(new AsyncIterator(async_worker)); } -ListIterator LocalFDB::list(const FDBToolRequest& request) { +ListIterator LocalFDB::list(const FDBToolRequest& request, const int level) { LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::list() : " << request << std::endl; - return queryInternal(request); + return queryInternal(request, level); } DumpIterator LocalFDB::dump(const FDBToolRequest &request, bool simple) { diff --git a/src/fdb5/api/LocalFDB.h b/src/fdb5/api/LocalFDB.h index 96f4a6914..bb654640b 100644 --- a/src/fdb5/api/LocalFDB.h +++ b/src/fdb5/api/LocalFDB.h @@ -41,7 +41,7 @@ class LocalFDB : public FDBBase { ListIterator inspect(const metkit::mars::MarsRequest& request) override; - ListIterator list(const FDBToolRequest& request) override; + ListIterator list(const FDBToolRequest& request, int level) override; DumpIterator dump(const FDBToolRequest& request, bool simple) override; diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc index c5e0b74e9..3f5359da1 100644 --- a/src/fdb5/api/RemoteFDB.cc +++ b/src/fdb5/api/RemoteFDB.cc @@ -62,7 +62,7 @@ class ConnectionError : public eckit::Exception { ConnectionError(const int); ConnectionError(const int, const eckit::net::Endpoint&); - bool retryOnClient() const override { return true; } + bool retryOnClient() const override { return true; } }; ConnectionError::ConnectionError(const int retries) { @@ -549,7 +549,14 @@ struct BaseAPIHelper { static ValueType valueFromStream(eckit::Stream& s, RemoteFDB* fdb) { return ValueType(s); } }; -using ListHelper = BaseAPIHelper; +struct ListHelper : public BaseAPIHelper { + + ListHelper(int level) : level_(level) {} + void encodeExtra(eckit::Stream& s) const { s << level_; } + +private: + int level_; +}; struct InspectHelper : BaseAPIHelper { @@ -625,7 +632,7 @@ struct WipeHelper : BaseAPIHelper { struct MoveHelper : BaseAPIHelper { - + MoveHelper(const eckit::URI& dest) : dest_(dest) {} @@ -715,8 +722,8 @@ auto RemoteFDB::forwardApiCall(const HelperClass& helper, const FDBToolRequest& ); } -ListIterator RemoteFDB::list(const FDBToolRequest& request) { - return forwardApiCall(ListHelper(), request); +ListIterator RemoteFDB::list(const FDBToolRequest& request, const int level) { + return forwardApiCall(ListHelper(level), request); } ListIterator RemoteFDB::inspect(const metkit::mars::MarsRequest& request) { diff --git a/src/fdb5/api/RemoteFDB.h b/src/fdb5/api/RemoteFDB.h index 38f83e4f2..084dd57e4 100644 --- a/src/fdb5/api/RemoteFDB.h +++ b/src/fdb5/api/RemoteFDB.h @@ -62,7 +62,7 @@ class RemoteFDB : public FDBBase { ListIterator inspect(const metkit::mars::MarsRequest& request) override; - ListIterator list(const FDBToolRequest& request) override; + ListIterator list(const FDBToolRequest& request, int level) override; DumpIterator dump(const FDBToolRequest& request, bool simple) override; diff --git a/src/fdb5/api/SelectFDB.cc b/src/fdb5/api/SelectFDB.cc index 586ee5b19..477255f8f 100644 --- a/src/fdb5/api/SelectFDB.cc +++ b/src/fdb5/api/SelectFDB.cc @@ -137,12 +137,10 @@ auto SelectFDB::queryInternal(const FDBToolRequest& request, const QueryFN& fn) return QueryIterator(new APIAggregateIterator(std::move(iterQueue))); } -ListIterator SelectFDB::list(const FDBToolRequest& request) { +ListIterator SelectFDB::list(const FDBToolRequest& request, const int level) { LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::list() >> " << request << std::endl; return queryInternal(request, - [](FDB& fdb, const FDBToolRequest& request) { - return fdb.list(request); - }); + [level](FDB& fdb, const FDBToolRequest& request) { return fdb.list(request, false, level); }); } DumpIterator SelectFDB::dump(const FDBToolRequest& request, bool simple) { diff --git a/src/fdb5/api/SelectFDB.h b/src/fdb5/api/SelectFDB.h index 78937c358..4407f7218 100644 --- a/src/fdb5/api/SelectFDB.h +++ b/src/fdb5/api/SelectFDB.h @@ -50,7 +50,7 @@ class SelectFDB : public FDBBase { ListIterator inspect(const metkit::mars::MarsRequest& request) override; - ListIterator list(const FDBToolRequest& request) override; + ListIterator list(const FDBToolRequest& request, int level) override; DumpIterator dump(const FDBToolRequest& request, bool simple) override; diff --git a/src/fdb5/api/local/ListVisitor.h b/src/fdb5/api/local/ListVisitor.h index 938eeec5e..8afefafac 100644 --- a/src/fdb5/api/local/ListVisitor.h +++ b/src/fdb5/api/local/ListVisitor.h @@ -23,6 +23,7 @@ #include "fdb5/database/Index.h" #include "fdb5/api/local/QueryVisitor.h" #include "fdb5/api/helpers/ListIterator.h" +#include "eckit/os/BackTrace.h" namespace fdb5 { namespace api { @@ -35,7 +36,8 @@ namespace local { struct ListVisitor : public QueryVisitor { public: - using QueryVisitor::QueryVisitor; + ListVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, int level): + QueryVisitor(queue, request), level_(level) { } /// Make a note of the current database. Subtract its key from the current /// request so we can test request is used in its entirety @@ -55,6 +57,12 @@ struct ListVisitor : public QueryVisitor { indexRequest_.unsetValues(kv.first); } + if (level_ == 1) { + eckit::Log::info() << eckit::BackTrace::dump() << std::endl; + queue_.emplace(ListElement({currentCatalogue_->key(), Key(), Key()}, FieldLocation::nullLocation(), 0)); + ret = false; + } + return ret; } @@ -73,9 +81,15 @@ struct ListVisitor : public QueryVisitor { } if (index.partialMatch(request_)) { - return true; // Explore contained entries + if (level_ == 2) { + const auto keyParts = std::vector {currentCatalogue_->key(), currentIndex_->key(), Key()}; + queue_.emplace(ListElement(keyParts, FieldLocation::nullLocation(), 0)); + return false; + } + return true; // Explore contained entries } - return false; // Skip contained entries + + return false; // Skip contained entries } /// Test if entry matches the current request. If so, add to the output queue. @@ -96,6 +110,7 @@ struct ListVisitor : public QueryVisitor { metkit::mars::MarsRequest indexRequest_; metkit::mars::MarsRequest datumRequest_; + const int level_; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/FieldLocation.cc b/src/fdb5/database/FieldLocation.cc index 9aaaeb79a..e7d5605e0 100644 --- a/src/fdb5/database/FieldLocation.cc +++ b/src/fdb5/database/FieldLocation.cc @@ -107,6 +107,32 @@ FieldLocationBuilderBase::~FieldLocationBuilderBase() { //---------------------------------------------------------------------------------------------------------------------- +namespace { + +struct NullFieldLocation: public FieldLocation { + eckit::DataHandle* dataHandle() const override { NOTIMP; } + + std::shared_ptr make_shared() const override; + + void visit(FieldLocationVisitor&) const override { NOTIMP; } + + void print(std::ostream& out) const override { out << "NullFieldLocation"; } +}; + +static auto emptyLocation = std::make_shared(); + +std::shared_ptr NullFieldLocation::make_shared() const { + return emptyLocation; +} + +} // namespace + +//---------------------------------------------------------------------------------------------------------------------- + +std::shared_ptr FieldLocation::nullLocation() { + return emptyLocation; +} + FieldLocation::FieldLocation(const eckit::URI& uri) : uri_(uri) { try { offset_ = eckit::Offset(std::stoll(uri.fragment())); diff --git a/src/fdb5/database/FieldLocation.h b/src/fdb5/database/FieldLocation.h index 09c9eb961..847df2b09 100644 --- a/src/fdb5/database/FieldLocation.h +++ b/src/fdb5/database/FieldLocation.h @@ -37,6 +37,8 @@ namespace fdb5 { class FieldLocationVisitor; class FieldLocation : public eckit::OwnedLock, public eckit::Streamable { +public: // factory + static std::shared_ptr nullLocation(); public: // methods diff --git a/src/fdb5/database/Manager.cc b/src/fdb5/database/Manager.cc index a9b033f19..3774268ae 100644 --- a/src/fdb5/database/Manager.cc +++ b/src/fdb5/database/Manager.cc @@ -298,7 +298,7 @@ std::vector Manager::visitableLocations(const metkit::mars::MarsRequ std::set engines = Manager::engines(rq, all); - LOG_DEBUG_LIB(LibFdb5) << "Matching engines for request " << rq << " -> " << engines << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "Matching engines for request " << rq << (all ? " ALL" : "") << " -> " << engines << std::endl; std::vector r; // union of all locations diff --git a/src/fdb5/tools/fdb-list.cc b/src/fdb5/tools/fdb-list.cc index 0c61129f0..e7a889265 100644 --- a/src/fdb5/tools/fdb-list.cc +++ b/src/fdb5/tools/fdb-list.cc @@ -54,6 +54,7 @@ class FDBList : public FDBVisitTool { options_.push_back(new SimpleOption("porcelain", "Streamlined and stable output for input into other tools")); options_.push_back(new SimpleOption("json", "Output available fields in JSON form")); options_.push_back(new SimpleOption("compact", "Aggregate available fields in MARS requests")); + options_.push_back(new SimpleOption("max-level", "Maximum (key) level to be searched")); } private: // methods @@ -68,8 +69,10 @@ class FDBList : public FDBVisitTool { bool porcelain_; bool json_; bool compact_; + int level_ {3}; }; +//---------------------------------------------------------------------------------------------------------------------- std::string keySignature(const fdb5::Key& key) { std::string signature; @@ -93,6 +96,7 @@ void FDBList::init(const CmdArgs& args) { porcelain_ = args.getBool("porcelain", false); json_ = args.getBool("json", false); compact_ = args.getBool("compact", false); + level_ = args.getInt("level", 3); if (json_) { porcelain_ = true; @@ -104,13 +108,13 @@ void FDBList::init(const CmdArgs& args) { if (compact_) { if (location_) { throw UserError("--compact and --location are not compatible", Here()); - } + } if (full_) { throw UserError("--compact and --full are not compatible", Here()); - } + } if (porcelain_) { throw UserError("--compact and --porcelain are not compatible", Here()); - } + } } /// @todo option ignore-errors @@ -135,7 +139,7 @@ void FDBList::execute(const CmdArgs& args) { } // If --full is supplied, then include all entries including duplicates. - auto listObject = fdb.list(request, !full_ && !compact_); + auto listObject = fdb.list(request, !full_ && !compact_, level_); std::map>>> requests; ListElement elem; @@ -216,4 +220,3 @@ int main(int argc, char **argv) { fdb5::tools::FDBList app(argc, argv); return app.start(); } - diff --git a/tests/fdb/api/ApiSpy.h b/tests/fdb/api/ApiSpy.h index c25efeebf..2c578157f 100644 --- a/tests/fdb/api/ApiSpy.h +++ b/tests/fdb/api/ApiSpy.h @@ -90,8 +90,9 @@ class ApiSpy : public fdb5::FDBBase { return fdb5::ListIterator(0); } - fdb5::ListIterator list(const fdb5::FDBToolRequest& request) override { + fdb5::ListIterator list(const fdb5::FDBToolRequest& /* request */, const int level) override { counts_.list += 1; + ASSERT(level == 3); return fdb5::ListIterator(0); } From d23778c00508f1a6d3933b8e4812956f3927dbcc Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 30 Jul 2024 10:44:35 +0200 Subject: [PATCH 03/61] test(fdb-list): add new --- tests/fdb/tools/CMakeLists.txt | 3 ++- tests/fdb/tools/fdb_list.sh.in | 42 ++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100755 tests/fdb/tools/fdb_list.sh.in diff --git a/tests/fdb/tools/CMakeLists.txt b/tests/fdb/tools/CMakeLists.txt index 6ff2df23a..d36fc20b7 100644 --- a/tests/fdb/tools/CMakeLists.txt +++ b/tests/fdb/tools/CMakeLists.txt @@ -1,9 +1,10 @@ list( APPEND fdb_tools_tests fdb_info + fdb_list fdb_axes ) foreach( _t ${fdb_tools_tests} ) - + configure_file( ${_t}.sh.in ${_t}.sh @ONLY ) ecbuild_add_test( diff --git a/tests/fdb/tools/fdb_list.sh.in b/tests/fdb/tools/fdb_list.sh.in new file mode 100755 index 000000000..f9496ff4d --- /dev/null +++ b/tests/fdb/tools/fdb_list.sh.in @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# set -eu + +yell() { echo "$(basename "$0"): $*" >&2; } +die() { yell "$*"; exit 1; } +try() { "$@" || die "Errored HERE => '$*'"; } + +function request() { + echo "RETRIEVE,CLASS=OD,TYPE=CF,STREAM=ENFO,EXPVER=0001,LEVTYPE=PL,DATE=20201106,TIME=1200,STEP=00,DOMAIN=G,$1" > req; +} + +export PATH=@CMAKE_BINARY_DIR@/bin:$PATH +export FDB5_CONFIG_FILE="local.yaml" +export FDB_HOME=@PROJECT_BINARY_DIR@ + +tname=list_test +srcdir=@CMAKE_CURRENT_SOURCE_DIR@ +bindir=@CMAKE_CURRENT_BINARY_DIR@ + +######################################################################################################################## + +try cd $bindir + +rm -rf $tname || true +mkdir -p $tname/localroot + +try cd $tname + +for f in local.yaml x.grib; do + cp $srcdir/$f ./ +done + +######################################################################################################################## +# Create a grib file with 4 steps and populate FDB + +try grib_set -s step=0 x.grib 0.grib +try grib_set -s step=6 x.grib 6.grib +try grib_set -s step=9 x.grib 9.grib +try cat 0.grib 6.grib 9.grib x.grib > "$tname.grib" + +try fdb-write "$tname.grib" From 3be1f228d5df6b22029243bb06df3cd5c5b8a74a Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 30 Jul 2024 16:57:07 +0200 Subject: [PATCH 04/61] test(fdb-list): prepare grib file --- tests/fdb/tools/fdb_list.sh.in | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/fdb/tools/fdb_list.sh.in b/tests/fdb/tools/fdb_list.sh.in index f9496ff4d..b284cc94d 100755 --- a/tests/fdb/tools/fdb_list.sh.in +++ b/tests/fdb/tools/fdb_list.sh.in @@ -28,7 +28,7 @@ mkdir -p $tname/localroot try cd $tname for f in local.yaml x.grib; do - cp $srcdir/$f ./ + cp "$srcdir/$f" ./ done ######################################################################################################################## @@ -37,6 +37,13 @@ done try grib_set -s step=0 x.grib 0.grib try grib_set -s step=6 x.grib 6.grib try grib_set -s step=9 x.grib 9.grib -try cat 0.grib 6.grib 9.grib x.grib > "$tname.grib" + +for f in *.grib; do + try grib_set -s type=cf "$f" "cf.$f" +done + +for f in *.grib; do + try cat "$f" >> "$tname.grib" +done try fdb-write "$tname.grib" From e0a0421dd3b55371a503dd3ca9d97d09692c433f Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 30 Jul 2024 17:11:34 +0200 Subject: [PATCH 05/61] fix(fdb-list): init class member --- src/fdb5/tools/fdb-list.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fdb5/tools/fdb-list.cc b/src/fdb5/tools/fdb-list.cc index e7a889265..8bf48ff8b 100644 --- a/src/fdb5/tools/fdb-list.cc +++ b/src/fdb5/tools/fdb-list.cc @@ -68,8 +68,8 @@ class FDBList : public FDBVisitTool { bool full_; bool porcelain_; bool json_; - bool compact_; int level_ {3}; + bool compact_ {false}; }; //---------------------------------------------------------------------------------------------------------------------- From c2703aba6afb9564943216ac24ae8aadad7f782a Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 30 Jul 2024 17:13:32 +0200 Subject: [PATCH 06/61] fix(fdb-list): unnecessary copy in loop --- src/fdb5/tools/fdb-list.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/fdb5/tools/fdb-list.cc b/src/fdb5/tools/fdb-list.cc index 8bf48ff8b..f87149fbd 100644 --- a/src/fdb5/tools/fdb-list.cc +++ b/src/fdb5/tools/fdb-list.cc @@ -77,9 +77,9 @@ class FDBList : public FDBVisitTool { std::string keySignature(const fdb5::Key& key) { std::string signature; std::string separator; - for (auto k : key.keys()) { - signature += separator+k; - separator=":"; + for (auto&& k : key.keys()) { + signature += separator + k; + separator = ":"; } return signature; } From 36e6a54c3a59a1502102e0af55a16e8fc21273ea Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 30 Jul 2024 17:14:44 +0200 Subject: [PATCH 07/61] fix(fdb-list): use make_unique not "new" --- src/fdb5/tools/fdb-list.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fdb5/tools/fdb-list.cc b/src/fdb5/tools/fdb-list.cc index f87149fbd..af280f2c3 100644 --- a/src/fdb5/tools/fdb-list.cc +++ b/src/fdb5/tools/fdb-list.cc @@ -126,7 +126,7 @@ void FDBList::execute(const CmdArgs& args) { std::unique_ptr json; if (json_) { - json.reset(new JSON(Log::info())); + json = std::make_unique(Log::info()); json->startList(); } From 77e3d77c76ab39b7e6579212afbf10dd51feacd2 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 30 Jul 2024 17:29:42 +0200 Subject: [PATCH 08/61] fix(fdb-list): use "override" (not virtual) --- src/fdb5/tools/fdb-list.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fdb5/tools/fdb-list.cc b/src/fdb5/tools/fdb-list.cc index af280f2c3..abc3180b5 100644 --- a/src/fdb5/tools/fdb-list.cc +++ b/src/fdb5/tools/fdb-list.cc @@ -59,8 +59,6 @@ class FDBList : public FDBVisitTool { private: // methods - virtual void execute(const CmdArgs& args); - virtual void init(const CmdArgs &args); bool location_; bool timestamp_; @@ -70,6 +68,8 @@ class FDBList : public FDBVisitTool { bool json_; int level_ {3}; bool compact_ {false}; + void execute(const CmdArgs& args) override; + void init(const CmdArgs& args) override; }; //---------------------------------------------------------------------------------------------------------------------- From 1e3381bfe8644091dc15c95ba446f6a0251b5b17 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 30 Jul 2024 17:44:12 +0200 Subject: [PATCH 09/61] fix(fdb-list): default member init and tab size --- src/fdb5/tools/fdb-list.cc | 53 +++++++++++++++----------------------- 1 file changed, 21 insertions(+), 32 deletions(-) diff --git a/src/fdb5/tools/fdb-list.cc b/src/fdb5/tools/fdb-list.cc index abc3180b5..1a09821c5 100644 --- a/src/fdb5/tools/fdb-list.cc +++ b/src/fdb5/tools/fdb-list.cc @@ -29,47 +29,37 @@ using namespace eckit; using namespace eckit::option; -namespace fdb5 { -namespace tools { +namespace fdb5::tools { //---------------------------------------------------------------------------------------------------------------------- -class FDBList : public FDBVisitTool { - - public: // methods - - FDBList(int argc, char **argv) : - FDBVisitTool(argc, argv, "class,expver"), - location_(false), - timestamp_(false), - length_(false), - full_(false), - porcelain_(false), - json_(false) { - +class FDBList: public FDBVisitTool { +public: // methods + FDBList(int argc, char** argv): FDBVisitTool(argc, argv, "class,expver") { options_.push_back(new SimpleOption("location", "Also print the location of each field")); options_.push_back(new SimpleOption("timestamp", "Also print the timestamp when the field was indexed")); options_.push_back(new SimpleOption("length", "Also print the field size")); options_.push_back(new SimpleOption("full", "Include all entries (including masked duplicates)")); - options_.push_back(new SimpleOption("porcelain", "Streamlined and stable output for input into other tools")); + options_.push_back(new SimpleOption("porcelain", + "Streamlined and stable output. " + "Useful as input for other tools or scripts")); options_.push_back(new SimpleOption("json", "Output available fields in JSON form")); options_.push_back(new SimpleOption("compact", "Aggregate available fields in MARS requests")); - options_.push_back(new SimpleOption("max-level", "Maximum (key) level to be searched")); + options_.push_back(new SimpleOption("depth", "Output entries up to 'depth' levels deep")); } - private: // methods - - - bool location_; - bool timestamp_; - bool length_; - bool full_; - bool porcelain_; - bool json_; - int level_ {3}; - bool compact_ {false}; +private: // methods void execute(const CmdArgs& args) override; void init(const CmdArgs& args) override; + + bool location_ {false}; + bool timestamp_ {false}; + bool length_ {false}; + bool full_ {false}; + bool porcelain_ {false}; + bool json_ {false}; + bool compact_ {false}; + int depth_ {3}; }; //---------------------------------------------------------------------------------------------------------------------- @@ -96,7 +86,7 @@ void FDBList::init(const CmdArgs& args) { porcelain_ = args.getBool("porcelain", false); json_ = args.getBool("json", false); compact_ = args.getBool("compact", false); - level_ = args.getInt("level", 3); + depth_ = args.getInt("depth", 3); if (json_) { porcelain_ = true; @@ -139,7 +129,7 @@ void FDBList::execute(const CmdArgs& args) { } // If --full is supplied, then include all entries including duplicates. - auto listObject = fdb.list(request, !full_ && !compact_, level_); + auto listObject = fdb.list(request, !full_ && !compact_, depth_); std::map>>> requests; ListElement elem; @@ -213,8 +203,7 @@ void FDBList::execute(const CmdArgs& args) { //---------------------------------------------------------------------------------------------------------------------- -} // namespace tools -} // namespace fdb5 +} // namespace fdb5::tools int main(int argc, char **argv) { fdb5::tools::FDBList app(argc, argv); From 7f6be88117d4c7f41db800d4a98becb7b4088224 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 30 Jul 2024 20:09:10 +0200 Subject: [PATCH 10/61] test(fdb-list): added --depth option --- tests/fdb/tools/fdb_list.sh.in | 42 ++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/fdb/tools/fdb_list.sh.in b/tests/fdb/tools/fdb_list.sh.in index b284cc94d..14034cfab 100755 --- a/tests/fdb/tools/fdb_list.sh.in +++ b/tests/fdb/tools/fdb_list.sh.in @@ -47,3 +47,45 @@ for f in *.grib; do done try fdb-write "$tname.grib" + +######################################################################################################################## +# test all +out=$(fdb-list --all --minimum-keys="" --porcelain) +exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=0,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=12,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=6,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=9,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=0,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=12,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=6,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=9,param=166}" +try test "$exp" = "$out" + +# test date depth=1 +out=$(fdb-list date=20201102 --minimum-keys="" --porcelain --depth=1) +exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{}{}" +try test "$exp" = "$out" + +# test date depth=2 +out=$(fdb-list date=20201102 --minimum-keys="" --porcelain --depth=2) +exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{}" +try test "$exp" = "$out" + +# test cf depth=1 +out=$(fdb-list type=cf --minimum-keys="" --porcelain --depth=1) +exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{}{}" +try test "$exp" = "$out" + +# test cf depth=2 +out=$(fdb-list type=cf --minimum-keys="" --porcelain --depth=2) +exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{}" +try test "$exp" = "$out" + +# test cf depth=3 +out=$(fdb-list type=cf --minimum-keys="" --porcelain --depth=3) +exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=0,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=12,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=6,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=9,param=166}" +try test "$exp" = "$out" From 5dfb6a05181cb8599059f04736fcf7ef3bbe3b13 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 30 Jul 2024 20:47:28 +0200 Subject: [PATCH 11/61] fix(ListVisitor): empty keys --- src/fdb5/api/local/ListVisitor.h | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/fdb5/api/local/ListVisitor.h b/src/fdb5/api/local/ListVisitor.h index 8afefafac..9fc709ce5 100644 --- a/src/fdb5/api/local/ListVisitor.h +++ b/src/fdb5/api/local/ListVisitor.h @@ -25,9 +25,7 @@ #include "fdb5/api/helpers/ListIterator.h" #include "eckit/os/BackTrace.h" -namespace fdb5 { -namespace api { -namespace local { +namespace fdb5::api::local { /// @note Helper classes for LocalFDB @@ -58,8 +56,9 @@ struct ListVisitor : public QueryVisitor { } if (level_ == 1) { - eckit::Log::info() << eckit::BackTrace::dump() << std::endl; - queue_.emplace(ListElement({currentCatalogue_->key(), Key(), Key()}, FieldLocation::nullLocation(), 0)); + /// @todo fix this so that we don't need to create an empty key + const auto emptyKey = Key {currentCatalogue_->schema().registry()}; + queue_.emplace(ListElement({currentCatalogue_->key(), emptyKey, emptyKey}, FieldLocation::nullLocation(), 0)); ret = false; } @@ -82,7 +81,9 @@ struct ListVisitor : public QueryVisitor { if (index.partialMatch(request_)) { if (level_ == 2) { - const auto keyParts = std::vector {currentCatalogue_->key(), currentIndex_->key(), Key()}; + /// @todo fix this so that we don't need to create an empty key + const auto emptyKey = Key {currentCatalogue_->schema().registry()}; + const auto keyParts = std::vector {currentCatalogue_->key(), currentIndex_->key(), emptyKey}; queue_.emplace(ListElement(keyParts, FieldLocation::nullLocation(), 0)); return false; } @@ -115,8 +116,6 @@ struct ListVisitor : public QueryVisitor { //---------------------------------------------------------------------------------------------------------------------- -} // namespace local -} // namespace api -} // namespace fdb5 +} // namespace fdb5::api::local #endif From 71bbf5103e0a82f028cd3d67355501c4dea714f0 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 30 Jul 2024 20:49:17 +0200 Subject: [PATCH 12/61] style(fdb-list): clean code --- src/fdb5/tools/fdb-list.cc | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/fdb5/tools/fdb-list.cc b/src/fdb5/tools/fdb-list.cc index 1a09821c5..a82b58698 100644 --- a/src/fdb5/tools/fdb-list.cc +++ b/src/fdb5/tools/fdb-list.cc @@ -159,19 +159,20 @@ void FDBList::execute(const CmdArgs& args) { it->second.emplace(signature, std::make_pair(keys[2].request(), std::unordered_set{keys[2]})); } } - } else { - if (json_) { - (*json) << elem; - } else { - if (porcelain_) { - elem.print(Log::info(), location_, false, false); - } else { - elem.print(Log::info(), location_, length_, timestamp_, ", "); - } - Log::info() << std::endl; - } + continue; } - } + + // JSON output + if (json) { + *json << elem; + continue; + } + + elem.print(Log::info(), location_, length_, timestamp_, ", "); + Log::info() << std::endl; + + } // while + if (compact_) { for (const auto& tree: requests) { for (const auto& leaf: tree.second) { @@ -194,11 +195,10 @@ void FDBList::execute(const CmdArgs& args) { } } // n.b. finding no data is not an error for fdb-list - } - if (json_) { - json->endList(); - } + } // requests + + if (json) { json->endList(); } } //---------------------------------------------------------------------------------------------------------------------- From c4ebc5b391667975cf7479954aa3dc49801f2857 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 30 Jul 2024 20:51:00 +0200 Subject: [PATCH 13/61] fix(fdb-list): porcelain defaults other options --- src/fdb5/tools/fdb-list.cc | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/src/fdb5/tools/fdb-list.cc b/src/fdb5/tools/fdb-list.cc index a82b58698..badc5c46f 100644 --- a/src/fdb5/tools/fdb-list.cc +++ b/src/fdb5/tools/fdb-list.cc @@ -88,23 +88,18 @@ void FDBList::init(const CmdArgs& args) { compact_ = args.getBool("compact", false); depth_ = args.getInt("depth", 3); - if (json_) { - porcelain_ = true; - if (location_ || timestamp_ || length_) { - throw UserError("--json and --location/--timestamp/--length not compatible", Here()); - } + if (json_) { porcelain_ = true; } + + if (porcelain_) { + location_ = false; + timestamp_ = false; + length_ = false; + compact_ = false; } if (compact_) { - if (location_) { - throw UserError("--compact and --location are not compatible", Here()); - } - if (full_) { - throw UserError("--compact and --full are not compatible", Here()); - } - if (porcelain_) { - throw UserError("--compact and --porcelain are not compatible", Here()); - } + if (location_) { throw UserError("--compact and --location are not compatible", Here()); } + if (full_) { throw UserError("--compact and --full are not compatible", Here()); } } /// @todo option ignore-errors From 8ce657e8fd551867778a01a1002f9a6d4dd40902 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 30 Jul 2024 20:52:46 +0200 Subject: [PATCH 14/61] chore(remote/Handler): todo list w/ level --- src/fdb5/remote/Handler.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/fdb5/remote/Handler.cc b/src/fdb5/remote/Handler.cc index 33fe984c6..ebb34361f 100644 --- a/src/fdb5/remote/Handler.cc +++ b/src/fdb5/remote/Handler.cc @@ -88,7 +88,10 @@ struct BaseHelper { }; struct ListHelper : public BaseHelper { + // void extraDecode(eckit::Stream& s) { s >> level_; } + ListIterator apiCall(FDB& fdb, const FDBToolRequest& request) const { + /// @todo return fdb.list(request, level_); return fdb.list(request); } }; From 8d9db4e64d23402a0dc10700a6a7dbc6d6b74c57 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 5 Aug 2024 18:16:07 +0200 Subject: [PATCH 15/61] feat(list)!: separate ListElement implements ctor for 1-,2-,3-level split key chains [skip-ci] --- src/fdb5/CMakeLists.txt | 3 +- src/fdb5/api/helpers/ListElement.cc | 136 +++++++++++++++++++++++++++ src/fdb5/api/helpers/ListElement.h | 110 ++++++++++++++++++++++ src/fdb5/api/helpers/ListIterator.cc | 69 -------------- src/fdb5/api/helpers/ListIterator.h | 74 +-------------- 5 files changed, 253 insertions(+), 139 deletions(-) create mode 100644 src/fdb5/api/helpers/ListElement.cc create mode 100644 src/fdb5/api/helpers/ListElement.h delete mode 100644 src/fdb5/api/helpers/ListIterator.cc diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt index 41f61c69b..f55713b59 100644 --- a/src/fdb5/CMakeLists.txt +++ b/src/fdb5/CMakeLists.txt @@ -39,7 +39,8 @@ list( APPEND fdb5_srcs api/helpers/FDBToolRequest.cc api/helpers/FDBToolRequest.h api/helpers/DumpIterator.h - api/helpers/ListIterator.cc + api/helpers/ListElement.cc + api/helpers/ListElement.h api/helpers/ListIterator.h api/helpers/LockIterator.h api/helpers/MoveIterator.h diff --git a/src/fdb5/api/helpers/ListElement.cc b/src/fdb5/api/helpers/ListElement.cc new file mode 100644 index 000000000..d24892265 --- /dev/null +++ b/src/fdb5/api/helpers/ListElement.cc @@ -0,0 +1,136 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "fdb5/api/helpers/ListElement.h" + +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "eckit/log/JSON.h" +#include "eckit/serialisation/Reanimator.h" +#include "eckit/serialisation/Stream.h" +#include "fdb5/database/FieldLocation.h" +#include "fdb5/database/Key.h" + +#include +#include +#include +#include +#include + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- +// HELPERS + +namespace { + +template 1 && N < 4)>> +auto combineKeys(const KeyChain& keys) -> Key { + /// @todo fixme: no API to get registry from Key; fix this copy hack + auto combined = keys[0]; + for (const auto& key : keys) { + for (auto&& [keyword, value] : key) { combined.set(keyword, value); } + } + return std::move(combined); +} + +} // namespace + +//---------------------------------------------------------------------------------------------------------------------- +// LIST ELEMENT + +ListElement::ListElement(Key key, const eckit::URI& uri, const TimeStamp timestamp): + key_ {std::move(key)}, keys_ {key_, Key {}, Key {}}, attributes_ {uri, timestamp} { } + +ListElement::ListElement(IndexKeys&& keys, const eckit::URI& uri, const TimeStamp timestamp): + key_ {combineKeys(keys)}, keys_ {std::move(keys[0]), std::move(keys[1]), Key {}}, attributes_ {uri, timestamp} { } + +ListElement::ListElement(DatumKeys&& keys, const FieldLocation& location, const TimeStamp timestamp): + key_ {combineKeys(keys)}, + keys_ {std::move(keys)}, + attributes_ {location.uri(), timestamp, location.offset(), location.length()}, + loc_ {location.make_shared()} { } + +ListElement::ListElement(const DatumKeys& keys, const FieldLocation& location, const TimeStamp timestamp): + key_ {combineKeys(keys)}, + keys_ {keys}, + attributes_ {location.uri(), timestamp, location.offset(), location.length()}, + loc_ {location.make_shared()} { } + +ListElement::ListElement(eckit::Stream& stream) { + for (auto& key : keys_) { stream >> key; } + stream >> key_; + stream >> attributes_; + loc_.reset(eckit::Reanimator::reanimate(stream)); +} + +auto ListElement::location() const -> const FieldLocation& { + ASSERT(loc_); + return *loc_; +} + +void ListElement::print(std::ostream& out, const bool location, const bool length, const bool timestamp, const char* sep) const { + if (location) { out << "host=" << (attributes_.uri.host().empty() ? "empty" : attributes_.uri.host()) << sep; } + for (const auto& key : keys_) { + if (!key.empty()) { out << key; } + } + if (length) { out << sep << "length=" << attributes_.length; } + if (timestamp) { out << sep << "timestamp=" << attributes_.timestamp; } + if (location && loc_) { out << sep << *loc_; } +} + +void ListElement::json(eckit::JSON& json) const { + json << key_.keyDict(); + json << "length" << attributes_.length; +} + +void ListElement::encode(eckit::Stream& stream) const { + stream << key_; + for (const auto& key : keys_) { stream << key; } + stream << attributes_; + if (loc_) { stream << *loc_; } +} + +//---------------------------------------------------------------------------------------------------------------------- +// friends + +std::ostream& operator<<(std::ostream& out, const ListElement& elem) { + elem.print(out, false, false, false, " "); + return out; +} + +eckit::Stream& operator<<(eckit::Stream& stream, const ListElement& elem) { + elem.encode(stream); + return stream; +} + +eckit::JSON& operator<<(eckit::JSON& json, const ListElement& elem) { + elem.json(json); + return json; +} + +void operator>>(eckit::Stream& stream, ListElement::Attributes& attrs) { + stream >> attrs.uri; + stream >> attrs.timestamp; + stream >> attrs.offset; + stream >> attrs.length; +} + +auto operator<<(eckit::Stream& stream, const ListElement::Attributes& attrs) -> eckit::Stream& { + stream << attrs.uri; + stream << attrs.timestamp; + stream << attrs.offset; + stream << attrs.length; + return stream; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/api/helpers/ListElement.h b/src/fdb5/api/helpers/ListElement.h new file mode 100644 index 000000000..681ebe554 --- /dev/null +++ b/src/fdb5/api/helpers/ListElement.h @@ -0,0 +1,110 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Simon Smart +/// @author Emanuele Danovaro +/// @author Metin Cakircali +/// @date October 2018 + +#pragma once + +#include "eckit/filesystem/URI.h" +#include "eckit/io/Length.h" +#include "eckit/io/Offset.h" +#include "fdb5/database/Key.h" + +#include +#include // std::size_t +#include // std::time_t +#include // std::ostream +#include + +namespace eckit { +class JSON; +class Stream; +} + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +class FieldLocation; + +template +using KeyChain = std::array; + +//---------------------------------------------------------------------------------------------------------------------- + +/// Define a standard object which can be used to iterate the results of a +/// list() call on an arbitrary FDB object + +class ListElement { +public: // types + using IndexKeys = KeyChain<2>; + using DatumKeys = KeyChain<3>; + using TimeStamp = std::time_t; + + struct Attributes { + eckit::URI uri; + TimeStamp timestamp {0}; + eckit::Offset offset {-1}; + eckit::Length length {-1}; + + friend void operator>>(eckit::Stream& stream, Attributes& attrs); + friend auto operator<<(eckit::Stream& stream, const Attributes& attrs) -> eckit::Stream&; + }; + +public: // methods + ListElement(Key key, const eckit::URI& uri, TimeStamp timestamp); + + ListElement(IndexKeys&& keys, const eckit::URI& uri, TimeStamp timestamp); + + ListElement(DatumKeys&& keys, const FieldLocation& location, TimeStamp timestamp); + + ListElement(const DatumKeys& keys, const FieldLocation& location, TimeStamp timestamp); + + explicit ListElement(eckit::Stream& stream); + + ListElement() = default; + + auto key() const -> const Key& { return key_; } + + auto keys() const -> const DatumKeys& { return keys_; } + + auto attributes() const -> const Attributes& { return attributes_; } + + auto location() const -> const FieldLocation&; + + void print(std::ostream& out, bool location, bool length, bool timestamp, const char* sep) const; + +private: // methods + void encode(eckit::Stream& stream) const; + + void json(eckit::JSON& json) const; + + friend std::ostream& operator<<(std::ostream& out, const ListElement& elem); + + friend eckit::Stream& operator<<(eckit::Stream& stream, const ListElement& elem); + + friend eckit::JSON& operator<<(eckit::JSON& json, const ListElement& elem); + +private: // members + Key key_; + + DatumKeys keys_; + + Attributes attributes_; + + std::shared_ptr loc_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/api/helpers/ListIterator.cc b/src/fdb5/api/helpers/ListIterator.cc deleted file mode 100644 index 811013899..000000000 --- a/src/fdb5/api/helpers/ListIterator.cc +++ /dev/null @@ -1,69 +0,0 @@ -/* - * (C) Copyright 1996- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - -/* - * This software was developed as part of the EC H2020 funded project NextGenIO - * (Project ID: 671951) www.nextgenio.eu - */ - -#include "fdb5/api/helpers/ListIterator.h" - -#include "eckit/log/JSON.h" - -namespace fdb5 { - -//---------------------------------------------------------------------------------------------------------------------- - -ListElement::ListElement(const std::vector& keyParts, std::shared_ptr location, time_t timestamp) : - keyParts_(keyParts), location_(location), timestamp_(timestamp) {} - -ListElement::ListElement(eckit::Stream &s) { - s >> keyParts_; - location_.reset(eckit::Reanimator::reanimate(s)); - s >> timestamp_; -} - -Key ListElement::combinedKey() const { - Key combined = keyParts_[2]; - - for (const Key& partKey : keyParts_) { - for (const auto& kv : partKey) { - combined.set(kv.first, kv.second); - } - } - return combined; -} - -void ListElement::print(std::ostream &out, bool withLocation, bool withLength, bool withTimestamp, const char* sep) const { - if (!withLocation && location_ && !location_->host().empty()) { - out << "host=" << location_->host() << ","; - } - for (const auto& bit : keyParts_) { - out << bit; - } - if (location_ && withLocation) out << sep << *location_; - if (withLength) out << sep << "length=" << location_->length(); - if (withTimestamp) out << sep << "timestamp=" << timestamp_; -} - -void ListElement::json(eckit::JSON& json) const { - json << combinedKey().keyDict(); - json << "length" << location_->length(); -} - -void ListElement::encode(eckit::Stream &s) const { - s << keyParts_; - s << *location_; - s << timestamp_; -} - -//---------------------------------------------------------------------------------------------------------------------- - -} // namespace fdb5 diff --git a/src/fdb5/api/helpers/ListIterator.h b/src/fdb5/api/helpers/ListIterator.h index 79788d0ef..59c1ccea9 100644 --- a/src/fdb5/api/helpers/ListIterator.h +++ b/src/fdb5/api/helpers/ListIterator.h @@ -19,75 +19,17 @@ #ifndef fdb5_ListIterator_H #define fdb5_ListIterator_H -#include #include -#include -#include -#include +#include #include "fdb5/database/Key.h" -#include "fdb5/database/FieldLocation.h" #include "fdb5/api/helpers/APIIterator.h" - -namespace eckit { - class Stream; - class JSON; -} +#include "fdb5/api/helpers/ListElement.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -/// Define a standard object which can be used to iterate the results of a -/// list() call on an arbitrary FDB object - -class ListElement { -public: // methods - - ListElement() = default; - ListElement(const std::vector& keyParts, std::shared_ptr location, time_t timestamp); - ListElement(eckit::Stream& s); - - const std::vector& key() const { return keyParts_; } - const FieldLocation& location() const { return *location_; } - const time_t& timestamp() const { return timestamp_; } - - Key combinedKey() const; - - void print(std::ostream& out, bool withLocation=false, bool withLength=false, bool withTimestamp=false, const char* sep = " ") const; - void json(eckit::JSON& json) const; - -private: // methods - - void encode(eckit::Stream& s) const; - - friend std::ostream& operator<<(std::ostream& os, const ListElement& e) { - e.print(os); - return os; - } - - friend eckit::Stream& operator<<(eckit::Stream& s, const ListElement& r) { - r.encode(s); - return s; - } - - friend eckit::JSON& operator<<(eckit::JSON& j, const ListElement& e) { - e.json(j); - return j; - } - -public: // members - - std::vector keyParts_; - -private: // members - - std::shared_ptr location_; - time_t timestamp_; -}; - -//---------------------------------------------------------------------------------------------------------------------- - using ListAggregateIterator = APIAggregateIterator; using ListAsyncIterator = APIAsyncIterator; @@ -113,16 +55,10 @@ class ListIterator : public APIIterator { ListElement tmp; while (APIIterator::next(tmp)) { if(deduplicate_) { - Key combinedKey = tmp.combinedKey(); - if (seenKeys_.find(combinedKey) == seenKeys_.end()) { - seenKeys_.emplace(std::move(combinedKey)); - std::swap(elem, tmp); - return true; - } - } else { - std::swap(elem, tmp); - return true; + if (const auto& [iter, success] = seenKeys_.insert(tmp.key()); !success) { continue; } } + std::swap(elem, tmp); + return true; } return false; } From fb4b8a086cf087adde58fe04068e1983ffa4d68c Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 5 Aug 2024 18:17:26 +0200 Subject: [PATCH 16/61] fix(ListVisitor): use discrete ListElement ctor for DB, Cat, and Datum --- src/fdb5/api/local/ListVisitor.h | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/fdb5/api/local/ListVisitor.h b/src/fdb5/api/local/ListVisitor.h index 9fc709ce5..50f56815a 100644 --- a/src/fdb5/api/local/ListVisitor.h +++ b/src/fdb5/api/local/ListVisitor.h @@ -19,11 +19,19 @@ #ifndef fdb5_api_local_ListVisitor_H #define fdb5_api_local_ListVisitor_H -#include "fdb5/database/DB.h" -#include "fdb5/database/Index.h" +#include "eckit/container/Queue.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "fdb5/api/helpers/ControlIterator.h" +#include "fdb5/api/helpers/ListElement.h" #include "fdb5/api/local/QueryVisitor.h" -#include "fdb5/api/helpers/ListIterator.h" -#include "eckit/os/BackTrace.h" +#include "fdb5/database/Catalogue.h" +#include "fdb5/database/EntryVisitMechanism.h" +#include "fdb5/database/Field.h" +#include "fdb5/database/Index.h" +#include "metkit/mars/MarsRequest.h" + +#include namespace fdb5::api::local { @@ -56,9 +64,7 @@ struct ListVisitor : public QueryVisitor { } if (level_ == 1) { - /// @todo fix this so that we don't need to create an empty key - const auto emptyKey = Key {currentCatalogue_->schema().registry()}; - queue_.emplace(ListElement({currentCatalogue_->key(), emptyKey, emptyKey}, FieldLocation::nullLocation(), 0)); + queue_.emplace(currentCatalogue_->key(), eckit::URI {}, 0); ret = false; } @@ -81,10 +87,7 @@ struct ListVisitor : public QueryVisitor { if (index.partialMatch(request_)) { if (level_ == 2) { - /// @todo fix this so that we don't need to create an empty key - const auto emptyKey = Key {currentCatalogue_->schema().registry()}; - const auto keyParts = std::vector {currentCatalogue_->key(), currentIndex_->key(), emptyKey}; - queue_.emplace(ListElement(keyParts, FieldLocation::nullLocation(), 0)); + queue_.emplace(KeyChain<2> {currentCatalogue_->key(), currentIndex_->key()}, eckit::URI {}, 0); return false; } return true; // Explore contained entries @@ -99,7 +102,9 @@ struct ListVisitor : public QueryVisitor { ASSERT(currentIndex_); if (key.match(datumRequest_)) { - queue_.emplace(ListElement({currentCatalogue_->key(), currentIndex_->key(), key}, field.stableLocation(), field.timestamp())); + queue_.emplace(KeyChain<3> {currentCatalogue_->key(), currentIndex_->key(), key}, + field.location(), + field.timestamp()); } } From 2c2b79dd94a7c0cdf66c96c34712f3616e14fb0b Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 5 Aug 2024 20:37:39 +0200 Subject: [PATCH 17/61] fix(FDB): missing include --- src/fdb5/api/FDB.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/fdb5/api/FDB.h b/src/fdb5/api/FDB.h index 32b1fdf16..f27c1e634 100644 --- a/src/fdb5/api/FDB.h +++ b/src/fdb5/api/FDB.h @@ -21,6 +21,7 @@ #include #include +#include #include "eckit/distributed/Transport.h" From 31edc16dc8522f33fe1298abc1e082c86c5e5cbe Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 5 Aug 2024 21:08:04 +0200 Subject: [PATCH 18/61] fix(FDB): name collisions --- src/fdb5/api/FDB.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/fdb5/api/FDB.cc b/src/fdb5/api/FDB.cc index 29bf72d4c..916073886 100644 --- a/src/fdb5/api/FDB.cc +++ b/src/fdb5/api/FDB.cc @@ -190,10 +190,10 @@ eckit::DataHandle* FDB::read(ListIterator& it, bool sorted) { } // checking all retrieved fields against the hypercube, to remove duplicates - ListElementDeduplicator dedup; - metkit::hypercube::HyperCubePayloaded cube(cubeRequest, dedup); - for(auto el: elements) { - cube.add(el.combinedKey().request(), el); + ListElementDeduplicator deduplicator; + metkit::hypercube::HyperCubePayloaded cube(cubeRequest, deduplicator); + for(auto elem: elements) { + cube.add(elem.combinedKey().request(), el); } if (cube.countVacant() > 0) { From 376d8422bef22e72aabdbae0de9c503591a9079b Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 5 Aug 2024 21:10:59 +0200 Subject: [PATCH 19/61] fix(FDB): unnecessary copy in loop --- src/fdb5/api/FDB.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fdb5/api/FDB.cc b/src/fdb5/api/FDB.cc index 916073886..cd5e6c38f 100644 --- a/src/fdb5/api/FDB.cc +++ b/src/fdb5/api/FDB.cc @@ -192,7 +192,7 @@ eckit::DataHandle* FDB::read(ListIterator& it, bool sorted) { // checking all retrieved fields against the hypercube, to remove duplicates ListElementDeduplicator deduplicator; metkit::hypercube::HyperCubePayloaded cube(cubeRequest, deduplicator); - for(auto elem: elements) { + for(const auto& elem: elements) { cube.add(elem.combinedKey().request(), el); } From 4a81dadb71c7875dcb61f3639f52bb19e2337e7e Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 5 Aug 2024 21:14:19 +0200 Subject: [PATCH 20/61] fix(FDB): after new ListElement --- src/fdb5/api/FDB.cc | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/fdb5/api/FDB.cc b/src/fdb5/api/FDB.cc index cd5e6c38f..4b0bbe618 100644 --- a/src/fdb5/api/FDB.cc +++ b/src/fdb5/api/FDB.cc @@ -149,7 +149,7 @@ bool FDB::sorted(const metkit::mars::MarsRequest &request) { class ListElementDeduplicator : public metkit::hypercube::Deduplicator { public: bool toReplace(const ListElement& existing, const ListElement& replacement) const override { - return existing.timestamp() < replacement.timestamp(); + return existing.attributes().timestamp < replacement.attributes().timestamp; } }; @@ -181,20 +181,18 @@ eckit::DataHandle* FDB::read(ListIterator& it, bool sorted) { if (dedup) { if (it.next(el)) { // build the request representing the tensor-product of all retrieved fields - metkit::mars::MarsRequest cubeRequest = el.combinedKey().request(); - std::vector elements{el}; + metkit::mars::MarsRequest cubeRequest = el.key().request(); + std::vector elements {el}; while (it.next(el)) { - cubeRequest.merge(el.combinedKey().request()); + cubeRequest.merge(el.key().request()); elements.push_back(el); } // checking all retrieved fields against the hypercube, to remove duplicates ListElementDeduplicator deduplicator; metkit::hypercube::HyperCubePayloaded cube(cubeRequest, deduplicator); - for(const auto& elem: elements) { - cube.add(elem.combinedKey().request(), el); - } + for (const auto& elem : elements) { cube.add(elem.key().request(), el); } if (cube.countVacant() > 0) { std::stringstream ss; @@ -205,7 +203,7 @@ eckit::DataHandle* FDB::read(ListIterator& it, bool sorted) { eckit::Log::warning() << ss.str() << std::endl; } - for (size_t i=0; i< cube.size(); i++) { + for (std::size_t i = 0; i < cube.size(); i++) { ListElement element; if (cube.find(i, element)) { result.add(element.location().dataHandle()); From e5edc6c4239dff9534044599b524410f3f2e82d7 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 5 Aug 2024 21:15:16 +0200 Subject: [PATCH 21/61] fix(FDB): 'some' include issues cleanup and add missing --- src/fdb5/api/FDB.cc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/fdb5/api/FDB.cc b/src/fdb5/api/FDB.cc index 4b0bbe618..1ff79bebe 100644 --- a/src/fdb5/api/FDB.cc +++ b/src/fdb5/api/FDB.cc @@ -13,22 +13,30 @@ * (Project ID: 671951) www.nextgenio.eu */ +#include +#include +#include +#include + #include "eckit/config/Resource.h" +#include "eckit/exception/Exceptions.h" #include "eckit/io/DataHandle.h" #include "eckit/io/MemoryHandle.h" #include "eckit/log/Log.h" #include "eckit/message/Message.h" #include "eckit/message/Reader.h" -#include "eckit/system/Plugin.h" #include "eckit/system/LibraryManager.h" +#include "metkit/hypercube/HyperCube.h" #include "metkit/hypercube/HyperCubePayloaded.h" #include "fdb5/LibFdb5.h" #include "fdb5/api/FDB.h" #include "fdb5/api/FDBFactory.h" #include "fdb5/api/helpers/FDBToolRequest.h" +#include "fdb5/api/helpers/ListIterator.h" +#include "fdb5/database/FieldLocation.h" #include "fdb5/database/Key.h" #include "fdb5/io/HandleGatherer.h" #include "fdb5/message/MessageDecoder.h" From 81a39241cb1e25f942e574a4f24616874d7db2ca Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 5 Aug 2024 21:18:46 +0200 Subject: [PATCH 22/61] chore(c api): param should be sha1 --- src/fdb5/api/fdb_c.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fdb5/api/fdb_c.h b/src/fdb5/api/fdb_c.h index e3371c61a..d3404c4f6 100644 --- a/src/fdb5/api/fdb_c.h +++ b/src/fdb5/api/fdb_c.h @@ -50,7 +50,7 @@ int fdb_version(const char** version); * \param version Return variable for version control checksum. Returned pointer valid throughout program lifetime. * \returns Return code (#FdbErrorValues) */ -int fdb_vcs_version(const char** version); +int fdb_vcs_version(const char** sha1); ///@} From 6d67ae63fbbab0392a1230df12e5b25fc3c4ed5c Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 5 Aug 2024 21:30:59 +0200 Subject: [PATCH 23/61] fix(c api): splitkey iterator reworked it to behave more like an iterator --- src/fdb5/api/fdb_c.cc | 94 +++++++++++++++++++------------------ src/fdb5/api/fdb_c.h | 9 +++- tests/fdb/api/test_fdb_c.cc | 59 ++++++++++++----------- 3 files changed, 86 insertions(+), 76 deletions(-) diff --git a/src/fdb5/api/fdb_c.cc b/src/fdb5/api/fdb_c.cc index 79aa7fc84..be20f9362 100644 --- a/src/fdb5/api/fdb_c.cc +++ b/src/fdb5/api/fdb_c.cc @@ -8,8 +8,9 @@ * does it submit to any jurisdiction. */ +#include + #include "eckit/io/MemoryHandle.h" -#include "eckit/io/FileDescHandle.h" #include "eckit/message/Message.h" #include "eckit/runtime/Main.h" @@ -20,6 +21,7 @@ #include "fdb5/fdb5_version.h" #include "fdb5/api/FDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" +#include "fdb5/api/helpers/ListElement.h" #include "fdb5/api/helpers/ListIterator.h" #include "fdb5/database/Key.h" @@ -88,50 +90,44 @@ struct fdb_request_t { }; struct fdb_split_key_t { -public: - fdb_split_key_t() : key_(nullptr), level_(-1) {} + using value_type = typename ListElement::DatumKeys; - void set(const std::vector& key) { - key_ = &key; - level_ = -1; + auto operator=(const value_type& keys) -> fdb_split_key_t& { + keys_ = &keys; + level_ = keys_->begin(); + curr_ = level_->begin(); + return *this; } - int next_metadata(const char** k, const char** v, size_t* level) { - if (key_ == nullptr) { - std::stringstream ss; - ss << "fdb_split_key_t not valid. Key not configured"; - throw eckit::UserError(ss.str(), Here()); - } - if (level_ == -1) { - if (0 < key_->size()) { - level_ = 0; - it_ = key_->at(0).begin(); - } else { - return FDB_ITERATION_COMPLETE; - } - } - while (it_ == key_->at(level_).end()) { - if (level_size()-1) { - level_++; - it_ = key_->at(level_).begin(); - } else { - return FDB_ITERATION_COMPLETE; - } + auto operator++() -> fdb_split_key_t& { + if (curr_ != level_->end()) { + ++curr_; + if (curr_ == level_->end() && level_ != keys_->end() - 1) { curr_ = (++level_)->begin(); } } + return *this; + } + + int metadata(const char** k, const char** v, size_t* level) const { + ASSERT_MSG(keys_, "keys are missing!"); + + if (curr_ == level_->end()) { return FDB_ITERATION_COMPLETE; } + + const auto& [key, val] = *curr_; + + *k = key.c_str(); + *v = val.c_str(); + + if (level) { *level = std::find(keys_->begin(), keys_->end(), *level_) - keys_->begin(); } - *k = it_->first.c_str(); - *v = it_->second.c_str(); - if (level != nullptr) { - *level = level_; - } - it_++; return FDB_SUCCESS; } -private: - const std::vector* key_; - int level_; - Key::const_iterator it_; +private: // members + const value_type* keys_ {nullptr}; + + value_type::const_iterator level_; + + value_type::value_type::const_iterator curr_; }; struct fdb_listiterator_t { @@ -147,17 +143,18 @@ struct fdb_listiterator_t { void attrs(const char** uri, size_t* off, size_t* len) { ASSERT(validEl_); - const FieldLocation& loc = el_.location(); - *uri = loc.uri().name().c_str(); - *off = loc.offset(); - *len = loc.length(); + const auto& attrs = el_.attributes(); + + *uri = attrs.uri.name().c_str(); + *off = attrs.offset; + *len = attrs.length; } void key(fdb_split_key_t* key) { ASSERT(validEl_); ASSERT(key); - key->set(el_.key()); + *key = el_.keys(); } private: @@ -197,7 +194,7 @@ struct fdb_datareader_t { delete dh_; dh_ = dh; } - + private: DataHandle* dh_; }; @@ -481,12 +478,19 @@ int fdb_new_splitkey(fdb_split_key_t** key) { }); } -int fdb_splitkey_next_metadata(fdb_split_key_t* it, const char** key, const char** value, size_t* level) { +int fdb_splitkey_next(fdb_split_key_t* it) { + return wrapApiFunction([it] { + ASSERT(it); + ++(*it); + }); +} + +int fdb_splitkey_metadata(fdb_split_key_t* it, const char** key, const char** value, size_t* level) { return wrapApiFunction(std::function {[it, key, value, level] { ASSERT(it); ASSERT(key); ASSERT(value); - return it->next_metadata(key, value, level); + return it->metadata(key, value, level); }}); } int fdb_delete_splitkey(fdb_split_key_t* key) { diff --git a/src/fdb5/api/fdb_c.h b/src/fdb5/api/fdb_c.h index d3404c4f6..6ac37cf5a 100644 --- a/src/fdb5/api/fdb_c.h +++ b/src/fdb5/api/fdb_c.h @@ -176,6 +176,13 @@ typedef struct fdb_split_key_t fdb_split_key_t; */ int fdb_new_splitkey(fdb_split_key_t** key); +/** Returns the next set of metadata in a SplitKey object. For a given ListElement, the SplitKey represents the Keys + * associated with each level of the FDB Key. + * \param it SplitKey instance + * \returns Return code (#FdbErrorValues) + */ +int fdb_splitkey_next(fdb_split_key_t* it); + /** Returns the next set of metadata in a SplitKey object. For a given ListElement, the SplitKey represents the Keys associated with each level of the FDB index. * Supports multiple fdb_split_key_t iterating over the same key. * \param it SplitKey instance @@ -184,7 +191,7 @@ int fdb_new_splitkey(fdb_split_key_t** key); * \param level level in the iondex of the current Key * \returns Return code (#FdbErrorValues) */ -int fdb_splitkey_next_metadata(fdb_split_key_t* it, const char** key, const char** value, size_t* level); +int fdb_splitkey_metadata(fdb_split_key_t* it, const char** key, const char** value, size_t* level); /** Deallocates SplitKey object and associated resources. * \param key SplitKey instance diff --git a/tests/fdb/api/test_fdb_c.cc b/tests/fdb/api/test_fdb_c.cc index 57c1f486c..cbb0d0568 100644 --- a/tests/fdb/api/test_fdb_c.cc +++ b/tests/fdb/api/test_fdb_c.cc @@ -40,25 +40,24 @@ void key_compare(const std::vector& keys, fdb_listiterator_t *it, boo fdb_split_key_t* sk = nullptr; fdb_new_splitkey(&sk); - err = fdb_listiterator_splitkey(it, sk); - EXPECT(err == FDB_SUCCESS); + EXPECT_EQUAL(fdb_listiterator_splitkey(it, sk), FDB_SUCCESS); size_t level = 0; - for (auto key: keys) { + for (const auto& key : keys) { for (auto k1: key) { - int err = fdb_splitkey_next_metadata(sk, &k, &v, checkLevel ? &l : nullptr); - EXPECT(err == FDB_SUCCESS); - EXPECT(k1.first == k); - EXPECT(k1.second == v); - if (checkLevel) { - EXPECT(level == l); - } + err = fdb_splitkey_metadata(sk, &k, &v, checkLevel ? &l : nullptr); + EXPECT_EQUAL(err, FDB_SUCCESS); + EXPECT_EQUAL(k1.first, k); + EXPECT_EQUAL(k1.second, v); + if (checkLevel) { EXPECT_EQUAL(level, l); } + fdb_splitkey_next(sk); } level++; } - err = fdb_splitkey_next_metadata(sk, &k, &v, &l); - EXPECT(err == FDB_ITERATION_COMPLETE); - + + fdb_splitkey_next(sk); + EXPECT_EQUAL(fdb_splitkey_metadata(sk, &k, &v, &l), FDB_ITERATION_COMPLETE); + err = fdb_delete_splitkey(sk); } @@ -113,7 +112,7 @@ CASE( "fdb_c - archive & list" ) { fdb_list(fdb, request, &it, true); int err = fdb_listiterator_next(it); ASSERT(err == FDB_SUCCESS); - + const char *uri; size_t off, attr_len; @@ -164,7 +163,7 @@ CASE( "fdb_c - archive & list" ) { fdb_list(fdb, request, &it, true); err = fdb_listiterator_next(it); ASSERT(err == FDB_SUCCESS); - + fdb_listiterator_attrs(it, &uri, &off, &attr_len); EXPECT(attr_len == 3280398); @@ -270,21 +269,21 @@ CASE( "fdb_c - multiple archive & list" ) { fdb_request_add1(req, "type", "an"); fdb_request_add1(req, "expver", "xxxx"); - EXPECT(FDB_ERROR_GENERAL_EXCEPTION == fdb_archive_multiple(fdb, req, buf, length1)); - EXPECT(FDB_SUCCESS == fdb_flush(fdb)); + EXPECT_EQUAL(FDB_ERROR_GENERAL_EXCEPTION, fdb_archive_multiple(fdb, req, buf, length1)); + EXPECT_EQUAL(FDB_SUCCESS, fdb_flush(fdb)); - EXPECT(FDB_SUCCESS == fdb_archive_multiple(fdb, req, buf, length1+length2)); - EXPECT(FDB_SUCCESS == fdb_flush(fdb)); + EXPECT_EQUAL(FDB_SUCCESS, fdb_archive_multiple(fdb, req, buf, length1 + length2)); + EXPECT_EQUAL(FDB_SUCCESS, fdb_flush(fdb)); fdb_request_add1(req, "levelist", "300"); - EXPECT(FDB_ERROR_GENERAL_EXCEPTION == fdb_archive_multiple(fdb, req, buf, length1+length2)); - EXPECT(FDB_SUCCESS == fdb_flush(fdb)); + EXPECT_EQUAL(FDB_ERROR_GENERAL_EXCEPTION, fdb_archive_multiple(fdb, req, buf, length1 + length2)); + EXPECT_EQUAL(FDB_SUCCESS, fdb_flush(fdb)); fdb_request_add(req, "levelist", levels, 2); - EXPECT(FDB_SUCCESS == fdb_archive_multiple(fdb, req, buf, length1+length2)); - EXPECT(FDB_SUCCESS == fdb_flush(fdb)); + EXPECT_EQUAL(FDB_SUCCESS, fdb_archive_multiple(fdb, req, buf, length1 + length2)); + EXPECT_EQUAL(FDB_SUCCESS, fdb_flush(fdb)); dh = grib3.fileHandle(); dh->openForRead(); @@ -297,11 +296,11 @@ CASE( "fdb_c - multiple archive & list" ) { const char* levels3[] = {"300", "400", "500"}; fdb_request_add(req, "levelist", levels3, 3); - EXPECT(FDB_ERROR_GENERAL_EXCEPTION == fdb_archive_multiple(fdb, req, buf, length1+length2+length3)); - EXPECT(FDB_SUCCESS == fdb_flush(fdb)); + EXPECT_EQUAL(FDB_ERROR_GENERAL_EXCEPTION, fdb_archive_multiple(fdb, req, buf, length1 + length2 + length3)); + EXPECT_EQUAL(FDB_SUCCESS, fdb_flush(fdb)); - EXPECT(FDB_SUCCESS == fdb_archive_multiple(fdb, nullptr, buf, length1+length2+length3)); - EXPECT(FDB_SUCCESS == fdb_flush(fdb)); + EXPECT_EQUAL(FDB_SUCCESS, fdb_archive_multiple(fdb, nullptr, buf, length1 + length2 + length3)); + EXPECT_EQUAL(FDB_SUCCESS, fdb_flush(fdb)); fdb_request_t* request; fdb_new_request(&request); @@ -342,12 +341,12 @@ CASE( "fdb_c - multiple archive & list" ) { fdb_list(fdb, request, &it, true); err = fdb_listiterator_next(it); ASSERT(err == FDB_SUCCESS); - + key_compare(k1, it); err = fdb_listiterator_next(it); ASSERT(err == FDB_SUCCESS); - + key_compare(k2, it); err = fdb_listiterator_next(it); @@ -480,7 +479,7 @@ CASE( "fdb_c - expand" ) { size_t numValues; char** values; - + fdb_request_get(request, "date", &values, &numValues); EXPECT_EQUAL(numValues, 2); EXPECT_EQUAL(0, strncmp(values[0], "20191110", 8)); From b68231bb5ea3113af7dffb831d5ada30592debfd Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 5 Aug 2024 21:31:46 +0200 Subject: [PATCH 24/61] fix(fdb-patch): after ListElement --- src/fdb5/tools/fdb-patch.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fdb5/tools/fdb-patch.cc b/src/fdb5/tools/fdb-patch.cc index ad73af583..3a99320f3 100644 --- a/src/fdb5/tools/fdb-patch.cc +++ b/src/fdb5/tools/fdb-patch.cc @@ -144,7 +144,7 @@ void FDBPatch::execute(const CmdArgs& args) { // (n.b. listed key is broken down as-per the schema) Key key; - for (const Key& k : elem.keyParts_) { + for (const Key& k : elem.keys()) { for (const auto& kv : k) { key.set(kv.first, kv.second); } From a0cb5fa029d16c5442f51073bee62559fcdcefd0 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 5 Aug 2024 21:37:40 +0200 Subject: [PATCH 25/61] fix(fdb-list): includes --- src/fdb5/tools/fdb-list.cc | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/fdb5/tools/fdb-list.cc b/src/fdb5/tools/fdb-list.cc index badc5c46f..3d2ef13a0 100644 --- a/src/fdb5/tools/fdb-list.cc +++ b/src/fdb5/tools/fdb-list.cc @@ -8,21 +8,26 @@ * does it submit to any jurisdiction. */ +#include +#include +#include #include +#include +#include +#include "eckit/exception/Exceptions.h" +#include "eckit/log/JSON.h" +#include "eckit/log/Log.h" #include "eckit/option/CmdArgs.h" -#include "eckit/config/Resource.h" #include "eckit/option/SimpleOption.h" -#include "eckit/option/VectorOption.h" -#include "eckit/option/CmdArgs.h" -#include "eckit/log/JSON.h" #include "metkit/hypercube/HyperCube.h" +#include "metkit/mars/MarsRequest.h" #include "fdb5/api/FDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" -#include "fdb5/database/DB.h" -#include "fdb5/database/Index.h" +#include "fdb5/api/helpers/ListElement.h" +#include "fdb5/api/helpers/ListIterator.h" #include "fdb5/rules/Schema.h" #include "fdb5/tools/FDBVisitTool.h" From dbf974e3f4a1e09f5f39f68b7f51f86b522d8a4b Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 5 Aug 2024 21:38:11 +0200 Subject: [PATCH 26/61] feat(fdb-list): options; assert depth and porcelain text --- src/fdb5/tools/fdb-list.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/fdb5/tools/fdb-list.cc b/src/fdb5/tools/fdb-list.cc index 3d2ef13a0..27543ce20 100644 --- a/src/fdb5/tools/fdb-list.cc +++ b/src/fdb5/tools/fdb-list.cc @@ -50,7 +50,7 @@ class FDBList: public FDBVisitTool { "Useful as input for other tools or scripts")); options_.push_back(new SimpleOption("json", "Output available fields in JSON form")); options_.push_back(new SimpleOption("compact", "Aggregate available fields in MARS requests")); - options_.push_back(new SimpleOption("depth", "Output entries up to 'depth' levels deep")); + options_.push_back(new SimpleOption("depth", "Output entries up to 'depth' levels deep [0-3]")); } private: // methods @@ -93,6 +93,8 @@ void FDBList::init(const CmdArgs& args) { compact_ = args.getBool("compact", false); depth_ = args.getInt("depth", 3); + ASSERT(depth_ > 0 && depth_ <= 3); + if (json_) { porcelain_ = true; } if (porcelain_) { From 4595fa794e80e48a9771bbc8865d17b5bad78657 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 5 Aug 2024 21:39:31 +0200 Subject: [PATCH 27/61] fix(fdb-list): keys api --- src/fdb5/tools/fdb-list.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fdb5/tools/fdb-list.cc b/src/fdb5/tools/fdb-list.cc index 27543ce20..ee0436e46 100644 --- a/src/fdb5/tools/fdb-list.cc +++ b/src/fdb5/tools/fdb-list.cc @@ -138,7 +138,7 @@ void FDBList::execute(const CmdArgs& args) { while (listObject.next(elem)) { if (compact_) { - std::vector keys = elem.key(); + const auto& keys = elem.keys(); ASSERT(keys.size() == 3); std::string treeAxes = keys[0]; From 7c428892fc8a7834d954f30fb87454d03a588f20 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 5 Aug 2024 21:40:39 +0200 Subject: [PATCH 28/61] fix(FieldLocation): remove nullLocation --- src/fdb5/database/FieldLocation.cc | 26 -------------------------- src/fdb5/database/FieldLocation.h | 5 +---- 2 files changed, 1 insertion(+), 30 deletions(-) diff --git a/src/fdb5/database/FieldLocation.cc b/src/fdb5/database/FieldLocation.cc index e7d5605e0..9aaaeb79a 100644 --- a/src/fdb5/database/FieldLocation.cc +++ b/src/fdb5/database/FieldLocation.cc @@ -107,32 +107,6 @@ FieldLocationBuilderBase::~FieldLocationBuilderBase() { //---------------------------------------------------------------------------------------------------------------------- -namespace { - -struct NullFieldLocation: public FieldLocation { - eckit::DataHandle* dataHandle() const override { NOTIMP; } - - std::shared_ptr make_shared() const override; - - void visit(FieldLocationVisitor&) const override { NOTIMP; } - - void print(std::ostream& out) const override { out << "NullFieldLocation"; } -}; - -static auto emptyLocation = std::make_shared(); - -std::shared_ptr NullFieldLocation::make_shared() const { - return emptyLocation; -} - -} // namespace - -//---------------------------------------------------------------------------------------------------------------------- - -std::shared_ptr FieldLocation::nullLocation() { - return emptyLocation; -} - FieldLocation::FieldLocation(const eckit::URI& uri) : uri_(uri) { try { offset_ = eckit::Offset(std::stoll(uri.fragment())); diff --git a/src/fdb5/database/FieldLocation.h b/src/fdb5/database/FieldLocation.h index 847df2b09..34a6d509d 100644 --- a/src/fdb5/database/FieldLocation.h +++ b/src/fdb5/database/FieldLocation.h @@ -36,10 +36,7 @@ namespace fdb5 { class FieldLocationVisitor; -class FieldLocation : public eckit::OwnedLock, public eckit::Streamable { -public: // factory - static std::shared_ptr nullLocation(); - +class FieldLocation: public eckit::OwnedLock, public eckit::Streamable { public: // methods FieldLocation() : offset_(eckit::Offset(0)), length_(eckit::Length(0)), remapKey_(Key()) {} From f41ca4dc044b3e0bcf5516f3f5b3c2300cb072ca Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 5 Aug 2024 21:45:38 +0200 Subject: [PATCH 29/61] fix(MultiRetrieveVisitor): include issues --- src/fdb5/database/MultiRetrieveVisitor.cc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/fdb5/database/MultiRetrieveVisitor.cc b/src/fdb5/database/MultiRetrieveVisitor.cc index cfed8c3a3..6c78a9f36 100644 --- a/src/fdb5/database/MultiRetrieveVisitor.cc +++ b/src/fdb5/database/MultiRetrieveVisitor.cc @@ -11,18 +11,19 @@ #include "fdb5/database/MultiRetrieveVisitor.h" #include +#include +#include +#include -#include "eckit/config/Resource.h" +#include "eckit/log/Log.h" #include "fdb5/LibFdb5.h" +#include "fdb5/api/helpers/ListElement.h" #include "fdb5/database/DB.h" #include "fdb5/database/Key.h" -#include "fdb5/io/HandleGatherer.h" #include "fdb5/types/Type.h" #include "fdb5/types/TypesRegistry.h" - - namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- @@ -108,7 +109,7 @@ bool MultiRetrieveVisitor::selectDatum(const Key& key, const Key& full) { simplifiedKey.set(k->first, k->second); } - iterator_.emplace(ListElement({db_->key(), db_->indexKey(), simplifiedKey}, field.stableLocation(), field.timestamp())); + iterator_.emplace({KeyChain<3> {db_->key(), db_->indexKey(), simplifiedKey}, field.location(), field.timestamp()}); return true; } From 70768b984b42525dde4e75368076fd5ff30b3d3a Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 5 Aug 2024 21:46:15 +0200 Subject: [PATCH 30/61] feat(remote): added level to Handler --- src/fdb5/remote/Handler.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/fdb5/remote/Handler.cc b/src/fdb5/remote/Handler.cc index ebb34361f..3083bbcba 100644 --- a/src/fdb5/remote/Handler.cc +++ b/src/fdb5/remote/Handler.cc @@ -88,12 +88,12 @@ struct BaseHelper { }; struct ListHelper : public BaseHelper { - // void extraDecode(eckit::Stream& s) { s >> level_; } + void extraDecode(eckit::Stream& s) { s >> level_; } - ListIterator apiCall(FDB& fdb, const FDBToolRequest& request) const { - /// @todo return fdb.list(request, level_); - return fdb.list(request); - } + ListIterator apiCall(FDB& fdb, const FDBToolRequest& request) const { return fdb.list(request, false, level_); } + +private: + int level_ {3}; }; struct InspectHelper : public BaseHelper { From d4ebfa98becfe5628ab139573b24cfc1a58eed9b Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 5 Aug 2024 21:51:31 +0200 Subject: [PATCH 31/61] fix(RemoteFDB): fix include issues --- src/fdb5/api/RemoteFDB.cc | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc index 3f5359da1..c4bce19a7 100644 --- a/src/fdb5/api/RemoteFDB.cc +++ b/src/fdb5/api/RemoteFDB.cc @@ -13,28 +13,33 @@ * (Project ID: 671951) www.nextgenio.eu */ -#include #include +#include +#include +#include +#include #include "fdb5/api/RemoteFDB.h" #include "fdb5/LibFdb5.h" +#include "fdb5/api/helpers/ListElement.h" #include "fdb5/io/HandleGatherer.h" #include "fdb5/remote/Messages.h" #include "fdb5/remote/RemoteFieldLocation.h" #include "fdb5/api/helpers/FDBToolRequest.h" #include "fdb5/database/Key.h" +#include "eckit/config/Configuration.h" #include "eckit/config/LocalConfiguration.h" +#include "eckit/config/Resource.h" +#include "eckit/exception/Exceptions.h" #include "eckit/io/Buffer.h" #include "eckit/log/Bytes.h" #include "eckit/log/Log.h" #include "eckit/message/Message.h" -#include "eckit/distributed/Transport.h" -#include "eckit/config/Resource.h" +#include "eckit/os/BackTrace.h" +#include "eckit/runtime/Main.h" #include "eckit/serialisation/MemoryStream.h" #include "eckit/utils/Translator.h" -#include "eckit/runtime/Main.h" -#include "eckit/os/BackTrace.h" #include "metkit/mars/MarsRequest.h" From 7ff9f399aec7456849b3450fd979ca7603e412f2 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 5 Aug 2024 21:51:43 +0200 Subject: [PATCH 32/61] fix(RemoteFDB): fix ListElement --- src/fdb5/api/RemoteFDB.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc index c4bce19a7..82decff94 100644 --- a/src/fdb5/api/RemoteFDB.cc +++ b/src/fdb5/api/RemoteFDB.cc @@ -567,7 +567,7 @@ struct InspectHelper : BaseAPIHelper Date: Mon, 5 Aug 2024 21:53:07 +0200 Subject: [PATCH 33/61] fix(LocalFDB): cleanup include --- src/fdb5/api/LocalFDB.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/fdb5/api/LocalFDB.cc b/src/fdb5/api/LocalFDB.cc index 11897ea72..8de5ef1f0 100644 --- a/src/fdb5/api/LocalFDB.cc +++ b/src/fdb5/api/LocalFDB.cc @@ -15,7 +15,6 @@ #include "eckit/container/Queue.h" #include "eckit/log/Log.h" -#include "eckit/message/Message.h" #include "fdb5/api/helpers/ListIterator.h" #include "fdb5/api/helpers/FDBToolRequest.h" @@ -34,7 +33,6 @@ #include "fdb5/api/local/DumpVisitor.h" #include "fdb5/api/local/ListVisitor.h" #include "fdb5/api/local/PurgeVisitor.h" -#include "fdb5/api/local/QueryVisitor.h" #include "fdb5/api/local/StatsVisitor.h" #include "fdb5/api/local/StatusVisitor.h" #include "fdb5/api/local/WipeVisitor.h" From 7cc6f334d205f2fbfbbbc029adeda805ef93deca Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 5 Aug 2024 21:54:05 +0200 Subject: [PATCH 34/61] test(service): fix after ListElement with levels --- tests/fdb/test_fdb5_service.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/fdb/test_fdb5_service.cc b/tests/fdb/test_fdb5_service.cc index 7c23287a4..66bc7ceab 100644 --- a/tests/fdb/test_fdb5_service.cc +++ b/tests/fdb/test_fdb5_service.cc @@ -134,7 +134,7 @@ CASE ( "test_fdb_stepunit_archive" ) { fdb5::ListIterator iter = fdb.list(r, true); fdb5::ListElement el; EXPECT(iter.next(el)); - EXPECT(el.combinedKey().get("step") == "60"); + EXPECT(el.key().get("step") == "60"); EXPECT(!iter.next(el)); } @@ -149,7 +149,7 @@ CASE ( "test_fdb_stepunit_archive" ) { fdb5::ListIterator iter = fdb.list(r, true); fdb5::ListElement el; EXPECT(iter.next(el)); - EXPECT(el.combinedKey().get("step") == "2"); + EXPECT(el.key().get("step") == "2"); EXPECT(!iter.next(el)); } @@ -159,9 +159,9 @@ CASE ( "test_fdb_stepunit_archive" ) { fdb5::ListIterator iter = fdb.list(r, true); fdb5::ListElement el; EXPECT(iter.next(el)); - EXPECT(el.combinedKey().get("step") == "2"); + EXPECT(el.key().get("step") == "2"); EXPECT(iter.next(el)); - EXPECT(el.combinedKey().get("step") == "60"); + EXPECT(el.key().get("step") == "60"); EXPECT(!iter.next(el)); } @@ -177,7 +177,7 @@ CASE ( "test_fdb_stepunit_archive" ) { fdb5::ListIterator iter = fdb.list(r, true); fdb5::ListElement el; EXPECT(iter.next(el)); - EXPECT(el.combinedKey().get("step") == "30m"); + EXPECT(el.key().get("step") == "30m"); EXPECT(!iter.next(el)); } @@ -189,9 +189,9 @@ CASE ( "test_fdb_stepunit_archive" ) { fdb5::ListIterator iter = fdb.list(r, true); fdb5::ListElement el; EXPECT(iter.next(el)); - EXPECT(el.combinedKey().get("step") == "30m"); + EXPECT(el.key().get("step") == "30m"); EXPECT(iter.next(el)); - EXPECT(el.combinedKey().get("step") == "2"); + EXPECT(el.key().get("step") == "2"); EXPECT(!iter.next(el)); }*/ } @@ -306,7 +306,7 @@ CASE ( "test_fdb_service" ) { fdb5::ListElement el; EXPECT(iter.next(el)); - eckit::PathName path = el.location().uri().path().dirName(); + eckit::PathName path = el.attributes().uri.path().dirName(); DIR* dirp = ::opendir(path.asString().c_str()); struct dirent* dp; @@ -489,7 +489,7 @@ CASE ( "test_fdb_service_subtoc" ) { fdb5::ListElement el; EXPECT(iter.next(el)); - eckit::PathName path = el.location().uri().path().dirName(); + eckit::PathName path = el.attributes().uri.path().dirName(); DIR* dirp = ::opendir(path.asString().c_str()); struct dirent* dp; From e085f57dce6b2effd5c03ca1d84485750c326e6b Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 5 Aug 2024 21:54:55 +0200 Subject: [PATCH 35/61] test(fdb-list): fix after new output format --- tests/fdb/tools/fdb_list.sh.in | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/fdb/tools/fdb_list.sh.in b/tests/fdb/tools/fdb_list.sh.in index 14034cfab..661518b85 100755 --- a/tests/fdb/tools/fdb_list.sh.in +++ b/tests/fdb/tools/fdb_list.sh.in @@ -50,7 +50,7 @@ try fdb-write "$tname.grib" ######################################################################################################################## # test all -out=$(fdb-list --all --minimum-keys="" --porcelain) +out=$(fdb-list --all --minimum-keys= --porcelain) exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=0,param=166} {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=12,param=166} {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=6,param=166} @@ -63,23 +63,23 @@ try test "$exp" = "$out" # test date depth=1 out=$(fdb-list date=20201102 --minimum-keys="" --porcelain --depth=1) -exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{}{}" +exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}" try test "$exp" = "$out" # test date depth=2 out=$(fdb-list date=20201102 --minimum-keys="" --porcelain --depth=2) -exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{} -{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{}" +exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}" try test "$exp" = "$out" # test cf depth=1 out=$(fdb-list type=cf --minimum-keys="" --porcelain --depth=1) -exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{}{}" +exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}" try test "$exp" = "$out" # test cf depth=2 out=$(fdb-list type=cf --minimum-keys="" --porcelain --depth=2) -exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{}" +exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}" try test "$exp" = "$out" # test cf depth=3 From bc0d333b14c5112c988aa7582790b81c291ee50f Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 6 Aug 2024 11:05:56 +0200 Subject: [PATCH 36/61] fix(fdb-list): update porcelain text --- src/fdb5/tools/fdb-list.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fdb5/tools/fdb-list.cc b/src/fdb5/tools/fdb-list.cc index ee0436e46..a964e4b78 100644 --- a/src/fdb5/tools/fdb-list.cc +++ b/src/fdb5/tools/fdb-list.cc @@ -46,8 +46,8 @@ class FDBList: public FDBVisitTool { options_.push_back(new SimpleOption("length", "Also print the field size")); options_.push_back(new SimpleOption("full", "Include all entries (including masked duplicates)")); options_.push_back(new SimpleOption("porcelain", - "Streamlined and stable output. " - "Useful as input for other tools or scripts")); + "Streamlined and stable output. Useful as input for other tools or scripts." + "Equivalent to compact=false, location=false, timestamp=false, length=false")); options_.push_back(new SimpleOption("json", "Output available fields in JSON form")); options_.push_back(new SimpleOption("compact", "Aggregate available fields in MARS requests")); options_.push_back(new SimpleOption("depth", "Output entries up to 'depth' levels deep [0-3]")); From 6d3fc155fd8f1e03fb2efa909866aba5407d77c2 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 6 Aug 2024 13:54:14 +0200 Subject: [PATCH 37/61] fix(c api): added depth to fdb_list fix(c api): test bug --- src/fdb5/api/fdb_c.cc | 7 ++++--- src/fdb5/api/fdb_c.h | 2 +- tests/fdb/api/test_fdb_c.cc | 18 ++++++++++-------- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/fdb5/api/fdb_c.cc b/src/fdb5/api/fdb_c.cc index be20f9362..141785f9c 100644 --- a/src/fdb5/api/fdb_c.cc +++ b/src/fdb5/api/fdb_c.cc @@ -352,17 +352,18 @@ int fdb_archive_multiple(fdb_handle_t* fdb, fdb_request_t* req, const char* data }); } -int fdb_list(fdb_handle_t* fdb, const fdb_request_t* req, fdb_listiterator_t** it, bool duplicates) { - return wrapApiFunction([fdb, req, it, duplicates] { +int fdb_list(fdb_handle_t* fdb, const fdb_request_t* req, fdb_listiterator_t** it, const bool duplicates, const int depth) { + return wrapApiFunction([fdb, req, it, duplicates, depth] { ASSERT(fdb); ASSERT(it); + ASSERT(depth >= 0 && depth <= 3); std::vector minKeySet; // we consider an empty set const FDBToolRequest toolRequest( req ? req->request() : metkit::mars::MarsRequest(), req == nullptr, minKeySet); - *it = new fdb_listiterator_t(fdb->list(toolRequest, duplicates)); + *it = new fdb_listiterator_t(fdb->list(toolRequest, duplicates, depth)); }); } int fdb_retrieve(fdb_handle_t* fdb, fdb_request_t* req, fdb_datareader_t* dr) { diff --git a/src/fdb5/api/fdb_c.h b/src/fdb5/api/fdb_c.h index 6ac37cf5a..a8825306e 100644 --- a/src/fdb5/api/fdb_c.h +++ b/src/fdb5/api/fdb_c.h @@ -343,7 +343,7 @@ int fdb_archive_multiple(fdb_handle_t* fdb, fdb_request_t* req, const char* data * \param duplicates Boolean flag used to specify if duplicated ListElements are to be reported or not. * \returns Return code (#FdbErrorValues) */ -int fdb_list(fdb_handle_t* fdb, const fdb_request_t* req, fdb_listiterator_t** it, bool duplicates); +int fdb_list(fdb_handle_t* fdb, const fdb_request_t* req, fdb_listiterator_t** it, bool duplicates, int depth); /** Return all available data whose metadata matches a given user request. * \param fdb FDB instance. diff --git a/tests/fdb/api/test_fdb_c.cc b/tests/fdb/api/test_fdb_c.cc index cbb0d0568..a62a96f42 100644 --- a/tests/fdb/api/test_fdb_c.cc +++ b/tests/fdb/api/test_fdb_c.cc @@ -62,6 +62,7 @@ void key_compare(const std::vector& keys, fdb_listiterator_t *it, boo } CASE( "fdb_c - archive & list" ) { + const int depth = 3; size_t length; DataHandle *dh; @@ -109,7 +110,7 @@ CASE( "fdb_c - archive & list" ) { const char **item= new const char*; fdb_listiterator_t* it; - fdb_list(fdb, request, &it, true); + fdb_list(fdb, request, &it, true, depth); int err = fdb_listiterator_next(it); ASSERT(err == FDB_SUCCESS); @@ -134,7 +135,7 @@ CASE( "fdb_c - archive & list" ) { fdb_request_add1(request, "param", "139"); - fdb_list(fdb, request, &it, true); + fdb_list(fdb, request, &it, true, depth); err = fdb_listiterator_next(it); ASSERT(err == FDB_ITERATION_COMPLETE); fdb_delete_listiterator(it); @@ -153,14 +154,14 @@ CASE( "fdb_c - archive & list" ) { EXPECT(FDB_SUCCESS == fdb_flush(fdb)); fdb_request_add1(request, "levelist", "400"); - fdb_list(fdb, request, &it, true); + fdb_list(fdb, request, &it, true, depth); err = fdb_listiterator_next(it); ASSERT(err == FDB_ITERATION_COMPLETE); fdb_delete_listiterator(it); fdb_request_add1(request, "param", "138"); - fdb_list(fdb, request, &it, true); + fdb_list(fdb, request, &it, true, depth); err = fdb_listiterator_next(it); ASSERT(err == FDB_SUCCESS); @@ -183,7 +184,7 @@ CASE( "fdb_c - archive & list" ) { const char* values[] = {"400", "300"}; fdb_request_add(request, "levelist", values, 2); - fdb_list(fdb, request, &it, true); + fdb_list(fdb, request, &it, true, depth); err = fdb_listiterator_next(it); ASSERT(err == FDB_SUCCESS); @@ -219,6 +220,7 @@ CASE( "fdb_c - archive & list" ) { #if fdb5_HAVE_GRIB CASE( "fdb_c - multiple archive & list" ) { + const int depth = 3; size_t length1, length2, length3; DataHandle *dh; @@ -319,7 +321,7 @@ CASE( "fdb_c - multiple archive & list" ) { const char **item= new const char*; bool exist; fdb_listiterator_t* it; - fdb_list(fdb, request, &it, true); + fdb_list(fdb, request, &it, true, depth); int err = fdb_listiterator_next(it); ASSERT(err == FDB_SUCCESS); @@ -330,7 +332,7 @@ CASE( "fdb_c - multiple archive & list" ) { fdb_delete_listiterator(it); fdb_request_add1(request, "step", "1"); - fdb_list(fdb, request, &it, true); + fdb_list(fdb, request, &it, true, depth); err = fdb_listiterator_next(it); ASSERT(err == FDB_ITERATION_COMPLETE); fdb_delete_listiterator(it); @@ -338,7 +340,7 @@ CASE( "fdb_c - multiple archive & list" ) { fdb_request_add1(request, "step", "0"); const char* values[] = {"400", "300"}; fdb_request_add(request, "levelist", values, 2); - fdb_list(fdb, request, &it, true); + fdb_list(fdb, request, &it, true, depth); err = fdb_listiterator_next(it); ASSERT(err == FDB_SUCCESS); From 713c8f387412daf8ba5d6d1a2d9fad8fe6bf6c55 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 6 Aug 2024 18:21:38 +0200 Subject: [PATCH 38/61] fix(ListElement): initial size = 0 --- src/fdb5/api/helpers/ListElement.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fdb5/api/helpers/ListElement.h b/src/fdb5/api/helpers/ListElement.h index 681ebe554..a82716bd1 100644 --- a/src/fdb5/api/helpers/ListElement.h +++ b/src/fdb5/api/helpers/ListElement.h @@ -54,8 +54,8 @@ class ListElement { struct Attributes { eckit::URI uri; TimeStamp timestamp {0}; - eckit::Offset offset {-1}; - eckit::Length length {-1}; + eckit::Offset offset {0}; + eckit::Length length {0}; friend void operator>>(eckit::Stream& stream, Attributes& attrs); friend auto operator<<(eckit::Stream& stream, const Attributes& attrs) -> eckit::Stream&; From f1cc6383c046a82df315bd487c944d94630fbcc4 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 6 Aug 2024 18:22:53 +0200 Subject: [PATCH 39/61] fix(c api): guard offset/len < 0 --- src/fdb5/api/fdb_c.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/fdb5/api/fdb_c.cc b/src/fdb5/api/fdb_c.cc index 141785f9c..ff1d2d81e 100644 --- a/src/fdb5/api/fdb_c.cc +++ b/src/fdb5/api/fdb_c.cc @@ -10,6 +10,7 @@ #include +#include "eckit/exception/Exceptions.h" #include "eckit/io/MemoryHandle.h" #include "eckit/message/Message.h" #include "eckit/runtime/Main.h" @@ -145,6 +146,10 @@ struct fdb_listiterator_t { const auto& attrs = el_.attributes(); + // guard against negative values + ASSERT(0 <= attrs.offset); + ASSERT(0 <= attrs.length); + *uri = attrs.uri.name().c_str(); *off = attrs.offset; *len = attrs.length; From 24b70dee385d11010004211685f09a7c91941593 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 7 Aug 2024 20:09:20 +0200 Subject: [PATCH 40/61] fix(fdb-list): updated doc for depth option --- src/fdb5/tools/fdb-list.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fdb5/tools/fdb-list.cc b/src/fdb5/tools/fdb-list.cc index a964e4b78..927b0e612 100644 --- a/src/fdb5/tools/fdb-list.cc +++ b/src/fdb5/tools/fdb-list.cc @@ -50,7 +50,7 @@ class FDBList: public FDBVisitTool { "Equivalent to compact=false, location=false, timestamp=false, length=false")); options_.push_back(new SimpleOption("json", "Output available fields in JSON form")); options_.push_back(new SimpleOption("compact", "Aggregate available fields in MARS requests")); - options_.push_back(new SimpleOption("depth", "Output entries up to 'depth' levels deep [0-3]")); + options_.push_back(new SimpleOption("depth", "Output entries up to 'depth' levels deep [0-2]")); } private: // methods From 39b27cd87d12060df24904155791586fd6897054 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 9 Aug 2024 13:10:15 +0200 Subject: [PATCH 41/61] test(fdb-list): rename cf -> an --- tests/fdb/tools/fdb_list.sh.in | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/fdb/tools/fdb_list.sh.in b/tests/fdb/tools/fdb_list.sh.in index 661518b85..020e5348a 100755 --- a/tests/fdb/tools/fdb_list.sh.in +++ b/tests/fdb/tools/fdb_list.sh.in @@ -39,7 +39,7 @@ try grib_set -s step=6 x.grib 6.grib try grib_set -s step=9 x.grib 9.grib for f in *.grib; do - try grib_set -s type=cf "$f" "cf.$f" + try grib_set -s type=an "$f" "cf.$f" done for f in *.grib; do @@ -55,10 +55,10 @@ exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=12,param=166} {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=6,param=166} {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=9,param=166} -{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=0,param=166} -{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=12,param=166} -{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=6,param=166} -{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=9,param=166}" +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=0,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=12,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=6,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=9,param=166}" try test "$exp" = "$out" # test date depth=1 @@ -69,23 +69,23 @@ try test "$exp" = "$out" # test date depth=2 out=$(fdb-list date=20201102 --minimum-keys="" --porcelain --depth=2) exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc} -{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}" +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}" try test "$exp" = "$out" # test cf depth=1 -out=$(fdb-list type=cf --minimum-keys="" --porcelain --depth=1) +out=$(fdb-list type=an --minimum-keys="" --porcelain --depth=1) exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}" try test "$exp" = "$out" # test cf depth=2 -out=$(fdb-list type=cf --minimum-keys="" --porcelain --depth=2) -exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}" +out=$(fdb-list type=an --minimum-keys="" --porcelain --depth=2) +exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}" try test "$exp" = "$out" # test cf depth=3 -out=$(fdb-list type=cf --minimum-keys="" --porcelain --depth=3) -exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=0,param=166} -{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=12,param=166} -{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=6,param=166} -{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=9,param=166}" +out=$(fdb-list type=an --minimum-keys="" --porcelain --depth=3) +exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=0,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=12,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=6,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=9,param=166}" try test "$exp" = "$out" From d8c3ec2c46ba3f1afe71938c62340da45e9c133e Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 9 Aug 2024 13:15:48 +0200 Subject: [PATCH 42/61] test(fdb-list): moved it to its own directory --- tests/fdb/tools/CMakeLists.txt | 3 ++- tests/fdb/tools/list/CMakeLists.txt | 8 +++++++ tests/fdb/tools/{ => list}/fdb_list.sh.in | 28 +++++++++++------------ tests/fdb/tools/list/local.yaml | 6 +++++ tests/fdb/tools/list/x.grib | 1 + 5 files changed, 31 insertions(+), 15 deletions(-) create mode 100644 tests/fdb/tools/list/CMakeLists.txt rename tests/fdb/tools/{ => list}/fdb_list.sh.in (82%) create mode 100644 tests/fdb/tools/list/local.yaml create mode 120000 tests/fdb/tools/list/x.grib diff --git a/tests/fdb/tools/CMakeLists.txt b/tests/fdb/tools/CMakeLists.txt index d36fc20b7..308c35b66 100644 --- a/tests/fdb/tools/CMakeLists.txt +++ b/tests/fdb/tools/CMakeLists.txt @@ -1,6 +1,5 @@ list( APPEND fdb_tools_tests fdb_info - fdb_list fdb_axes ) foreach( _t ${fdb_tools_tests} ) @@ -15,3 +14,5 @@ foreach( _t ${fdb_tools_tests} ) endforeach() add_subdirectory( auxiliary ) + +add_subdirectory( list ) diff --git a/tests/fdb/tools/list/CMakeLists.txt b/tests/fdb/tools/list/CMakeLists.txt new file mode 100644 index 000000000..07d3fa583 --- /dev/null +++ b/tests/fdb/tools/list/CMakeLists.txt @@ -0,0 +1,8 @@ +set(test_name fdb_list) + +configure_file( ${test_name}.sh.in ${test_name}.sh @ONLY ) + +ecbuild_add_test( + TYPE SCRIPT + CONDITION HAVE_FDB_BUILD_TOOLS + COMMAND ${test_name}.sh) diff --git a/tests/fdb/tools/fdb_list.sh.in b/tests/fdb/tools/list/fdb_list.sh.in similarity index 82% rename from tests/fdb/tools/fdb_list.sh.in rename to tests/fdb/tools/list/fdb_list.sh.in index 020e5348a..661518b85 100755 --- a/tests/fdb/tools/fdb_list.sh.in +++ b/tests/fdb/tools/list/fdb_list.sh.in @@ -39,7 +39,7 @@ try grib_set -s step=6 x.grib 6.grib try grib_set -s step=9 x.grib 9.grib for f in *.grib; do - try grib_set -s type=an "$f" "cf.$f" + try grib_set -s type=cf "$f" "cf.$f" done for f in *.grib; do @@ -55,10 +55,10 @@ exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=12,param=166} {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=6,param=166} {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=9,param=166} -{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=0,param=166} -{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=12,param=166} -{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=6,param=166} -{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=9,param=166}" +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=0,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=12,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=6,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=9,param=166}" try test "$exp" = "$out" # test date depth=1 @@ -69,23 +69,23 @@ try test "$exp" = "$out" # test date depth=2 out=$(fdb-list date=20201102 --minimum-keys="" --porcelain --depth=2) exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc} -{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}" +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}" try test "$exp" = "$out" # test cf depth=1 -out=$(fdb-list type=an --minimum-keys="" --porcelain --depth=1) +out=$(fdb-list type=cf --minimum-keys="" --porcelain --depth=1) exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}" try test "$exp" = "$out" # test cf depth=2 -out=$(fdb-list type=an --minimum-keys="" --porcelain --depth=2) -exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}" +out=$(fdb-list type=cf --minimum-keys="" --porcelain --depth=2) +exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}" try test "$exp" = "$out" # test cf depth=3 -out=$(fdb-list type=an --minimum-keys="" --porcelain --depth=3) -exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=0,param=166} -{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=12,param=166} -{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=6,param=166} -{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=9,param=166}" +out=$(fdb-list type=cf --minimum-keys="" --porcelain --depth=3) +exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=0,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=12,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=6,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=9,param=166}" try test "$exp" = "$out" diff --git a/tests/fdb/tools/list/local.yaml b/tests/fdb/tools/list/local.yaml new file mode 100644 index 000000000..8e43d2e8f --- /dev/null +++ b/tests/fdb/tools/list/local.yaml @@ -0,0 +1,6 @@ +--- +type: local +engine: toc +spaces: + - roots: + - path: ./localroot diff --git a/tests/fdb/tools/list/x.grib b/tests/fdb/tools/list/x.grib new file mode 120000 index 000000000..635e33b3e --- /dev/null +++ b/tests/fdb/tools/list/x.grib @@ -0,0 +1 @@ +../../../regressions/FDB-307/x.grib \ No newline at end of file From 38c24d3b5e04cf9af86ab2d51dc9a8360b3f072f Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 13 Aug 2024 16:52:52 +0200 Subject: [PATCH 43/61] fix(c api): revert fdb_splitkey_t API --- src/fdb5/api/fdb_c.cc | 11 ++----- src/fdb5/api/fdb_c.h | 9 +----- tests/fdb/api/test_fdb_c.cc | 59 +++++++++++++++++++------------------ 3 files changed, 33 insertions(+), 46 deletions(-) diff --git a/src/fdb5/api/fdb_c.cc b/src/fdb5/api/fdb_c.cc index ff1d2d81e..209a553fd 100644 --- a/src/fdb5/api/fdb_c.cc +++ b/src/fdb5/api/fdb_c.cc @@ -484,19 +484,12 @@ int fdb_new_splitkey(fdb_split_key_t** key) { }); } -int fdb_splitkey_next(fdb_split_key_t* it) { - return wrapApiFunction([it] { - ASSERT(it); - ++(*it); - }); -} - -int fdb_splitkey_metadata(fdb_split_key_t* it, const char** key, const char** value, size_t* level) { +int fdb_splitkey_next_metadata(fdb_split_key_t* it, const char** key, const char** value, size_t* level) { return wrapApiFunction(std::function {[it, key, value, level] { ASSERT(it); ASSERT(key); ASSERT(value); - return it->metadata(key, value, level); + return it->next_metadata(key, value, level); }}); } int fdb_delete_splitkey(fdb_split_key_t* key) { diff --git a/src/fdb5/api/fdb_c.h b/src/fdb5/api/fdb_c.h index a8825306e..8cd2ae5e5 100644 --- a/src/fdb5/api/fdb_c.h +++ b/src/fdb5/api/fdb_c.h @@ -176,13 +176,6 @@ typedef struct fdb_split_key_t fdb_split_key_t; */ int fdb_new_splitkey(fdb_split_key_t** key); -/** Returns the next set of metadata in a SplitKey object. For a given ListElement, the SplitKey represents the Keys - * associated with each level of the FDB Key. - * \param it SplitKey instance - * \returns Return code (#FdbErrorValues) - */ -int fdb_splitkey_next(fdb_split_key_t* it); - /** Returns the next set of metadata in a SplitKey object. For a given ListElement, the SplitKey represents the Keys associated with each level of the FDB index. * Supports multiple fdb_split_key_t iterating over the same key. * \param it SplitKey instance @@ -191,7 +184,7 @@ int fdb_splitkey_next(fdb_split_key_t* it); * \param level level in the iondex of the current Key * \returns Return code (#FdbErrorValues) */ -int fdb_splitkey_metadata(fdb_split_key_t* it, const char** key, const char** value, size_t* level); +int fdb_splitkey_next_metadata(fdb_split_key_t* it, const char** key, const char** value, size_t* level); /** Deallocates SplitKey object and associated resources. * \param key SplitKey instance diff --git a/tests/fdb/api/test_fdb_c.cc b/tests/fdb/api/test_fdb_c.cc index a62a96f42..3d18b6352 100644 --- a/tests/fdb/api/test_fdb_c.cc +++ b/tests/fdb/api/test_fdb_c.cc @@ -40,24 +40,25 @@ void key_compare(const std::vector& keys, fdb_listiterator_t *it, boo fdb_split_key_t* sk = nullptr; fdb_new_splitkey(&sk); - EXPECT_EQUAL(fdb_listiterator_splitkey(it, sk), FDB_SUCCESS); + err = fdb_listiterator_splitkey(it, sk); + EXPECT(err == FDB_SUCCESS); size_t level = 0; - for (const auto& key : keys) { + for (auto key: keys) { for (auto k1: key) { - err = fdb_splitkey_metadata(sk, &k, &v, checkLevel ? &l : nullptr); - EXPECT_EQUAL(err, FDB_SUCCESS); - EXPECT_EQUAL(k1.first, k); - EXPECT_EQUAL(k1.second, v); - if (checkLevel) { EXPECT_EQUAL(level, l); } - fdb_splitkey_next(sk); + int err = fdb_splitkey_next_metadata(sk, &k, &v, checkLevel ? &l : nullptr); + EXPECT(err == FDB_SUCCESS); + EXPECT(k1.first == k); + EXPECT(k1.second == v); + if (checkLevel) { + EXPECT(level == l); + } } level++; } - - fdb_splitkey_next(sk); - EXPECT_EQUAL(fdb_splitkey_metadata(sk, &k, &v, &l), FDB_ITERATION_COMPLETE); - + err = fdb_splitkey_next_metadata(sk, &k, &v, &l); + EXPECT(err == FDB_ITERATION_COMPLETE); + err = fdb_delete_splitkey(sk); } @@ -113,7 +114,7 @@ CASE( "fdb_c - archive & list" ) { fdb_list(fdb, request, &it, true, depth); int err = fdb_listiterator_next(it); ASSERT(err == FDB_SUCCESS); - + const char *uri; size_t off, attr_len; @@ -164,7 +165,7 @@ CASE( "fdb_c - archive & list" ) { fdb_list(fdb, request, &it, true, depth); err = fdb_listiterator_next(it); ASSERT(err == FDB_SUCCESS); - + fdb_listiterator_attrs(it, &uri, &off, &attr_len); EXPECT(attr_len == 3280398); @@ -271,21 +272,21 @@ CASE( "fdb_c - multiple archive & list" ) { fdb_request_add1(req, "type", "an"); fdb_request_add1(req, "expver", "xxxx"); - EXPECT_EQUAL(FDB_ERROR_GENERAL_EXCEPTION, fdb_archive_multiple(fdb, req, buf, length1)); - EXPECT_EQUAL(FDB_SUCCESS, fdb_flush(fdb)); + EXPECT(FDB_ERROR_GENERAL_EXCEPTION == fdb_archive_multiple(fdb, req, buf, length1)); + EXPECT(FDB_SUCCESS == fdb_flush(fdb)); - EXPECT_EQUAL(FDB_SUCCESS, fdb_archive_multiple(fdb, req, buf, length1 + length2)); - EXPECT_EQUAL(FDB_SUCCESS, fdb_flush(fdb)); + EXPECT(FDB_SUCCESS == fdb_archive_multiple(fdb, req, buf, length1+length2)); + EXPECT(FDB_SUCCESS == fdb_flush(fdb)); fdb_request_add1(req, "levelist", "300"); - EXPECT_EQUAL(FDB_ERROR_GENERAL_EXCEPTION, fdb_archive_multiple(fdb, req, buf, length1 + length2)); - EXPECT_EQUAL(FDB_SUCCESS, fdb_flush(fdb)); + EXPECT(FDB_ERROR_GENERAL_EXCEPTION == fdb_archive_multiple(fdb, req, buf, length1+length2)); + EXPECT(FDB_SUCCESS == fdb_flush(fdb)); fdb_request_add(req, "levelist", levels, 2); - EXPECT_EQUAL(FDB_SUCCESS, fdb_archive_multiple(fdb, req, buf, length1 + length2)); - EXPECT_EQUAL(FDB_SUCCESS, fdb_flush(fdb)); + EXPECT(FDB_SUCCESS == fdb_archive_multiple(fdb, req, buf, length1+length2)); + EXPECT(FDB_SUCCESS == fdb_flush(fdb)); dh = grib3.fileHandle(); dh->openForRead(); @@ -298,11 +299,11 @@ CASE( "fdb_c - multiple archive & list" ) { const char* levels3[] = {"300", "400", "500"}; fdb_request_add(req, "levelist", levels3, 3); - EXPECT_EQUAL(FDB_ERROR_GENERAL_EXCEPTION, fdb_archive_multiple(fdb, req, buf, length1 + length2 + length3)); - EXPECT_EQUAL(FDB_SUCCESS, fdb_flush(fdb)); + EXPECT(FDB_ERROR_GENERAL_EXCEPTION == fdb_archive_multiple(fdb, req, buf, length1+length2+length3)); + EXPECT(FDB_SUCCESS == fdb_flush(fdb)); - EXPECT_EQUAL(FDB_SUCCESS, fdb_archive_multiple(fdb, nullptr, buf, length1 + length2 + length3)); - EXPECT_EQUAL(FDB_SUCCESS, fdb_flush(fdb)); + EXPECT(FDB_SUCCESS == fdb_archive_multiple(fdb, nullptr, buf, length1+length2+length3)); + EXPECT(FDB_SUCCESS == fdb_flush(fdb)); fdb_request_t* request; fdb_new_request(&request); @@ -343,12 +344,12 @@ CASE( "fdb_c - multiple archive & list" ) { fdb_list(fdb, request, &it, true, depth); err = fdb_listiterator_next(it); ASSERT(err == FDB_SUCCESS); - + key_compare(k1, it); err = fdb_listiterator_next(it); ASSERT(err == FDB_SUCCESS); - + key_compare(k2, it); err = fdb_listiterator_next(it); @@ -481,7 +482,7 @@ CASE( "fdb_c - expand" ) { size_t numValues; char** values; - + fdb_request_get(request, "date", &values, &numValues); EXPECT_EQUAL(numValues, 2); EXPECT_EQUAL(0, strncmp(values[0], "20191110", 8)); From 12b398f4eb33909f5232b01512f71d12b23b9db9 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 13 Aug 2024 16:54:56 +0200 Subject: [PATCH 44/61] fix(c api): make fdb_splitkey_t work introduces --- src/fdb5/api/fdb_c.cc | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/fdb5/api/fdb_c.cc b/src/fdb5/api/fdb_c.cc index 209a553fd..ce9589dad 100644 --- a/src/fdb5/api/fdb_c.cc +++ b/src/fdb5/api/fdb_c.cc @@ -95,12 +95,17 @@ struct fdb_split_key_t { auto operator=(const value_type& keys) -> fdb_split_key_t& { keys_ = &keys; - level_ = keys_->begin(); - curr_ = level_->begin(); + level_ = keys_->end(); return *this; } auto operator++() -> fdb_split_key_t& { + /// @todo the following "if" is an unfortuante consequense of a flaw in this iterator + if (level_ == keys_->end()) { + level_ = keys_->begin(); + curr_ = level_->begin(); + return *this; + } if (curr_ != level_->end()) { ++curr_; if (curr_ == level_->end() && level_ != keys_->end() - 1) { curr_ = (++level_)->begin(); } @@ -108,10 +113,14 @@ struct fdb_split_key_t { return *this; } - int metadata(const char** k, const char** v, size_t* level) const { - ASSERT_MSG(keys_, "keys are missing!"); - + int next() { + ++(*this); if (curr_ == level_->end()) { return FDB_ITERATION_COMPLETE; } + return FDB_SUCCESS; + } + + void metadata(const char** k, const char** v, size_t* level) const { + ASSERT_MSG(keys_, "keys are missing!"); const auto& [key, val] = *curr_; @@ -119,8 +128,6 @@ struct fdb_split_key_t { *v = val.c_str(); if (level) { *level = std::find(keys_->begin(), keys_->end(), *level_) - keys_->begin(); } - - return FDB_SUCCESS; } private: // members @@ -128,7 +135,7 @@ struct fdb_split_key_t { value_type::const_iterator level_; - value_type::value_type::const_iterator curr_; + Key::const_iterator curr_; }; struct fdb_listiterator_t { @@ -489,7 +496,9 @@ int fdb_splitkey_next_metadata(fdb_split_key_t* it, const char** key, const char ASSERT(it); ASSERT(key); ASSERT(value); - return it->next_metadata(key, value, level); + const auto stat = it->next(); + it->metadata(key, value, level); + return stat; }}); } int fdb_delete_splitkey(fdb_split_key_t* key) { From 8cd9711b44b44ce818790e3017d5b15dca9878fd Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 13 Aug 2024 17:01:02 +0200 Subject: [PATCH 45/61] fix(c api): removed junk find() --- src/fdb5/api/fdb_c.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fdb5/api/fdb_c.cc b/src/fdb5/api/fdb_c.cc index ce9589dad..c102108ee 100644 --- a/src/fdb5/api/fdb_c.cc +++ b/src/fdb5/api/fdb_c.cc @@ -100,7 +100,7 @@ struct fdb_split_key_t { } auto operator++() -> fdb_split_key_t& { - /// @todo the following "if" is an unfortuante consequense of a flaw in this iterator + /// @todo the following "if" is an unfortunate consequense of a flaw in this iterator if (level_ == keys_->end()) { level_ = keys_->begin(); curr_ = level_->begin(); @@ -127,7 +127,7 @@ struct fdb_split_key_t { *k = key.c_str(); *v = val.c_str(); - if (level) { *level = std::find(keys_->begin(), keys_->end(), *level_) - keys_->begin(); } + if (level) { *level = level_ - keys_->begin(); } } private: // members From 376d511f987ba81fbd915db0746efd39614a75b1 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 13 Aug 2024 17:02:52 +0200 Subject: [PATCH 46/61] style(c api): typo --- src/fdb5/api/fdb_c.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fdb5/api/fdb_c.cc b/src/fdb5/api/fdb_c.cc index c102108ee..0fda2b002 100644 --- a/src/fdb5/api/fdb_c.cc +++ b/src/fdb5/api/fdb_c.cc @@ -100,7 +100,7 @@ struct fdb_split_key_t { } auto operator++() -> fdb_split_key_t& { - /// @todo the following "if" is an unfortunate consequense of a flaw in this iterator + /// @todo the following "if" is an unfortunate consequence of a flaw in this iterator if (level_ == keys_->end()) { level_ = keys_->begin(); curr_ = level_->begin(); From de79888a6fbb8b065a8a131c9fbefb9c0f3c5537 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 13 Aug 2024 18:40:08 +0200 Subject: [PATCH 47/61] feat(ListElement): throw with a msg only datum elements support location --- src/fdb5/api/helpers/ListElement.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fdb5/api/helpers/ListElement.cc b/src/fdb5/api/helpers/ListElement.cc index d24892265..b5328e96b 100644 --- a/src/fdb5/api/helpers/ListElement.cc +++ b/src/fdb5/api/helpers/ListElement.cc @@ -72,7 +72,7 @@ ListElement::ListElement(eckit::Stream& stream) { } auto ListElement::location() const -> const FieldLocation& { - ASSERT(loc_); + if (!loc_) { throw eckit::SeriousBug("Only datum (3-level) elements have FieldLocation.", Here()); } return *loc_; } From 59fdb36e8410227ecc407767326bb3896d2fd76d Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 14 Aug 2024 01:13:52 +0200 Subject: [PATCH 48/61] feat(ListElement): added KeyChain (aka SplitKey) --- src/fdb5/CMakeLists.txt | 1 + src/fdb5/api/FDB.cc | 8 +- src/fdb5/api/RemoteFDB.cc | 2 +- src/fdb5/api/fdb_c.cc | 21 +++-- src/fdb5/api/helpers/ListElement.cc | 98 +++++++++-------------- src/fdb5/api/helpers/ListElement.h | 51 +++++------- src/fdb5/api/helpers/ListIterator.h | 2 +- src/fdb5/api/local/ListVisitor.h | 6 +- src/fdb5/database/KeyChain.h | 81 +++++++++++++++++++ src/fdb5/database/MultiRetrieveVisitor.cc | 2 +- tests/fdb/test_fdb5_service.cc | 27 +++---- 11 files changed, 167 insertions(+), 132 deletions(-) create mode 100644 src/fdb5/database/KeyChain.h diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt index f55713b59..e4074d1e2 100644 --- a/src/fdb5/CMakeLists.txt +++ b/src/fdb5/CMakeLists.txt @@ -127,6 +127,7 @@ list( APPEND fdb5_srcs database/IndexFactory.h database/Key.cc database/Key.h + database/KeyChain.h database/ReadVisitor.cc database/ReadVisitor.h database/Report.cc diff --git a/src/fdb5/api/FDB.cc b/src/fdb5/api/FDB.cc index 1ff79bebe..67f2b44a9 100644 --- a/src/fdb5/api/FDB.cc +++ b/src/fdb5/api/FDB.cc @@ -157,7 +157,7 @@ bool FDB::sorted(const metkit::mars::MarsRequest &request) { class ListElementDeduplicator : public metkit::hypercube::Deduplicator { public: bool toReplace(const ListElement& existing, const ListElement& replacement) const override { - return existing.attributes().timestamp < replacement.attributes().timestamp; + return existing.timestamp() < replacement.timestamp(); } }; @@ -189,18 +189,18 @@ eckit::DataHandle* FDB::read(ListIterator& it, bool sorted) { if (dedup) { if (it.next(el)) { // build the request representing the tensor-product of all retrieved fields - metkit::mars::MarsRequest cubeRequest = el.key().request(); + metkit::mars::MarsRequest cubeRequest = el.keys().combine().request(); std::vector elements {el}; while (it.next(el)) { - cubeRequest.merge(el.key().request()); + cubeRequest.merge(el.keys().combine().request()); elements.push_back(el); } // checking all retrieved fields against the hypercube, to remove duplicates ListElementDeduplicator deduplicator; metkit::hypercube::HyperCubePayloaded cube(cubeRequest, deduplicator); - for (const auto& elem : elements) { cube.add(elem.key().request(), el); } + for (const auto& elem : elements) { cube.add(elem.keys().combine().request(), el); } if (cube.countVacant() > 0) { std::stringstream ss; diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc index 82decff94..2e079f648 100644 --- a/src/fdb5/api/RemoteFDB.cc +++ b/src/fdb5/api/RemoteFDB.cc @@ -567,7 +567,7 @@ struct InspectHelper : BaseAPIHelper - #include "eckit/exception/Exceptions.h" #include "eckit/io/MemoryHandle.h" #include "eckit/message/Message.h" #include "eckit/runtime/Main.h" +#include "eckit/utils/Tokenizer.h" #include "metkit/mars/MarsRequest.h" #include "metkit/mars/MarsExpension.h" -#include "eckit/utils/Tokenizer.h" -#include "fdb5/fdb5_version.h" #include "fdb5/api/FDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" #include "fdb5/api/helpers/ListElement.h" #include "fdb5/api/helpers/ListIterator.h" #include "fdb5/database/Key.h" +#include "fdb5/database/KeyChain.h" +#include "fdb5/fdb5_version.h" #include "fdb5/api/fdb_c.h" @@ -91,7 +90,7 @@ struct fdb_request_t { }; struct fdb_split_key_t { - using value_type = typename ListElement::DatumKeys; + using value_type = KeyChain; auto operator=(const value_type& keys) -> fdb_split_key_t& { keys_ = &keys; @@ -151,15 +150,13 @@ struct fdb_listiterator_t { void attrs(const char** uri, size_t* off, size_t* len) { ASSERT(validEl_); - const auto& attrs = el_.attributes(); - // guard against negative values - ASSERT(0 <= attrs.offset); - ASSERT(0 <= attrs.length); + ASSERT(0 <= el_.offset()); + ASSERT(0 <= el_.length()); - *uri = attrs.uri.name().c_str(); - *off = attrs.offset; - *len = attrs.length; + *uri = el_.uri().name().c_str(); + *off = el_.offset(); + *len = el_.length(); } void key(fdb_split_key_t* key) { diff --git a/src/fdb5/api/helpers/ListElement.cc b/src/fdb5/api/helpers/ListElement.cc index b5328e96b..3fc95313f 100644 --- a/src/fdb5/api/helpers/ListElement.cc +++ b/src/fdb5/api/helpers/ListElement.cc @@ -17,58 +17,37 @@ #include "eckit/serialisation/Stream.h" #include "fdb5/database/FieldLocation.h" #include "fdb5/database/Key.h" +#include "fdb5/database/KeyChain.h" #include #include #include -#include #include namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -// HELPERS -namespace { +ListElement::ListElement(Key dbKey, const eckit::URI& uri, const TimeStamp& timestamp): + keys_ {std::move(dbKey)}, uri_ {uri}, timestamp_ {timestamp} { } -template 1 && N < 4)>> -auto combineKeys(const KeyChain& keys) -> Key { - /// @todo fixme: no API to get registry from Key; fix this copy hack - auto combined = keys[0]; - for (const auto& key : keys) { - for (auto&& [keyword, value] : key) { combined.set(keyword, value); } - } - return std::move(combined); -} - -} // namespace - -//---------------------------------------------------------------------------------------------------------------------- -// LIST ELEMENT - -ListElement::ListElement(Key key, const eckit::URI& uri, const TimeStamp timestamp): - key_ {std::move(key)}, keys_ {key_, Key {}, Key {}}, attributes_ {uri, timestamp} { } +ListElement::ListElement(Key dbKey, Key indexKey, const eckit::URI& uri, const TimeStamp& timestamp): + keys_ {std::move(dbKey), std::move(indexKey)}, uri_ {uri}, timestamp_ {timestamp} { } -ListElement::ListElement(IndexKeys&& keys, const eckit::URI& uri, const TimeStamp timestamp): - key_ {combineKeys(keys)}, keys_ {std::move(keys[0]), std::move(keys[1]), Key {}}, attributes_ {uri, timestamp} { } +ListElement::ListElement(Key dbKey, Key indexKey, Key datumKey, const FieldLocation& location, const TimeStamp& timestamp): + keys_ {std::move(dbKey), std::move(indexKey), std::move(datumKey)}, + uri_ {location.uri()}, + loc_ {location.make_shared()}, + timestamp_ {timestamp} { } -ListElement::ListElement(DatumKeys&& keys, const FieldLocation& location, const TimeStamp timestamp): - key_ {combineKeys(keys)}, - keys_ {std::move(keys)}, - attributes_ {location.uri(), timestamp, location.offset(), location.length()}, - loc_ {location.make_shared()} { } - -ListElement::ListElement(const DatumKeys& keys, const FieldLocation& location, const TimeStamp timestamp): - key_ {combineKeys(keys)}, - keys_ {keys}, - attributes_ {location.uri(), timestamp, location.offset(), location.length()}, - loc_ {location.make_shared()} { } +ListElement::ListElement(const KeyChain& keys, const FieldLocation& location, const TimeStamp& timestamp): + ListElement(keys[0], keys[1], keys[2], location, timestamp) { } ListElement::ListElement(eckit::Stream& stream) { - for (auto& key : keys_) { stream >> key; } - stream >> key_; - stream >> attributes_; + stream >> keys_; + stream >> uri_; loc_.reset(eckit::Reanimator::reanimate(stream)); + stream >> timestamp_; } auto ListElement::location() const -> const FieldLocation& { @@ -76,26 +55,38 @@ auto ListElement::location() const -> const FieldLocation& { return *loc_; } +auto ListElement::offset() const -> eckit::Offset { + return location().offset(); +} + +auto ListElement::length() const -> eckit::Length { + return location().length(); +} + void ListElement::print(std::ostream& out, const bool location, const bool length, const bool timestamp, const char* sep) const { - if (location) { out << "host=" << (attributes_.uri.host().empty() ? "empty" : attributes_.uri.host()) << sep; } - for (const auto& key : keys_) { - if (!key.empty()) { out << key; } + out << keys_; + if (location) { + out << sep; + if (loc_) { + out << *loc_; + } else { + out << "host=" << (uri_.host().empty() ? "empty" : uri_.host()); + } } - if (length) { out << sep << "length=" << attributes_.length; } - if (timestamp) { out << sep << "timestamp=" << attributes_.timestamp; } - if (location && loc_) { out << sep << *loc_; } + if (length && loc_) { out << sep << "length=" << loc_->length(); } + if (timestamp) { out << sep << "timestamp=" << timestamp_; } } void ListElement::json(eckit::JSON& json) const { - json << key_.keyDict(); - json << "length" << attributes_.length; + json << keys_.combine().keyDict(); + if (loc_) { json << "length" << loc_->length(); } } void ListElement::encode(eckit::Stream& stream) const { - stream << key_; for (const auto& key : keys_) { stream << key; } - stream << attributes_; + stream << uri_; if (loc_) { stream << *loc_; } + stream << timestamp_; } //---------------------------------------------------------------------------------------------------------------------- @@ -116,21 +107,6 @@ eckit::JSON& operator<<(eckit::JSON& json, const ListElement& elem) { return json; } -void operator>>(eckit::Stream& stream, ListElement::Attributes& attrs) { - stream >> attrs.uri; - stream >> attrs.timestamp; - stream >> attrs.offset; - stream >> attrs.length; -} - -auto operator<<(eckit::Stream& stream, const ListElement::Attributes& attrs) -> eckit::Stream& { - stream << attrs.uri; - stream << attrs.timestamp; - stream << attrs.offset; - stream << attrs.length; - return stream; -} - //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 diff --git a/src/fdb5/api/helpers/ListElement.h b/src/fdb5/api/helpers/ListElement.h index a82716bd1..00a6e8be9 100644 --- a/src/fdb5/api/helpers/ListElement.h +++ b/src/fdb5/api/helpers/ListElement.h @@ -18,12 +18,10 @@ #include "eckit/filesystem/URI.h" #include "eckit/io/Length.h" #include "eckit/io/Offset.h" -#include "fdb5/database/Key.h" +#include "fdb5/database/KeyChain.h" -#include -#include // std::size_t -#include // std::time_t -#include // std::ostream +#include +#include #include namespace eckit { @@ -33,13 +31,8 @@ class Stream; namespace fdb5 { -//---------------------------------------------------------------------------------------------------------------------- - class FieldLocation; -template -using KeyChain = std::array; - //---------------------------------------------------------------------------------------------------------------------- /// Define a standard object which can be used to iterate the results of a @@ -47,41 +40,33 @@ using KeyChain = std::array; class ListElement { public: // types - using IndexKeys = KeyChain<2>; - using DatumKeys = KeyChain<3>; using TimeStamp = std::time_t; - struct Attributes { - eckit::URI uri; - TimeStamp timestamp {0}; - eckit::Offset offset {0}; - eckit::Length length {0}; - - friend void operator>>(eckit::Stream& stream, Attributes& attrs); - friend auto operator<<(eckit::Stream& stream, const Attributes& attrs) -> eckit::Stream&; - }; - public: // methods - ListElement(Key key, const eckit::URI& uri, TimeStamp timestamp); + ListElement(Key dbKey, const eckit::URI& uri, const TimeStamp& timestamp); - ListElement(IndexKeys&& keys, const eckit::URI& uri, TimeStamp timestamp); + ListElement(Key dbKey, Key indexKey, const eckit::URI& uri, const TimeStamp& timestamp); - ListElement(DatumKeys&& keys, const FieldLocation& location, TimeStamp timestamp); + ListElement(Key dbKey, Key indexKey, Key datumKey, const FieldLocation& location, const TimeStamp& timestamp); - ListElement(const DatumKeys& keys, const FieldLocation& location, TimeStamp timestamp); + ListElement(const KeyChain& keys, const FieldLocation& location, const TimeStamp& timestamp); explicit ListElement(eckit::Stream& stream); ListElement() = default; - auto key() const -> const Key& { return key_; } + auto keys() const -> const KeyChain& { return keys_; } - auto keys() const -> const DatumKeys& { return keys_; } + auto uri() const -> const eckit::URI& { return uri_; } - auto attributes() const -> const Attributes& { return attributes_; } + auto timestamp() const -> const TimeStamp& { return timestamp_; } auto location() const -> const FieldLocation&; + auto offset() const -> eckit::Offset; + + auto length() const -> eckit::Length; + void print(std::ostream& out, bool location, bool length, bool timestamp, const char* sep) const; private: // methods @@ -96,13 +81,13 @@ class ListElement { friend eckit::JSON& operator<<(eckit::JSON& json, const ListElement& elem); private: // members - Key key_; + KeyChain keys_; - DatumKeys keys_; - - Attributes attributes_; + eckit::URI uri_; std::shared_ptr loc_; + + TimeStamp timestamp_ {0}; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/api/helpers/ListIterator.h b/src/fdb5/api/helpers/ListIterator.h index 59c1ccea9..72fbd99de 100644 --- a/src/fdb5/api/helpers/ListIterator.h +++ b/src/fdb5/api/helpers/ListIterator.h @@ -55,7 +55,7 @@ class ListIterator : public APIIterator { ListElement tmp; while (APIIterator::next(tmp)) { if(deduplicate_) { - if (const auto& [iter, success] = seenKeys_.insert(tmp.key()); !success) { continue; } + if (const auto& [iter, success] = seenKeys_.insert(tmp.keys().combine()); !success) { continue; } } std::swap(elem, tmp); return true; diff --git a/src/fdb5/api/local/ListVisitor.h b/src/fdb5/api/local/ListVisitor.h index 50f56815a..aadd93967 100644 --- a/src/fdb5/api/local/ListVisitor.h +++ b/src/fdb5/api/local/ListVisitor.h @@ -87,7 +87,7 @@ struct ListVisitor : public QueryVisitor { if (index.partialMatch(request_)) { if (level_ == 2) { - queue_.emplace(KeyChain<2> {currentCatalogue_->key(), currentIndex_->key()}, eckit::URI {}, 0); + queue_.emplace(currentCatalogue_->key(), currentIndex_->key(), eckit::URI {}, 0); return false; } return true; // Explore contained entries @@ -102,9 +102,7 @@ struct ListVisitor : public QueryVisitor { ASSERT(currentIndex_); if (key.match(datumRequest_)) { - queue_.emplace(KeyChain<3> {currentCatalogue_->key(), currentIndex_->key(), key}, - field.location(), - field.timestamp()); + queue_.emplace(currentCatalogue_->key(), currentIndex_->key(), key, field.location(), field.timestamp()); } } diff --git a/src/fdb5/database/KeyChain.h b/src/fdb5/database/KeyChain.h new file mode 100644 index 000000000..c11ab734c --- /dev/null +++ b/src/fdb5/database/KeyChain.h @@ -0,0 +1,81 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @file KeyChain.h +/// @author Metin Cakircali +/// @date Aug 2024 + +#pragma once + +#include "eckit/serialisation/Stream.h" +#include "fdb5/database/Key.h" + +#include +#include +#include +#include + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- +/// @brief KeyChain is a set of 3-level keys, where each level is a Key object. +/// Level 1 is the database key, level 2 is the index key, and level 3 is the datum key. + +class KeyChain: public std::array { +public: // methods + KeyChain() = default; + + KeyChain(Key&& dbKey, Key&& indexKey, Key&& datumKey): + std::array {std::move(dbKey), std::move(indexKey), std::move(datumKey)} { } + + KeyChain(Key&& dbKey, Key&& indexKey): KeyChain(std::move(dbKey), std::move(indexKey), Key {}) { } + + explicit KeyChain(Key&& dbKey): KeyChain(std::move(dbKey), Key {}, Key {}) { } + + auto combine() const -> const Key& { + if (!key_) { + /// @todo fixme: no API to get registry from Key; fix this copy hack + key_.emplace((*this)[0]); + for (auto&& key : *this) { + for (auto&& [keyword, value] : key) { key_->set(keyword, value); } + } + } + return *key_; + } + +private: // methods + void print(std::ostream& out) const { + for (auto&& key : *this) { + if (!key.empty()) { out << key; } + } + } + + friend std::ostream& operator<<(std::ostream& out, const KeyChain& keys) { + keys.print(out); + return out; + } + + friend eckit::Stream& operator<<(eckit::Stream& out, const KeyChain& keys) { + for (auto&& key : keys) { out << key; } + return out; + } + + friend eckit::Stream& operator>>(eckit::Stream& out, KeyChain& keys) { + for (auto&& key : keys) { out >> key; } + return out; + } + +private: // members + mutable std::optional key_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/database/MultiRetrieveVisitor.cc b/src/fdb5/database/MultiRetrieveVisitor.cc index 6c78a9f36..dd6cd4025 100644 --- a/src/fdb5/database/MultiRetrieveVisitor.cc +++ b/src/fdb5/database/MultiRetrieveVisitor.cc @@ -109,7 +109,7 @@ bool MultiRetrieveVisitor::selectDatum(const Key& key, const Key& full) { simplifiedKey.set(k->first, k->second); } - iterator_.emplace({KeyChain<3> {db_->key(), db_->indexKey(), simplifiedKey}, field.location(), field.timestamp()}); + iterator_.emplace({db_->key(), db_->indexKey(), simplifiedKey, field.location(), field.timestamp()}); return true; } diff --git a/tests/fdb/test_fdb5_service.cc b/tests/fdb/test_fdb5_service.cc index 66bc7ceab..eebdf076c 100644 --- a/tests/fdb/test_fdb5_service.cc +++ b/tests/fdb/test_fdb5_service.cc @@ -18,15 +18,12 @@ #include #include "eckit/io/DataHandle.h" -#include "eckit/io/FileHandle.h" #include "eckit/io/MemoryHandle.h" -#include "eckit/io/MultiHandle.h" #include "eckit/runtime/Main.h" #include "eckit/types/Types.h" #include "eckit/utils/Translator.h" #include "metkit/mars/MarsRequest.h" -#include "metkit/mars/MarsExpension.h" #include "metkit/mars/TypeAny.h" #include "fdb5/database/Key.h" @@ -134,9 +131,9 @@ CASE ( "test_fdb_stepunit_archive" ) { fdb5::ListIterator iter = fdb.list(r, true); fdb5::ListElement el; EXPECT(iter.next(el)); - EXPECT(el.key().get("step") == "60"); - EXPECT(!iter.next(el)); - } + EXPECT(el.keys().combine().get("step") == "60"); + EXPECT(!iter.next(el)); + } key.set("step","2"); key.set("stepunits","h"); @@ -149,8 +146,8 @@ CASE ( "test_fdb_stepunit_archive" ) { fdb5::ListIterator iter = fdb.list(r, true); fdb5::ListElement el; EXPECT(iter.next(el)); - EXPECT(el.key().get("step") == "2"); - EXPECT(!iter.next(el)); + EXPECT(el.keys().combine().get("step") == "2"); + EXPECT(!iter.next(el)); } req.setValuesTyped(new metkit::mars::TypeAny("step"), std::vector{"2","60"}); @@ -159,11 +156,11 @@ CASE ( "test_fdb_stepunit_archive" ) { fdb5::ListIterator iter = fdb.list(r, true); fdb5::ListElement el; EXPECT(iter.next(el)); - EXPECT(el.key().get("step") == "2"); - EXPECT(iter.next(el)); - EXPECT(el.key().get("step") == "60"); - EXPECT(!iter.next(el)); - } + EXPECT(el.keys().combine().get("step") == "2"); + EXPECT(iter.next(el)); + EXPECT(el.keys().combine().get("step") == "60"); + EXPECT(!iter.next(el)); + } // sub-hourly data are not yet supported in metkit /* key.set("step","30"); @@ -306,7 +303,7 @@ CASE ( "test_fdb_service" ) { fdb5::ListElement el; EXPECT(iter.next(el)); - eckit::PathName path = el.attributes().uri.path().dirName(); + eckit::PathName path = el.uri().path().dirName(); DIR* dirp = ::opendir(path.asString().c_str()); struct dirent* dp; @@ -489,7 +486,7 @@ CASE ( "test_fdb_service_subtoc" ) { fdb5::ListElement el; EXPECT(iter.next(el)); - eckit::PathName path = el.attributes().uri.path().dirName(); + eckit::PathName path = el.uri().path().dirName(); DIR* dirp = ::opendir(path.asString().c_str()); struct dirent* dp; From 8e209ea02b69b949e36651b497c570a4888f4015 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 14 Aug 2024 09:04:11 +0200 Subject: [PATCH 49/61] style: preserve api --- src/fdb5/api/FDB.cc | 10 +++++----- src/fdb5/api/helpers/ListElement.h | 3 +++ src/fdb5/api/helpers/ListIterator.h | 2 +- tests/fdb/test_fdb5_service.cc | 12 ++++++------ 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/fdb5/api/FDB.cc b/src/fdb5/api/FDB.cc index 67f2b44a9..25317d531 100644 --- a/src/fdb5/api/FDB.cc +++ b/src/fdb5/api/FDB.cc @@ -23,11 +23,10 @@ #include "eckit/io/DataHandle.h" #include "eckit/io/MemoryHandle.h" #include "eckit/log/Log.h" +#include "eckit/log/Timer.h" #include "eckit/message/Message.h" #include "eckit/message/Reader.h" -#include "eckit/system/LibraryManager.h" - #include "metkit/hypercube/HyperCube.h" #include "metkit/hypercube/HyperCubePayloaded.h" @@ -35,6 +34,7 @@ #include "fdb5/api/FDB.h" #include "fdb5/api/FDBFactory.h" #include "fdb5/api/helpers/FDBToolRequest.h" +#include "fdb5/api/helpers/ListElement.h" #include "fdb5/api/helpers/ListIterator.h" #include "fdb5/database/FieldLocation.h" #include "fdb5/database/Key.h" @@ -189,18 +189,18 @@ eckit::DataHandle* FDB::read(ListIterator& it, bool sorted) { if (dedup) { if (it.next(el)) { // build the request representing the tensor-product of all retrieved fields - metkit::mars::MarsRequest cubeRequest = el.keys().combine().request(); + metkit::mars::MarsRequest cubeRequest = el.combinedKey().request(); std::vector elements {el}; while (it.next(el)) { - cubeRequest.merge(el.keys().combine().request()); + cubeRequest.merge(el.combinedKey().request()); elements.push_back(el); } // checking all retrieved fields against the hypercube, to remove duplicates ListElementDeduplicator deduplicator; metkit::hypercube::HyperCubePayloaded cube(cubeRequest, deduplicator); - for (const auto& elem : elements) { cube.add(elem.keys().combine().request(), el); } + for (const auto& elem : elements) { cube.add(elem.combinedKey().request(), el); } if (cube.countVacant() > 0) { std::stringstream ss; diff --git a/src/fdb5/api/helpers/ListElement.h b/src/fdb5/api/helpers/ListElement.h index 00a6e8be9..9c2e3791a 100644 --- a/src/fdb5/api/helpers/ListElement.h +++ b/src/fdb5/api/helpers/ListElement.h @@ -18,6 +18,7 @@ #include "eckit/filesystem/URI.h" #include "eckit/io/Length.h" #include "eckit/io/Offset.h" +#include "fdb5/database/Key.h" #include "fdb5/database/KeyChain.h" #include @@ -57,6 +58,8 @@ class ListElement { auto keys() const -> const KeyChain& { return keys_; } + auto combinedKey() const -> const Key& { return keys_.combine(); } + auto uri() const -> const eckit::URI& { return uri_; } auto timestamp() const -> const TimeStamp& { return timestamp_; } diff --git a/src/fdb5/api/helpers/ListIterator.h b/src/fdb5/api/helpers/ListIterator.h index 72fbd99de..341afeba9 100644 --- a/src/fdb5/api/helpers/ListIterator.h +++ b/src/fdb5/api/helpers/ListIterator.h @@ -55,7 +55,7 @@ class ListIterator : public APIIterator { ListElement tmp; while (APIIterator::next(tmp)) { if(deduplicate_) { - if (const auto& [iter, success] = seenKeys_.insert(tmp.keys().combine()); !success) { continue; } + if (const auto& [iter, success] = seenKeys_.insert(tmp.combinedKey()); !success) { continue; } } std::swap(elem, tmp); return true; diff --git a/tests/fdb/test_fdb5_service.cc b/tests/fdb/test_fdb5_service.cc index eebdf076c..a7c13a08c 100644 --- a/tests/fdb/test_fdb5_service.cc +++ b/tests/fdb/test_fdb5_service.cc @@ -131,7 +131,7 @@ CASE ( "test_fdb_stepunit_archive" ) { fdb5::ListIterator iter = fdb.list(r, true); fdb5::ListElement el; EXPECT(iter.next(el)); - EXPECT(el.keys().combine().get("step") == "60"); + EXPECT(el.combinedKey().get("step") == "60"); EXPECT(!iter.next(el)); } @@ -146,7 +146,7 @@ CASE ( "test_fdb_stepunit_archive" ) { fdb5::ListIterator iter = fdb.list(r, true); fdb5::ListElement el; EXPECT(iter.next(el)); - EXPECT(el.keys().combine().get("step") == "2"); + EXPECT(el.combinedKey().get("step") == "2"); EXPECT(!iter.next(el)); } @@ -156,9 +156,9 @@ CASE ( "test_fdb_stepunit_archive" ) { fdb5::ListIterator iter = fdb.list(r, true); fdb5::ListElement el; EXPECT(iter.next(el)); - EXPECT(el.keys().combine().get("step") == "2"); + EXPECT(el.combinedKey().get("step") == "2"); EXPECT(iter.next(el)); - EXPECT(el.keys().combine().get("step") == "60"); + EXPECT(el.combinedKey().get("step") == "60"); EXPECT(!iter.next(el)); } @@ -374,8 +374,8 @@ CASE ( "test_fdb_service_subtoc" ) { eckit::LocalConfiguration userConf; userConf.set("useSubToc", true); - - fdb5::Config expanded = fdb5::Config().expandConfig(); + + fdb5::Config expanded = fdb5::Config().expandConfig(); fdb5::Config config(expanded, userConf); From bea681d97be574daa8a687d45386ffffb6cc2aec Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 14 Aug 2024 09:08:26 +0200 Subject: [PATCH 50/61] style: replace auto --- src/fdb5/api/helpers/ListElement.cc | 6 +++--- src/fdb5/api/helpers/ListElement.h | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/fdb5/api/helpers/ListElement.cc b/src/fdb5/api/helpers/ListElement.cc index 3fc95313f..6d778f157 100644 --- a/src/fdb5/api/helpers/ListElement.cc +++ b/src/fdb5/api/helpers/ListElement.cc @@ -50,16 +50,16 @@ ListElement::ListElement(eckit::Stream& stream) { stream >> timestamp_; } -auto ListElement::location() const -> const FieldLocation& { +const FieldLocation& ListElement::location() const { if (!loc_) { throw eckit::SeriousBug("Only datum (3-level) elements have FieldLocation.", Here()); } return *loc_; } -auto ListElement::offset() const -> eckit::Offset { +eckit::Offset ListElement::offset() const { return location().offset(); } -auto ListElement::length() const -> eckit::Length { +eckit::Length ListElement::length() const { return location().length(); } diff --git a/src/fdb5/api/helpers/ListElement.h b/src/fdb5/api/helpers/ListElement.h index 9c2e3791a..54d29a50c 100644 --- a/src/fdb5/api/helpers/ListElement.h +++ b/src/fdb5/api/helpers/ListElement.h @@ -56,19 +56,19 @@ class ListElement { ListElement() = default; - auto keys() const -> const KeyChain& { return keys_; } + const KeyChain& keys() const { return keys_; } - auto combinedKey() const -> const Key& { return keys_.combine(); } + const Key& combinedKey() const { return keys_.combine(); } - auto uri() const -> const eckit::URI& { return uri_; } + const eckit::URI& uri() const { return uri_; } - auto timestamp() const -> const TimeStamp& { return timestamp_; } + const TimeStamp& timestamp() const { return timestamp_; } - auto location() const -> const FieldLocation&; + const FieldLocation& location() const; - auto offset() const -> eckit::Offset; + eckit::Offset offset() const; - auto length() const -> eckit::Length; + eckit::Length length() const; void print(std::ostream& out, bool location, bool length, bool timestamp, const char* sep) const; From 7e7311691d6214dd5f64d0514e9f0fe60d5b6e47 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 14 Aug 2024 09:49:40 +0200 Subject: [PATCH 51/61] fix(fdb-list): param defaults and throw for porcelain option --- src/fdb5/tools/fdb-list.cc | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/src/fdb5/tools/fdb-list.cc b/src/fdb5/tools/fdb-list.cc index 927b0e612..350ebfe07 100644 --- a/src/fdb5/tools/fdb-list.cc +++ b/src/fdb5/tools/fdb-list.cc @@ -21,6 +21,7 @@ #include "eckit/option/CmdArgs.h" #include "eckit/option/SimpleOption.h" +#include "fdb5/LibFdb5.h" #include "metkit/hypercube/HyperCube.h" #include "metkit/mars/MarsRequest.h" @@ -45,9 +46,10 @@ class FDBList: public FDBVisitTool { options_.push_back(new SimpleOption("timestamp", "Also print the timestamp when the field was indexed")); options_.push_back(new SimpleOption("length", "Also print the field size")); options_.push_back(new SimpleOption("full", "Include all entries (including masked duplicates)")); - options_.push_back(new SimpleOption("porcelain", + options_.push_back( + new SimpleOption("porcelain", "Streamlined and stable output. Useful as input for other tools or scripts." - "Equivalent to compact=false, location=false, timestamp=false, length=false")); + "Incompatible with options: location, timestamp, and length")); options_.push_back(new SimpleOption("json", "Output available fields in JSON form")); options_.push_back(new SimpleOption("compact", "Aggregate available fields in MARS requests")); options_.push_back(new SimpleOption("depth", "Output entries up to 'depth' levels deep [0-2]")); @@ -84,24 +86,27 @@ void FDBList::init(const CmdArgs& args) { FDBVisitTool::init(args); - location_ = args.getBool("location", false); - timestamp_ = args.getBool("timestamp", false); - length_ = args.getBool("length", false); - full_ = args.getBool("full", false); - porcelain_ = args.getBool("porcelain", false); - json_ = args.getBool("json", false); - compact_ = args.getBool("compact", false); - depth_ = args.getInt("depth", 3); + location_ = args.getBool("location", location_); + timestamp_ = args.getBool("timestamp", timestamp_); + length_ = args.getBool("length", length_); + full_ = args.getBool("full", full_); + porcelain_ = args.getBool("porcelain", porcelain_); + json_ = args.getBool("json", json_); + compact_ = args.getBool("compact", compact_); + depth_ = args.getInt("depth", depth_); - ASSERT(depth_ > 0 && depth_ <= 3); + ASSERT(depth_ > 0 && depth_ < 4); - if (json_) { porcelain_ = true; } + if (json_) { + eckit::Log::debug() << "Setting porcelain=true" << '\n'; + porcelain_ = true; + } if (porcelain_) { - location_ = false; - timestamp_ = false; - length_ = false; - compact_ = false; + if (location_) { throw UserError("--porcelain and --location are not compatible", Here()); } + if (timestamp_) { throw UserError("--porcelain and --timestamp are not compatible", Here()); } + if (length_) { throw UserError("--porcelain and --length are not compatible", Here()); } + if (compact_) { throw UserError("--porcelain and --compact are not compatible", Here()); } } if (compact_) { From 346a47fd57f814af4545c00ce26981ae4a232c90 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 14 Aug 2024 12:29:13 +0200 Subject: [PATCH 52/61] fix(c api): fdb_list assert bug --- src/fdb5/api/fdb_c.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fdb5/api/fdb_c.cc b/src/fdb5/api/fdb_c.cc index 216d914f5..dc075fcb8 100644 --- a/src/fdb5/api/fdb_c.cc +++ b/src/fdb5/api/fdb_c.cc @@ -365,7 +365,7 @@ int fdb_list(fdb_handle_t* fdb, const fdb_request_t* req, fdb_listiterator_t** i return wrapApiFunction([fdb, req, it, duplicates, depth] { ASSERT(fdb); ASSERT(it); - ASSERT(depth >= 0 && depth <= 3); + ASSERT(depth >= 1 && depth <= 3); std::vector minKeySet; // we consider an empty set const FDBToolRequest toolRequest( From 7f72cc319a5d203ee35d345024c87bfd1ce5fb27 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 14 Aug 2024 12:51:01 +0200 Subject: [PATCH 53/61] fix(ListElement): offset/length returns 0 for non-datum elements --- src/fdb5/api/helpers/ListElement.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fdb5/api/helpers/ListElement.cc b/src/fdb5/api/helpers/ListElement.cc index 6d778f157..18bbeab4d 100644 --- a/src/fdb5/api/helpers/ListElement.cc +++ b/src/fdb5/api/helpers/ListElement.cc @@ -56,11 +56,11 @@ const FieldLocation& ListElement::location() const { } eckit::Offset ListElement::offset() const { - return location().offset(); + return loc_ ? loc_->offset() : eckit::Offset(0); } eckit::Length ListElement::length() const { - return location().length(); + return loc_ ? loc_->length() : eckit::Length(0); } void ListElement::print(std::ostream& out, const bool location, const bool length, const bool timestamp, const char* sep) const { From 14d0f7432dc5730d53885e7c855aa04b0b27aacc Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 14 Aug 2024 15:38:06 +0200 Subject: [PATCH 54/61] test(c api): added fdb_list with depth=1,2,3 --- tests/fdb/api/test_fdb_c.cc | 95 +++++++++++++++++++++++++++++++++---- 1 file changed, 85 insertions(+), 10 deletions(-) diff --git a/tests/fdb/api/test_fdb_c.cc b/tests/fdb/api/test_fdb_c.cc index 3d18b6352..7a9876570 100644 --- a/tests/fdb/api/test_fdb_c.cc +++ b/tests/fdb/api/test_fdb_c.cc @@ -8,9 +8,10 @@ * does it submit to any jurisdiction. */ -#include +#include +#include +#include -#include "eckit/config/Resource.h" #include "eckit/filesystem/PathName.h" #include "eckit/io/Buffer.h" #include "eckit/io/DataHandle.h" @@ -44,9 +45,10 @@ void key_compare(const std::vector& keys, fdb_listiterator_t *it, boo EXPECT(err == FDB_SUCCESS); size_t level = 0; - for (auto key: keys) { - for (auto k1: key) { + for (const auto& key : keys) { + for (const auto& k1 : key) { int err = fdb_splitkey_next_metadata(sk, &k, &v, checkLevel ? &l : nullptr); + std::cerr << "k=" << k << " v=" << v << " l=" << l << std::endl; EXPECT(err == FDB_SUCCESS); EXPECT(k1.first == k); EXPECT(k1.second == v); @@ -58,7 +60,7 @@ void key_compare(const std::vector& keys, fdb_listiterator_t *it, boo } err = fdb_splitkey_next_metadata(sk, &k, &v, &l); EXPECT(err == FDB_ITERATION_COMPLETE); - + err = fdb_delete_splitkey(sk); } @@ -114,7 +116,7 @@ CASE( "fdb_c - archive & list" ) { fdb_list(fdb, request, &it, true, depth); int err = fdb_listiterator_next(it); ASSERT(err == FDB_SUCCESS); - + const char *uri; size_t off, attr_len; @@ -165,7 +167,7 @@ CASE( "fdb_c - archive & list" ) { fdb_list(fdb, request, &it, true, depth); err = fdb_listiterator_next(it); ASSERT(err == FDB_SUCCESS); - + fdb_listiterator_attrs(it, &uri, &off, &attr_len); EXPECT(attr_len == 3280398); @@ -344,18 +346,91 @@ CASE( "fdb_c - multiple archive & list" ) { fdb_list(fdb, request, &it, true, depth); err = fdb_listiterator_next(it); ASSERT(err == FDB_SUCCESS); - + key_compare(k1, it); err = fdb_listiterator_next(it); ASSERT(err == FDB_SUCCESS); - + key_compare(k2, it); err = fdb_listiterator_next(it); ASSERT(err == FDB_ITERATION_COMPLETE); fdb_delete_listiterator(it); } + +CASE("fdb_c - list depth=1,2,3") { + fdb_handle_t* fdb = nullptr; + fdb_new_handle(&fdb); + + std::vector key300d1 { + {{"class", "rd"}, {"expver", "xxxx"}, {"stream", "oper"}, {"date", "20191110"}, {"time", "0000"}, {"domain", "g"}}, + }; + + std::vector key300d2 { + {{"class", "rd"}, {"expver", "xxxx"}, {"stream", "oper"}, {"date", "20191110"}, {"time", "0000"}, {"domain", "g"}}, + {{"type", "an"}, {"levtype", "pl"}}, + }; + + std::vector key300d3 { + {{"class", "rd"}, {"expver", "xxxx"}, {"stream", "oper"}, {"date", "20191110"}, {"time", "0000"}, {"domain", "g"}}, + {{"type", "an"}, {"levtype", "pl"}}, + {{"step", "0"}, {"levelist", "300"}, {"param", "138"}}, + }; + + fdb_request_t* request = nullptr; + fdb_new_request(&request); + fdb_request_add1(request, "domain", "g"); + fdb_request_add1(request, "stream", "oper"); + fdb_request_add1(request, "levtype", "pl"); + fdb_request_add1(request, "levelist", "300"); + fdb_request_add1(request, "date", "20191110"); + fdb_request_add1(request, "time", "0000"); + fdb_request_add1(request, "step", "0"); + fdb_request_add1(request, "param", "138"); + fdb_request_add1(request, "class", "rd"); + fdb_request_add1(request, "type", "an"); + fdb_request_add1(request, "expver", "xxxx"); + + { // depth=1 + fdb_listiterator_t* iter = nullptr; + fdb_list(fdb, request, &iter, true, 1); + int err = fdb_listiterator_next(iter); + ASSERT(err == FDB_SUCCESS); + + key_compare(key300d1, iter); + + err = fdb_listiterator_next(iter); + ASSERT(err == FDB_ITERATION_COMPLETE); + fdb_delete_listiterator(iter); + } + + { // depth=2 + fdb_listiterator_t* iter = nullptr; + fdb_list(fdb, request, &iter, true, 2); + int err = fdb_listiterator_next(iter); + ASSERT(err == FDB_SUCCESS); + + key_compare(key300d2, iter); + + err = fdb_listiterator_next(iter); + ASSERT(err == FDB_ITERATION_COMPLETE); + fdb_delete_listiterator(iter); + } + + { // depth=3 + fdb_listiterator_t* iter = nullptr; + fdb_list(fdb, request, &iter, true, 3); + int err = fdb_listiterator_next(iter); + ASSERT(err == FDB_SUCCESS); + + key_compare(key300d3, iter); + + err = fdb_listiterator_next(iter); + ASSERT(err == FDB_ITERATION_COMPLETE); + fdb_delete_listiterator(iter); + } +} #endif CASE( "fdb_c - retrieve bad request" ) { @@ -482,7 +557,7 @@ CASE( "fdb_c - expand" ) { size_t numValues; char** values; - + fdb_request_get(request, "date", &values, &numValues); EXPECT_EQUAL(numValues, 2); EXPECT_EQUAL(0, strncmp(values[0], "20191110", 8)); From cbef3e907be459221e8529aeb2e9e0776fac26f1 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 14 Aug 2024 18:31:18 +0200 Subject: [PATCH 55/61] fix(ListElement): bug print length --- src/fdb5/api/helpers/ListElement.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fdb5/api/helpers/ListElement.cc b/src/fdb5/api/helpers/ListElement.cc index 18bbeab4d..1546bdcaa 100644 --- a/src/fdb5/api/helpers/ListElement.cc +++ b/src/fdb5/api/helpers/ListElement.cc @@ -73,7 +73,7 @@ void ListElement::print(std::ostream& out, const bool location, const bool lengt out << "host=" << (uri_.host().empty() ? "empty" : uri_.host()); } } - if (length && loc_) { out << sep << "length=" << loc_->length(); } + if (length) { out << sep << "length=" << this->length(); } if (timestamp) { out << sep << "timestamp=" << timestamp_; } } From 75bcf1c65a5f3de5c841fc5e0e56d485a82ac8c3 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Thu, 15 Aug 2024 13:48:41 +0200 Subject: [PATCH 56/61] fix(ListElement): api issues around FieldLocation shared_ptr --- src/fdb5/api/RemoteFDB.cc | 3 ++- src/fdb5/api/helpers/ListElement.cc | 15 ++++++++++----- src/fdb5/api/helpers/ListElement.h | 9 ++++++--- src/fdb5/api/local/ListVisitor.h | 2 +- src/fdb5/database/MultiRetrieveVisitor.cc | 2 +- 5 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc index 2e079f648..044d222f7 100644 --- a/src/fdb5/api/RemoteFDB.cc +++ b/src/fdb5/api/RemoteFDB.cc @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -567,7 +568,7 @@ struct InspectHelper : BaseAPIHelper(fdb, elem.location()), elem.timestamp()}; } }; diff --git a/src/fdb5/api/helpers/ListElement.cc b/src/fdb5/api/helpers/ListElement.cc index 1546bdcaa..ce4fd6d4d 100644 --- a/src/fdb5/api/helpers/ListElement.cc +++ b/src/fdb5/api/helpers/ListElement.cc @@ -34,14 +34,14 @@ ListElement::ListElement(Key dbKey, const eckit::URI& uri, const TimeStamp& time ListElement::ListElement(Key dbKey, Key indexKey, const eckit::URI& uri, const TimeStamp& timestamp): keys_ {std::move(dbKey), std::move(indexKey)}, uri_ {uri}, timestamp_ {timestamp} { } -ListElement::ListElement(Key dbKey, Key indexKey, Key datumKey, const FieldLocation& location, const TimeStamp& timestamp): +ListElement::ListElement(Key dbKey, Key indexKey, Key datumKey, std::shared_ptr location, + const TimeStamp& timestamp): keys_ {std::move(dbKey), std::move(indexKey), std::move(datumKey)}, - uri_ {location.uri()}, - loc_ {location.make_shared()}, + loc_ {std::move(location)}, timestamp_ {timestamp} { } -ListElement::ListElement(const KeyChain& keys, const FieldLocation& location, const TimeStamp& timestamp): - ListElement(keys[0], keys[1], keys[2], location, timestamp) { } +ListElement::ListElement(const KeyChain& keys, std::shared_ptr location, const TimeStamp& timestamp): + ListElement(keys[0], keys[1], keys[2], std::move(location), timestamp) { } ListElement::ListElement(eckit::Stream& stream) { stream >> keys_; @@ -55,6 +55,11 @@ const FieldLocation& ListElement::location() const { return *loc_; } +const eckit::URI& ListElement::uri() const { + if (loc_) { return loc_->uri(); } + return uri_; +} + eckit::Offset ListElement::offset() const { return loc_ ? loc_->offset() : eckit::Offset(0); } diff --git a/src/fdb5/api/helpers/ListElement.h b/src/fdb5/api/helpers/ListElement.h index 54d29a50c..625868d6b 100644 --- a/src/fdb5/api/helpers/ListElement.h +++ b/src/fdb5/api/helpers/ListElement.h @@ -48,9 +48,10 @@ class ListElement { ListElement(Key dbKey, Key indexKey, const eckit::URI& uri, const TimeStamp& timestamp); - ListElement(Key dbKey, Key indexKey, Key datumKey, const FieldLocation& location, const TimeStamp& timestamp); + ListElement(Key dbKey, Key indexKey, Key datumKey, std::shared_ptr location, + const TimeStamp& timestamp); - ListElement(const KeyChain& keys, const FieldLocation& location, const TimeStamp& timestamp); + ListElement(const KeyChain& keys, std::shared_ptr location, const TimeStamp& timestamp); explicit ListElement(eckit::Stream& stream); @@ -60,10 +61,12 @@ class ListElement { const Key& combinedKey() const { return keys_.combine(); } - const eckit::URI& uri() const { return uri_; } + const eckit::URI& uri() const; const TimeStamp& timestamp() const { return timestamp_; } + std::shared_ptr sharedLocation() const { return loc_; } + const FieldLocation& location() const; eckit::Offset offset() const; diff --git a/src/fdb5/api/local/ListVisitor.h b/src/fdb5/api/local/ListVisitor.h index aadd93967..32ad23f8b 100644 --- a/src/fdb5/api/local/ListVisitor.h +++ b/src/fdb5/api/local/ListVisitor.h @@ -102,7 +102,7 @@ struct ListVisitor : public QueryVisitor { ASSERT(currentIndex_); if (key.match(datumRequest_)) { - queue_.emplace(currentCatalogue_->key(), currentIndex_->key(), key, field.location(), field.timestamp()); + queue_.emplace(currentCatalogue_->key(), currentIndex_->key(), key, field.stableLocation(), field.timestamp()); } } diff --git a/src/fdb5/database/MultiRetrieveVisitor.cc b/src/fdb5/database/MultiRetrieveVisitor.cc index dd6cd4025..e1d1b8c70 100644 --- a/src/fdb5/database/MultiRetrieveVisitor.cc +++ b/src/fdb5/database/MultiRetrieveVisitor.cc @@ -109,7 +109,7 @@ bool MultiRetrieveVisitor::selectDatum(const Key& key, const Key& full) { simplifiedKey.set(k->first, k->second); } - iterator_.emplace({db_->key(), db_->indexKey(), simplifiedKey, field.location(), field.timestamp()}); + iterator_.emplace({db_->key(), db_->indexKey(), simplifiedKey, field.stableLocation(), field.timestamp()}); return true; } From d638c98f8c3026728dad846673eda7d8740e7f91 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Thu, 15 Aug 2024 14:12:56 +0200 Subject: [PATCH 57/61] docs(fdb-list): corrected depth range --- src/fdb5/tools/fdb-list.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fdb5/tools/fdb-list.cc b/src/fdb5/tools/fdb-list.cc index 350ebfe07..85628912d 100644 --- a/src/fdb5/tools/fdb-list.cc +++ b/src/fdb5/tools/fdb-list.cc @@ -52,7 +52,7 @@ class FDBList: public FDBVisitTool { "Incompatible with options: location, timestamp, and length")); options_.push_back(new SimpleOption("json", "Output available fields in JSON form")); options_.push_back(new SimpleOption("compact", "Aggregate available fields in MARS requests")); - options_.push_back(new SimpleOption("depth", "Output entries up to 'depth' levels deep [0-2]")); + options_.push_back(new SimpleOption("depth", "Output entries up to 'depth' levels deep [1-3]")); } private: // methods From 25e03d1ef09707f0983c84597af6c03298f25546 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 9 Aug 2024 13:10:15 +0200 Subject: [PATCH 58/61] test(fdb-list): rename cf -> an --- tests/fdb/tools/list/fdb_list.sh.in | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/fdb/tools/list/fdb_list.sh.in b/tests/fdb/tools/list/fdb_list.sh.in index 661518b85..020e5348a 100755 --- a/tests/fdb/tools/list/fdb_list.sh.in +++ b/tests/fdb/tools/list/fdb_list.sh.in @@ -39,7 +39,7 @@ try grib_set -s step=6 x.grib 6.grib try grib_set -s step=9 x.grib 9.grib for f in *.grib; do - try grib_set -s type=cf "$f" "cf.$f" + try grib_set -s type=an "$f" "cf.$f" done for f in *.grib; do @@ -55,10 +55,10 @@ exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=12,param=166} {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=6,param=166} {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=9,param=166} -{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=0,param=166} -{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=12,param=166} -{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=6,param=166} -{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=9,param=166}" +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=0,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=12,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=6,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=9,param=166}" try test "$exp" = "$out" # test date depth=1 @@ -69,23 +69,23 @@ try test "$exp" = "$out" # test date depth=2 out=$(fdb-list date=20201102 --minimum-keys="" --porcelain --depth=2) exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc} -{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}" +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}" try test "$exp" = "$out" # test cf depth=1 -out=$(fdb-list type=cf --minimum-keys="" --porcelain --depth=1) +out=$(fdb-list type=an --minimum-keys="" --porcelain --depth=1) exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}" try test "$exp" = "$out" # test cf depth=2 -out=$(fdb-list type=cf --minimum-keys="" --porcelain --depth=2) -exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}" +out=$(fdb-list type=an --minimum-keys="" --porcelain --depth=2) +exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}" try test "$exp" = "$out" # test cf depth=3 -out=$(fdb-list type=cf --minimum-keys="" --porcelain --depth=3) -exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=0,param=166} -{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=12,param=166} -{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=6,param=166} -{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=cf,levtype=sfc}{step=9,param=166}" +out=$(fdb-list type=an --minimum-keys="" --porcelain --depth=3) +exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=0,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=12,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=6,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=9,param=166}" try test "$exp" = "$out" From d82131f44f1214b2036026898da9aed798c590f3 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 13 Sep 2024 01:06:15 +0200 Subject: [PATCH 59/61] test(fdb-list): add marsflow tests --- tests/fdb/tools/list/CMakeLists.txt | 33 +++- tests/fdb/tools/list/list_all.sh.in | 154 ++++++++++++++++++ tests/fdb/tools/list/list_all_full.sh.in | 104 ++++++++++++ .../list/{fdb_list.sh.in => list_depth.sh.in} | 36 ++-- tests/fdb/tools/list/list_location.sh.in | 67 ++++++++ tests/fdb/tools/list/list_masking.sh.in | 82 ++++++++++ tests/fdb/tools/list/list_minimum_keys.sh.in | 93 +++++++++++ tests/fdb/tools/list/list_porcelain.sh.in | 114 +++++++++++++ tests/fdb/tools/list/list_ranges.sh.in | 91 +++++++++++ 9 files changed, 752 insertions(+), 22 deletions(-) create mode 100755 tests/fdb/tools/list/list_all.sh.in create mode 100755 tests/fdb/tools/list/list_all_full.sh.in rename tests/fdb/tools/list/{fdb_list.sh.in => list_depth.sh.in} (85%) create mode 100755 tests/fdb/tools/list/list_location.sh.in create mode 100755 tests/fdb/tools/list/list_masking.sh.in create mode 100755 tests/fdb/tools/list/list_minimum_keys.sh.in create mode 100755 tests/fdb/tools/list/list_porcelain.sh.in create mode 100755 tests/fdb/tools/list/list_ranges.sh.in diff --git a/tests/fdb/tools/list/CMakeLists.txt b/tests/fdb/tools/list/CMakeLists.txt index 07d3fa583..37ea2b3a6 100644 --- a/tests/fdb/tools/list/CMakeLists.txt +++ b/tests/fdb/tools/list/CMakeLists.txt @@ -1,8 +1,29 @@ -set(test_name fdb_list) +# get test data for the list tests +ecbuild_get_test_multidata( TARGET get_fdb_test_data_list LABELS list NAMES "od.oper.grib" ) -configure_file( ${test_name}.sh.in ${test_name}.sh @ONLY ) +# list of tests +list( APPEND fdb_tools_list_tests + all + all_full + location + masking + minimum_keys + porcelain + ranges + depth +) -ecbuild_add_test( - TYPE SCRIPT - CONDITION HAVE_FDB_BUILD_TOOLS - COMMAND ${test_name}.sh) +# add the tests +foreach( _test ${fdb_tools_list_tests} ) + + configure_file( list_${_test}.sh.in list_${_test}.sh @ONLY ) + + ecbuild_add_test( + TARGET test_fdb_tools_list_${_test} + TYPE SCRIPT + LABELS list + CONDITION HAVE_FDB_BUILD_TOOLS + TEST_DEPENDS get_fdb_test_data_list + COMMAND list_${_test}.sh ) + +endforeach( ) diff --git a/tests/fdb/tools/list/list_all.sh.in b/tests/fdb/tools/list/list_all.sh.in new file mode 100755 index 000000000..8a2f1b2e1 --- /dev/null +++ b/tests/fdb/tools/list/list_all.sh.in @@ -0,0 +1,154 @@ +#!/usr/bin/env bash + +set -ux + +yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; } +die() { yell "$*"; exit 1; } +try() { "$@" || die "Errored HERE => '$*'"; } + +export PATH=@CMAKE_BINARY_DIR@/bin:$PATH +export FDB5_CONFIG_FILE="local.yaml" +export FDB_HOME=@PROJECT_BINARY_DIR@ + +test_name=all +src_data="od.oper.grib" +src_dir=@CMAKE_CURRENT_SOURCE_DIR@ +bin_dir=@CMAKE_CURRENT_BINARY_DIR@ + +######################################################################################################################## + +echo "running test '$test_name' on $(hostname)" + +try cd $bin_dir + +rm -rf $test_name || true +mkdir -p $test_name/localroot + +try cd $test_name + +try cp "$src_dir/local.yaml" ./ +try cp "$bin_dir/$src_data" ./ + +######################################################################################################################## + +count() { + [[ $# -eq 1 ]] || die "count requires 1 argument; expected count" + val=$(wc -l < out) && val=$((val + 0)) + [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" +} + +grep_count() { + [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count" + val=$(grep -E "$1" out | wc -l) && val=$((val + 0)) + [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]" +} + +######################################################################################################################## + +# Ensure that listing finds the correct data, but excludes duplicates. + +# Set up the data for the test + +try grib_set -s class=rd,expver=xxxx,type=fc,step=0 "$src_data" data.xxxx.0.grib +try grib_set -s class=rd,expver=xxxx,type=fc,step=1 "$src_data" data.xxxx.1.grib +try grib_set -s class=rd,expver=xxxx,type=fc,step=2 "$src_data" data.xxxx.2.grib +try grib_set -s class=rd,expver=xxxy,type=fc,step=0 "$src_data" data.xxxy.0.grib +try grib_set -s class=rd,expver=xxxy,type=fc,step=1 "$src_data" data.xxxy.1.grib +try grib_set -s class=rd,expver=xxxy,type=fc,step=2 "$src_data" data.xxxy.2.grib + +# Set up some regexes for later testing + +regex_x0="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}" +regex_x1="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=1,levelist=[0-9]+,param=[0-9]+}" +regex_x2="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=2,levelist=[0-9]+,param=[0-9]+}" +regex_y0="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}" +regex_y1="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=1,levelist=[0-9]+,param=[0-9]+}" +regex_y2="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=2,levelist=[0-9]+,param=[0-9]+}" + +# We loop over all these tests twice. We should get the same listing results both times +# as the default behaviour is to only show the data that would be retrieved (i.e. that matches a MARS request.) + +sum=0 +for i in 1 2; do + + echo "============ Loop $i ============" + + try fdb-write data.xxxx.0.grib + try fdb-list --all --minimum-keys="" --porcelain > out + if [[ $i = 1 ]]; then + sum=$((sum + 24)) + count $sum + grep_count "$regex_x1" 0 + grep_count "$regex_x2" 0 + grep_count "$regex_y0" 0 + grep_count "$regex_y1" 0 + grep_count "$regex_y2" 0 + fi + grep_count "$regex_x0" 24 + + try fdb-write data.xxxy.0.grib + try fdb-list --all --minimum-keys="" --porcelain > out + if [[ $i = 1 ]]; then + sum=$((sum + 24)) + count $sum + grep_count "$regex_x1" 0 + grep_count "$regex_x2" 0 + grep_count "$regex_y1" 0 + grep_count "$regex_y2" 0 + fi + grep_count "$regex_x0" 24 + grep_count "$regex_y0" 24 + + + try fdb-write data.xxxx.1.grib + try fdb-list --all --minimum-keys="" --porcelain > out + if [[ $i = 1 ]]; then + sum=$((sum + 24)) + count $sum + grep_count "$regex_x2" 0 + grep_count "$regex_y1" 0 + grep_count "$regex_y2" 0 + fi + grep_count "$regex_x0" 24 + grep_count "$regex_y0" 24 + grep_count "$regex_x1" 24 + + try fdb-write data.xxxy.1.grib + try fdb-list --all --minimum-keys="" --porcelain > out + if [[ $i = 1 ]]; then + sum=$((sum + 24)) + count $sum + grep_count "$regex_x2" 0 + grep_count "$regex_y2" 0 + fi + grep_count "$regex_x0" 24 + grep_count "$regex_y0" 24 + grep_count "$regex_x1" 24 + grep_count "$regex_y1" 24 + + + try fdb-write data.xxxx.2.grib + try fdb-list --all --minimum-keys="" --porcelain > out + if [[ $i = 1 ]]; then + sum=$((sum + 24)) + count $sum + grep_count "$regex_y2" 0 + fi + grep_count "$regex_x0" 24 + grep_count "$regex_y0" 24 + grep_count "$regex_x1" 24 + grep_count "$regex_y1" 24 + grep_count "$regex_x2" 24 + + try fdb-write data.xxxy.2.grib + try fdb-list --all --minimum-keys="" --porcelain > out + [[ $i = 1 ]] && sum=$((sum + 24)) + count $sum + grep_count "$regex_x0" 24 + grep_count "$regex_y0" 24 + grep_count "$regex_x1" 24 + grep_count "$regex_y1" 24 + grep_count "$regex_x2" 24 + grep_count "$regex_y2" 24 + +done diff --git a/tests/fdb/tools/list/list_all_full.sh.in b/tests/fdb/tools/list/list_all_full.sh.in new file mode 100755 index 000000000..51300eb46 --- /dev/null +++ b/tests/fdb/tools/list/list_all_full.sh.in @@ -0,0 +1,104 @@ +#!/usr/bin/env bash + +set -ux + +yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; } +die() { yell "$*"; exit 1; } +try() { "$@" || die "Errored HERE => '$*'"; } + +export PATH=@CMAKE_BINARY_DIR@/bin:$PATH +export FDB5_CONFIG_FILE="local.yaml" +export FDB_HOME=@PROJECT_BINARY_DIR@ + +test_name=all_full +src_data="od.oper.grib" +src_dir=@CMAKE_CURRENT_SOURCE_DIR@ +bin_dir=@CMAKE_CURRENT_BINARY_DIR@ + +######################################################################################################################## + +echo "running test '$test_name' on $(hostname)" + +try cd $bin_dir + +rm -rf $test_name || true +mkdir -p $test_name/localroot + +try cd $test_name + +try cp "$src_dir/local.yaml" ./ +try cp "$bin_dir/$src_data" ./ + +######################################################################################################################## + +count() { + [[ $# -eq 1 ]] || die "count requires 1 argument; expected count" + val=$(wc -l < out) && val=$((val + 0)) + [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" +} + +grep_count() { + [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count" + val=$(grep -E "$1" out | wc -l) && val=$((val + 0)) + [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]" +} + +######################################################################################################################## + +# Ensure that listing finds the correct data, but excludes duplicates. +# This is the same data as in the all test, but with a full enumeration + +# Set up the data for the test + +try grib_set -s class=rd,expver=xxxx,type=fc,step=0 "$src_data" data.xxxx.0.grib +try grib_set -s class=rd,expver=xxxx,type=fc,step=1 "$src_data" data.xxxx.1.grib +try grib_set -s class=rd,expver=xxxy,type=fc,step=0 "$src_data" data.xxxy.0.grib +try grib_set -s class=rd,expver=xxxy,type=fc,step=1 "$src_data" data.xxxy.1.grib + +# Set up some regexes for later testing + +regex_x0="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}" +regex_x1="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=1,levelist=[0-9]+,param=[0-9]+}" +regex_y0="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}" +regex_y1="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=1,levelist=[0-9]+,param=[0-9]+}" + +# We loop over all these tests twice. We should get the same listing results both times +# as the default behaviour is to only show the data that would be retrieved (i.e. that matches a MARS request.) + +for i in 0 1; do + + try fdb-write data.xxxx.0.grib + try fdb-list --all --minimum-keys="" --porcelain --full > out + count $((24+(i*96))) + grep_count "$regex_x0" $((24+(24*i))) + grep_count "$regex_x1" $((0+(24*i))) + grep_count "$regex_y0" $((0+(24*i))) + grep_count "$regex_y1" $((0+(24*i))) + + try fdb-write data.xxxy.0.grib + try fdb-list --all --minimum-keys="" --porcelain --full > out + count $((48+(i*96))) + grep_count "$regex_x0" $((24+(24*i))) + grep_count "$regex_x1" $((0+(24*i))) + grep_count "$regex_y0" $((24+(24*i))) + grep_count "$regex_y1" $((0+(24*i))) + + try fdb-write data.xxxx.1.grib + try fdb-list --all --minimum-keys="" --porcelain --full > out + count $((72+(i*96))) + grep_count "$regex_x0" $((24+(24*i))) + grep_count "$regex_x1" $((24+(24*i))) + grep_count "$regex_y0" $((24+(24*i))) + grep_count "$regex_y1" $((0+(24*i))) + + try fdb-write data.xxxy.1.grib + try fdb-list --all --minimum-keys="" --porcelain --full > out + count $((96+(i*96))) + grep_count "$regex_x0" $((24+(24*i))) + grep_count "$regex_x1" $((24+(24*i))) + grep_count "$regex_y0" $((24+(24*i))) + grep_count "$regex_y1" $((24+(24*i))) + +done + +exit 0 diff --git a/tests/fdb/tools/list/fdb_list.sh.in b/tests/fdb/tools/list/list_depth.sh.in similarity index 85% rename from tests/fdb/tools/list/fdb_list.sh.in rename to tests/fdb/tools/list/list_depth.sh.in index 020e5348a..fbb6c12c2 100755 --- a/tests/fdb/tools/list/fdb_list.sh.in +++ b/tests/fdb/tools/list/list_depth.sh.in @@ -1,8 +1,8 @@ #!/usr/bin/env bash -# set -eu +set -ux -yell() { echo "$(basename "$0"): $*" >&2; } +yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; } die() { yell "$*"; exit 1; } try() { "$@" || die "Errored HERE => '$*'"; } @@ -14,42 +14,46 @@ export PATH=@CMAKE_BINARY_DIR@/bin:$PATH export FDB5_CONFIG_FILE="local.yaml" export FDB_HOME=@PROJECT_BINARY_DIR@ -tname=list_test -srcdir=@CMAKE_CURRENT_SOURCE_DIR@ -bindir=@CMAKE_CURRENT_BINARY_DIR@ +test_name=depth +src_data=x.grib +src_dir=@CMAKE_CURRENT_SOURCE_DIR@ +bin_dir=@CMAKE_CURRENT_BINARY_DIR@ ######################################################################################################################## -try cd $bindir +echo "running test '$test_name' on $(hostname)" -rm -rf $tname || true -mkdir -p $tname/localroot +try cd $bin_dir -try cd $tname +rm -rf $test_name || true +mkdir -p $test_name/localroot -for f in local.yaml x.grib; do - cp "$srcdir/$f" ./ +try cd $test_name + +for f in local.yaml $src_data; do + try cp "$src_dir/$f" ./ done ######################################################################################################################## # Create a grib file with 4 steps and populate FDB -try grib_set -s step=0 x.grib 0.grib -try grib_set -s step=6 x.grib 6.grib -try grib_set -s step=9 x.grib 9.grib +try grib_set -s step=0 $src_data 0.grib +try grib_set -s step=6 $src_data 6.grib +try grib_set -s step=9 $src_data 9.grib for f in *.grib; do try grib_set -s type=an "$f" "cf.$f" done for f in *.grib; do - try cat "$f" >> "$tname.grib" + try cat "$f" >> "$test_name.grib" done -try fdb-write "$tname.grib" +try fdb-write "$test_name.grib" ######################################################################################################################## # test all + out=$(fdb-list --all --minimum-keys= --porcelain) exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=0,param=166} {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=12,param=166} diff --git a/tests/fdb/tools/list/list_location.sh.in b/tests/fdb/tools/list/list_location.sh.in new file mode 100755 index 000000000..5a8903888 --- /dev/null +++ b/tests/fdb/tools/list/list_location.sh.in @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +set -ux + +yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; } +die() { yell "$*"; exit 1; } +try() { "$@" || die "Errored HERE => '$*'"; } + +export PATH=@CMAKE_BINARY_DIR@/bin:$PATH +export FDB5_CONFIG_FILE="local.yaml" +export FDB_HOME=@PROJECT_BINARY_DIR@ + +test_name=location +src_data="od.oper.grib" +src_dir=@CMAKE_CURRENT_SOURCE_DIR@ +bin_dir=@CMAKE_CURRENT_BINARY_DIR@ + +######################################################################################################################## + +echo "running test '$test_name' on $(hostname)" + +try cd $bin_dir + +rm -rf $test_name || true +mkdir -p $test_name/localroot + +try cd $test_name + +try cp "$src_dir/local.yaml" ./ +try cp "$bin_dir/$src_data" ./ + +######################################################################################################################## + +count() { + [[ $# -eq 1 ]] || die "count requires 1 argument; expected count" + val=$(wc -l < out) && val=$((val + 0)) + [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" +} + +grep_count() { + [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count" + val=$(grep -E "$1" out | wc -l) && val=$((val + 0)) + [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]" +} + +######################################################################################################################## + +# Check that we can obtain the location of the data + +try grib_set -s class=rd,expver=xxxx "$src_data" data.xxxx.grib +try grib_ls -m data.xxxx.grib +try fdb-write data.xxxx.grib + +try fdb-list --all --minimum-keys= --porcelain --full > out +count 24 + +# Specify the location flag + +try fdb-list class=rd,expver=xxxx,time=0000 --location > out +grep_count "localroot/rd:xxxx" 12 + +# And without the location flag, none of the location info is included + +fdb-list class=rd,expver=xxxx,time=0000 --porcelain +try fdb-list class=rd,expver=xxxx,time=0000 --porcelain > out +count 12 +grep_count "{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=0000,domain=g}{type=an,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}" 12 diff --git a/tests/fdb/tools/list/list_masking.sh.in b/tests/fdb/tools/list/list_masking.sh.in new file mode 100755 index 000000000..d73c681b3 --- /dev/null +++ b/tests/fdb/tools/list/list_masking.sh.in @@ -0,0 +1,82 @@ +#!/usr/bin/env bash + +set -ux + +yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; } +die() { yell "$*"; exit 1; } +try() { "$@" || die "Errored HERE => '$*'"; } + +export PATH=@CMAKE_BINARY_DIR@/bin:$PATH +export FDB5_CONFIG_FILE="local.yaml" +export FDB_HOME=@PROJECT_BINARY_DIR@ + +test_name=masking +src_data="od.oper.grib" +src_dir=@CMAKE_CURRENT_SOURCE_DIR@ +bin_dir=@CMAKE_CURRENT_BINARY_DIR@ + +######################################################################################################################## + +echo "running test '$test_name' on $(hostname)" + +try cd $bin_dir + +rm -rf $test_name || true +mkdir -p $test_name/localroot + +try cd $test_name + +try cp "$src_dir/local.yaml" ./ +try cp "$bin_dir/$src_data" ./ + +######################################################################################################################## + +count() { + [[ $# -eq 1 ]] || die "count requires 1 argument; expected count" + val=$(wc -l < out) && val=$((val + 0)) + [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" +} + +# grep_count() { +# [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count" +# val=$(grep -E "$1" out | wc -l) && val=$((val + 0)) +# [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]" +# } + +######################################################################################################################## + +# Ensure that we can wipe specified databases with ranges in the requests +# --> Ensures the MARS requests are working correctly + +try grib_set -s class=rd,expver=xxxx "$src_data" data.xxxx.d1.grib +try grib_set -s class=rd,expver=xxxx,date=20170101 "$src_data" data.xxxx.d2.grib +try grib_ls -m data.xxxx.d1.grib + +try fdb-write data.xxxx.d1.grib +try fdb-write data.xxxx.d2.grib +try fdb-write data.xxxx.d1.grib +try fdb-write data.xxxx.d2.grib + +echo "Incorrect number of entries written" +try fdb-list --all --minimum-keys="" --porcelain > out +count 48 + +echo "Incorrect number of entries written" +fdb-list --all --minimum-keys="" --porcelain --full > out +count 96 + +echo "Entries should be masked without --full" +fdb-list class=rd,expver=xxxx,date=-1/20170101,stream=oper,type=an,levtype=pl,param=155/138,levelist=300/400/500/700/850/1000 --porcelain > out +count 48 + +echo "All entries should be visible with --full" +try fdb-list class=rd,expver=xxxx,date=-1/20170101,stream=oper,type=an,levtype=pl,param=155/138,levelist=300/400/500/700/850/1000 --porcelain --full > out +count 96 + +echo "Entries should be masked without full" +try fdb-list class=rd,expver=xxxx,date=-1/20170101,stream=oper,type=an,levtype=pl,param=130/138,levelist=300/123/1000 --porcelain > out +count 8 + +echo "All entries should be visible with --full" +try fdb-list class=rd,expver=xxxx,date=-1/20170101,stream=oper,type=an,levtype=pl,param=130/138,levelist=300/123/1000 --porcelain --full > out +count 16 diff --git a/tests/fdb/tools/list/list_minimum_keys.sh.in b/tests/fdb/tools/list/list_minimum_keys.sh.in new file mode 100755 index 000000000..325ca5a5c --- /dev/null +++ b/tests/fdb/tools/list/list_minimum_keys.sh.in @@ -0,0 +1,93 @@ +#!/usr/bin/env bash + +set -ux + +yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; } +die() { yell "$*"; exit 1; } +try() { "$@" || die "Errored HERE => '$*'"; } + +export PATH=@CMAKE_BINARY_DIR@/bin:$PATH +export FDB5_CONFIG_FILE="local.yaml" +export FDB_HOME=@PROJECT_BINARY_DIR@ + +test_name=minimum_keys +src_data="od.oper.grib" +src_dir=@CMAKE_CURRENT_SOURCE_DIR@ +bin_dir=@CMAKE_CURRENT_BINARY_DIR@ + +######################################################################################################################## + +echo "running test '$test_name' on $(hostname)" + +try cd $bin_dir + +rm -rf $test_name || true +mkdir -p $test_name/localroot + +try cd $test_name + +try cp "$src_dir/local.yaml" ./ +try cp "$bin_dir/$src_data" ./ + +######################################################################################################################## + +count() { + [[ $# -eq 1 ]] || die "count requires 1 argument; expected count" + val=$(wc -l < out) && val=$((val + 0)) + [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" +} + +# grep_count() { +# [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count" +# val=$(grep -E "$1" out | wc -l) && val=$((val + 0)) +# [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]" +# } + +######################################################################################################################## + +echo "Ensure that we can wipe specified databases, but only when fully specified" + +try grib_set -s class=rd,expver=xxxx "$src_data" data.xxxx.grib +try grib_ls -m data.xxxx.grib + +try fdb-write data.xxxx.grib +try fdb-list --all --full --minimum-keys= --porcelain > out +count 24 + + +echo "Check that if any of the keys are missing, then nothing happens" + +for invalid_key in class=rd expver=xxxx; do + # set +e + fdb-list $invalid_key + rc=$? + # set -e + + [[ $rc = 0 ]] && die "Should not have succeeded: $invalid_key" +done + + +echo "Check that supplying the key makes it work" + +try fdb-list class=rd,expver=xxxx --porcelain > out +count 24 + + +echo "Check that we can change the minimum keys" + +for invalid_key in class=rd,expver=xxxx \ + class=rd,time=1200 \ + expver=xxxx,time=1200; do + set +e + fdb-list --minimum-keys="class,expver,time" $invalid_key + rc=$? + set -e + + [[ $rc = 0 ]] && echo "Should not have succeeded: $invalid_key" && exit 1 +done + + +echo "And that the purge works now this is supplied" + +fdb-list --minimum-keys="class,expver,time" class=rd,expver=xxxx,time=1200 --porcelain > out +count 12 diff --git a/tests/fdb/tools/list/list_porcelain.sh.in b/tests/fdb/tools/list/list_porcelain.sh.in new file mode 100755 index 000000000..932eb396c --- /dev/null +++ b/tests/fdb/tools/list/list_porcelain.sh.in @@ -0,0 +1,114 @@ +#!/usr/bin/env bash + +set -ux + +yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; } +die() { yell "$*"; exit 1; } +try() { "$@" || die "Errored HERE => '$*'"; } + +export PATH=@CMAKE_BINARY_DIR@/bin:$PATH +export FDB5_CONFIG_FILE="local.yaml" +export FDB_HOME=@PROJECT_BINARY_DIR@ + +test_name=porcelain +src_data="od.oper.grib" +src_dir=@CMAKE_CURRENT_SOURCE_DIR@ +bin_dir=@CMAKE_CURRENT_BINARY_DIR@ + +######################################################################################################################## + +echo "running test '$test_name' on $(hostname)" + +try cd $bin_dir + +rm -rf $test_name || true +mkdir -p $test_name/localroot + +try cd $test_name + +try cp "$src_dir/local.yaml" ./ +try cp "$bin_dir/$src_data" ./ + +######################################################################################################################## + +count() { + [[ $# -eq 1 ]] || die "count requires 1 argument; expected count" + val=$(wc -l < out) && val=$((val + 0)) + [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" +} + +grep_count() { + [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count" + val=$(grep -E "$1" out | wc -l) && val=$((val + 0)) + [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]" +} + +######################################################################################################################## + +# If we don't specify --porcelain, then the expanded request is prepended to the output request. +# Measure the difference! + +try grib_set -s class=rd,expver=xxxx "$src_data" data.xxxx.grib +try grib_ls -m data.xxxx.grib +try fdb-write data.xxxx.grib + + +echo "Test all output" + +try fdb-list --all --minimum-keys="" --porcelain --full > out +count 24 + + +echo "Test normal output" + +try fdb-list class=rd,expver=xxxx > out +grep_count '{class=rd,expver=xxxx' 24 +grep_count '^retrieve,$' 1 +grep_count '^\t.*=.*' 2 +grep_count '^\tclass=rd' 1 +grep_count '^\texpver=xxxx' 1 + + +echo "Test porcelain output" + +try fdb-list class=rd,expver=xxxx --porcelain > out +grep_count '{class=rd,expver=xxxx' 24 +count 24 + + +echo "Test expansion of date and param without porcelain" + +try fdb-list class=rd,expver=xxxx,date=-1,stream=oper,type=an,levtype=pl,param=t > out +grep_count '{class=rd,expver=xxxx' 0 +grep_count '^retrieve,$' 1 +grep_count '^\t.*=.*' 7 +grep_count '^\tclass=rd' 1 +grep_count '^\texpver=xxxx' 1 +grep_count '^\tparam=130' 1 +grep_count '^\tdate=[0-9]{8}' 1 + + +echo "Test expansion of date and param with porcelain" + +try fdb-list class=rd,expver=xxxx,date=-1,stream=oper,type=an,levtype=pl,param=t --porcelain > out +[[ "$(grep '{class=rd,expver=xxxx' out | grep 'date=[0-9]{8}' | wc -l)" -eq 0 ]] || die "Incorrect number of entries reported" +count 0 + + +echo "Test expansion of date and param without porcelain" + +try fdb-list class=rd,expver=xxxx,date=-1,stream=oper,type=an,levtype=pl,param=138 > out +[[ $(grep '{class=rd,expver=xxxx' out | grep -E 'date=[0-9]{8}' | grep param=138 | wc -l) =~ "12" ]] || die "Incorrect number of entries reported" +grep_count '^retrieve,$' 1 +grep_count '^\t.*=.*' 7 +grep_count '^\tclass=rd' 1 +grep_count '^\texpver=xxxx' 1 +grep_count '^\tparam=138' 1 +grep_count '^\tdate=[0-9]{8}' 1 + + +echo "Test expansion of date and param with porcelain" + +try fdb-list class=rd,expver=xxxx,date=-1,stream=oper,type=an,levtype=pl,param=138 --porcelain > out +[[ $(grep '{class=rd,expver=xxxx' out | grep -E 'date=[0-9]{8}' | grep param=138 | wc -l) =~ "12" ]] || die "Incorrect number of entries reported" +count 12 diff --git a/tests/fdb/tools/list/list_ranges.sh.in b/tests/fdb/tools/list/list_ranges.sh.in new file mode 100755 index 000000000..eeea01bb5 --- /dev/null +++ b/tests/fdb/tools/list/list_ranges.sh.in @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +set -ux + +yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; } +die() { yell "$*"; exit 1; } +try() { "$@" || die "Errored HERE => '$*'"; } + +export PATH=@CMAKE_BINARY_DIR@/bin:$PATH +export FDB5_CONFIG_FILE="local.yaml" +export FDB_HOME=@PROJECT_BINARY_DIR@ + +test_name=ranges +src_data="od.oper.grib" +src_dir=@CMAKE_CURRENT_SOURCE_DIR@ +bin_dir=@CMAKE_CURRENT_BINARY_DIR@ + +######################################################################################################################## + +echo "running test '$test_name' on $(hostname)" + +try cd $bin_dir + +rm -rf $test_name || true +mkdir -p $test_name/localroot + +try cd $test_name + +try cp "$src_dir/local.yaml" ./ +try cp "$bin_dir/$src_data" ./ + +######################################################################################################################## + +count() { + [[ $# -ne 1 ]] && die "count requires 1 argument; expected count" + val=$(wc -l < out) && val=$((val + 0)) + [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" +} + +# grep_count() { +# [[ $# -ne 2 ]] && die "grep_count requires 2 arguments; regex and expected count" +# val=$(grep -E "$1" out | wc -l) && val=$((val + 0)) +# [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]" +# } + +######################################################################################################################## + +# Ensure that we can wipe specified databases with ranges in the requests +# --> Ensures the MARS requests are working correctly + +try grib_set -s class=rd,expver=xxxx "$src_data" data.xxxx.d1.grib +try grib_set -s class=rd,expver=xxxx,date=20170101 "$src_data" data.xxxx.d2.grib +try grib_set -s class=rd,expver=xxxx,date=20180103 "$src_data" data.xxxx.d3.grib +try grib_ls -m data.xxxx.d1.grib + +try fdb-write data.xxxx.d1.grib +try fdb-write data.xxxx.d2.grib +try fdb-write data.xxxx.d3.grib + +try fdb-list --all --minimum-keys="" --porcelain --full > out +count 72 + +try fdb-list class=rd,expver=xxxx --porcelain > out +count 72 + +try fdb-list class=rd,expver=xxxx,date=-1 --porcelain > out +count 24 + +try fdb-list class=rd,expver=xxxx,date=-1/20170101 --porcelain > out +count 48 + +try fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103 --porcelain > out +count 72 + +try fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103/20060101,stream=oper,type=an,levtype=pl,param=60 --porcelain > out +count 0 + +try fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103,stream=oper,type=an,levtype=pl,param=155 --porcelain > out +count 36 + +try fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103,stream=oper,type=an,levtype=pl,param=60/155 --porcelain > out +count 36 + +try fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103,stream=oper,type=an,levtype=pl,param=60/155/138 --porcelain > out +count 72 + +try fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103,stream=oper,type=an,levtype=pl,param=60/155/138,levelist=300/400/500/700/850/1000 --porcelain > out +count 72 + +try fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103,stream=oper,type=an,levtype=pl,param=60/155/138,levelist=300/123/1000 --porcelain > out +count 24 From 13eeff94c0c11a29de969e0734341db9b1663855 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 13 Sep 2024 11:48:54 +0200 Subject: [PATCH 60/61] test(fdb-list): fix macos grep issue and several minor issues --- tests/fdb/tools/list/list_all.sh.in | 58 +++++------ tests/fdb/tools/list/list_all_full.sh.in | 52 +++++----- tests/fdb/tools/list/list_depth.sh.in | 34 +++---- tests/fdb/tools/list/list_location.sh.in | 50 ++++----- tests/fdb/tools/list/list_masking.sh.in | 67 ++++++------ tests/fdb/tools/list/list_minimum_keys.sh.in | 68 +++++-------- tests/fdb/tools/list/list_porcelain.sh.in | 101 ++++++++++--------- tests/fdb/tools/list/list_ranges.sh.in | 76 +++++++------- 8 files changed, 240 insertions(+), 266 deletions(-) diff --git a/tests/fdb/tools/list/list_all.sh.in b/tests/fdb/tools/list/list_all.sh.in index 8a2f1b2e1..d714c4b59 100755 --- a/tests/fdb/tools/list/list_all.sh.in +++ b/tests/fdb/tools/list/list_all.sh.in @@ -2,10 +2,24 @@ set -ux -yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; } +yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; } die() { yell "$*"; exit 1; } try() { "$@" || die "Errored HERE => '$*'"; } +line_count() { + [[ $# -eq 1 ]] || die "line_count requires 1 argument; expected line count" + val=$(wc -l < out) && val=$((val + 0)) + [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" +} + +grep_count() { + [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count" + val=$(grep -cE "$1" out) + [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]" +} + +######################################################################################################################## + export PATH=@CMAKE_BINARY_DIR@/bin:$PATH export FDB5_CONFIG_FILE="local.yaml" export FDB_HOME=@PROJECT_BINARY_DIR@ @@ -21,8 +35,8 @@ echo "running test '$test_name' on $(hostname)" try cd $bin_dir -rm -rf $test_name || true -mkdir -p $test_name/localroot +try rm -rf $test_name +try mkdir -p $test_name/localroot try cd $test_name @@ -31,20 +45,6 @@ try cp "$bin_dir/$src_data" ./ ######################################################################################################################## -count() { - [[ $# -eq 1 ]] || die "count requires 1 argument; expected count" - val=$(wc -l < out) && val=$((val + 0)) - [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" -} - -grep_count() { - [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count" - val=$(grep -E "$1" out | wc -l) && val=$((val + 0)) - [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]" -} - -######################################################################################################################## - # Ensure that listing finds the correct data, but excludes duplicates. # Set up the data for the test @@ -74,10 +74,10 @@ for i in 1 2; do echo "============ Loop $i ============" try fdb-write data.xxxx.0.grib - try fdb-list --all --minimum-keys="" --porcelain > out + try fdb-list --all --minimum-keys="" --porcelain | tee out if [[ $i = 1 ]]; then sum=$((sum + 24)) - count $sum + line_count $sum grep_count "$regex_x1" 0 grep_count "$regex_x2" 0 grep_count "$regex_y0" 0 @@ -87,10 +87,10 @@ for i in 1 2; do grep_count "$regex_x0" 24 try fdb-write data.xxxy.0.grib - try fdb-list --all --minimum-keys="" --porcelain > out + try fdb-list --all --minimum-keys="" --porcelain | tee out if [[ $i = 1 ]]; then sum=$((sum + 24)) - count $sum + line_count $sum grep_count "$regex_x1" 0 grep_count "$regex_x2" 0 grep_count "$regex_y1" 0 @@ -101,10 +101,10 @@ for i in 1 2; do try fdb-write data.xxxx.1.grib - try fdb-list --all --minimum-keys="" --porcelain > out + try fdb-list --all --minimum-keys="" --porcelain | tee out if [[ $i = 1 ]]; then sum=$((sum + 24)) - count $sum + line_count $sum grep_count "$regex_x2" 0 grep_count "$regex_y1" 0 grep_count "$regex_y2" 0 @@ -114,10 +114,10 @@ for i in 1 2; do grep_count "$regex_x1" 24 try fdb-write data.xxxy.1.grib - try fdb-list --all --minimum-keys="" --porcelain > out + try fdb-list --all --minimum-keys="" --porcelain | tee out if [[ $i = 1 ]]; then sum=$((sum + 24)) - count $sum + line_count $sum grep_count "$regex_x2" 0 grep_count "$regex_y2" 0 fi @@ -128,10 +128,10 @@ for i in 1 2; do try fdb-write data.xxxx.2.grib - try fdb-list --all --minimum-keys="" --porcelain > out + try fdb-list --all --minimum-keys="" --porcelain | tee out if [[ $i = 1 ]]; then sum=$((sum + 24)) - count $sum + line_count $sum grep_count "$regex_y2" 0 fi grep_count "$regex_x0" 24 @@ -141,9 +141,9 @@ for i in 1 2; do grep_count "$regex_x2" 24 try fdb-write data.xxxy.2.grib - try fdb-list --all --minimum-keys="" --porcelain > out + try fdb-list --all --minimum-keys="" --porcelain | tee out [[ $i = 1 ]] && sum=$((sum + 24)) - count $sum + line_count $sum grep_count "$regex_x0" 24 grep_count "$regex_y0" 24 grep_count "$regex_x1" 24 diff --git a/tests/fdb/tools/list/list_all_full.sh.in b/tests/fdb/tools/list/list_all_full.sh.in index 51300eb46..79963ebda 100755 --- a/tests/fdb/tools/list/list_all_full.sh.in +++ b/tests/fdb/tools/list/list_all_full.sh.in @@ -2,10 +2,24 @@ set -ux -yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; } +yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; } die() { yell "$*"; exit 1; } try() { "$@" || die "Errored HERE => '$*'"; } +line_count() { + [[ $# -eq 1 ]] || die "line_count requires 1 argument; expected line count" + val=$(wc -l < out) && val=$((val + 0)) + [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" +} + +grep_count() { + [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count" + val=$(grep -cEE "$1" out) + [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]" +} + +######################################################################################################################## + export PATH=@CMAKE_BINARY_DIR@/bin:$PATH export FDB5_CONFIG_FILE="local.yaml" export FDB_HOME=@PROJECT_BINARY_DIR@ @@ -21,8 +35,8 @@ echo "running test '$test_name' on $(hostname)" try cd $bin_dir -rm -rf $test_name || true -mkdir -p $test_name/localroot +try rm -rf $test_name +try mkdir -p $test_name/localroot try cd $test_name @@ -31,20 +45,6 @@ try cp "$bin_dir/$src_data" ./ ######################################################################################################################## -count() { - [[ $# -eq 1 ]] || die "count requires 1 argument; expected count" - val=$(wc -l < out) && val=$((val + 0)) - [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" -} - -grep_count() { - [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count" - val=$(grep -E "$1" out | wc -l) && val=$((val + 0)) - [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]" -} - -######################################################################################################################## - # Ensure that listing finds the correct data, but excludes duplicates. # This is the same data as in the all test, but with a full enumeration @@ -67,33 +67,35 @@ regex_y1="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g} for i in 0 1; do + echo "============ Loop $i ============" + try fdb-write data.xxxx.0.grib - try fdb-list --all --minimum-keys="" --porcelain --full > out - count $((24+(i*96))) + try fdb-list --all --minimum-keys="" --porcelain --full | tee out + line_count $((24+(i*96))) grep_count "$regex_x0" $((24+(24*i))) grep_count "$regex_x1" $((0+(24*i))) grep_count "$regex_y0" $((0+(24*i))) grep_count "$regex_y1" $((0+(24*i))) try fdb-write data.xxxy.0.grib - try fdb-list --all --minimum-keys="" --porcelain --full > out - count $((48+(i*96))) + try fdb-list --all --minimum-keys="" --porcelain --full | tee out + line_count $((48+(i*96))) grep_count "$regex_x0" $((24+(24*i))) grep_count "$regex_x1" $((0+(24*i))) grep_count "$regex_y0" $((24+(24*i))) grep_count "$regex_y1" $((0+(24*i))) try fdb-write data.xxxx.1.grib - try fdb-list --all --minimum-keys="" --porcelain --full > out - count $((72+(i*96))) + try fdb-list --all --minimum-keys="" --porcelain --full | tee out + line_count $((72+(i*96))) grep_count "$regex_x0" $((24+(24*i))) grep_count "$regex_x1" $((24+(24*i))) grep_count "$regex_y0" $((24+(24*i))) grep_count "$regex_y1" $((0+(24*i))) try fdb-write data.xxxy.1.grib - try fdb-list --all --minimum-keys="" --porcelain --full > out - count $((96+(i*96))) + try fdb-list --all --minimum-keys="" --porcelain --full | tee out + line_count $((96+(i*96))) grep_count "$regex_x0" $((24+(24*i))) grep_count "$regex_x1" $((24+(24*i))) grep_count "$regex_y0" $((24+(24*i))) diff --git a/tests/fdb/tools/list/list_depth.sh.in b/tests/fdb/tools/list/list_depth.sh.in index fbb6c12c2..9a40e52be 100755 --- a/tests/fdb/tools/list/list_depth.sh.in +++ b/tests/fdb/tools/list/list_depth.sh.in @@ -2,13 +2,11 @@ set -ux -yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; } +yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; } die() { yell "$*"; exit 1; } try() { "$@" || die "Errored HERE => '$*'"; } -function request() { - echo "RETRIEVE,CLASS=OD,TYPE=CF,STREAM=ENFO,EXPVER=0001,LEVTYPE=PL,DATE=20201106,TIME=1200,STEP=00,DOMAIN=G,$1" > req; -} +######################################################################################################################## export PATH=@CMAKE_BINARY_DIR@/bin:$PATH export FDB5_CONFIG_FILE="local.yaml" @@ -25,8 +23,8 @@ echo "running test '$test_name' on $(hostname)" try cd $bin_dir -rm -rf $test_name || true -mkdir -p $test_name/localroot +try rm -rf $test_name +try mkdir -p $test_name/localroot try cd $test_name @@ -52,9 +50,8 @@ done try fdb-write "$test_name.grib" ######################################################################################################################## -# test all -out=$(fdb-list --all --minimum-keys= --porcelain) +echo "Test: all" exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=0,param=166} {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=12,param=166} {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=6,param=166} @@ -63,33 +60,34 @@ exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=12,param=166} {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=6,param=166} {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=9,param=166}" +out=$(try fdb-list --all --minimum-keys= --porcelain) try test "$exp" = "$out" -# test date depth=1 -out=$(fdb-list date=20201102 --minimum-keys="" --porcelain --depth=1) +echo "Test: date depth=1" exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}" +out=$(try fdb-list date=20201102 --minimum-keys="" --porcelain --depth=1) try test "$exp" = "$out" -# test date depth=2 -out=$(fdb-list date=20201102 --minimum-keys="" --porcelain --depth=2) +echo "Test: date depth=2" exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc} {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}" +out=$(try fdb-list date=20201102 --minimum-keys="" --porcelain --depth=2) try test "$exp" = "$out" -# test cf depth=1 -out=$(fdb-list type=an --minimum-keys="" --porcelain --depth=1) +echo "Test: cf depth=1" exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}" +out=$(try fdb-list type=an --minimum-keys="" --porcelain --depth=1) try test "$exp" = "$out" -# test cf depth=2 -out=$(fdb-list type=an --minimum-keys="" --porcelain --depth=2) +echo "Test: cf depth=2" exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}" +out=$(try fdb-list type=an --minimum-keys="" --porcelain --depth=2) try test "$exp" = "$out" -# test cf depth=3 -out=$(fdb-list type=an --minimum-keys="" --porcelain --depth=3) +echo "Test: cf depth=3" exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=0,param=166} {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=12,param=166} {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=6,param=166} {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=9,param=166}" +out=$(try fdb-list type=an --minimum-keys="" --porcelain --depth=3) try test "$exp" = "$out" diff --git a/tests/fdb/tools/list/list_location.sh.in b/tests/fdb/tools/list/list_location.sh.in index 5a8903888..063a6a29a 100755 --- a/tests/fdb/tools/list/list_location.sh.in +++ b/tests/fdb/tools/list/list_location.sh.in @@ -2,10 +2,24 @@ set -ux -yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; } +yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; } die() { yell "$*"; exit 1; } try() { "$@" || die "Errored HERE => '$*'"; } +line_count() { + [[ $# -eq 1 ]] || die "line_count requires 1 argument; expected line count" + val=$(wc -l < out) && val=$((val + 0)) + [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" +} + +grep_count() { + [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count" + val=$(grep -cE "$1" out) + [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]" +} + +######################################################################################################################## + export PATH=@CMAKE_BINARY_DIR@/bin:$PATH export FDB5_CONFIG_FILE="local.yaml" export FDB_HOME=@PROJECT_BINARY_DIR@ @@ -21,8 +35,8 @@ echo "running test '$test_name' on $(hostname)" try cd $bin_dir -rm -rf $test_name || true -mkdir -p $test_name/localroot +try rm -rf $test_name +try mkdir -p $test_name/localroot try cd $test_name @@ -31,37 +45,23 @@ try cp "$bin_dir/$src_data" ./ ######################################################################################################################## -count() { - [[ $# -eq 1 ]] || die "count requires 1 argument; expected count" - val=$(wc -l < out) && val=$((val + 0)) - [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" -} - -grep_count() { - [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count" - val=$(grep -E "$1" out | wc -l) && val=$((val + 0)) - [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]" -} - -######################################################################################################################## - # Check that we can obtain the location of the data try grib_set -s class=rd,expver=xxxx "$src_data" data.xxxx.grib try grib_ls -m data.xxxx.grib try fdb-write data.xxxx.grib -try fdb-list --all --minimum-keys= --porcelain --full > out -count 24 +echo "Test: --all --minimum-keys= --porcelain --full" +try fdb-list --all --minimum-keys= --porcelain --full | tee out +line_count 24 -# Specify the location flag -try fdb-list class=rd,expver=xxxx,time=0000 --location > out +echo "Test: Specify the location flag" +try fdb-list class=rd,expver=xxxx,time=0000 --location | tee out grep_count "localroot/rd:xxxx" 12 -# And without the location flag, none of the location info is included -fdb-list class=rd,expver=xxxx,time=0000 --porcelain -try fdb-list class=rd,expver=xxxx,time=0000 --porcelain > out -count 12 +echo "Test: without the location flag, none of the location info is included" +try fdb-list class=rd,expver=xxxx,time=0000 --porcelain | tee out +line_count 12 grep_count "{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=0000,domain=g}{type=an,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}" 12 diff --git a/tests/fdb/tools/list/list_masking.sh.in b/tests/fdb/tools/list/list_masking.sh.in index d73c681b3..76beb44a6 100755 --- a/tests/fdb/tools/list/list_masking.sh.in +++ b/tests/fdb/tools/list/list_masking.sh.in @@ -2,10 +2,18 @@ set -ux -yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; } +yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; } die() { yell "$*"; exit 1; } try() { "$@" || die "Errored HERE => '$*'"; } +line_count() { + [[ $# -eq 1 ]] || die "line_count requires 1 argument; expected line count" + val=$(wc -l < out) && val=$((val + 0)) + [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" +} + +######################################################################################################################## + export PATH=@CMAKE_BINARY_DIR@/bin:$PATH export FDB5_CONFIG_FILE="local.yaml" export FDB_HOME=@PROJECT_BINARY_DIR@ @@ -21,8 +29,8 @@ echo "running test '$test_name' on $(hostname)" try cd $bin_dir -rm -rf $test_name || true -mkdir -p $test_name/localroot +try rm -rf $test_name +try mkdir -p $test_name/localroot try cd $test_name @@ -31,52 +39,39 @@ try cp "$bin_dir/$src_data" ./ ######################################################################################################################## -count() { - [[ $# -eq 1 ]] || die "count requires 1 argument; expected count" - val=$(wc -l < out) && val=$((val + 0)) - [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" -} - -# grep_count() { -# [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count" -# val=$(grep -E "$1" out | wc -l) && val=$((val + 0)) -# [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]" -# } - -######################################################################################################################## - # Ensure that we can wipe specified databases with ranges in the requests # --> Ensures the MARS requests are working correctly try grib_set -s class=rd,expver=xxxx "$src_data" data.xxxx.d1.grib -try grib_set -s class=rd,expver=xxxx,date=20170101 "$src_data" data.xxxx.d2.grib +try grib_set -s class=rd,expver=xxxx,date=20240912 "$src_data" data.xxxx.d2.grib try grib_ls -m data.xxxx.d1.grib try fdb-write data.xxxx.d1.grib try fdb-write data.xxxx.d2.grib +# write the data twice try fdb-write data.xxxx.d1.grib try fdb-write data.xxxx.d2.grib -echo "Incorrect number of entries written" -try fdb-list --all --minimum-keys="" --porcelain > out -count 48 +echo "Test: Incorrect number of entries written" +try fdb-list --all --minimum-keys="" --porcelain | tee out +line_count 48 -echo "Incorrect number of entries written" -fdb-list --all --minimum-keys="" --porcelain --full > out -count 96 +echo "Test: Incorrect number of entries written" +try fdb-list --all --minimum-keys="" --porcelain --full | tee out +line_count 96 -echo "Entries should be masked without --full" -fdb-list class=rd,expver=xxxx,date=-1/20170101,stream=oper,type=an,levtype=pl,param=155/138,levelist=300/400/500/700/850/1000 --porcelain > out -count 48 +echo "Test: Entries should be masked without --full" +try fdb-list class=rd,expver=xxxx,date=20240911/20240912,stream=oper,type=an,levtype=pl,param=155/138,levelist=300/400/500/700/850/1000 --porcelain | tee out +line_count 48 -echo "All entries should be visible with --full" -try fdb-list class=rd,expver=xxxx,date=-1/20170101,stream=oper,type=an,levtype=pl,param=155/138,levelist=300/400/500/700/850/1000 --porcelain --full > out -count 96 +echo "Test: All entries should be visible with --full" +try fdb-list class=rd,expver=xxxx,date=20240911/20240912,stream=oper,type=an,levtype=pl,param=155/138,levelist=300/400/500/700/850/1000 --porcelain --full | tee out +line_count 96 -echo "Entries should be masked without full" -try fdb-list class=rd,expver=xxxx,date=-1/20170101,stream=oper,type=an,levtype=pl,param=130/138,levelist=300/123/1000 --porcelain > out -count 8 +echo "Test: Entries should be masked without full" +try fdb-list class=rd,expver=xxxx,date=20240911/20240912,stream=oper,type=an,levtype=pl,param=130/138,levelist=300/123/1000 --porcelain | tee out +line_count 8 -echo "All entries should be visible with --full" -try fdb-list class=rd,expver=xxxx,date=-1/20170101,stream=oper,type=an,levtype=pl,param=130/138,levelist=300/123/1000 --porcelain --full > out -count 16 +echo "Test: All entries should be visible with --full" +try fdb-list class=rd,expver=xxxx,date=20240911/20240912,stream=oper,type=an,levtype=pl,param=130/138,levelist=300/123/1000 --porcelain --full | tee out +line_count 16 diff --git a/tests/fdb/tools/list/list_minimum_keys.sh.in b/tests/fdb/tools/list/list_minimum_keys.sh.in index 325ca5a5c..9c62683ca 100755 --- a/tests/fdb/tools/list/list_minimum_keys.sh.in +++ b/tests/fdb/tools/list/list_minimum_keys.sh.in @@ -2,10 +2,18 @@ set -ux -yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; } +yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; } die() { yell "$*"; exit 1; } try() { "$@" || die "Errored HERE => '$*'"; } +line_count() { + [[ $# -eq 1 ]] || die "line_count requires 1 argument; expected line count" + val=$(wc -l < out) && val=$((val + 0)) + [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" +} + +######################################################################################################################## + export PATH=@CMAKE_BINARY_DIR@/bin:$PATH export FDB5_CONFIG_FILE="local.yaml" export FDB_HOME=@PROJECT_BINARY_DIR@ @@ -21,8 +29,8 @@ echo "running test '$test_name' on $(hostname)" try cd $bin_dir -rm -rf $test_name || true -mkdir -p $test_name/localroot +try rm -rf $test_name +try mkdir -p $test_name/localroot try cd $test_name @@ -31,63 +39,39 @@ try cp "$bin_dir/$src_data" ./ ######################################################################################################################## -count() { - [[ $# -eq 1 ]] || die "count requires 1 argument; expected count" - val=$(wc -l < out) && val=$((val + 0)) - [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" -} - -# grep_count() { -# [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count" -# val=$(grep -E "$1" out | wc -l) && val=$((val + 0)) -# [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]" -# } - -######################################################################################################################## - -echo "Ensure that we can wipe specified databases, but only when fully specified" +# Ensure that we can wipe specified databases, but only when fully specified try grib_set -s class=rd,expver=xxxx "$src_data" data.xxxx.grib try grib_ls -m data.xxxx.grib - try fdb-write data.xxxx.grib -try fdb-list --all --full --minimum-keys= --porcelain > out -count 24 -echo "Check that if any of the keys are missing, then nothing happens" +echo "Test: check all output" +try fdb-list --all --full --minimum-keys= --porcelain | tee out +line_count 24 + +echo "Test: nothing happens if any of the keys are missing" for invalid_key in class=rd expver=xxxx; do - # set +e fdb-list $invalid_key rc=$? - # set -e - [[ $rc = 0 ]] && die "Should not have succeeded: $invalid_key" done -echo "Check that supplying the key makes it work" +echo "Test: supplying the key" +try fdb-list class=rd,expver=xxxx --porcelain | tee out +line_count 24 -try fdb-list class=rd,expver=xxxx --porcelain > out -count 24 - -echo "Check that we can change the minimum keys" - -for invalid_key in class=rd,expver=xxxx \ - class=rd,time=1200 \ - expver=xxxx,time=1200; do - set +e +echo "Test: changing the minimum keys" +for invalid_key in class=rd,expver=xxxx class=rd,time=1200 expver=xxxx,time=1200; do fdb-list --minimum-keys="class,expver,time" $invalid_key rc=$? - set -e - - [[ $rc = 0 ]] && echo "Should not have succeeded: $invalid_key" && exit 1 + [[ $rc = 0 ]] && die "Should not have succeeded: $invalid_key" done -echo "And that the purge works now this is supplied" - -fdb-list --minimum-keys="class,expver,time" class=rd,expver=xxxx,time=1200 --porcelain > out -count 12 +echo "Test: the purge works now this is supplied" +try fdb-list --minimum-keys="class,expver,time" class=rd,expver=xxxx,time=1200 --porcelain | tee out +line_count 12 diff --git a/tests/fdb/tools/list/list_porcelain.sh.in b/tests/fdb/tools/list/list_porcelain.sh.in index 932eb396c..e06f152ec 100755 --- a/tests/fdb/tools/list/list_porcelain.sh.in +++ b/tests/fdb/tools/list/list_porcelain.sh.in @@ -2,10 +2,24 @@ set -ux -yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; } +yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; } die() { yell "$*"; exit 1; } try() { "$@" || die "Errored HERE => '$*'"; } +line_count() { + [[ $# -eq 1 ]] || die "line_count requires 1 argument; expected line count" + val=$(wc -l < out) && val=$((val + 0)) + [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" +} + +grep_count() { + [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count" + val=$(grep -cE "$1" out) + [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]" +} + +######################################################################################################################## + export PATH=@CMAKE_BINARY_DIR@/bin:$PATH export FDB5_CONFIG_FILE="local.yaml" export FDB_HOME=@PROJECT_BINARY_DIR@ @@ -21,8 +35,8 @@ echo "running test '$test_name' on $(hostname)" try cd $bin_dir -rm -rf $test_name || true -mkdir -p $test_name/localroot +try rm -rf $test_name +try mkdir -p $test_name/localroot try cd $test_name @@ -31,20 +45,6 @@ try cp "$bin_dir/$src_data" ./ ######################################################################################################################## -count() { - [[ $# -eq 1 ]] || die "count requires 1 argument; expected count" - val=$(wc -l < out) && val=$((val + 0)) - [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" -} - -grep_count() { - [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count" - val=$(grep -E "$1" out | wc -l) && val=$((val + 0)) - [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]" -} - -######################################################################################################################## - # If we don't specify --porcelain, then the expanded request is prepended to the output request. # Measure the difference! @@ -52,63 +52,64 @@ try grib_set -s class=rd,expver=xxxx "$src_data" data.xxxx.grib try grib_ls -m data.xxxx.grib try fdb-write data.xxxx.grib +tab=$'\t' # macos doesn't support '\t' in grep echo "Test all output" -try fdb-list --all --minimum-keys="" --porcelain --full > out -count 24 +try fdb-list --all --minimum-keys="" --porcelain --full | tee out +line_count 24 echo "Test normal output" -try fdb-list class=rd,expver=xxxx > out -grep_count '{class=rd,expver=xxxx' 24 -grep_count '^retrieve,$' 1 -grep_count '^\t.*=.*' 2 -grep_count '^\tclass=rd' 1 -grep_count '^\texpver=xxxx' 1 +try fdb-list class=rd,expver=xxxx | tee out +grep_count "{class=rd,expver=xxxx" 24 +grep_count "^retrieve,$" 1 +grep_count "^${tab}.*=.*" 2 +grep_count "^${tab}class=rd" 1 +grep_count "^${tab}expver=xxxx" 1 echo "Test porcelain output" -try fdb-list class=rd,expver=xxxx --porcelain > out -grep_count '{class=rd,expver=xxxx' 24 -count 24 +try fdb-list class=rd,expver=xxxx --porcelain | tee out +line_count 24 +grep_count "{class=rd,expver=xxxx" 24 echo "Test expansion of date and param without porcelain" -try fdb-list class=rd,expver=xxxx,date=-1,stream=oper,type=an,levtype=pl,param=t > out -grep_count '{class=rd,expver=xxxx' 0 -grep_count '^retrieve,$' 1 -grep_count '^\t.*=.*' 7 -grep_count '^\tclass=rd' 1 -grep_count '^\texpver=xxxx' 1 -grep_count '^\tparam=130' 1 -grep_count '^\tdate=[0-9]{8}' 1 +try fdb-list class=rd,expver=xxxx,date=20240911,stream=oper,type=an,levtype=pl,param=t | tee out +grep_count "{class=rd,expver=xxxx" 0 +grep_count "^retrieve,$" 1 +grep_count "^${tab}.*=.*" 7 +grep_count "^${tab}class=rd" 1 +grep_count "^${tab}expver=xxxx" 1 +grep_count "^${tab}param=130" 1 +grep_count "^${tab}date=20240911" 1 echo "Test expansion of date and param with porcelain" -try fdb-list class=rd,expver=xxxx,date=-1,stream=oper,type=an,levtype=pl,param=t --porcelain > out -[[ "$(grep '{class=rd,expver=xxxx' out | grep 'date=[0-9]{8}' | wc -l)" -eq 0 ]] || die "Incorrect number of entries reported" -count 0 +try fdb-list class=rd,expver=xxxx,date=20240911,stream=oper,type=an,levtype=pl,param=t --porcelain | tee out +line_count 0 +[[ "$(grep '{class=rd,expver=xxxx' out | grep 'date=20240911' | wc -l)" -eq 0 ]] || die "Incorrect number of entries reported" echo "Test expansion of date and param without porcelain" -try fdb-list class=rd,expver=xxxx,date=-1,stream=oper,type=an,levtype=pl,param=138 > out -[[ $(grep '{class=rd,expver=xxxx' out | grep -E 'date=[0-9]{8}' | grep param=138 | wc -l) =~ "12" ]] || die "Incorrect number of entries reported" -grep_count '^retrieve,$' 1 -grep_count '^\t.*=.*' 7 -grep_count '^\tclass=rd' 1 -grep_count '^\texpver=xxxx' 1 -grep_count '^\tparam=138' 1 -grep_count '^\tdate=[0-9]{8}' 1 +try fdb-list class=rd,expver=xxxx,date=20240911,stream=oper,type=an,levtype=pl,param=138 | tee out +grep_count "^retrieve,$" 1 +grep_count "^${tab}.*=.*" 7 +grep_count "^${tab}class=rd" 1 +grep_count "^${tab}expver=xxxx" 1 +grep_count "^${tab}param=138" 1 +grep_count "^${tab}date=20240911" 1 +[[ $(grep '{class=rd,expver=xxxx' out | grep 'date=20240911' | grep param=138 | wc -l) -eq 12 ]] || die "Incorrect number of entries reported" echo "Test expansion of date and param with porcelain" -try fdb-list class=rd,expver=xxxx,date=-1,stream=oper,type=an,levtype=pl,param=138 --porcelain > out -[[ $(grep '{class=rd,expver=xxxx' out | grep -E 'date=[0-9]{8}' | grep param=138 | wc -l) =~ "12" ]] || die "Incorrect number of entries reported" -count 12 +try fdb-list class=rd,expver=xxxx,date=20240911,stream=oper,type=an,levtype=pl,param=138 --porcelain | tee out +line_count 12 +[[ $(grep '{class=rd,expver=xxxx' out | grep 'date=20240911' | grep param=138 | wc -l) -eq 12 ]] || die "Incorrect number of entries reported" diff --git a/tests/fdb/tools/list/list_ranges.sh.in b/tests/fdb/tools/list/list_ranges.sh.in index eeea01bb5..bb1812131 100755 --- a/tests/fdb/tools/list/list_ranges.sh.in +++ b/tests/fdb/tools/list/list_ranges.sh.in @@ -2,10 +2,18 @@ set -ux -yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; } +yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; } die() { yell "$*"; exit 1; } try() { "$@" || die "Errored HERE => '$*'"; } +line_count() { + [[ $# -eq 1 ]] || die "line_count requires 1 argument; expected line count" + val=$(wc -l < out) && val=$((val + 0)) + [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" +} + +######################################################################################################################## + export PATH=@CMAKE_BINARY_DIR@/bin:$PATH export FDB5_CONFIG_FILE="local.yaml" export FDB_HOME=@PROJECT_BINARY_DIR@ @@ -21,8 +29,8 @@ echo "running test '$test_name' on $(hostname)" try cd $bin_dir -rm -rf $test_name || true -mkdir -p $test_name/localroot +try rm -rf $test_name +try mkdir -p $test_name/localroot try cd $test_name @@ -31,61 +39,47 @@ try cp "$bin_dir/$src_data" ./ ######################################################################################################################## -count() { - [[ $# -ne 1 ]] && die "count requires 1 argument; expected count" - val=$(wc -l < out) && val=$((val + 0)) - [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" -} - -# grep_count() { -# [[ $# -ne 2 ]] && die "grep_count requires 2 arguments; regex and expected count" -# val=$(grep -E "$1" out | wc -l) && val=$((val + 0)) -# [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]" -# } - -######################################################################################################################## - # Ensure that we can wipe specified databases with ranges in the requests # --> Ensures the MARS requests are working correctly try grib_set -s class=rd,expver=xxxx "$src_data" data.xxxx.d1.grib -try grib_set -s class=rd,expver=xxxx,date=20170101 "$src_data" data.xxxx.d2.grib -try grib_set -s class=rd,expver=xxxx,date=20180103 "$src_data" data.xxxx.d3.grib +try grib_set -s class=rd,expver=xxxx,date=20240912 "$src_data" data.xxxx.d2.grib +try grib_set -s class=rd,expver=xxxx,date=20241001 "$src_data" data.xxxx.d3.grib try grib_ls -m data.xxxx.d1.grib try fdb-write data.xxxx.d1.grib try fdb-write data.xxxx.d2.grib try fdb-write data.xxxx.d3.grib -try fdb-list --all --minimum-keys="" --porcelain --full > out -count 72 +try fdb-list --all --minimum-keys="" --porcelain --full | tee out +line_count 72 -try fdb-list class=rd,expver=xxxx --porcelain > out -count 72 +try fdb-list class=rd,expver=xxxx --porcelain | tee out +line_count 72 -try fdb-list class=rd,expver=xxxx,date=-1 --porcelain > out -count 24 +try fdb-list class=rd,expver=xxxx,date=20240911 --porcelain | tee out +line_count 24 -try fdb-list class=rd,expver=xxxx,date=-1/20170101 --porcelain > out -count 48 +try fdb-list class=rd,expver=xxxx,date=20240911/20240912 --porcelain | tee out +line_count 48 -try fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103 --porcelain > out -count 72 +try fdb-list class=rd,expver=xxxx,date=20240911/20240912/20241001 --porcelain | tee out +line_count 72 -try fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103/20060101,stream=oper,type=an,levtype=pl,param=60 --porcelain > out -count 0 +try fdb-list class=rd,expver=xxxx,date=20240911/20240912/20241001/20060101,stream=oper,type=an,levtype=pl,param=60 --porcelain | tee out +line_count 0 -try fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103,stream=oper,type=an,levtype=pl,param=155 --porcelain > out -count 36 +try fdb-list class=rd,expver=xxxx,date=20240911/20240912/20241001,stream=oper,type=an,levtype=pl,param=155 --porcelain | tee out +line_count 36 -try fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103,stream=oper,type=an,levtype=pl,param=60/155 --porcelain > out -count 36 +try fdb-list class=rd,expver=xxxx,date=20240911/20240912/20241001,stream=oper,type=an,levtype=pl,param=60/155 --porcelain | tee out +line_count 36 -try fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103,stream=oper,type=an,levtype=pl,param=60/155/138 --porcelain > out -count 72 +try fdb-list class=rd,expver=xxxx,date=20240911/20240912/20241001,stream=oper,type=an,levtype=pl,param=60/155/138 --porcelain | tee out +line_count 72 -try fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103,stream=oper,type=an,levtype=pl,param=60/155/138,levelist=300/400/500/700/850/1000 --porcelain > out -count 72 +try fdb-list class=rd,expver=xxxx,date=20240911/20240912/20241001,stream=oper,type=an,levtype=pl,param=60/155/138,levelist=300/400/500/700/850/1000 --porcelain | tee out +line_count 72 -try fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103,stream=oper,type=an,levtype=pl,param=60/155/138,levelist=300/123/1000 --porcelain > out -count 24 +try fdb-list class=rd,expver=xxxx,date=20240911/20240912/20241001,stream=oper,type=an,levtype=pl,param=60/155/138,levelist=300/123/1000 --porcelain | tee out +line_count 24 From bf0392145bbbb1ea586e0d7ad684dc3d287cfc25 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 13 Sep 2024 12:44:50 +0200 Subject: [PATCH 61/61] test(fdb-list): cleanup after success --- tests/fdb/tools/list/list_all.sh.in | 8 +++++++- tests/fdb/tools/list/list_all_full.sh.in | 8 ++++++-- tests/fdb/tools/list/list_depth.sh.in | 19 +++++++++++++------ tests/fdb/tools/list/list_location.sh.in | 8 +++++++- tests/fdb/tools/list/list_masking.sh.in | 8 +++++++- tests/fdb/tools/list/list_minimum_keys.sh.in | 8 +++++++- tests/fdb/tools/list/list_porcelain.sh.in | 8 +++++++- tests/fdb/tools/list/list_ranges.sh.in | 8 +++++++- 8 files changed, 61 insertions(+), 14 deletions(-) diff --git a/tests/fdb/tools/list/list_all.sh.in b/tests/fdb/tools/list/list_all.sh.in index d714c4b59..f54007e15 100755 --- a/tests/fdb/tools/list/list_all.sh.in +++ b/tests/fdb/tools/list/list_all.sh.in @@ -41,7 +41,10 @@ try mkdir -p $test_name/localroot try cd $test_name try cp "$src_dir/local.yaml" ./ -try cp "$bin_dir/$src_data" ./ +try ln -s "$bin_dir/$src_data" ./ + +work_dir=$(pwd) +echo "Working directory: $work_dir" ######################################################################################################################## @@ -152,3 +155,6 @@ for i in 1 2; do grep_count "$regex_y2" 24 done + +echo "cleanup" +try rm -rf "$work_dir/localroot" "$work_dir"/data.*.grib diff --git a/tests/fdb/tools/list/list_all_full.sh.in b/tests/fdb/tools/list/list_all_full.sh.in index 79963ebda..b20ed0f0e 100755 --- a/tests/fdb/tools/list/list_all_full.sh.in +++ b/tests/fdb/tools/list/list_all_full.sh.in @@ -41,7 +41,10 @@ try mkdir -p $test_name/localroot try cd $test_name try cp "$src_dir/local.yaml" ./ -try cp "$bin_dir/$src_data" ./ +try ln -s "$bin_dir/$src_data" ./ + +work_dir=$(pwd) +echo "Working directory: $work_dir" ######################################################################################################################## @@ -103,4 +106,5 @@ for i in 0 1; do done -exit 0 +echo "cleanup" +try rm -rf "$work_dir/localroot" "$work_dir"/data.*.grib diff --git a/tests/fdb/tools/list/list_depth.sh.in b/tests/fdb/tools/list/list_depth.sh.in index 9a40e52be..70ba3a833 100755 --- a/tests/fdb/tools/list/list_depth.sh.in +++ b/tests/fdb/tools/list/list_depth.sh.in @@ -35,19 +35,22 @@ done ######################################################################################################################## # Create a grib file with 4 steps and populate FDB -try grib_set -s step=0 $src_data 0.grib -try grib_set -s step=6 $src_data 6.grib -try grib_set -s step=9 $src_data 9.grib +try grib_set -s step=0 $src_data data.0.grib +try grib_set -s step=6 $src_data data.6.grib +try grib_set -s step=9 $src_data data.9.grib for f in *.grib; do - try grib_set -s type=an "$f" "cf.$f" + try grib_set -s type=an "$f" "data.an.${f#*.}" done for f in *.grib; do - try cat "$f" >> "$test_name.grib" + try cat "$f" >> "data.$test_name.grib" done -try fdb-write "$test_name.grib" +try fdb-write "data.$test_name.grib" + +work_dir=$(pwd) +echo "Working directory: $work_dir" ######################################################################################################################## @@ -91,3 +94,7 @@ exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=9,param=166}" out=$(try fdb-list type=an --minimum-keys="" --porcelain --depth=3) try test "$exp" = "$out" + + +echo "cleanup" +try rm -rf "$work_dir/localroot" "$work_dir"/*data.*.grib diff --git a/tests/fdb/tools/list/list_location.sh.in b/tests/fdb/tools/list/list_location.sh.in index 063a6a29a..fe3e058bf 100755 --- a/tests/fdb/tools/list/list_location.sh.in +++ b/tests/fdb/tools/list/list_location.sh.in @@ -41,7 +41,10 @@ try mkdir -p $test_name/localroot try cd $test_name try cp "$src_dir/local.yaml" ./ -try cp "$bin_dir/$src_data" ./ +try ln -s "$bin_dir/$src_data" ./ + +work_dir=$(pwd) +echo "Working directory: $work_dir" ######################################################################################################################## @@ -65,3 +68,6 @@ echo "Test: without the location flag, none of the location info is included" try fdb-list class=rd,expver=xxxx,time=0000 --porcelain | tee out line_count 12 grep_count "{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=0000,domain=g}{type=an,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}" 12 + +echo "cleanup" +try rm -rf "$work_dir/localroot" "$work_dir"/data.*.grib diff --git a/tests/fdb/tools/list/list_masking.sh.in b/tests/fdb/tools/list/list_masking.sh.in index 76beb44a6..5c5e8ec8e 100755 --- a/tests/fdb/tools/list/list_masking.sh.in +++ b/tests/fdb/tools/list/list_masking.sh.in @@ -35,7 +35,10 @@ try mkdir -p $test_name/localroot try cd $test_name try cp "$src_dir/local.yaml" ./ -try cp "$bin_dir/$src_data" ./ +try ln -s "$bin_dir/$src_data" ./ + +work_dir=$(pwd) +echo "Working directory: $work_dir" ######################################################################################################################## @@ -75,3 +78,6 @@ line_count 8 echo "Test: All entries should be visible with --full" try fdb-list class=rd,expver=xxxx,date=20240911/20240912,stream=oper,type=an,levtype=pl,param=130/138,levelist=300/123/1000 --porcelain --full | tee out line_count 16 + +echo "cleanup" +try rm -rf "$work_dir/localroot" "$work_dir"/data.*.grib diff --git a/tests/fdb/tools/list/list_minimum_keys.sh.in b/tests/fdb/tools/list/list_minimum_keys.sh.in index 9c62683ca..e8aad4f0a 100755 --- a/tests/fdb/tools/list/list_minimum_keys.sh.in +++ b/tests/fdb/tools/list/list_minimum_keys.sh.in @@ -35,7 +35,10 @@ try mkdir -p $test_name/localroot try cd $test_name try cp "$src_dir/local.yaml" ./ -try cp "$bin_dir/$src_data" ./ +try ln -s "$bin_dir/$src_data" ./ + +work_dir=$(pwd) +echo "Working directory: $work_dir" ######################################################################################################################## @@ -75,3 +78,6 @@ done echo "Test: the purge works now this is supplied" try fdb-list --minimum-keys="class,expver,time" class=rd,expver=xxxx,time=1200 --porcelain | tee out line_count 12 + +echo "cleanup" +try rm -rf "$work_dir/localroot" "$work_dir"/data.*.grib diff --git a/tests/fdb/tools/list/list_porcelain.sh.in b/tests/fdb/tools/list/list_porcelain.sh.in index e06f152ec..75a6d9440 100755 --- a/tests/fdb/tools/list/list_porcelain.sh.in +++ b/tests/fdb/tools/list/list_porcelain.sh.in @@ -41,7 +41,10 @@ try mkdir -p $test_name/localroot try cd $test_name try cp "$src_dir/local.yaml" ./ -try cp "$bin_dir/$src_data" ./ +try ln -s "$bin_dir/$src_data" ./ + +work_dir=$(pwd) +echo "Working directory: $work_dir" ######################################################################################################################## @@ -113,3 +116,6 @@ echo "Test expansion of date and param with porcelain" try fdb-list class=rd,expver=xxxx,date=20240911,stream=oper,type=an,levtype=pl,param=138 --porcelain | tee out line_count 12 [[ $(grep '{class=rd,expver=xxxx' out | grep 'date=20240911' | grep param=138 | wc -l) -eq 12 ]] || die "Incorrect number of entries reported" + +echo "cleanup" +try rm -rf "$work_dir/localroot" "$work_dir"/data.*.grib diff --git a/tests/fdb/tools/list/list_ranges.sh.in b/tests/fdb/tools/list/list_ranges.sh.in index bb1812131..0cd71222f 100755 --- a/tests/fdb/tools/list/list_ranges.sh.in +++ b/tests/fdb/tools/list/list_ranges.sh.in @@ -35,7 +35,10 @@ try mkdir -p $test_name/localroot try cd $test_name try cp "$src_dir/local.yaml" ./ -try cp "$bin_dir/$src_data" ./ +try ln -s "$bin_dir/$src_data" ./ + +work_dir=$(pwd) +echo "Working directory: $work_dir" ######################################################################################################################## @@ -83,3 +86,6 @@ line_count 72 try fdb-list class=rd,expver=xxxx,date=20240911/20240912/20241001,stream=oper,type=an,levtype=pl,param=60/155/138,levelist=300/123/1000 --porcelain | tee out line_count 24 + +echo "cleanup" +try rm -rf "$work_dir/localroot" "$work_dir"/data.*.grib