From 7a593abbe2d8ac2a31a18f558a13cb48ac2a2629 Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Fri, 6 Oct 2023 14:25:01 +0100 Subject: [PATCH 001/186] wip --- src/fdb5/CMakeLists.txt | 11 + src/fdb5/config/Config.cc | 10 +- src/fdb5/config/Config.h | 1 + src/fdb5/database/ArchiveVisitor.cc | 9 +- src/fdb5/database/ArchiveVisitor.h | 1 - src/fdb5/database/Archiver.cc | 32 +- src/fdb5/database/Archiver.h | 7 +- src/fdb5/database/BaseArchiveVisitor.cc | 9 +- src/fdb5/database/BaseArchiveVisitor.h | 3 +- src/fdb5/database/Catalogue.cc | 4 +- src/fdb5/database/DB.cc | 26 +- src/fdb5/database/DB.h | 4 +- src/fdb5/database/Key.cc | 12 +- src/fdb5/database/Key.h | 2 +- src/fdb5/database/Store.cc | 8 +- src/fdb5/database/Store.h | 27 +- src/fdb5/rados/RadosStore.cc | 14 +- src/fdb5/rados/RadosStore.h | 6 +- src/fdb5/remote/CatalogueHandler.cc | 263 ++++++++ src/fdb5/remote/CatalogueHandler.h | 46 ++ src/fdb5/remote/ClientConnection.cc | 350 +++++++++++ src/fdb5/remote/ClientConnection.h | 133 ++++ src/fdb5/remote/DecoupledHandler.cc | 355 +++++++++++ src/fdb5/remote/DecoupledHandler.h | 113 ++++ src/fdb5/remote/FdbServer.cc | 44 +- src/fdb5/remote/RemoteStore.cc | 775 ++++++++++++++++++++++++ src/fdb5/remote/RemoteStore.h | 108 ++++ src/fdb5/remote/StoreHandler.cc | 430 +++++++++++++ src/fdb5/remote/StoreHandler.h | 47 ++ src/fdb5/rules/Schema.cc | 8 +- src/fdb5/toc/AdoptVisitor.cc | 13 +- src/fdb5/toc/RootManager.cc | 5 + src/fdb5/toc/TocCommon.cc | 1 - src/fdb5/toc/TocCommon.h | 1 - src/fdb5/toc/TocHandler.cc | 2 + src/fdb5/toc/TocHandler.h | 1 + src/fdb5/toc/TocStore.cc | 25 +- src/fdb5/toc/TocStore.h | 6 +- 38 files changed, 2810 insertions(+), 102 deletions(-) create mode 100644 src/fdb5/remote/CatalogueHandler.cc create mode 100644 src/fdb5/remote/CatalogueHandler.h create mode 100644 src/fdb5/remote/ClientConnection.cc create mode 100644 src/fdb5/remote/ClientConnection.h create mode 100644 src/fdb5/remote/DecoupledHandler.cc create mode 100644 src/fdb5/remote/DecoupledHandler.h create mode 100644 src/fdb5/remote/RemoteStore.cc create mode 100644 src/fdb5/remote/RemoteStore.h create mode 100644 src/fdb5/remote/StoreHandler.cc create mode 100644 src/fdb5/remote/StoreHandler.h diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt index 8ccf68784..64fa282e4 100644 --- a/src/fdb5/CMakeLists.txt +++ b/src/fdb5/CMakeLists.txt @@ -212,7 +212,11 @@ if(HAVE_FDB_REMOTE) list( APPEND fdb5_srcs api/RemoteFDB.cc api/RemoteFDB.h + remote/RemoteStore.cc + remote/RemoteStore.h + remote/ClientConnection.h + remote/ClientConnection.cc remote/RemoteConfiguration.h remote/RemoteConfiguration.cc remote/RemoteFieldLocation.h @@ -225,6 +229,13 @@ if(HAVE_FDB_REMOTE) remote/AvailablePortList.h remote/FdbServer.h remote/FdbServer.cc + + # remote/CatalogueHandler.h + # remote/CatalogueHandler.cc + remote/StoreHandler.h + remote/StoreHandler.cc + remote/DecoupledHandler.h + remote/DecoupledHandler.cc ) endif() diff --git a/src/fdb5/config/Config.cc b/src/fdb5/config/Config.cc index b7cba90a9..f59df7fae 100644 --- a/src/fdb5/config/Config.cc +++ b/src/fdb5/config/Config.cc @@ -57,7 +57,7 @@ class SchemaRegistry { //---------------------------------------------------------------------------------------------------------------------- -Config::Config() : schemaPath_("") { +Config::Config() : schemaPath_(""), schemaPathInitialised_(false) { userConfig_ = std::make_shared(eckit::LocalConfiguration()); } @@ -71,8 +71,8 @@ Config Config::make(const eckit::PathName& path, const eckit::Configuration& use } Config::Config(const Configuration& config, const eckit::Configuration& userConfig) : - LocalConfiguration(config) { - initializeSchemaPath(); + LocalConfiguration(config), schemaPathInitialised_(false) { + // initializeSchemaPath(); userConfig_ = std::make_shared(userConfig); } @@ -178,6 +178,9 @@ const PathName& Config::schemaPath() const { void Config::initializeSchemaPath() const { + if (schemaPathInitialised_) { + return; + } // If the user has specified the schema location in the FDB config, use that, // otherwise use the library-wide schema path. @@ -201,6 +204,7 @@ PathName Config::configPath() const { } const Schema& Config::schema() const { + initializeSchemaPath(); return SchemaRegistry::instance().get(schemaPath()); } diff --git a/src/fdb5/config/Config.h b/src/fdb5/config/Config.h index e82046021..f8d8e1d94 100644 --- a/src/fdb5/config/Config.h +++ b/src/fdb5/config/Config.h @@ -64,6 +64,7 @@ class Config : public eckit::LocalConfiguration { private: // members mutable eckit::PathName schemaPath_; + mutable bool schemaPathInitialised_; std::shared_ptr userConfig_; }; diff --git a/src/fdb5/database/ArchiveVisitor.cc b/src/fdb5/database/ArchiveVisitor.cc index 9e07ea2d6..76595627f 100644 --- a/src/fdb5/database/ArchiveVisitor.cc +++ b/src/fdb5/database/ArchiveVisitor.cc @@ -27,10 +27,13 @@ bool ArchiveVisitor::selectDatum(const Key &key, const Key &full) { // eckit::Log::info() << "selectDatum " << key << ", " << full << " " << size_ << std::endl; checkMissingKeys(full); - ASSERT(current()); - - current()->archive(key, data_, size_); + CatalogueWriter* writeCatalogue = dynamic_cast(current()); + const Index& idx = writeCatalogue->currentIndex(); + // here we could create a queue... and keep accepting archival request until the queue is full + auto futureLocation = store()->archive(idx.key(), data_, size_); + writeCatalogue->archive(key, futureLocation.get()); +// writeCatalogue->archive(catalogue->key(), idx.key(), key, futureLocation.get()); return true; } diff --git a/src/fdb5/database/ArchiveVisitor.h b/src/fdb5/database/ArchiveVisitor.h index 0534dd4ec..f6213465f 100644 --- a/src/fdb5/database/ArchiveVisitor.h +++ b/src/fdb5/database/ArchiveVisitor.h @@ -42,7 +42,6 @@ class ArchiveVisitor : public BaseArchiveVisitor { const void *data_; size_t size_; - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/Archiver.cc b/src/fdb5/database/Archiver.cc index 682fdc9b0..702920fce 100644 --- a/src/fdb5/database/Archiver.cc +++ b/src/fdb5/database/Archiver.cc @@ -56,19 +56,21 @@ void Archiver::archive(const Key &key, BaseArchiveVisitor& visitor) { void Archiver::flush() { for (store_t::iterator i = databases_.begin(); i != databases_.end(); ++i) { - i->second.second->flush(); + i->second.second.second->flush(); // flush the store + i->second.second.first->flush(); // flush the catalogue } } - -DB& Archiver::database(const Key &key) { +void Archiver::selectDatabase(const Key &key) { store_t::iterator i = databases_.find(key); if (i != databases_.end() ) { - DB& db = *(i->second.second); + std::cout << "found key: " << key << std::endl; + current_ = i->second.second.first.get(); + store_ = i->second.second.second.get(); i->second.first = ::time(0); - return db; + return; } static size_t fdbMaxNbDBsOpen = eckit::Resource("fdbMaxNbDBsOpen", 64); @@ -85,25 +87,27 @@ DB& Archiver::database(const Key &key) { } } if (found) { - eckit::Log::info() << "Closing database " << *databases_[oldK].second << std::endl; + // what if the catalogue/store are not flashed ??? + eckit::Log::warning() << "Closing database " << *databases_[oldK].second.first << std::endl; databases_.erase(oldK); } } - std::unique_ptr db = DB::buildWriter(key, dbConfig_); - - ASSERT(db); + std::unique_ptr cat = CatalogueFactory::instance().build(key, dbConfig_, false); + ASSERT(cat); // If this database is locked for writing then this is an error - if (!db->enabled(ControlIdentifier::Archive)) { + if (!cat->enabled(ControlIdentifier::Archive)) { std::ostringstream ss; - ss << "Database " << *db << " matched for archived is LOCKED against archiving"; + ss << "Database " << *cat << " matched for archived is LOCKED against archiving"; throw eckit::UserError(ss.str(), Here()); } - DB& out = *db; - databases_[key] = std::make_pair(::time(0), std::move(db)); - return out; + std::unique_ptr str = cat->buildStore(); + current_ = cat.get(); + store_ = str.get(); + + databases_[key] = std::make_pair(::time(0), std::make_pair(std::move(cat), std::move(str))); } void Archiver::print(std::ostream &out) const { diff --git a/src/fdb5/database/Archiver.h b/src/fdb5/database/Archiver.h index 38454f23c..c3d72bc35 100644 --- a/src/fdb5/database/Archiver.h +++ b/src/fdb5/database/Archiver.h @@ -62,13 +62,13 @@ class Archiver : public eckit::NonCopyable { void print(std::ostream &out) const; - DB& database(const Key &key); + void selectDatabase(const Key &key); private: // members friend class BaseArchiveVisitor; - typedef std::map< Key, std::pair > > store_t; + typedef std::map< Key, std::pair, std::unique_ptr > > > store_t; Config dbConfig_; @@ -76,7 +76,8 @@ class Archiver : public eckit::NonCopyable { std::vector prev_; - DB* current_; + Catalogue* current_; + Store* store_; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/BaseArchiveVisitor.cc b/src/fdb5/database/BaseArchiveVisitor.cc index 0622f8852..40dde3c2a 100644 --- a/src/fdb5/database/BaseArchiveVisitor.cc +++ b/src/fdb5/database/BaseArchiveVisitor.cc @@ -26,7 +26,8 @@ BaseArchiveVisitor::BaseArchiveVisitor(Archiver &owner, const Key &field) : bool BaseArchiveVisitor::selectDatabase(const Key &key, const Key&) { eckit::Log::debug() << "selectDatabase " << key << std::endl; - owner_.current_ = &owner_.database(key); + owner_.selectDatabase(key); + ASSERT(owner_.current_); owner_.current_->deselectIndex(); return true; @@ -49,10 +50,14 @@ const Schema& BaseArchiveVisitor::databaseSchema() const { return current()->schema(); } -DB* BaseArchiveVisitor::current() const { +Catalogue* BaseArchiveVisitor::current() const { return owner_.current_; } +Store* BaseArchiveVisitor::store() const { + return owner_.store_; +} + //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 diff --git a/src/fdb5/database/BaseArchiveVisitor.h b/src/fdb5/database/BaseArchiveVisitor.h index eac41cd59..29733b7fa 100644 --- a/src/fdb5/database/BaseArchiveVisitor.h +++ b/src/fdb5/database/BaseArchiveVisitor.h @@ -44,7 +44,8 @@ class BaseArchiveVisitor : public WriteVisitor { virtual const Schema& databaseSchema() const; - fdb5::DB* current() const; + fdb5::Catalogue* current() const; + fdb5::Store* store() const; private: // members diff --git a/src/fdb5/database/Catalogue.cc b/src/fdb5/database/Catalogue.cc index 5efb2857a..317ad75fe 100644 --- a/src/fdb5/database/Catalogue.cc +++ b/src/fdb5/database/Catalogue.cc @@ -26,11 +26,11 @@ namespace fdb5 { std::unique_ptr Catalogue::buildStore() { if (buildByKey_) - return StoreFactory::instance().build(schema(), key(), config_); + return StoreFactory::instance().build(key(), config_); else { std::string name = config_.getString("store", "file"); - return StoreFactory::instance().build(schema(), eckit::URI(name, uri()), config_); + return StoreFactory::instance().build(eckit::URI(name, uri()), config_); } } diff --git a/src/fdb5/database/DB.cc b/src/fdb5/database/DB.cc index 579095730..aa30b9413 100644 --- a/src/fdb5/database/DB.cc +++ b/src/fdb5/database/DB.cc @@ -103,14 +103,14 @@ eckit::DataHandle *DB::retrieve(const Key& key) { return nullptr; } -void DB::archive(const Key& key, const void* data, eckit::Length length) { +// void DB::archive(const Key& key, const void* data, eckit::Length length) { - CatalogueWriter* cat = dynamic_cast(catalogue_.get()); - ASSERT(cat); +// CatalogueWriter* cat = dynamic_cast(catalogue_.get()); +// ASSERT(cat); - const Index& idx = cat->currentIndex(); - cat->archive(key, store().archive(idx.key(), data, length)); -} +// const Index& idx = cat->currentIndex(); +// cat->archive(key, store().archive(idx.key(), data, length)); +// } bool DB::open() { bool ret = catalogue_->open(); @@ -164,14 +164,14 @@ void DB::reconsolidate() { cat->reconsolidate(); } -void DB::index(const Key &key, const eckit::PathName &path, eckit::Offset offset, eckit::Length length) { - if (catalogue_->type() == TocEngine::typeName()) { - CatalogueWriter* cat = dynamic_cast(catalogue_.get()); - ASSERT(cat); +// void DB::index(const Key &key, const eckit::PathName &path, eckit::Offset offset, eckit::Length length) { +// if (catalogue_->type() == TocEngine::typeName()) { +// CatalogueWriter* cat = dynamic_cast(catalogue_.get()); +// ASSERT(cat); - cat->index(key, eckit::URI("file", path), offset, length); - } -} +// cat->index(key, eckit::URI("file", path), offset, length); +// } +// } void DB::dump(std::ostream& out, bool simple, const eckit::Configuration& conf) const { catalogue_->dump(out, simple, conf); diff --git a/src/fdb5/database/DB.h b/src/fdb5/database/DB.h index c7e35f2d5..fd593c31f 100644 --- a/src/fdb5/database/DB.h +++ b/src/fdb5/database/DB.h @@ -56,7 +56,7 @@ class DB final : public eckit::OwnedLock { bool axis(const std::string &keyword, eckit::StringSet &s) const; bool inspect(const Key& key, Field& field); eckit::DataHandle *retrieve(const Key &key); - void archive(const Key &key, const void *data, eckit::Length length); +// void archive(const Key &key, const void *data, eckit::Length length); bool open(); void flush(); @@ -79,7 +79,7 @@ class DB final : public eckit::OwnedLock { void visitEntries(EntryVisitor& visitor, bool sorted = false); /// Used for adopting & indexing external data to the TOC dir - void index(const Key &key, const eckit::PathName &path, eckit::Offset offset, eckit::Length length); +// void index(const Key &key, const eckit::PathName &path, eckit::Offset offset, eckit::Length length); // Control access properties of the DB void control(const ControlAction& action, const ControlIdentifiers& identifiers) const; diff --git a/src/fdb5/database/Key.cc b/src/fdb5/database/Key.cc index db9f9c7c6..76a3103a2 100644 --- a/src/fdb5/database/Key.cc +++ b/src/fdb5/database/Key.cc @@ -316,7 +316,7 @@ std::string Key::canonicalValue(const std::string& keyword) const { return canonicalise(keyword, it->second); } -std::string Key::valuesToString() const { +std::string Key::valuesToString(bool ruleOrDefault) const { ASSERT(names_.size() == keys_.size()); @@ -328,11 +328,15 @@ std::string Key::valuesToString() const { ASSERT(i != keys_.end()); oss << sep; - oss << canonicalise(*j, i->second); - + if (rule_ || ruleOrDefault) { + oss << canonicalise(*j, i->second); + } + else { + oss << i->second; + } + sep = ":"; } - return oss.str(); } diff --git a/src/fdb5/database/Key.h b/src/fdb5/database/Key.h index 5e889907f..27b6d24b5 100644 --- a/src/fdb5/database/Key.h +++ b/src/fdb5/database/Key.h @@ -108,7 +108,7 @@ class Key { const Rule *rule() const; const TypesRegistry& registry() const; - std::string valuesToString() const; + std::string valuesToString(bool ruleOrDefault = true) const; const eckit::StringList& names() const; diff --git a/src/fdb5/database/Store.cc b/src/fdb5/database/Store.cc index 35058f274..a87eb68ab 100644 --- a/src/fdb5/database/Store.cc +++ b/src/fdb5/database/Store.cc @@ -72,7 +72,7 @@ void StoreFactory::list(std::ostream& out) { } } -std::unique_ptr StoreFactory::build(const Schema& schema, const Key& key, const Config& config) { +std::unique_ptr StoreFactory::build(const Key& key, const Config& config) { std::string name = config.getString("store", "file"); std::string nameLowercase = eckit::StringTools::lower(name); @@ -89,10 +89,10 @@ std::unique_ptr StoreFactory::build(const Schema& schema, const Key& key, throw eckit::SeriousBug(std::string("No StoreBuilder called ") + nameLowercase); } - return (*j).second->make(schema, key, config); + return (*j).second->make(key, config); } -std::unique_ptr StoreFactory::build(const Schema& schema, const eckit::URI& uri, const Config& config) { +std::unique_ptr StoreFactory::build(const eckit::URI& uri, const Config& config) { std::string name = uri.scheme(); std::string nameLowercase = eckit::StringTools::lower(name); @@ -109,7 +109,7 @@ std::unique_ptr StoreFactory::build(const Schema& schema, const eckit::UR throw eckit::SeriousBug(std::string("No StoreBuilder called ") + nameLowercase); } - return (*j).second->make(schema, uri, config); + return (*j).second->make(uri, config); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/Store.h b/src/fdb5/database/Store.h index 268359b39..2af38484c 100644 --- a/src/fdb5/database/Store.h +++ b/src/fdb5/database/Store.h @@ -12,8 +12,9 @@ /// @author Emanuele Danovaro /// @date August 2019 -#ifndef fdb5_Store_H -#define fdb5_Store_H +#pragma once + +#include #include "eckit/distributed/Transport.h" #include "eckit/filesystem/URI.h" @@ -31,12 +32,12 @@ namespace fdb5 { class Store { public: - Store(const Schema& schema) : schema_(schema) {} + Store() {} virtual ~Store() {} virtual eckit::DataHandle* retrieve(Field& field) const = 0; - virtual std::unique_ptr archive(const Key &key, const void *data, eckit::Length length) = 0; + virtual std::future > archive(const Key& key, const void *data, eckit::Length length) = 0; virtual void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit = true) const = 0; @@ -57,9 +58,6 @@ class Store { virtual void remove(const Key& key) const { NOTIMP; } virtual eckit::URI uri() const = 0; - -protected: // members - const Schema& schema_; //<< schema is owned by catalogue which always outlives the store }; @@ -71,14 +69,14 @@ class StoreBuilderBase { public: StoreBuilderBase(const std::string&); virtual ~StoreBuilderBase(); - virtual std::unique_ptr make(const Schema& schema, const Key& key, const Config& config) = 0; - virtual std::unique_ptr make(const Schema& schema, const eckit::URI& uri, const Config& config) = 0; + virtual std::unique_ptr make(const Key& key, const Config& config) = 0; + virtual std::unique_ptr make(const eckit::URI& uri, const Config& config) = 0; }; template class StoreBuilder : public StoreBuilderBase { - virtual std::unique_ptr make(const Schema& schema, const Key& key, const Config& config) override { return std::unique_ptr(new T(schema, key, config)); } - virtual std::unique_ptr make(const Schema& schema, const eckit::URI& uri, const Config& config) override { return std::unique_ptr(new T(schema, uri, config)); } + virtual std::unique_ptr make(const Key& key, const Config& config) override { return std::unique_ptr(new T(key, config)); } + virtual std::unique_ptr make(const eckit::URI& uri, const Config& config) override { return std::unique_ptr(new T(uri, config)); } public: StoreBuilder(const std::string& name) : StoreBuilderBase(name) {} @@ -95,17 +93,15 @@ class StoreFactory { bool has(const std::string& name); void list(std::ostream&); - /// @param schema the schema read by the catalog /// @param key the user-specified key /// @param config the fdb config /// @returns store built by specified builder - std::unique_ptr build(const Schema& schema, const Key& key, const Config& config); + std::unique_ptr build(const Key& key, const Config& config); - /// @param schema the schema read by the catalog /// @param uri search uri /// @param config the fdb config /// @returns store built by specified builder - std::unique_ptr build(const Schema& schema, const eckit::URI& uri, const Config& config); + std::unique_ptr build(const eckit::URI& uri, const Config& config); private: StoreFactory(); @@ -115,4 +111,3 @@ class StoreFactory { }; } -#endif // fdb5_Store_H diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc index c65c34fac..e27e89595 100644 --- a/src/fdb5/rados/RadosStore.cc +++ b/src/fdb5/rados/RadosStore.cc @@ -28,11 +28,11 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -RadosStore::RadosStore(const Schema& schema, const Key& key, const Config& config) : - Store(schema), directory_("mars:"+key.valuesToString()) {} +RadosStore::RadosStore(const Key& key, const Config& config) : + Store(), directory_("mars:"+key.valuesToString()) {} -RadosStore::RadosStore(const Schema& schema, const eckit::URI& uri, const Config& config) : - Store(schema), directory_("mars:"+uri.path().dirName()) {} +RadosStore::RadosStore(const eckit::URI& uri, const Config& config) : + Store(), directory_("mars:"+uri.path().dirName()) {} eckit::URI RadosStore::uri() const { return URI("rados", directory_); @@ -48,7 +48,7 @@ eckit::DataHandle* RadosStore::retrieve(Field& field, Key& remapKey) const { field.dataHandle(remapKey); } -FieldLocation* RadosStore::archive(const Key &key, const void *data, eckit::Length length) { +std::future > RadosStore::archive(const Key& key, const void *data, eckit::Length length) { dirty_ = true; eckit::PathName dataPath = getDataPath(key); @@ -62,7 +62,9 @@ FieldLocation* RadosStore::archive(const Key &key, const void *data, eckit::Leng ASSERT(len == length); - return new RadosFieldLocation(dataUri, position, length); + std::promise > loc; + loc.set_value(std::unique_ptr(new RadosFieldLocation(dataUri, position, length))); + return loc.get_future(); } void RadosStore::flush() { diff --git a/src/fdb5/rados/RadosStore.h b/src/fdb5/rados/RadosStore.h index b35189be6..367e78825 100644 --- a/src/fdb5/rados/RadosStore.h +++ b/src/fdb5/rados/RadosStore.h @@ -30,8 +30,8 @@ class RadosStore : public Store { public: // methods - RadosStore(const Schema& schema, const Key& key, const Config& config); - RadosStore(const Schema& schema, const eckit::URI& uri, const Config& config); + RadosStore(const Key& key, const Config& config); + RadosStore(const eckit::URI& uri, const Config& config); ~RadosStore() override {} @@ -50,7 +50,7 @@ class RadosStore : public Store { bool exists() const override; eckit::DataHandle* retrieve(Field& field, Key& remapKey) const override; - FieldLocation* archive(const Key &key, const void *data, eckit::Length length) override; + std::future > archive(const Key& key, const void *data, eckit::Length length) override; void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override; diff --git a/src/fdb5/remote/CatalogueHandler.cc b/src/fdb5/remote/CatalogueHandler.cc new file mode 100644 index 000000000..f466e1bad --- /dev/null +++ b/src/fdb5/remote/CatalogueHandler.cc @@ -0,0 +1,263 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "fdb5/remote/CatalogueHandler.h" + +#include "eckit/serialisation/MemoryStream.h" + +#include "fdb5/LibFdb5.h" +#include "fdb5/api/FDBFactory.h" +#include "fdb5/api/helpers/FDBToolRequest.h" + +using namespace eckit; +using metkit::mars::MarsRequest; + +namespace fdb5::remote { + +namespace { + +// *************************************************************************************** +// All of the standard API functions behave in roughly the same manner. The Helper Classes +// described here capture the manner in which their behaviours differ. +// +// See forwardApiCall() for how these helpers are used to forward an API call using a +// worker thread. +// +// *************************************************************************************** +// +// Note that we use the "control" and "data" connections in a specific way, although these +// may not be the optimal names for them. "control" is used for blocking requests, +// and "data" is used for non-blocking activity. +// +// *************************************************************************************** + +template +struct BaseHelper { + virtual size_t encodeBufferSize(const ValueType&) const { return 4096; } + void extraDecode(eckit::Stream&) {} + ValueType apiCall(FDBCatalogueBase&, const FDBToolRequest&) const { NOTIMP; } + + struct Encoded { + size_t position; + eckit::Buffer buf; + }; + + Encoded encode(const ValueType& elem, const DecoupledHandler&) const { + eckit::Buffer encodeBuffer(encodeBufferSize(elem)); + MemoryStream s(encodeBuffer); + s << elem; + return {s.position(), std::move(encodeBuffer)}; + } +}; + +struct ListHelper : public BaseHelper { + ListIterator apiCall(FDBCatalogueBase& fdb, const FDBToolRequest& request) const { + return fdb.list(request); + } +}; + +struct InspectHelper : public BaseHelper { + ListIterator apiCall(FDBCatalogueBase& fdb, const FDBToolRequest& request) const { + return fdb.inspect(request.request()); + } +}; + +} + + +CatalogueHandler::CatalogueHandler(eckit::net::TCPSocket& socket, const Config& config): + DecoupledHandler(socket, config) { + +} +CatalogueHandler::~CatalogueHandler() {} + +void CatalogueHandler::initialiseConnections() { + DecoupledHandler::initialiseConnections(); + + std::string storeHost = ::getenv("FDB_STORE_HOST") ? ::getenv("FDB_STORE_HOST") : "localhost"; + std::string storePort = ::getenv("FDB_STORE_PORT") ? ::getenv("FDB_STORE_PORT") : "7000"; + net::Endpoint storeEndpoint(storeHost, std::stoi(storePort)); + + + // Log::info() << "Sending store endpoint to client: " << storeEndpoint << std::endl; + // { + // Buffer startupBuffer(1024); + // MemoryStream s(startupBuffer); + + // s << clientSession; + // s << sessionID_; + // s << storeEndpoint; + + // // s << storeEndpoint; // xxx single-store case only: we cant do this with multiple stores // For now, dont send the store endpoint to the client + + // Log::debug() << "Protocol negotiation - configuration: " << agreedConf_ < tail; + + // listen loop + while (true) { + tidyWorkers(); + + socketRead(&hdr, sizeof(hdr), controlSocket_); + + ASSERT(hdr.marker == StartMarker); + ASSERT(hdr.version == CurrentVersion); + Log::debug() << "Got message with request ID: " << hdr.requestID << std::endl; + + try { + switch (hdr.message) { + case Message::Exit: + Log::status() << "Exiting" << std::endl; + Log::info() << "Exiting" << std::endl; + return; + + case Message::List: + forwardApiCall(hdr); + break; + + case Message::Dump: + NOTIMP; + break; + + case Message::Purge: + NOTIMP; + break; + + case Message::Stats: + NOTIMP; + break; + + case Message::Status: + NOTIMP; + break; + + case Message::Wipe: + NOTIMP; + break; + + case Message::Control: + NOTIMP; + break; + + case Message::Inspect: + forwardApiCall(hdr); + break; + + case Message::Read: + read(hdr); + break; + + case Message::Flush: + flush(hdr); + break; + + case Message::Archive: + archive(hdr); + break; + + case Message::Store: + store(hdr); + break; + + default: { + std::stringstream ss; + ss << "ERROR: Unexpected message recieved (" << static_cast(hdr.message) + << "). ABORTING"; + Log::status() << ss.str() << std::endl; + Log::error() << "Retrieving... " << ss.str() << std::endl; + throw SeriousBug(ss.str(), Here()); + } + } + + // Ensure we have consumed exactly the correct amount from the socket. + socketRead(&tail, sizeof(tail), controlSocket_); + ASSERT(tail == EndMarker); + + // Acknowledge receipt of command + controlWrite(Message::Received, hdr.requestID); + } + catch (std::exception& e) { + // n.b. more general than eckit::Exception + std::string what(e.what()); + controlWrite(Message::Error, hdr.requestID, what.c_str(), what.length()); + } + catch (...) { + std::string what("Caught unexpected and unknown error"); + controlWrite(Message::Error, hdr.requestID, what.c_str(), what.length()); + } + } +} + +void CatalogueHandler::index(const MessageHeader& hdr) { + NOTIMP; +} + + +// API + +template +void CatalogueHandler::forwardApiCall(const MessageHeader& hdr) { + HelperClass helper; + + std::cout << "CatalogueHandler::forwardApiCall" << std::endl; + + Buffer payload(receivePayload(hdr, controlSocket_)); + MemoryStream s(payload); + + FDBToolRequest request(s); + helper.extraDecode(s); + + // Construct worker thread to feed responses back to client + + ASSERT(catalogue_); + ASSERT(workerThreads_.find(hdr.requestID) == workerThreads_.end()); + + workerThreads_.emplace( + hdr.requestID, std::async(std::launch::async, [request, hdr, helper, this]() { + try { + auto iterator = helper.apiCall(*catalogue_, request); + + typename decltype(iterator)::value_type elem; + while (iterator.next(elem)) { + auto encoded(helper.encode(elem, *this)); + dataWrite(Message::Blob, hdr.requestID, encoded.buf, encoded.position); + } + + dataWrite(Message::Complete, hdr.requestID); + } + catch (std::exception& e) { + // n.b. more general than eckit::Exception + std::string what(e.what()); + dataWrite(Message::Error, hdr.requestID, what.c_str(), what.length()); + } + catch (...) { + // We really don't want to std::terminate the thread + std::string what("Caught unexpected, unknown exception in worker"); + dataWrite(Message::Error, hdr.requestID, what.c_str(), what.length()); + } + })); +} + +} // namespace fdb5::remote diff --git a/src/fdb5/remote/CatalogueHandler.h b/src/fdb5/remote/CatalogueHandler.h new file mode 100644 index 000000000..a334fc9ff --- /dev/null +++ b/src/fdb5/remote/CatalogueHandler.h @@ -0,0 +1,46 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#ifndef fdb5_remote_CatalogueHandler_H +#define fdb5_remote_CatalogueHandler_H + +#include "fdb5/remote/DecoupledHandler.h" + +namespace fdb5 { +namespace remote { +//---------------------------------------------------------------------------------------------------------------------- +class CatalogueHandler : public DecoupledHandler { +public: // methods + + CatalogueHandler(eckit::net::TCPSocket& socket, const Config& config); + ~CatalogueHandler(); + + void handle() override; + +private: // methods + + void initialiseConnections() override; + void index(const MessageHeader& hdr); + + // API functionality + template + void forwardApiCall(const MessageHeader& hdr); + +private: // member + +// std::unique_ptr catalogue_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace remote +} // namespace fdb5 + +#endif // fdb5_remote_CatalogueHandler_H diff --git a/src/fdb5/remote/ClientConnection.cc b/src/fdb5/remote/ClientConnection.cc new file mode 100644 index 000000000..c64d86e55 --- /dev/null +++ b/src/fdb5/remote/ClientConnection.cc @@ -0,0 +1,350 @@ + + +#include +#include + +#include "fdb5/remote/ClientConnection.h" + +#include "fdb5/LibFdb5.h" +#include "fdb5/io/HandleGatherer.h" +#include "fdb5/remote/Messages.h" +#include "fdb5/remote/RemoteFieldLocation.h" +#include "fdb5/api/helpers/FDBToolRequest.h" +#include "fdb5/database/Key.h" + +#include "eckit/config/LocalConfiguration.h" +#include "eckit/io/Buffer.h" +#include "eckit/log/Bytes.h" +#include "eckit/log/Log.h" +#include "eckit/message/Message.h" +#include "eckit/config/Resource.h" +#include "eckit/serialisation/MemoryStream.h" +#include "eckit/utils/Translator.h" +#include "eckit/runtime/Main.h" +#include "eckit/os/BackTrace.h" + +#include "metkit/mars/MarsRequest.h" + +using namespace eckit; +using namespace eckit::net; +// using namespace fdb5::remote; + +namespace fdb5 { +namespace remote { + +//---------------------------------------------------------------------------------------------------------------------- + +namespace{ + +class ConnectionError : public eckit::Exception { +public: + ConnectionError(const int); + ConnectionError(const int, const eckit::net::Endpoint&); + + bool retryOnClient() const override { return true; } +}; + +ConnectionError::ConnectionError(const int retries) { + std::ostringstream s; + s << "Unable to create a connection with the FDB server after " << retries << " retries"; + reason(s.str()); + Log::status() << what() << std::endl; +} + +ConnectionError::ConnectionError(const int retries, const eckit::net::Endpoint& endpoint) { + std::ostringstream s; + s << "Unable to create a connection with the FDB endpoint " << endpoint << " after " << retries << " retries"; + reason(s.str()); + Log::status() << what() << std::endl; +} + +class TCPException : public Exception { +public: + TCPException(const std::string& msg, const CodeLocation& here) : + Exception(std::string("TCPException: ") + msg, here) { + + eckit::Log::error() << "TCP Exception; backtrace(): " << std::endl; + eckit::Log::error() << eckit::BackTrace::dump() << std::endl; + } +}; + +} + +//---------------------------------------------------------------------------------------------------------------------- + + +ClientConnection::ClientConnection(const eckit::net::Endpoint& controlEndpoint, const eckit::Configuration& config): + controlEndpoint_(controlEndpoint), + connected_(false) { + Log::debug() << "ClientConnection::ClientConnection() controlEndpoint: " << controlEndpoint_ << std::endl; + } + + +ClientConnection::~ClientConnection() { + disconnect(); +} + +void ClientConnection::connect() { + + if (connected_) { + Log::warning() << "ClientConnection::connect() called when already connected" << std::endl; + return; + } + + static int fdbMaxConnectRetries = eckit::Resource("fdbMaxConnectRetries", 5); + + try { + // Connect to server, and check that the server is happy on the response + + Log::debug() << "Connecting to host: " << controlEndpoint_ << std::endl; + controlClient_.connect(controlEndpoint_, fdbMaxConnectRetries); + writeControlStartupMessage(); + SessionID serverSession = verifyServerStartupResponse(); + + // Connect to the specified data port + Log::debug() << "Received data endpoint from host: " << dataEndpoint_ << std::endl; + dataClient_.connect(dataEndpoint_, fdbMaxConnectRetries); + writeDataStartupMessage(serverSession); + + // And the connections are set up. Let everything start up! + listeningThread_ = std::thread([this] { listeningThreadLoop(); }); + connected_ = true; + } catch(TooManyRetries& e) { + if (controlClient_.isConnected()) { + controlClient_.close(); + throw ConnectionError(fdbMaxConnectRetries, dataEndpoint_); + } else { + throw ConnectionError(fdbMaxConnectRetries, controlEndpoint_); + } + } +} +SessionID ClientConnection::verifyServerStartupResponse() { + + MessageHeader hdr; + controlRead(&hdr, sizeof(hdr)); + + ASSERT(hdr.marker == StartMarker); + ASSERT(hdr.version == CurrentVersion); + ASSERT(hdr.message == Message::Startup); + ASSERT(hdr.requestID == 0); + + Buffer payload(hdr.payloadSize); + eckit::FixedString<4> tail; + controlRead(payload, hdr.payloadSize); + controlRead(&tail, sizeof(tail)); + ASSERT(tail == EndMarker); + + MemoryStream s(payload); + SessionID clientSession(s); + SessionID serverSession(s); + Endpoint dataEndpoint(s); + LocalConfiguration serverFunctionality(s); + + dataEndpoint_ = dataEndpoint; + + if (dataEndpoint_.hostname() != controlEndpoint_.hostname()) { + Log::warning() << "Data and control interface hostnames do not match. " + << dataEndpoint_.hostname() << " /= " + << controlEndpoint_.hostname() << std::endl; + } + + if (clientSession != sessionID_) { + std::stringstream ss; + ss << "Session ID does not match session received from server: " + << sessionID_ << " != " << clientSession; + throw BadValue(ss.str(), Here()); + } + + return serverSession; +} + +void ClientConnection::writeControlStartupMessage() { + + Buffer payload(4096); + MemoryStream s(payload); + s << sessionID_; + s << controlEndpoint_; + s << LibFdb5::instance().remoteProtocolVersion().used(); + + // TODO: Abstract this dictionary into a RemoteConfiguration object, which + // understands how to do the negotiation, etc, but uses Value (i.e. + // essentially JSON) over the wire for flexibility. + s << availableFunctionality().get(); + + controlWrite(Message::Startup, 0, payload.data(), s.position()); +} + +eckit::LocalConfiguration ClientConnection::availableFunctionality() const { + eckit::LocalConfiguration conf; + std::vector remoteFieldLocationVersions = {1}; + conf.set("RemoteFieldLocation", remoteFieldLocationVersions); + return conf; +} + +void ClientConnection::disconnect() { + if (connected_) { + + // Send termination message + controlWrite(Message::Exit, generateRequestID()); //xxx why do we generate a request ID here? + + listeningThread_.join(); + + // Close both the control and data connections + controlClient_.close(); + dataClient_.close(); + connected_ = false; + } +} + +void ClientConnection::writeDataStartupMessage(const eckit::SessionID& serverSession) { + + Buffer payload(1024); + MemoryStream s(payload); + + s << sessionID_; + s << serverSession; + + dataWrite(Message::Startup, 0, payload.data(), s.position()); +} + +// ----------------------------------------------------------------------------------------------------- + +void ClientConnection::controlWriteCheckResponse(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) { + + controlWrite(msg, requestID, payload, payloadLength); + + // Wait for the receipt acknowledgement + + MessageHeader response; + controlRead(&response, sizeof(MessageHeader)); + + handleError(response); + + ASSERT(response.marker == StartMarker); + ASSERT(response.version == CurrentVersion); + ASSERT(response.message == Message::Received); + + eckit::FixedString<4> tail; + controlRead(&tail, sizeof(tail)); + ASSERT(tail == EndMarker); +} + +void ClientConnection::controlWrite(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) { + + ASSERT((payload == nullptr) == (payloadLength == 0)); + + MessageHeader message(msg, requestID, payloadLength); + controlWrite(&message, sizeof(message)); + if (payload) { + controlWrite(payload, payloadLength); + } + controlWrite(&EndMarker, sizeof(EndMarker)); +} + +void ClientConnection::controlWrite(const void* data, size_t length) { + size_t written = controlClient_.write(data, length); + if (length != written) { + std::stringstream ss; + ss << "Write error. Expected " << length << " bytes, wrote " << written; + throw TCPException(ss.str(), Here()); + } +} + +void ClientConnection::controlRead(void* data, size_t length) { + size_t read = controlClient_.read(data, length); + if (length != read) { + std::stringstream ss; + ss << "Read error. Expected " << length << " bytes, read " << read; + throw TCPException(ss.str(), Here()); + } +} + +void ClientConnection::dataWrite(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) { + + ASSERT((payload == nullptr) == (payloadLength == 0)); + MessageHeader message(msg, requestID, payloadLength); + dataWrite(&message, sizeof(message)); + if (payload) { + dataWrite(payload, payloadLength); + } + dataWrite(&EndMarker, sizeof(EndMarker)); +} + +void ClientConnection::dataWrite(const void* data, size_t length) { + size_t written = dataClient_.write(data, length); + if (length != written) { + std::stringstream ss; + ss << "Write error. Expected " << length << " bytes, wrote " << written; + throw TCPException(ss.str(), Here()); + } +} + +void ClientConnection::dataRead(void* data, size_t length) { + size_t read = dataClient_.read(data, length); + if (length != read) { + std::stringstream ss; + ss << "Read error. Expected " << length << " bytes, read " << read; + throw TCPException(ss.str(), Here()); + } +} + +void ClientConnection::handleError(const MessageHeader& hdr) { + + ASSERT(hdr.marker == StartMarker); + ASSERT(hdr.version == CurrentVersion); + + if (hdr.message == Message::Error) { + ASSERT(hdr.payloadSize > 9); + + std::string what(hdr.payloadSize, ' '); + controlRead(&what[0], hdr.payloadSize); + what[hdr.payloadSize] = 0; // Just in case + + try { + eckit::FixedString<4> tail; + controlRead(&tail, sizeof(tail)); + } catch (...) {} + + throw DecoupledFDBException(what, controlEndpoint_); + } +} + +void ClientConnection::index(const Key& key, const FieldLocation& location) { + NOTIMP; +} + +// void ClientConnection::store(const Key& key, const void* data, size_t length) { + +// // if there is no archiving thread active, then start one. +// // n.b. reset the archiveQueue_ after a potential flush() cycle. + +// if (!archiveFuture_.valid()) { + +// // Start the archival request on the remote side +// ASSERT(archiveID_ == 0); +// uint32_t id = generateRequestID(); +// controlWriteCheckResponse(Message::Archive, id); +// archiveID_ = id; + +// // Reset the queue after previous done/errors +// { +// std::lock_guard lock(archiveQueuePtrMutex_); +// ASSERT(!archiveQueue_); +// archiveQueue_.reset(new ArchiveQueue(maxArchiveQueueLength_)); +// } + +// archiveFuture_ = std::async(std::launch::async, [this, id] { return archiveThreadLoop(id); }); +// } + +// ASSERT(archiveFuture_.valid()); +// ASSERT(archiveID_ != 0); +// { +// std::lock_guard lock(archiveQueuePtrMutex_); +// ASSERT(archiveQueue_); +// archiveQueue_->emplace(std::make_pair(key, Buffer(reinterpret_cast(data), length))); +// } +// } + + +} // namespace remote +} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/remote/ClientConnection.h b/src/fdb5/remote/ClientConnection.h new file mode 100644 index 000000000..64debb695 --- /dev/null +++ b/src/fdb5/remote/ClientConnection.h @@ -0,0 +1,133 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#pragma once + +#include +#include + +#include "eckit/container/Queue.h" +#include "eckit/io/Buffer.h" +#include "eckit/net/Endpoint.h" +#include "eckit/net/TCPClient.h" +#include "eckit/net/TCPStream.h" +#include "eckit/runtime/SessionID.h" + +#include "fdb5/api/FDB.h" +#include "fdb5/api/FDBFactory.h" +#include "fdb5/remote/Messages.h" +#include "eckit/utils/Translator.h" + + +namespace eckit { +// xxx can we give this code a better home? +template<> struct Translator { + std::string operator()(const net::Endpoint& e) { + std::stringstream ss; + ss << e; + return ss.str(); + } +}; +} + +namespace fdb5 { + +//class DecoupledFDB; + +namespace remote { + +//---------------------------------------------------------------------------------------------------------------------- + +class ClientConnection : eckit::NonCopyable { + +public: // types + +public: + + ClientConnection(const eckit::net::Endpoint& controlEndpoint, const eckit::Configuration& config); + ~ClientConnection(); + + void setDataEndpoint(const eckit::net::Endpoint& dataEndpoint); + void connect(); + void disconnect(); + + const eckit::net::Endpoint& controlEndpoint() const { + return controlEndpoint_; + } + const eckit::net::Endpoint& dataEndpoint() const { + return dataEndpoint_; + } + // const eckit::net::Endpoint& dataEndpoint() const { + // return dataEndpoint_; + // } +protected: + + virtual void listeningThreadLoop() = 0; + + // Construct dictionary for protocol negotiation + // im not sure if this belongs here, or in the above class + eckit::LocalConfiguration availableFunctionality() const; + + void writeControlStartupMessage(); + void writeDataStartupMessage(const eckit::SessionID& serverSession); + eckit::SessionID verifyServerStartupResponse(); + + void controlWriteCheckResponse(remote::Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0); + void controlWrite(remote::Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0); + void controlWrite(const void* data, size_t length); + void controlRead(void* data, size_t length); + void dataWrite(remote::Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0); + void dataWrite(const void* data, size_t length); + void dataRead(void* data, size_t length); + void handleError(const remote::MessageHeader& hdr); + + void index(const Key& key, const FieldLocation& location); + // void store(const Key& key, const void* data, size_t length); + // void archive(const Key& key, const void* data, size_t length); + // long sendArchiveData(uint32_t id, const std::vector>& elements, size_t count); + // void sendArchiveData(uint32_t id, const Key& key, const void* data, size_t length); + // void flush(FDBStats& stats); + +private: + eckit::SessionID sessionID_; + + eckit::net::Endpoint controlEndpoint_; + eckit::net::Endpoint dataEndpoint_; + eckit::net::TCPClient controlClient_; + eckit::net::TCPClient dataClient_; + std::thread listeningThread_; + + bool connected_; + +// friend class fdb5::DecoupledFDB; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +// // n.b. if we get integer overflow, we reuse the IDs. This is not a +// // big deal. The idea that we could be on the 2.1 billionth (successful) +// // request, and still have an ongoing request 0 is ... laughable. +static uint32_t generateRequestID() { + + static std::mutex m; + static uint32_t id = 0; + + std::lock_guard lock(m); + return ++id; +} + +class DecoupledFDBException : public eckit::RemoteException { +public: + DecoupledFDBException(const std::string& msg, const eckit::net::Endpoint& endpoint): + eckit::RemoteException(msg, eckit::Translator()(endpoint)) {} +}; + +} // namespace remote +} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/remote/DecoupledHandler.cc b/src/fdb5/remote/DecoupledHandler.cc new file mode 100644 index 000000000..d41adc56a --- /dev/null +++ b/src/fdb5/remote/DecoupledHandler.cc @@ -0,0 +1,355 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project NextGenIO + * (Project ID: 671951) www.nextgenio.eu + */ + +#include + +#include "eckit/config/Resource.h" +#include "eckit/maths/Functions.h" +#include "eckit/net/Endpoint.h" +#include "eckit/runtime/Main.h" +#include "eckit/runtime/SessionID.h" +#include "eckit/serialisation/MemoryStream.h" + +#include "metkit/mars/MarsRequest.h" + +#include "fdb5/LibFdb5.h" +#include "fdb5/fdb5_version.h" +#include "fdb5/api/helpers/FDBToolRequest.h" +#include "fdb5/database/Key.h" +#include "fdb5/remote/AvailablePortList.h" +#include "fdb5/remote/Messages.h" +#include "fdb5/remote/RemoteFieldLocation.h" +#include "fdb5/api/FDB.h" + +#include "fdb5/remote/DecoupledHandler.h" + +using namespace eckit; +using metkit::mars::MarsRequest; + +namespace fdb5 { +namespace remote { + +// helpers +namespace { + +class TCPException : public eckit::Exception { +public: + TCPException(const std::string& msg, const eckit::CodeLocation& here) : + eckit::Exception(std::string("TCPException: ") + msg, here) {} +}; + +std::vector intersection(const LocalConfiguration& c1, const LocalConfiguration& c2, const std::string& field){ + + std::vector v1 = c1.getIntVector(field); + std::vector v2 = c2.getIntVector(field); + std::vector v3; + + std::sort(v1.begin(), v1.end()); + std::sort(v2.begin(), v2.end()); + + std::set_intersection(v1.begin(),v1.end(), + v2.begin(),v2.end(), + back_inserter(v3)); + return v3; +} +} // namespace + +DecoupledHandler::DecoupledHandler(eckit::net::TCPSocket& socket, const Config& config) : + config_(config), + controlSocket_(socket), + dataSocket_(selectDataPort()), + dataListenHostname_(config.getString("dataListenHostname", "")), + // fdb_(config), + readLocationQueue_(eckit::Resource("fdbRetrieveQueueSize", 10000)) { + Log::debug() << "DecoupledHandler::DecoupledHandler initialized" << std::endl; + } + +DecoupledHandler::~DecoupledHandler() { + // We don't want to die before the worker threads are cleaned up + waitForWorkers(); + + // And notify the client that we are done. + Log::info() << "Sending exit message to client" << std::endl; + dataWrite(Message::Exit, 0); + Log::info() << "Done" << std::endl; +} + +eckit::LocalConfiguration DecoupledHandler::availableFunctionality() const { + eckit::LocalConfiguration conf; +// Add to the configuration all the components that require to be versioned, as in the following example, with a vector of supported version numbers + std::vector remoteFieldLocationVersions = {1}; + conf.set("RemoteFieldLocation", remoteFieldLocationVersions); + return conf; +} + +void DecoupledHandler::initialiseConnections() { + // Read the startup message from the client. Check that it all checks out. + + MessageHeader hdr; + socketRead(&hdr, sizeof(hdr), controlSocket_); + + ASSERT(hdr.marker == StartMarker); + ASSERT(hdr.version == CurrentVersion); + ASSERT(hdr.message == Message::Startup); + ASSERT(hdr.requestID == 0); + + Buffer payload1 = receivePayload(hdr, controlSocket_); + eckit::FixedString<4> tail; + socketRead(&tail, sizeof(tail), controlSocket_); + ASSERT(tail == EndMarker); + + MemoryStream s1(payload1); + SessionID clientSession(s1); + net::Endpoint endpointFromClient(s1); + unsigned int remoteProtocolVersion = 0; + std::string errorMsg; + + try { + s1 >> remoteProtocolVersion; + } catch (...) { + errorMsg = "Error retrieving client protocol version"; + } + + if (errorMsg.empty() && !LibFdb5::instance().remoteProtocolVersion().check(remoteProtocolVersion, false)) { + std::stringstream ss; + ss << "FDB server version " << fdb5_version_str() << " - remote protocol version not supported:" << std::endl; + ss << " versions supported by server: " << LibFdb5::instance().remoteProtocolVersion().supportedStr() << std::endl; + ss << " version requested by client: " << remoteProtocolVersion << std::endl; + errorMsg = ss.str(); + } + + if (errorMsg.empty()) { + LocalConfiguration clientAvailableFunctionality(s1); + LocalConfiguration serverConf = availableFunctionality(); + agreedConf_ = LocalConfiguration(); + + // agree on a common functionality by intersecting server and client version numbers + std::vector rflCommon = intersection(clientAvailableFunctionality, serverConf, "RemoteFieldLocation"); + if (rflCommon.size() > 0) { + Log::debug() << "Protocol negotiation - RemoteFieldLocation version " << rflCommon.back() << std::endl; + agreedConf_.set("RemoteFieldLocation", rflCommon.back()); + } + else { + std::stringstream ss; + ss << "FDB server version " << fdb5_version_str() << " - failed protocol negotiation with FDB client" << std::endl; + ss << " server functionality: " << serverConf << std::endl; + ss << " client functionality: " << clientAvailableFunctionality << std::endl; + errorMsg = ss.str(); + } + } + + // We want a data connection too. Send info to RemoteFDB, and wait for connection + // n.b. FDB-192: we use the host communicated from the client endpoint. This + // ensures that if a specific interface has been selected and the + // server has multiple, then we use that on, whilst retaining + // the capacity in the protocol for the server to make a choice. + + int dataport = dataSocket_.localPort(); + net::Endpoint dataEndpoint(endpointFromClient.hostname(), dataport); + + Log::info() << "Sending data endpoint to client: " << dataEndpoint << std::endl; + { + Buffer startupBuffer(1024); + MemoryStream s(startupBuffer); + + s << clientSession; + s << sessionID_; + s << dataEndpoint; + s << agreedConf_.get(); + // s << storeEndpoint; // xxx single-store case only: we cant do this with multiple stores // For now, dont send the store endpoint to the client + + Log::debug() << "Protocol negotiation - configuration: " << agreedConf_ <() << "DecoupledHandler::dataWrite. requestID: " << requestID << " payloadLength: " << payloadLength + << " msg: " << static_cast(msg) << std::endl; + + ASSERT((payload == nullptr) == (payloadLength == 0)); + + MessageHeader message(msg, requestID, payloadLength); + + std::lock_guard lock(dataWriteMutex_); + + dataWriteUnsafe(&message, sizeof(message)); + if (payload) { + dataWriteUnsafe(payload, payloadLength); + } + dataWriteUnsafe(&EndMarker, sizeof(EndMarker)); +} + +void DecoupledHandler::dataWriteUnsafe(const void* data, size_t length) { + size_t written = dataSocket_.write(data, length); + if (length != written) { + std::stringstream ss; + ss << "Write error. Expected " << length << " bytes, wrote " << written; + throw TCPException(ss.str(), Here()); + } +} + +Buffer DecoupledHandler::receivePayload(const MessageHeader& hdr, net::TCPSocket& socket) { + Buffer payload(hdr.payloadSize); + + ASSERT(hdr.payloadSize > 0); + socketRead(payload, hdr.payloadSize, socket); + + return payload; +} + +void DecoupledHandler::tidyWorkers() { + std::map>::iterator it = workerThreads_.begin(); + + for (; it != workerThreads_.end(); /* empty */) { + std::future_status stat = it->second.wait_for(std::chrono::milliseconds(0)); + + if (stat == std::future_status::ready) { + Log::info() << "Tidying up worker for request ID: " << it->first << std::endl; + workerThreads_.erase(it++); + } + else { + ++it; + } + } +} + +void DecoupledHandler::waitForWorkers() { + readLocationQueue_.close(); + + tidyWorkers(); + + for (auto& it : workerThreads_) { + Log::error() << "Worker thread still alive for request ID: " << it.first << std::endl; + Log::error() << "Joining ..." << std::endl; + it.second.get(); + Log::error() << "Thread complete" << std::endl; + } + + if (readLocationWorker_.joinable()) { + readLocationWorker_.join(); + } +} + + +void DecoupledHandler::read(const MessageHeader& hdr) { + // store only + NOTIMP; +} + +void DecoupledHandler::archive(const MessageHeader& hdr) { + // catalogue only + NOTIMP; +} + +// void DecoupledHandler::store(const MessageHeader& hdr) { +// // store only +// NOTIMP; +// } + +void DecoupledHandler::flush(const MessageHeader& hdr) { + // store only + NOTIMP; +} + + +} // namespace remote +} // namespace fdb5 diff --git a/src/fdb5/remote/DecoupledHandler.h b/src/fdb5/remote/DecoupledHandler.h new file mode 100644 index 000000000..63c044b65 --- /dev/null +++ b/src/fdb5/remote/DecoupledHandler.h @@ -0,0 +1,113 @@ + + +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project NextGenIO + * (Project ID: 671951) www.nextgenio.eu + */ + +#ifndef fdb5_remote_DecoupledHandler_H +#define fdb5_remote_DecoupledHandler_H + +#include +#include + +#include "eckit/io/Buffer.h" +#include "eckit/io/DataHandle.h" +#include "eckit/net/TCPServer.h" +#include "eckit/net/TCPSocket.h" +#include "eckit/runtime/SessionID.h" + +#include "metkit/mars/MarsRequest.h" + +#include "fdb5/api/FDBFactory.h" +#include "fdb5/config/Config.h" +#include "fdb5/database/Key.h" +#include "fdb5/remote/Messages.h" + +namespace fdb5 { + +class Config; + +namespace remote { + +struct MessageHeader; + +//---------------------------------------------------------------------------------------------------------------------- +// Base class for CatalogueHandler and StoreHandler + +class DecoupledHandler : private eckit::NonCopyable { +public: // methods + DecoupledHandler(eckit::net::TCPSocket& socket, const Config& config); + ~DecoupledHandler(); + + virtual void handle() = 0; + + std::string host() const { return controlSocket_.localHost(); } + int port() const { return controlSocket_.localPort(); } + const eckit::LocalConfiguration& agreedConf() const { return agreedConf_; } + +protected: + + // socket methods + int selectDataPort(); + virtual void initialiseConnections(); + eckit::LocalConfiguration availableFunctionality() const; + + void controlWrite(Message msg, uint32_t requestID, const void* payload = nullptr, uint32_t payloadLength = 0); + void controlWrite(const void* data, size_t length); + void socketRead(void* data, size_t length, eckit::net::TCPSocket& socket); + + // dataWrite is protected using a mutex, as we may have multiple workers. + void dataWrite(Message msg, uint32_t requestID, const void* payload = nullptr, uint32_t payloadLength = 0); + eckit::Buffer receivePayload(const MessageHeader& hdr, eckit::net::TCPSocket& socket); + + // socket methods + void dataWriteUnsafe(const void* data, size_t length); + + // Worker functionality + void tidyWorkers(); + void waitForWorkers(); + + + virtual void read(const MessageHeader& hdr); + virtual void archive(const MessageHeader& hdr); + // virtual void store(const MessageHeader& hdr); + virtual void flush(const MessageHeader& hdr); + +protected: + + Config config_; + eckit::net::TCPSocket controlSocket_; + eckit::net::EphemeralTCPServer dataSocket_; + std::string dataListenHostname_; + // FDB fdb_; + eckit::Queue>> readLocationQueue_; + + eckit::SessionID sessionID_; + eckit::LocalConfiguration agreedConf_; + std::mutex dataWriteMutex_; + std::map> workerThreads_; + std::future archiveFuture_; + std::thread readLocationWorker_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace remote +} // namespace fdb5 + + +//---------------------------------------------------------------------------------------------------------------------- + + +#endif // fdb5_remote_DecoupledHandler_H diff --git a/src/fdb5/remote/FdbServer.cc b/src/fdb5/remote/FdbServer.cc index e45b0ca20..21194870f 100644 --- a/src/fdb5/remote/FdbServer.cc +++ b/src/fdb5/remote/FdbServer.cc @@ -19,7 +19,10 @@ #include "fdb5/remote/FdbServer.h" #include "fdb5/remote/AvailablePortList.h" -#include "fdb5/remote/Handler.h" +// #include "fdb5/remote/Handler.h" +#include "fdb5/remote/CatalogueHandler.h" +#include "fdb5/remote/StoreHandler.h" +#include "eckit/config/Resource.h" using namespace eckit; @@ -46,8 +49,21 @@ void FDBForker::run() { eckit::Log::info() << "FDB forked pid " << ::getpid() << std::endl; - RemoteHandler handler(socket_, config_); - handler.handle(); + if (config_.getString("type", "local") == "catalogue" || (::getenv("FDB_IS_CAT") && ::getenv("FDB_IS_CAT")[0] == '1')) { + eckit::Log::info() << "FDB using Catalogue Handler" << std::endl; + // CatalogueHandler handler(socket_, config_); + // handler.handle(); + } + else if (config_.getString("type", "local") == "store" || (::getenv("FDB_IS_STORE") && ::getenv("FDB_IS_STORE")[0] == '1')) { + eckit::Log::info() << "FDB using Store Handler" << std::endl; + StoreHandler handler(socket_, config_); + handler.handle(); + } + // else { + // eckit::Log::info() << "FDB using Remote Handler" << std::endl; + // RemoteHandler handler(socket_, config_); + // handler.handle(); + // } } //---------------------------------------------------------------------------------------------------------------------- @@ -76,10 +92,28 @@ FDBServerThread::FDBServerThread(net::TCPSocket& socket, const Config& config) : config_(config) {} void FDBServerThread::run() { + std::cout << "FDBServerThread::run()" << std::endl; eckit::Log::info() << "FDB started handler thread" << std::endl; - RemoteHandler handler(socket_, config_); - handler.handle(); + + if (config_.getString("type", "local") == "catalogue" || (::getenv("FDB_IS_CAT") && ::getenv("FDB_IS_CAT")[0] == '1')) { + eckit::Log::info() << "FDB using Catalogue Handler" << std::endl; + // CatalogueHandler handler(socket_, config_); + // handler.handle(); + } + else if (config_.getString("type", "local") == "store" || (::getenv("FDB_IS_STORE") && ::getenv("FDB_IS_STORE")[0] == '1')) { + eckit::Log::info() << "FDB using Store Handler" << std::endl; + StoreHandler handler(socket_, config_); + handler.handle(); + } + // else { + // eckit::Log::info() << "FDB using Remote Handler" << std::endl; + // RemoteHandler handler(socket_, config_); + // handler.handle(); + // } + + // RemoteHandler handler(socket_, config_); + // handler.handle(); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc new file mode 100644 index 000000000..c36a608ca --- /dev/null +++ b/src/fdb5/remote/RemoteStore.cc @@ -0,0 +1,775 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include +#include + +#include "eckit/log/Timer.h" + +#include "eckit/config/Resource.h" +#include "eckit/io/AIOHandle.h" +#include "eckit/io/EmptyHandle.h" +#include "eckit/runtime/Main.h" +#include "eckit/serialisation/MemoryStream.h" + +#include "fdb5/LibFdb5.h" +#include "fdb5/rules/Rule.h" +#include "fdb5/database/FieldLocation.h" +#include "fdb5/remote/RemoteStore.h" +#include "fdb5/io/FDBFileHandle.h" + +using namespace eckit; + +namespace { + // // n.b. if we get integer overflow, we reuse the IDs. This is not a +// // big deal. The idea that we could be on the 2.1 billionth (successful) +// // request, and still have an ongoing request 0 is ... laughable. +static uint32_t generateRequestID() { + + static std::mutex m; + static uint32_t id = 0; + + std::lock_guard lock(m); + return ++id; +} + +} +namespace fdb5::remote { + +//---------------------------------------------------------------------------------------------------------------------- + +RemoteStore::RemoteStore(const Key& key, const Config& config) : + ClientConnection(eckit::net::Endpoint(config.getString("storeHost"), config.getInt("storePort")), config), + key_(key), config_(config), dirty_(false), archiveID_(0), + maxArchiveQueueLength_(eckit::Resource("fdbRemoteArchiveQueueLength;$FDB_REMOTE_ARCHIVE_QUEUE_LENGTH", 200)), + retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)), + maxArchiveBatchSize_(config.getInt("maxBatchSize", 1)) {} + +RemoteStore::RemoteStore(const Key& key, const Config& config, const eckit::net::Endpoint& controlEndpoint) : + ClientConnection(controlEndpoint, config), + key_(key), config_(config), dirty_(false), archiveID_(0), + maxArchiveQueueLength_(eckit::Resource("fdbRemoteArchiveQueueLength;$FDB_REMOTE_ARCHIVE_QUEUE_LENGTH", 200)), + retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)), + maxArchiveBatchSize_(config.getInt("maxBatchSize", 1)) {} + +RemoteStore::RemoteStore(const eckit::URI& uri, const Config& config) : + ClientConnection(eckit::net::Endpoint(uri.hostport()), config), + key_(Key()), config_(config), dirty_(false), archiveID_(0), + maxArchiveQueueLength_(eckit::Resource("fdbRemoteArchiveQueueLength;$FDB_REMOTE_ARCHIVE_QUEUE_LENGTH", 200)), + retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)), + maxArchiveBatchSize_(config.getInt("maxBatchSize", 1)) { + ASSERT(uri.scheme() == "fdb"); +} + +RemoteStore::~RemoteStore() { + // If we have launched a thread with an async and we manage to get here, this is + // an error. n.b. if we don't do something, we will block in the destructor + // of std::future. + if (archiveFuture_.valid()) { + Log::error() << "Attempting to destruct DecoupledFDB with active archive thread" << std::endl; + eckit::Main::instance().terminate(); + } + + ASSERT(!dirty_); + disconnect(); +} + +eckit::URI RemoteStore::uri() const { + return URI("fdb", ""); +} + +bool RemoteStore::exists() const { + return true; // directory_.exists(); +} + +eckit::DataHandle* RemoteStore::retrieve(Field& field) const { + //return field.dataHandle(); + return nullptr; +} + +std::future > RemoteStore::archive(const Key& key, const void *data, eckit::Length length) { + // dirty_ = true; + // uint32_t id = generateRequestID(); + + connect(); + + // if there is no archiving thread active, then start one. + // n.b. reset the archiveQueue_ after a potential flush() cycle. + + if (!archiveFuture_.valid()) { + + // Start the archival request on the remote side + ASSERT(archiveID_ == 0); + uint32_t id = generateRequestID(); + archiveID_ = id; + controlWriteCheckResponse(fdb5::remote::Message::Archive, id); + + // Reset the queue after previous done/errors + { + std::lock_guard lock(archiveQueuePtrMutex_); + ASSERT(!archiveQueue_); + archiveQueue_.reset(new ArchiveQueue(maxArchiveQueueLength_)); + } + + archiveFuture_ = std::async(std::launch::async, [this, id] { return archiveThreadLoop(id); }); + } + + ASSERT(archiveFuture_.valid()); + ASSERT(archiveID_ != 0); + { + std::lock_guard lock(archiveQueuePtrMutex_); + ASSERT(archiveQueue_); + // std::vector combinedKey = {key_, key}; + // std::cout << "archiveID_: " << archiveID_ << " Archiving " << combinedKey << std::endl; + archiveQueue_->emplace(std::make_pair(key, Buffer(reinterpret_cast(data), length))); + } + + std::promise > loc; + auto futureLocation = loc.get_future(); + locations_[archiveID_] = std::move(loc); + return futureLocation; +} + +bool RemoteStore::open() { + connect(); + return true; +} + +void RemoteStore::flush() { +// if (!dirty_) { +// return; +// } + +// // ensure consistent state before writing Toc entry + +// flushDataHandles(); + +// dirty_ = false; +} + +void RemoteStore::close() { + disconnect(); +} + +void RemoteStore::remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const { + NOTIMP; +} + +void RemoteStore::remove(const Key& key) const { + NOTIMP; +} + + + + +// eckit::PathName src_db = directory_ / key.valuesToString(); + +// DIR* dirp = ::opendir(src_db.asString().c_str()); +// struct dirent* dp; +// while ((dp = ::readdir(dirp)) != NULL) { +// if (strstr( dp->d_name, ".data")) { +// eckit::PathName dataFile = src_db / dp->d_name; +// eckit::Log::debug() << "Removing " << dataFile << std::endl; +// dataFile.unlink(false); +// } +// } +// closedir(dirp); +// } + +// void RemoteStore::connect() { +// ASSERT(remote_); +// remote_->connect(); +// } + +// void RemoteStore::disconnect() { +// if (remote_) { +// remote_->disconnect(); +// } +// } + +void RemoteStore::print(std::ostream &out) const { + out << "RemoteStore(host=" << controlEndpoint() << ", data=" << dataEndpoint() << ")"; +} + + + + + +// void RemoteFDB::flush() { + +// Timer timer; + +// timer.start(); + +// // Flush only does anything if there is an ongoing archive(); +// if (archiveFuture_.valid()) { + +// ASSERT(archiveID_ != 0); +// { +// ASSERT(archiveQueue_); +// std::lock_guard lock(archiveQueuePtrMutex_); +// archiveQueue_->close(); +// } +// FDBStats stats = archiveFuture_.get(); +// ASSERT(!archiveQueue_); +// archiveID_ = 0; + +// ASSERT(stats.numFlush() == 0); +// size_t numArchive = stats.numArchive(); + +// Buffer sendBuf(4096); +// MemoryStream s(sendBuf); +// s << numArchive; + +// // The flush call is blocking +// controlWriteCheckResponse(fdb5::remote::Message::Flush, generateRequestID(), sendBuf, s.position()); + +// internalStats_ += stats; +// } + +// timer.stop(); +// internalStats_.addFlush(timer); +// } + + + +FDBStats RemoteStore::archiveThreadLoop(uint32_t requestID) { + FDBStats localStats; + eckit::Timer timer; + + // We can pop multiple elements off the archive queue simultaneously, if + // configured + std::vector> elements; + for (size_t i = 0; i < maxArchiveBatchSize_; ++i) { + elements.emplace_back(std::make_pair(Key{}, Buffer{0})); + } + + try { + + ASSERT(archiveQueue_); + ASSERT(archiveID_ != 0); + + long popped; + while ((popped = archiveQueue_->pop(elements)) != -1) { + + timer.start(); + long dataSent = sendArchiveData(requestID, elements, popped); + timer.stop(); + localStats.addArchive(dataSent, timer, popped); + } + + // And note that we are done. (don't time this, as already being blocked + // on by the ::flush() routine) + + MessageHeader hdr(Message::Flush, requestID); + dataWrite(&hdr, sizeof(hdr)); + dataWrite(&EndMarker, sizeof(EndMarker)); + + archiveID_ = 0; + archiveQueue_.reset(); + + } catch (...) { + archiveQueue_->interrupt(std::current_exception()); + throw; + } + + return localStats; + + // We are inside an async, so don't need to worry about exceptions escaping. + // They will be released when flush() is called. +} + + + +void RemoteStore::listeningThreadLoop() { + + /// @note This routine retrieves BOTH normal API asynchronously returned data, AND + /// fields that are being returned by a retrieve() call. These need to go into different + /// queues + /// --> Test if the requestID is a known API request, otherwise push onto the retrieve queue + + + /// @note messageQueues_ is a map of requestID:MessageQueue. At the point that + /// a request is complete, errored or otherwise killed, it needs to be removed + /// from the map. The shared_ptr allows this removal to be asynchronous with + /// the actual task cleaning up and returning to the client. + + try { + + MessageHeader hdr; + eckit::FixedString<4> tail; + + while (true) { + + dataRead(&hdr, sizeof(hdr)); + + ASSERT(hdr.marker == StartMarker); + ASSERT(hdr.version == CurrentVersion); + + switch (hdr.message) { + + case Message::Exit: + return; + + case Message::Blob: { + Buffer payload(hdr.payloadSize); + if (hdr.payloadSize > 0) dataRead(payload, hdr.payloadSize); + + auto it = messageQueues_.find(hdr.requestID); + if (it != messageQueues_.end()) { + it->second->emplace(std::make_pair(hdr, std::move(payload))); + } else { + retrieveMessageQueue_.emplace(std::make_pair(hdr, std::move(payload))); + } + break; + } + + case Message::Complete: { + auto it = messageQueues_.find(hdr.requestID); + if (it != messageQueues_.end()) { + it->second->close(); + + // Remove entry (shared_ptr --> message queue will be destroyed when it + // goes out of scope in the worker thread). + messageQueues_.erase(it); + + } else { + retrieveMessageQueue_.emplace(std::make_pair(hdr, Buffer(0))); + } + break; + } + + case Message::Error: { + + auto it = messageQueues_.find(hdr.requestID); + if (it != messageQueues_.end()) { + std::string msg; + if (hdr.payloadSize > 0) { + msg.resize(hdr.payloadSize, ' '); + dataRead(&msg[0], hdr.payloadSize); + } + it->second->interrupt(std::make_exception_ptr(DecoupledFDBException(msg, dataEndpoint()))); + + // Remove entry (shared_ptr --> message queue will be destroyed when it + // goes out of scope in the worker thread). + messageQueues_.erase(it); + + } else if (hdr.requestID == archiveID_) { + + std::lock_guard lock(archiveQueuePtrMutex_); + + if (archiveQueue_) { + std::string msg; + if (hdr.payloadSize > 0) { + msg.resize(hdr.payloadSize, ' '); + dataRead(&msg[0], hdr.payloadSize); + } + + archiveQueue_->interrupt(std::make_exception_ptr(DecoupledFDBException(msg, dataEndpoint()))); + } + } else { + Buffer payload(hdr.payloadSize); + if (hdr.payloadSize > 0) dataRead(payload, hdr.payloadSize); + retrieveMessageQueue_.emplace(std::make_pair(hdr, std::move(payload))); + } + break; + } + + default: { + std::stringstream ss; + ss << "ERROR: Unexpected message recieved (" << static_cast(hdr.message) << "). ABORTING"; + Log::status() << ss.str() << std::endl; + Log::error() << "Retrieving... " << ss.str() << std::endl; + throw SeriousBug(ss.str(), Here()); + } + }; + + // Ensure we have consumed exactly the correct amount from the socket. + + dataRead(&tail, sizeof(tail)); + ASSERT(tail == EndMarker); + } + + // We don't want to let exceptions escape inside a worker thread. + + } catch (const std::exception& e) { + for (auto& it : messageQueues_) { + it.second->interrupt(std::make_exception_ptr(e)); + } + messageQueues_.clear(); + retrieveMessageQueue_.interrupt(std::make_exception_ptr(e)); + { + std::lock_guard lock(archiveQueuePtrMutex_); + if (archiveQueue_) archiveQueue_->interrupt(std::make_exception_ptr(e)); + } + } catch (...) { + for (auto& it : messageQueues_) { + it.second->interrupt(std::current_exception()); + } + messageQueues_.clear(); + retrieveMessageQueue_.interrupt(std::current_exception()); + { + std::lock_guard lock(archiveQueuePtrMutex_); + if (archiveQueue_) archiveQueue_->interrupt(std::current_exception()); + } + } +} + + +void RemoteStore::flush(FDBStats& internalStats) { + // Flush only does anything if there is an ongoing archive(); + if (! archiveFuture_.valid()) return; + + ASSERT(archiveID_ != 0); + { + ASSERT(archiveQueue_); + std::lock_guard lock(archiveQueuePtrMutex_); + archiveQueue_->close(); + } + FDBStats stats = archiveFuture_.get(); + ASSERT(!archiveQueue_); + archiveID_ = 0; + + ASSERT(stats.numFlush() == 0); + size_t numArchive = stats.numArchive(); + + Buffer sendBuf(4096); + MemoryStream s(sendBuf); + s << numArchive; + + // The flush call is blocking + controlWriteCheckResponse(Message::Flush, generateRequestID(), sendBuf, s.position()); + + internalStats += stats; +} + + +// long RemoteFDB::sendArchiveData(uint32_t id, const std::vector>& elements, size_t count) { + +// if (count == 1) { +// sendArchiveData(id, elements[0].first, elements[0].second.data(), elements[0].second.size()); +// return elements[0].second.size(); +// } + +// // Serialise the keys + +// std::vector keyBuffers; +// std::vector keySizes; +// keyBuffers.reserve(count); +// keySizes.reserve(count); + +// size_t containedSize = 0; + +// for (size_t i = 0; i < count; ++i) { +// keyBuffers.emplace_back(Buffer {4096}); +// MemoryStream keyStream(keyBuffers.back()); +// keyStream << elements[i].first; +// keySizes.push_back(keyStream.position()); +// containedSize += (keyStream.position() + elements[i].second.size() + +// sizeof(MessageHeader) + sizeof(EndMarker)); +// } + +// // Construct the containing message + +// MessageHeader message(fdb5::remote::Message::MultiBlob, id, containedSize); +// dataWrite(&message, sizeof(message)); + +// long dataSent = 0; + +// for (size_t i = 0; i < count; ++i) { +// MessageHeader containedMessage(fdb5::remote::Message::Blob, id, elements[i].second.size() + keySizes[i]); +// dataWrite(&containedMessage, sizeof(containedMessage)); +// dataWrite(keyBuffers[i], keySizes[i]); +// dataWrite(elements[i].second.data(), elements[i].second.size()); +// dataWrite(&EndMarker, sizeof(EndMarker)); +// dataSent += elements[i].second.size(); +// } + +// dataWrite(&EndMarker, sizeof(EndMarker)); +// return dataSent; +// } + + +// void RemoteFDB::sendArchiveData(uint32_t id, const Key& key, const void* data, size_t length) { + +// ASSERT(data); +// ASSERT(length != 0); + +// Buffer keyBuffer(4096); +// MemoryStream keyStream(keyBuffer); +// keyStream << key; + +// MessageHeader message(fdb5::remote::Message::Blob, id, length + keyStream.position()); +// dataWrite(&message, sizeof(message)); +// dataWrite(keyBuffer, keyStream.position()); +// dataWrite(data, length); +// dataWrite(&EndMarker, sizeof(EndMarker)); +// } + +// // ----------------------------------------------------------------------------------------------------- + +// // +// /// @note The DataHandles returned by retrieve() MUST STRICTLY be read in order. +// /// We do not create multiple message queues, one for each requestID, even +// /// though that would be nice. This is because a commonly used retrieve +// /// pattern uses many retrieve() calls aggregated into a MultiHandle, and +// /// if we created lots of queues we would just run out of memory receiving +// /// from the stream. Further, if we curcumvented this by blocking, then we +// /// could get deadlocked if we try and read a message that is further back +// /// in the stream +// /// +// /// --> Retrieve is a _streaming_ service. + +// namespace { + +// class FDBRemoteDataHandle : public DataHandle { + +// public: // methods + +// FDBRemoteDataHandle(uint32_t requestID, +// RemoteFDB::MessageQueue& queue, +// const net::Endpoint& remoteEndpoint) : +// requestID_(requestID), +// queue_(queue), +// remoteEndpoint_(remoteEndpoint), +// pos_(0), +// overallPosition_(0), +// currentBuffer_(0), +// complete_(false) {} +// virtual bool canSeek() const override { return false; } + +// private: // methods + +// void print(std::ostream& s) const override { +// s << "FDBRemoteDataHandle(id=" << requestID_ << ")"; +// } + +// Length openForRead() override { +// return estimate(); +// } +// void openForWrite(const Length&) override { NOTIMP; } +// void openForAppend(const Length&) override { NOTIMP; } +// long write(const void*, long) override { NOTIMP; } +// void close() override {} + +// long read(void* pos, long sz) override { + +// if (complete_) return 0; + +// if (currentBuffer_.size() != 0) return bufferRead(pos, sz); + +// // If we are in the DataHandle, then there MUST be data to read + +// RemoteFDB::StoredMessage msg = std::make_pair(remote::MessageHeader{}, eckit::Buffer{0}); +// ASSERT(queue_.pop(msg) != -1); + +// // TODO; Error handling in the retrieve pathway + +// const MessageHeader& hdr(msg.first); + +// ASSERT(hdr.marker == StartMarker); +// ASSERT(hdr.version == CurrentVersion); + +// // Handle any remote errors communicated from the server + +// if (hdr.message == fdb5::remote::Message::Error) { +// std::string errmsg(static_cast(msg.second), msg.second.size()); +// throw RemoteFDBException(errmsg, remoteEndpoint_); +// } + +// // Are we now complete + +// if (hdr.message == fdb5::remote::Message::Complete) { +// complete_ = 0; +// return 0; +// } + +// ASSERT(hdr.message == fdb5::remote::Message::Blob); + +// // Otherwise return the data! + +// std::swap(currentBuffer_, msg.second); + +// return bufferRead(pos, sz); +// } + +// // A helper function that returns some, or all, of a buffer that has +// // already been retrieved. + +// long bufferRead(void* pos, long sz) { + +// ASSERT(currentBuffer_.size() != 0); +// ASSERT(pos_ < currentBuffer_.size()); + +// long read = std::min(sz, long(currentBuffer_.size() - pos_)); + +// ::memcpy(pos, ¤tBuffer_[pos_], read); +// pos_ += read; +// overallPosition_ += read; + +// // If we have exhausted this buffer, free it up. + +// if (pos_ >= currentBuffer_.size()) { +// Buffer nullBuffer(0); +// std::swap(currentBuffer_, nullBuffer); +// pos_ = 0; +// ASSERT(currentBuffer_.size() == 0); +// } + +// return read; +// } + +// Length estimate() override { +// return 0; +// } + +// Offset position() override { +// return overallPosition_; +// } + +// private: // members + +// uint32_t requestID_; +// RemoteFDB::MessageQueue& queue_; +// net::Endpoint remoteEndpoint_; +// size_t pos_; +// Offset overallPosition_; +// Buffer currentBuffer_; +// bool complete_; +// }; + +// } + +// // Here we do (asynchronous) retrieving related stuff + +// //DataHandle* RemoteFDB::retrieve(const metkit::mars::MarsRequest& request) { + +// // connect(); + +// // Buffer encodeBuffer(4096); +// // MemoryStream s(encodeBuffer); +// // s << request; + +// // uint32_t id = generateRequestID(); + +// // controlWriteCheckResponse(fdb5::remote::Message::Retrieve, id, encodeBuffer, s.position()); + +// // return new FDBRemoteDataHandle(id, retrieveMessageQueue_, controlEndpoint_); +// //} + + +// // Here we do (asynchronous) read related stuff + + +// eckit::DataHandle* RemoteFDB::dataHandle(const FieldLocation& fieldLocation) { +// return dataHandle(fieldLocation, Key()); +// } + +// eckit::DataHandle* RemoteFDB::dataHandle(const FieldLocation& fieldLocation, const Key& remapKey) { + +// connect(); + +// Buffer encodeBuffer(4096); +// MemoryStream s(encodeBuffer); +// s << fieldLocation; +// s << remapKey; + +// uint32_t id = generateRequestID(); + +// controlWriteCheckResponse(fdb5::remote::Message::Read, id, encodeBuffer, s.position()); + +// return new FDBRemoteDataHandle(id, retrieveMessageQueue_, controlEndpoint_); +// } + + + +long RemoteStore::sendArchiveData(uint32_t id, const std::vector>& elements, size_t count) { + if (count == 1) { + sendArchiveData(id, elements[0].first, elements[0].second.data(), elements[0].second.size()); + return elements[0].second.size(); + } + + // Serialise the keys + + std::vector keyBuffers; + std::vector keySizes; + keyBuffers.reserve(count); + keySizes.reserve(count); + + size_t containedSize = 0; + + for (size_t i = 0; i < count; ++i) { + keyBuffers.emplace_back(Buffer {4096}); + MemoryStream keyStream(keyBuffers.back()); + keyStream << elements[i].first; + keySizes.push_back(keyStream.position()); + containedSize += (keyStream.position() + elements[i].second.size() + + sizeof(MessageHeader) + sizeof(EndMarker)); + } + + // Construct the containing message + + MessageHeader message(Message::MultiBlob, id, containedSize); + dataWrite(&message, sizeof(message)); + + long dataSent = 0; + + for (size_t i = 0; i < count; ++i) { + MessageHeader containedMessage(Message::Blob, id, elements[i].second.size() + keySizes[i]); + dataWrite(&containedMessage, sizeof(containedMessage)); + dataWrite(keyBuffers[i], keySizes[i]); + dataWrite(elements[i].second.data(), elements[i].second.size()); + dataWrite(&EndMarker, sizeof(EndMarker)); + dataSent += elements[i].second.size(); + } + + dataWrite(&EndMarker, sizeof(EndMarker)); + return dataSent; +} + + +void RemoteStore::sendArchiveData(uint32_t id, const Key& key, const void* data, size_t length) { + + ASSERT(data); + ASSERT(length != 0); + + Buffer keyBuffer(4096); + MemoryStream keyStream(keyBuffer); + + if (!key_.rule()) { + std::cout << "RemoteStore::sendArchiveData - missing rule in " << key_ << std::endl; + } else { + std::cout << "Good so far... RemoteStore::sendArchiveData - existing rule in " << key_ << std::endl; + + } + if (!key.rule()) { + std::cout << "RemoteStore::sendArchiveData - missing rule in " << key << std::endl; + } else { + std::cout << "Good so far... RemoteStore::sendArchiveData - existing rule in " << key << std::endl; + } + + keyStream << key_; + keyStream << key; + + MessageHeader message(Message::Blob, id, length + keyStream.position()); + dataWrite(&message, sizeof(message)); + dataWrite(keyBuffer, keyStream.position()); + dataWrite(data, length); + dataWrite(&EndMarker, sizeof(EndMarker)); +} + + + + +static StoreBuilder builder("remote"); + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5::remote diff --git a/src/fdb5/remote/RemoteStore.h b/src/fdb5/remote/RemoteStore.h new file mode 100644 index 000000000..d46d04d21 --- /dev/null +++ b/src/fdb5/remote/RemoteStore.h @@ -0,0 +1,108 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @file RemoteStore.h +/// @author Emanuele Danovaro +/// @date October 2023 + +#pragma once + +#include "fdb5/database/DB.h" +#include "fdb5/database/Index.h" +#include "fdb5/database/Store.h" +#include "fdb5/remote/ClientConnection.h" + +namespace fdb5::remote { + +//---------------------------------------------------------------------------------------------------------------------- + +/// Store that connects to a remote Store + +class RemoteStore : public Store, public ClientConnection { + +public: // types + + using ArchiveQueue = eckit::Queue>; + using StoredMessage = std::pair; + using MessageQueue = eckit::Queue; + +public: // methods + + RemoteStore(const Key& key, const Config& config); + RemoteStore(const Key& key, const Config& config, const eckit::net::Endpoint& controlEndpoint); + RemoteStore(const eckit::URI& uri, const Config& config); + + ~RemoteStore() override; + + eckit::URI uri() const override; + + bool open() override; + void flush() override; + void close() override; + + void checkUID() const override { } + + bool canMoveTo(const Key& key, const Config& config, const eckit::URI& dest) const override { return false; } + void moveTo(const Key& key, const Config& config, const eckit::URI& dest, eckit::Queue& queue) const override { NOTIMP; } + void remove(const Key& key) const override; + +protected: // methods + + std::string type() const override { return "remote"; } + + bool exists() const override; + + eckit::DataHandle* retrieve(Field& field) const override; + std::future > archive(const Key& key, const void *data, eckit::Length length) override; + + void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override; + + void print( std::ostream &out ) const override; + +private: // methods + + FDBStats archiveThreadLoop(uint32_t requestID); + // Listen to the dataClient for incoming messages, and push them onto + // appropriate queues. + void listeningThreadLoop() override; + long sendArchiveData(uint32_t id, const std::vector>& elements, size_t count); + void sendArchiveData(uint32_t id, const Key& key, const void* data, size_t length); + void flush(FDBStats& stats); + +// void connect(); +// void disconnect(); + +private: // members + + Key key_; + const Config& config_; + + bool dirty_; + std::future archiveFuture_; + uint32_t archiveID_; + std::mutex archiveQueuePtrMutex_; + std::unique_ptr archiveQueue_; + size_t maxArchiveQueueLength_; + size_t maxArchiveBatchSize_; + // @note This is a map of requestID:MessageQueue. At the point that a request is + // complete, errored or otherwise killed, it needs to be removed from the map. + // The shared_ptr allows this removal to be asynchronous with the actual task + // cleaning up and returning to the client. + + std::map> messageQueues_; + MessageQueue retrieveMessageQueue_; + + std::map > > locations_; + +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5::remote diff --git a/src/fdb5/remote/StoreHandler.cc b/src/fdb5/remote/StoreHandler.cc new file mode 100644 index 000000000..4be48e13a --- /dev/null +++ b/src/fdb5/remote/StoreHandler.cc @@ -0,0 +1,430 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "fdb5/remote/StoreHandler.h" +#include "eckit/serialisation/MemoryStream.h" +#include "fdb5/LibFdb5.h" +#include "eckit/config/Resource.h" + + +using namespace eckit; +using metkit::mars::MarsRequest; + +namespace fdb5 { +namespace remote { + +StoreHandler::StoreHandler(eckit::net::TCPSocket& socket, const Config& config): + DecoupledHandler(socket, config) {} + +StoreHandler::~StoreHandler() {} + +void StoreHandler::initialiseConnections() { + DecoupledHandler::initialiseConnections(); + + Log::info() << "StoreHandler::initialiseConnections" << std::endl; +} + + +void StoreHandler::handle() { + initialiseConnections(); + + + Log::info() << "Server started ..." << std::endl; + + MessageHeader hdr; + eckit::FixedString<4> tail; + + // listen loop + while (true) { + tidyWorkers(); + + socketRead(&hdr, sizeof(hdr), controlSocket_); + + ASSERT(hdr.marker == StartMarker); + ASSERT(hdr.version == CurrentVersion); + Log::debug() << "Got message with request ID: " << hdr.requestID << std::endl; + + try { + switch (hdr.message) { + case Message::Exit: + Log::status() << "Exiting" << std::endl; + Log::info() << "Exiting" << std::endl; + return; + + case Message::List: + NOTIMP; + break; + + case Message::Dump: + NOTIMP; + break; + + case Message::Purge: + NOTIMP; + break; + + case Message::Stats: + NOTIMP; + break; + + case Message::Status: + NOTIMP; + break; + + case Message::Wipe: + NOTIMP; + break; + + case Message::Control: + NOTIMP; + break; + + case Message::Inspect: + NOTIMP; + break; + + case Message::Read: + read(hdr); + break; + + case Message::Flush: + flush(hdr); + break; + + case Message::Archive: + archive(hdr); + break; + + default: { + std::stringstream ss; + ss << "ERROR: Unexpected message recieved (" << static_cast(hdr.message) + << "). ABORTING"; + Log::status() << ss.str() << std::endl; + Log::error() << "Retrieving... " << ss.str() << std::endl; + throw SeriousBug(ss.str(), Here()); + } + } + + // Ensure we have consumed exactly the correct amount from the socket. + socketRead(&tail, sizeof(tail), controlSocket_); + ASSERT(tail == EndMarker); + + // Acknowledge receipt of command + controlWrite(Message::Received, hdr.requestID); + } + catch (std::exception& e) { + // n.b. more general than eckit::Exception + std::string what(e.what()); + controlWrite(Message::Error, hdr.requestID, what.c_str(), what.length()); + } + catch (...) { + std::string what("Caught unexpected and unknown error"); + controlWrite(Message::Error, hdr.requestID, what.c_str(), what.length()); + } + } +} + +void StoreHandler::read(const MessageHeader& hdr) { + + if (!readLocationWorker_.joinable()) { + readLocationWorker_ = std::thread([this] { readLocationThreadLoop(); }); + } + + Buffer payload(receivePayload(hdr, controlSocket_)); + MemoryStream s(payload); + + std::unique_ptr location(eckit::Reanimator::reanimate(s)); + + Log::debug() << "Queuing for read: " << hdr.requestID << " " << *location << std::endl; + + std::unique_ptr dh; + dh.reset(location->dataHandle()); + + readLocationQueue_.emplace(std::make_pair(hdr.requestID, std::move(dh))); +} + +void StoreHandler::readLocationThreadLoop() { + std::pair> elem; + + while (readLocationQueue_.pop(elem) != -1) { + // Get the next MarsRequest in sequence to work on, do the retrieve, and + // send the data back to the client. + + const uint32_t requestID(elem.first); + + // Forward the API call + writeToParent(elem.first, std::move(elem.second)); + } +} + +void StoreHandler::writeToParent(const uint32_t requestID, std::unique_ptr dh) { + try { + Log::status() << "Reading: " << requestID << std::endl; + // Write the data to the parent, in chunks if necessary. + + Buffer writeBuffer(10 * 1024 * 1024); + long dataRead; + + dh->openForRead(); + Log::debug() << "Reading: " << requestID << " dh size: " << dh->size() + << std::endl; + + while ((dataRead = dh->read(writeBuffer, writeBuffer.size())) != 0) { + dataWrite(Message::Blob, requestID, writeBuffer, dataRead); + } + + // And when we are done, add a complete message. + + Log::debug() << "Writing retrieve complete message: " << requestID + << std::endl; + + dataWrite(Message::Complete, requestID); + + Log::status() << "Done retrieve: " << requestID << std::endl; + Log::debug() << "Done retrieve: " << requestID << std::endl; + } + catch (std::exception& e) { + // n.b. more general than eckit::Exception + std::string what(e.what()); + dataWrite(Message::Error, requestID, what.c_str(), what.length()); + } + catch (...) { + // We really don't want to std::terminate the thread + std::string what("Caught unexpected, unknown exception in retrieve worker"); + dataWrite(Message::Error, requestID, what.c_str(), what.length()); + } +} + +void StoreHandler::archive(const MessageHeader& hdr) { + + ASSERT(hdr.payloadSize == 0); + + // Ensure that we aren't already running a store() + + ASSERT(!archiveFuture_.valid()); + + // Start archive worker thread + + uint32_t id = hdr.requestID; + archiveFuture_ = std::async(std::launch::async, [this, id] { return archiveThreadLoop(id); }); +} + +Store& StoreHandler::store(Key dbKey) { + auto it = stores_.find(dbKey); + if (it != stores_.end()) { + return *(it->second); + } + + return *(stores_[dbKey] = StoreFactory::instance().build(dbKey, config_)); +} + +// A helper function to make archiveThreadLoop a bit cleaner +void StoreHandler::archiveBlobPayload(uint32_t id, const void* data, size_t length) { + MemoryStream s(data, length); + + fdb5::Key dbKey(s); + fdb5::Key idxKey(s); + + if (!dbKey.rule()) { + std::cout << "StoreHandler::archiveBlobPayload - missing rule in " << dbKey << std::endl; + } + if (!idxKey.rule()) { + std::cout << "StoreHandler::archiveBlobPayload - missing rule in " << idxKey << std::endl; + } + + + std::stringstream ss_key; + ss_key << dbKey << idxKey; + + const char* charData = static_cast(data); // To allow pointer arithmetic + Log::status() << "Archiving data: " << ss_key.str() << std::endl; + + std::cout << "StoreHandler::archiveBlobPayload - Archiving data: " << ss_key.str() << " - size: " << length << std::endl; + Store& ss = store(dbKey); + std::cout << "StoreHandler::archiveBlobPayload - Created store " << dbKey << std::endl; + + auto futureLocation = ss.archive(idxKey, charData + s.position(), length - s.position()); + std::cout << "StoreHandler::archiveBlobPayload - Archiving done " << std::endl; + Log::status() << "Archiving done: " << ss_key.str() << std::endl; + + auto loc = futureLocation.get(); + std::cout << "FieldLocation: " << (*loc) << std::endl; + + ss.flush(); + + // eckit::Buffer locBuffer(2048); + // MemoryStream locStream(locBuffer); + // locStream << *(futureLocation.get()); + // dataWrite(Message::Blob, id, locBuffer.data(), locStream.position()); +} + +size_t StoreHandler::archiveThreadLoop(uint32_t id) { + size_t totalArchived = 0; + + // Create a worker that will do the actual archiving + + static size_t queueSize(eckit::Resource("fdbServerMaxQueueSize", 32)); + eckit::Queue> queue(queueSize); + + std::future worker = std::async(std::launch::async, [this, &queue, id] { + size_t totalArchived = 0; + + std::pair elem = std::make_pair(Buffer{0}, false); + + try { + long queuelen; + while ((queuelen = queue.pop(elem)) != -1) { + if (elem.second) { + // Handle MultiBlob + + const char* firstData = static_cast(elem.first.data()); // For pointer arithmetic + const char* charData = firstData; + while (size_t(charData - firstData) < elem.first.size()) { + const MessageHeader* hdr = static_cast(static_cast(charData)); + ASSERT(hdr->marker == StartMarker); + ASSERT(hdr->version == CurrentVersion); + ASSERT(hdr->message == Message::Blob); + ASSERT(hdr->requestID == id); + charData += sizeof(MessageHeader); + + const void* payloadData = charData; + charData += hdr->payloadSize; + + const decltype(EndMarker)* e = static_cast(static_cast(charData)); + ASSERT(*e == EndMarker); + charData += sizeof(EndMarker); + + archiveBlobPayload(id, payloadData, hdr->payloadSize); + totalArchived += 1; + } + } + else { + // Handle single blob + archiveBlobPayload(id, elem.first.data(), elem.first.size()); + totalArchived += 1; + } + } + + dataWrite(Message::Complete, id); + } + catch (...) { + // Ensure exception propagates across the queue back to the parent thread. + queue.interrupt(std::current_exception()); + throw; + } + + return totalArchived; + }); + + try { + // The archive loop is the only thing that can listen on the data socket, + // so we don't need to to anything clever here. + + // n.b. we also don't need to lock on read. We are the only thing that reads. + + while (true) { + MessageHeader hdr; + socketRead(&hdr, sizeof(hdr), dataSocket_); + + ASSERT(hdr.marker == StartMarker); + ASSERT(hdr.version == CurrentVersion); + ASSERT(hdr.requestID == id); + + // Have we been told that we are done yet? + if (hdr.message == Message::Flush) + break; + + ASSERT(hdr.message == Message::Blob || hdr.message == Message::MultiBlob); + + Buffer payload(receivePayload(hdr, dataSocket_)); + MemoryStream s(payload); + + eckit::FixedString<4> tail; + socketRead(&tail, sizeof(tail), dataSocket_); + ASSERT(tail == EndMarker); + + // Queueing payload + + size_t sz = payload.size(); + Log::debug() << "Queueing data: " << sz << std::endl; + size_t queuelen = queue.emplace( + std::make_pair(std::move(payload), hdr.message == Message::MultiBlob)); + Log::status() << "Queued data (" << queuelen << ", size=" << sz << ")" << std::endl; + ; + Log::debug() << "Queued data (" << queuelen << ", size=" << sz << ")" + << std::endl; + ; + } + + // Trigger cleanup of the workers + queue.close(); + + // Complete reading the Flush instruction + + eckit::FixedString<4> tail; + socketRead(&tail, sizeof(tail), dataSocket_); + ASSERT(tail == EndMarker); + + // Ensure worker is done + + ASSERT(worker.valid()); + totalArchived = worker.get(); // n.b. use of async, get() propagates any exceptions. + } + catch (std::exception& e) { + // n.b. more general than eckit::Exception + std::string what(e.what()); + dataWrite(Message::Error, id, what.c_str(), what.length()); + queue.interrupt(std::current_exception()); + throw; + } + catch (...) { + std::string what("Caught unexpected, unknown exception in retrieve worker"); + dataWrite(Message::Error, id, what.c_str(), what.length()); + queue.interrupt(std::current_exception()); + throw; + } + + return totalArchived; +} + +void StoreHandler::flush(const MessageHeader& hdr) { + Buffer payload(receivePayload(hdr, controlSocket_)); + MemoryStream s(payload); + + fdb5::Key dbKey(s); + + size_t numArchived; + s >> numArchived; + + ASSERT(numArchived == 0 || archiveFuture_.valid()); + + if (archiveFuture_.valid()) { + // Ensure that the expected number of fields have been written, and that the + // archive worker processes have been cleanly wound up. + size_t n = archiveFuture_.get(); + ASSERT(numArchived == n); + + // Do the actual flush! + Log::info() << "Flushing" << std::endl; + Log::status() << "Flushing" << std::endl; + if (dbKey.empty()) { + for (auto it = stores_.begin(); it != stores_.end(); it++) { + it->second->flush(); + } + } else { + store(dbKey).flush(); + } + Log::info() << "Flush complete" << std::endl; + Log::status() << "Flush complete" << std::endl; + } +} + +} // namespace remote +} // namespace fdb5 diff --git a/src/fdb5/remote/StoreHandler.h b/src/fdb5/remote/StoreHandler.h new file mode 100644 index 000000000..bcfc20bb7 --- /dev/null +++ b/src/fdb5/remote/StoreHandler.h @@ -0,0 +1,47 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#pragma once + +#include "fdb5/remote/DecoupledHandler.h" + +namespace fdb5::remote { + +//---------------------------------------------------------------------------------------------------------------------- +class StoreHandler : public DecoupledHandler { +public: // methods + StoreHandler(eckit::net::TCPSocket& socket, const Config& config); + ~StoreHandler(); + + void handle() override; + +private: // methods + + void read(const MessageHeader& hdr) override; + + void initialiseConnections() override; + void readLocationThreadLoop(); + void writeToParent(const uint32_t requestID, std::unique_ptr dh); + + void archive(const MessageHeader& hdr) override; + size_t archiveThreadLoop(uint32_t id); + void archiveBlobPayload(uint32_t id, const void* data, size_t length); + + void flush(const MessageHeader& hdr) override; + + Store& store(Key dbKey); + +private: // members + std::map> stores_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5::remote diff --git a/src/fdb5/rules/Schema.cc b/src/fdb5/rules/Schema.cc index c70896ba3..16ac2bf92 100644 --- a/src/fdb5/rules/Schema.cc +++ b/src/fdb5/rules/Schema.cc @@ -148,9 +148,11 @@ void Schema::load(const eckit::PathName &path, bool replace) { eckit::Log::debug() << "Loading FDB rules from " << path << std::endl; std::ifstream in(path.localPath()); - if (!in) - throw eckit::CantOpenFile(path); - + if (!in) { + auto ex = eckit::CantOpenFile(path); + ex.dumpStackTrace(); + throw ex; + } load(in, replace); } diff --git a/src/fdb5/toc/AdoptVisitor.cc b/src/fdb5/toc/AdoptVisitor.cc index a5c26d8e1..3aff5340a 100644 --- a/src/fdb5/toc/AdoptVisitor.cc +++ b/src/fdb5/toc/AdoptVisitor.cc @@ -13,6 +13,7 @@ #include "fdb5/database/DB.h" #include "fdb5/toc/AdoptVisitor.h" +#include "fdb5/toc/TocEngine.h" using namespace eckit; @@ -34,11 +35,17 @@ bool AdoptVisitor::selectDatum(const Key &key, const Key &full) { // Log::info() << "selectDatum " << key << ", " << full << " " << length_ << std::endl; checkMissingKeys(full); - ASSERT(current()); + Catalogue* catalogue = current(); + ASSERT(catalogue); - current()->index(key, path_, offset_, length_); + if (catalogue->type() == TocEngine::typeName()) { + CatalogueWriter* cat = dynamic_cast(catalogue); + ASSERT(cat); - return true; + cat->index(key, eckit::URI("file", path_), offset_, length_); + return true; + } + return false; } void AdoptVisitor::print(std::ostream &out) const { diff --git a/src/fdb5/toc/RootManager.cc b/src/fdb5/toc/RootManager.cc index 5e3db718d..ae2408b0e 100644 --- a/src/fdb5/toc/RootManager.cc +++ b/src/fdb5/toc/RootManager.cc @@ -575,6 +575,11 @@ std::string RootManager::dbPathName(const Key& key) } } + if (!key.rule()) { + std::cout << "RootManager::dbPathName - missing rule in " << key << std::endl; + } + + // default naming convention for DB's dbpath = key.valuesToString(); eckit::Log::debug() << "Using default naming convention for key " << key << " -> " << dbpath << std::endl; diff --git a/src/fdb5/toc/TocCommon.cc b/src/fdb5/toc/TocCommon.cc index 5795dcfab..e913bade4 100644 --- a/src/fdb5/toc/TocCommon.cc +++ b/src/fdb5/toc/TocCommon.cc @@ -34,7 +34,6 @@ eckit::PathName TocCommon::findRealPath(const eckit::PathName& path) { TocCommon::TocCommon(const eckit::PathName& directory) : directory_(findRealPath(directory)), - schemaPath_(directory_ / "schema"), dbUID_(static_cast(-1)), userUID_(::getuid()), dirty_(false) {} diff --git a/src/fdb5/toc/TocCommon.h b/src/fdb5/toc/TocCommon.h index 52251a082..d6caf21b4 100644 --- a/src/fdb5/toc/TocCommon.h +++ b/src/fdb5/toc/TocCommon.h @@ -47,7 +47,6 @@ class TocCommon { protected: // members const eckit::PathName directory_; - const eckit::PathName schemaPath_; mutable uid_t dbUID_; uid_t userUID_; diff --git a/src/fdb5/toc/TocHandler.cc b/src/fdb5/toc/TocHandler.cc index 7ea74e371..55676791e 100644 --- a/src/fdb5/toc/TocHandler.cc +++ b/src/fdb5/toc/TocHandler.cc @@ -120,6 +120,7 @@ class CachedFDProxy { TocHandler::TocHandler(const eckit::PathName& directory, const Config& config) : TocCommon(directory), + schemaPath_(directory_ / "schema"), tocPath_(directory_ / "toc"), dbConfig_(config), serialisationVersion_(TocSerialisationVersion(config)), @@ -144,6 +145,7 @@ TocHandler::TocHandler(const eckit::PathName& directory, const Config& config) : TocHandler::TocHandler(const eckit::PathName& path, const Key& parentKey) : TocCommon(path.dirName()), parentKey_(parentKey), + schemaPath_(directory_ / "schema"), tocPath_(TocCommon::findRealPath(path)), serialisationVersion_(TocSerialisationVersion(dbConfig_)), useSubToc_(false), diff --git a/src/fdb5/toc/TocHandler.h b/src/fdb5/toc/TocHandler.h index f03792985..222306c09 100644 --- a/src/fdb5/toc/TocHandler.h +++ b/src/fdb5/toc/TocHandler.h @@ -173,6 +173,7 @@ class TocHandler : public TocCommon, private eckit::NonCopyable { protected: // members mutable Key parentKey_; // Contains the key of the first TOC explored in subtoc chain + const eckit::PathName schemaPath_; uid_t dbUID() const override; diff --git a/src/fdb5/toc/TocStore.cc b/src/fdb5/toc/TocStore.cc index d4ef8abc9..8832cf267 100644 --- a/src/fdb5/toc/TocStore.cc +++ b/src/fdb5/toc/TocStore.cc @@ -34,11 +34,11 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -TocStore::TocStore(const Schema& schema, const Key& key, const Config& config) : - Store(schema), TocCommon(StoreRootManager(config).directory(key).directory_) {} +TocStore::TocStore(const Key& key, const Config& config) : + Store(), TocCommon(StoreRootManager(config).directory(key).directory_) {} -TocStore::TocStore(const Schema& schema, const eckit::URI& uri, const Config& config) : - Store(schema), TocCommon(uri.path().dirName()) {} +TocStore::TocStore(const eckit::URI& uri, const Config& config) : + Store(), TocCommon(uri.path().dirName()) {} eckit::URI TocStore::uri() const { return URI("file", directory_); @@ -52,7 +52,7 @@ eckit::DataHandle* TocStore::retrieve(Field& field) const { return field.dataHandle(); } -std::unique_ptr TocStore::archive(const Key &key, const void *data, eckit::Length length) { +std::future > TocStore::archive(const Key& key, const void *data, eckit::Length length) { dirty_ = true; eckit::PathName dataPath = getDataPath(key); @@ -65,7 +65,9 @@ std::unique_ptr TocStore::archive(const Key &key, const void *dat ASSERT(len == length); - return std::unique_ptr(new TocFieldLocation(dataPath, position, length, Key())); + std::promise > loc; + loc.set_value(std::unique_ptr(new TocFieldLocation(dataPath, position, length, Key()))); + return loc.get_future(); } void TocStore::flush() { @@ -181,7 +183,10 @@ eckit::DataHandle& TocStore::getDataHandle( const eckit::PathName &path ) { eckit::PathName TocStore::generateDataPath(const Key &key) const { eckit::PathName dpath ( directory_ ); - dpath /= key.valuesToString(); + if (!key.rule()) { + std::cout << "TocStore::generateDataPath - missing rule in " << key << std::endl; + } + dpath /= key.valuesToString(false); dpath = eckit::PathName::unique(dpath) + ".data"; return dpath; } @@ -260,7 +265,11 @@ void TocStore::moveTo(const Key& key, const Config& config, const eckit::URI& de void TocStore::remove(const Key& key) const { - eckit::PathName src_db = directory_ / key.valuesToString(); + if (!key.rule()) { + std::cout << "TocStore::remove - missing rule in " << key << std::endl; + } + + eckit::PathName src_db = directory_ / key.valuesToString(false); DIR* dirp = ::opendir(src_db.asString().c_str()); struct dirent* dp; diff --git a/src/fdb5/toc/TocStore.h b/src/fdb5/toc/TocStore.h index 10500748b..3fc2c1b5f 100644 --- a/src/fdb5/toc/TocStore.h +++ b/src/fdb5/toc/TocStore.h @@ -33,8 +33,8 @@ class TocStore : public Store, public TocCommon { public: // methods - TocStore(const Schema& schema, const Key& key, const Config& config); - TocStore(const Schema& schema, const eckit::URI& uri, const Config& config); + TocStore(const Key& key, const Config& config); + TocStore(const eckit::URI& uri, const Config& config); ~TocStore() override {} @@ -57,7 +57,7 @@ class TocStore : public Store, public TocCommon { bool exists() const override; eckit::DataHandle* retrieve(Field& field) const override; - std::unique_ptr archive(const Key &key, const void *data, eckit::Length length) override; + std::future > archive(const Key& key, const void *data, eckit::Length length) override; void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override; From 8fc61a7cdb34b757ce73c154e5b56c35fe8b3498 Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Fri, 6 Oct 2023 14:33:45 +0100 Subject: [PATCH 002/186] Schema streamable --- src/fdb5/database/Archiver.cc | 5 +- src/fdb5/remote/Handler.cc | 6 ++- src/fdb5/remote/Handler.h | 5 +- src/fdb5/remote/Messages.h | 1 + src/fdb5/remote/StoreHandler.cc | 2 +- src/fdb5/remote/StoreHandler.h | 2 +- src/fdb5/rules/MatchAlways.cc | 12 +++++ src/fdb5/rules/MatchAlways.h | 16 +++++-- src/fdb5/rules/MatchAny.cc | 25 ++++++++++ src/fdb5/rules/MatchAny.h | 16 +++++-- src/fdb5/rules/MatchHidden.cc | 25 ++++++++++ src/fdb5/rules/MatchHidden.h | 12 ++++- src/fdb5/rules/MatchOptional.cc | 25 ++++++++++ src/fdb5/rules/MatchOptional.h | 12 ++++- src/fdb5/rules/MatchValue.cc | 14 ++++++ src/fdb5/rules/MatchValue.h | 11 ++++- src/fdb5/rules/Matcher.cc | 11 +++++ src/fdb5/rules/Matcher.h | 16 ++++++- src/fdb5/rules/Predicate.cc | 14 ++++++ src/fdb5/rules/Predicate.h | 14 +++++- src/fdb5/rules/Rule.cc | 51 ++++++++++++++++++++ src/fdb5/rules/Rule.h | 18 +++++-- src/fdb5/rules/Schema.cc | 84 +++++++++++++++++++++------------ src/fdb5/rules/Schema.h | 13 ++++- src/fdb5/types/TypesRegistry.cc | 31 +++++++++++- src/fdb5/types/TypesRegistry.h | 12 ++++- tests/fdb/test_fdb5_service.cc | 39 +++++++++++++++ 27 files changed, 433 insertions(+), 59 deletions(-) diff --git a/src/fdb5/database/Archiver.cc b/src/fdb5/database/Archiver.cc index 702920fce..9538f202f 100644 --- a/src/fdb5/database/Archiver.cc +++ b/src/fdb5/database/Archiver.cc @@ -47,11 +47,14 @@ void Archiver::archive(const Key &key, BaseArchiveVisitor& visitor) { dbConfig_.schema().expand(key, visitor); - if (visitor.rule() == nullptr) { // Make sure we did find a rule that matched + const Rule* rule = visitor.rule(); + if (rule == nullptr) { // Make sure we did find a rule that matched std::ostringstream oss; oss << "FDB: Could not find a rule to archive " << key; throw eckit::SeriousBug(oss.str()); } + // validate metadata + rule->check(key); } void Archiver::flush() { diff --git a/src/fdb5/remote/Handler.cc b/src/fdb5/remote/Handler.cc index d78364080..a9983c7e3 100644 --- a/src/fdb5/remote/Handler.cc +++ b/src/fdb5/remote/Handler.cc @@ -419,6 +419,10 @@ void RemoteHandler::handle() { archive(hdr); break; + // case Message::Store: + // store(hdr); + // break; + default: { std::stringstream ss; ss << "ERROR: Unexpected message recieved (" << static_cast(hdr.message) @@ -607,7 +611,7 @@ void RemoteHandler::forwardApiCall(const MessageHeader& hdr) { void RemoteHandler::archive(const MessageHeader& hdr) { ASSERT(hdr.payloadSize == 0); - // Ensure that we aren't already running an archive() + // Ensure that we aren't already running an store() ASSERT(!archiveFuture_.valid()); diff --git a/src/fdb5/remote/Handler.h b/src/fdb5/remote/Handler.h index 991d521d5..01b819293 100644 --- a/src/fdb5/remote/Handler.h +++ b/src/fdb5/remote/Handler.h @@ -86,10 +86,11 @@ class RemoteHandler : private eckit::NonCopyable { template void forwardApiCall(const MessageHeader& hdr); - void list(const MessageHeader& hdr); - void dump(const MessageHeader& hdr); +// void list(const MessageHeader& hdr); +// void dump(const MessageHeader& hdr); void flush(const MessageHeader& hdr); +// void store(const MessageHeader& hdr); void archive(const MessageHeader& hdr); void retrieve(const MessageHeader& hdr); void read(const MessageHeader& hdr); diff --git a/src/fdb5/remote/Messages.h b/src/fdb5/remote/Messages.h index eb89e3cc0..b7f927c15 100644 --- a/src/fdb5/remote/Messages.h +++ b/src/fdb5/remote/Messages.h @@ -62,6 +62,7 @@ enum class Message : uint16_t { Inspect, Read, Move, + Store, // Responses Received = 200, diff --git a/src/fdb5/remote/StoreHandler.cc b/src/fdb5/remote/StoreHandler.cc index 4be48e13a..8fec55dbe 100644 --- a/src/fdb5/remote/StoreHandler.cc +++ b/src/fdb5/remote/StoreHandler.cc @@ -202,7 +202,7 @@ void StoreHandler::writeToParent(const uint32_t requestID, std::unique_ptr dh); - void archive(const MessageHeader& hdr) override; + void store(const MessageHeader& hdr) override; size_t archiveThreadLoop(uint32_t id); void archiveBlobPayload(uint32_t id, const void* data, size_t length); diff --git a/src/fdb5/rules/MatchAlways.cc b/src/fdb5/rules/MatchAlways.cc index 333cf6965..af0554a49 100644 --- a/src/fdb5/rules/MatchAlways.cc +++ b/src/fdb5/rules/MatchAlways.cc @@ -17,10 +17,22 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- +eckit::ClassSpec MatchAlways::classSpec_ = { &Matcher::classSpec(), "MatchAlways", }; + +eckit::Reanimator MatchAlways::reanimator_; + + MatchAlways::MatchAlways() : Matcher() { } +MatchAlways::MatchAlways(eckit::Stream&) : + Matcher() { +} + +void MatchAlways::encode(eckit::Stream& s) const { +} + MatchAlways::~MatchAlways() { } diff --git a/src/fdb5/rules/MatchAlways.h b/src/fdb5/rules/MatchAlways.h index 23cb35f06..3148e00b1 100644 --- a/src/fdb5/rules/MatchAlways.h +++ b/src/fdb5/rules/MatchAlways.h @@ -13,8 +13,7 @@ /// @author Tiago Quintino /// @date Mar 2016 -#ifndef fdb5_MatchAlways_H -#define fdb5_MatchAlways_H +#pragma once #include @@ -29,6 +28,7 @@ class MatchAlways : public Matcher { public: // methods MatchAlways(); + MatchAlways(eckit::Stream& s); virtual ~MatchAlways() override; @@ -36,14 +36,22 @@ class MatchAlways : public Matcher { virtual void dump(std::ostream &s, const std::string &keyword, const TypesRegistry ®istry) const override; + const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } + static const eckit::ClassSpec& classSpec() { return classSpec_; } + private: // methods + void encode(eckit::Stream&) const override; + virtual void print( std::ostream &out ) const override; +private: // members + + static eckit::ClassSpec classSpec_; + static eckit::Reanimator reanimator_; + }; //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 - -#endif diff --git a/src/fdb5/rules/MatchAny.cc b/src/fdb5/rules/MatchAny.cc index f84cf84f6..394cf5d21 100644 --- a/src/fdb5/rules/MatchAny.cc +++ b/src/fdb5/rules/MatchAny.cc @@ -16,11 +16,36 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- +eckit::ClassSpec MatchAny::classSpec_ = { &Matcher::classSpec(), "MatchAny", }; + +eckit::Reanimator MatchAny::reanimator_; + + MatchAny::MatchAny(const std::set &values) : Matcher(), values_(values) { } +MatchAny::MatchAny(eckit::Stream& s) : + Matcher() { + + size_t numValues; + std::string value; + + s >> numValues; + for (size_t i=0; i < numValues; i++) { + s >> value; + values_.insert(value); + } +} + +void MatchAny::encode(eckit::Stream& s) const { + s << values_.size(); + for (const std::string& value : values_) { + s << value; + } +} + MatchAny::~MatchAny() { } diff --git a/src/fdb5/rules/MatchAny.h b/src/fdb5/rules/MatchAny.h index 74d0a5a41..ab3fe68be 100644 --- a/src/fdb5/rules/MatchAny.h +++ b/src/fdb5/rules/MatchAny.h @@ -13,8 +13,7 @@ /// @author Tiago Quintino /// @date Mar 2016 -#ifndef fdb5_MatchAny_H -#define fdb5_MatchAny_H +#pragma once #include #include @@ -25,11 +24,12 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -class MatchAny : public Matcher { +class MatchAny : public Matcher{ public: // methods MatchAny(const std::set &values); + MatchAny(eckit::Stream& s); virtual ~MatchAny() override; @@ -37,12 +37,20 @@ class MatchAny : public Matcher { virtual void dump(std::ostream &s, const std::string &keyword, const TypesRegistry ®istry) const override; + const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } + static const eckit::ClassSpec& classSpec() { return classSpec_; } + private: // methods + void encode(eckit::Stream&) const override; + virtual void print( std::ostream &out ) const override; private: // members + static eckit::ClassSpec classSpec_; + static eckit::Reanimator reanimator_; + std::set values_; }; @@ -50,5 +58,3 @@ class MatchAny : public Matcher { //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 - -#endif diff --git a/src/fdb5/rules/MatchHidden.cc b/src/fdb5/rules/MatchHidden.cc index 5cd6990be..8dc5fb8d6 100644 --- a/src/fdb5/rules/MatchHidden.cc +++ b/src/fdb5/rules/MatchHidden.cc @@ -21,11 +21,36 @@ static std::string empty; //---------------------------------------------------------------------------------------------------------------------- +eckit::ClassSpec MatchHidden::classSpec_ = { &Matcher::classSpec(), "MatchHidden", }; + +eckit::Reanimator MatchHidden::reanimator_; + + MatchHidden::MatchHidden(const std::string &def) : Matcher() { default_.push_back(def); } +MatchHidden::MatchHidden(eckit::Stream& s) : + Matcher() { + + size_t numValues; + std::string value; + + s >> numValues; + for (size_t i=0; i < numValues; i++) { + s >> value; + default_.push_back(value); + } +} + +void MatchHidden::encode(eckit::Stream& s) const { + s << default_.size(); + for (const std::string& value : default_) { + s << value; + } +} + MatchHidden::~MatchHidden() { } diff --git a/src/fdb5/rules/MatchHidden.h b/src/fdb5/rules/MatchHidden.h index d40217ee4..41003e990 100644 --- a/src/fdb5/rules/MatchHidden.h +++ b/src/fdb5/rules/MatchHidden.h @@ -25,11 +25,12 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -class MatchHidden : public Matcher { +class MatchHidden : public Matcher{ public: // methods MatchHidden(const std::string &def); + MatchHidden(eckit::Stream& s); virtual ~MatchHidden() override; @@ -37,14 +38,23 @@ class MatchHidden : public Matcher { virtual void dump(std::ostream &s, const std::string &keyword, const TypesRegistry ®istry) const override; + const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } + static const eckit::ClassSpec& classSpec() { return classSpec_; } + private: // methods + void encode(eckit::Stream&) const override; + virtual bool optional() const override; virtual const std::string &value(const Key &, const std::string &keyword) const override; virtual const std::vector& values(const metkit::mars::MarsRequest& rq, const std::string& keyword) const override; virtual void print( std::ostream &out ) const override; virtual const std::string &defaultValue() const override; +private: // members + + static eckit::ClassSpec classSpec_; + static eckit::Reanimator reanimator_; std::vector default_; diff --git a/src/fdb5/rules/MatchOptional.cc b/src/fdb5/rules/MatchOptional.cc index 12f795b22..9e4eb4c70 100644 --- a/src/fdb5/rules/MatchOptional.cc +++ b/src/fdb5/rules/MatchOptional.cc @@ -24,11 +24,36 @@ static std::string empty; //---------------------------------------------------------------------------------------------------------------------- +eckit::ClassSpec MatchOptional::classSpec_ = { &Matcher::classSpec(), "MatchOptional", }; + +eckit::Reanimator MatchOptional::reanimator_; + + MatchOptional::MatchOptional(const std::string &def) : Matcher() { default_.push_back(def); } +MatchOptional::MatchOptional(eckit::Stream& s) : + Matcher() { + + size_t numValues; + std::string value; + + s >> numValues; + for (size_t i=0; i < numValues; i++) { + s >> value; + default_.push_back(value); + } +} + +void MatchOptional::encode(eckit::Stream& s) const { + s << default_.size(); + for (const std::string& value : default_) { + s << value; + } +} + MatchOptional::~MatchOptional() { } diff --git a/src/fdb5/rules/MatchOptional.h b/src/fdb5/rules/MatchOptional.h index 8b76fec88..3fd8066cb 100644 --- a/src/fdb5/rules/MatchOptional.h +++ b/src/fdb5/rules/MatchOptional.h @@ -25,11 +25,12 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -class MatchOptional : public Matcher { +class MatchOptional : public Matcher{ public: // methods MatchOptional(const std::string &def); + MatchOptional(eckit::Stream& s); virtual ~MatchOptional() override; @@ -37,8 +38,13 @@ class MatchOptional : public Matcher { virtual void dump(std::ostream &s, const std::string &keyword, const TypesRegistry ®istry) const override; + const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } + static const eckit::ClassSpec& classSpec() { return classSpec_; } + private: // methods + void encode(eckit::Stream&) const override; + virtual bool optional() const override; virtual const std::string &value(const Key &, const std::string &keyword) const override; virtual const std::vector& values(const metkit::mars::MarsRequest& rq, const std::string& keyword) const override; @@ -46,6 +52,10 @@ class MatchOptional : public Matcher { virtual const std::string &defaultValue() const override; virtual void fill(Key &key, const std::string &keyword, const std::string& value) const override; +private: // members + + static eckit::ClassSpec classSpec_; + static eckit::Reanimator reanimator_; std::vector default_; diff --git a/src/fdb5/rules/MatchValue.cc b/src/fdb5/rules/MatchValue.cc index 3c1d24d3b..6073dc219 100644 --- a/src/fdb5/rules/MatchValue.cc +++ b/src/fdb5/rules/MatchValue.cc @@ -16,11 +16,25 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- +eckit::ClassSpec MatchValue::classSpec_ = { &Matcher::classSpec(), "MatchValue", }; + +eckit::Reanimator MatchValue::reanimator_; + + MatchValue::MatchValue(const std::string &value) : Matcher(), value_(value) { } +MatchValue::MatchValue(eckit::Stream& s) : + Matcher() { + + s >> value_; +} + +void MatchValue::encode(eckit::Stream& s) const { + s << value_; +} MatchValue::~MatchValue() { } diff --git a/src/fdb5/rules/MatchValue.h b/src/fdb5/rules/MatchValue.h index 037346653..4ee25ca85 100644 --- a/src/fdb5/rules/MatchValue.h +++ b/src/fdb5/rules/MatchValue.h @@ -25,11 +25,12 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -class MatchValue : public Matcher { +class MatchValue : public Matcher{ public: // methods MatchValue(const std::string &value); + MatchValue(eckit::Stream& s); virtual ~MatchValue() override; @@ -37,12 +38,20 @@ class MatchValue : public Matcher { virtual void dump(std::ostream &s, const std::string &keyword, const TypesRegistry ®istry) const override; + const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } + static const eckit::ClassSpec& classSpec() { return classSpec_; } + private: // methods + void encode(eckit::Stream&) const override; + virtual void print( std::ostream &out ) const override; private: // members + static eckit::ClassSpec classSpec_; + static eckit::Reanimator reanimator_; + std::string value_; }; diff --git a/src/fdb5/rules/Matcher.cc b/src/fdb5/rules/Matcher.cc index 697d78610..ad2d2b526 100644 --- a/src/fdb5/rules/Matcher.cc +++ b/src/fdb5/rules/Matcher.cc @@ -19,9 +19,20 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- +eckit::ClassSpec Matcher::classSpec_ = { &eckit::Streamable::classSpec(), "Matcher", }; + +//eckit::Reanimator Matcher::reanimator_; + + Matcher::Matcher() { } +Matcher::Matcher(eckit::Stream&) { +} + +void Matcher::encode(eckit::Stream& s) const { +} + Matcher::~Matcher() { } diff --git a/src/fdb5/rules/Matcher.h b/src/fdb5/rules/Matcher.h index a15291186..33937a637 100644 --- a/src/fdb5/rules/Matcher.h +++ b/src/fdb5/rules/Matcher.h @@ -19,7 +19,8 @@ #include #include -#include "eckit/memory/NonCopyable.h" +//#include "eckit/memory/NonCopyable.h" +#include "eckit/serialisation/Streamable.h" class MarsTask; namespace metkit { @@ -35,11 +36,12 @@ class TypesRegistry; //---------------------------------------------------------------------------------------------------------------------- -class Matcher : public eckit::NonCopyable { +class Matcher : public eckit::Streamable { public: // methods Matcher(); + Matcher(eckit::Stream& s); virtual ~Matcher(); @@ -57,10 +59,20 @@ class Matcher : public eckit::NonCopyable { friend std::ostream &operator<<(std::ostream &s, const Matcher &x); +// const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } + static const eckit::ClassSpec& classSpec() { return classSpec_; } + private: // methods + void encode(eckit::Stream&) const override; + virtual void print( std::ostream &out ) const = 0; +private: // members + + static eckit::ClassSpec classSpec_; + static eckit::Reanimator reanimator_; + }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/rules/Predicate.cc b/src/fdb5/rules/Predicate.cc index e002b71c8..5f932d880 100644 --- a/src/fdb5/rules/Predicate.cc +++ b/src/fdb5/rules/Predicate.cc @@ -17,6 +17,10 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- +eckit::ClassSpec Predicate::classSpec_ = { &eckit::Streamable::classSpec(), "Predicate", }; + +eckit::Reanimator Predicate::reanimator_; + Predicate::Predicate(const std::string &keyword, Matcher *matcher) : matcher_(matcher), keyword_(keyword) { @@ -24,6 +28,16 @@ Predicate::Predicate(const std::string &keyword, Matcher *matcher) : // eckit::Log::debug() << std::endl; } +Predicate::Predicate(eckit::Stream& s) { + s >> keyword_; + matcher_.reset(eckit::Reanimator::reanimate(s)); +} + +void Predicate::encode(eckit::Stream& s) const { + s << keyword_; + s << *matcher_; +} + Predicate::~Predicate() { } diff --git a/src/fdb5/rules/Predicate.h b/src/fdb5/rules/Predicate.h index 6c7a390bc..04461b898 100644 --- a/src/fdb5/rules/Predicate.h +++ b/src/fdb5/rules/Predicate.h @@ -20,7 +20,8 @@ #include #include -#include "eckit/memory/NonCopyable.h" +#include "eckit/serialisation/Streamable.h" +#include "eckit/serialisation/Reanimator.h" namespace metkit { class MarsRequest; } @@ -32,11 +33,12 @@ class TypesRegistry; //---------------------------------------------------------------------------------------------------------------------- -class Predicate : public eckit::NonCopyable { +class Predicate : public eckit::Streamable { public: // methods Predicate(const std::string &keyword, Matcher *matcher); + Predicate(eckit::Stream& s); ~Predicate(); @@ -53,14 +55,22 @@ class Predicate : public eckit::NonCopyable { std::string keyword() const; + const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } + static const eckit::ClassSpec& classSpec() { return classSpec_; } + private: // methods friend std::ostream &operator<<(std::ostream &s, const Predicate &x); + void encode(eckit::Stream& s) const override; + void print( std::ostream &out ) const; private: // members + static eckit::ClassSpec classSpec_; + static eckit::Reanimator reanimator_; + std::unique_ptr matcher_; std::string keyword_; diff --git a/src/fdb5/rules/Rule.cc b/src/fdb5/rules/Rule.cc index 1d627c8fb..8a1f9aac1 100644 --- a/src/fdb5/rules/Rule.cc +++ b/src/fdb5/rules/Rule.cc @@ -20,12 +20,17 @@ #include "fdb5/rules/Schema.h" #include "fdb5/database/ReadVisitor.h" #include "fdb5/database/WriteVisitor.h" +#include "fdb5/types/Type.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- +eckit::ClassSpec Rule::classSpec_ = { &eckit::Streamable::classSpec(), "Rule", }; + +eckit::Reanimator Rule::reanimator_; + Rule::Rule(const Schema &schema, size_t line, std::vector &predicates, std::vector &rules, @@ -39,6 +44,41 @@ Rule::Rule(const Schema &schema, } } +Rule::Rule(eckit::Stream& s): + Rule(Schema(), s) { +} + +Rule::Rule(const Schema &schema, eckit::Stream& s): + schema_(schema), registry_(s) { + + size_t numPredicates; + size_t numRules; + + s >> line_; + s >> numPredicates; + for (size_t i=0; i < numPredicates; i++) { + predicates_.push_back(eckit::Reanimator::reanimate(s)); + } + + s >> numRules; + for (size_t i=0; i < numRules; i++) { + rules_.push_back(new Rule(schema, s)); + } +} + +void Rule::encode(eckit::Stream& s) const { + registry_.encode(s); + s << line_; + s << predicates_.size(); + for (const Predicate* predicate : predicates_) { + s << *predicate; + } + s << rules_.size(); + for (const Rule* rule : rules_) { + rule->encode(s); + } +} + Rule::~Rule() { for (std::vector::iterator i = predicates_.begin(); i != predicates_.end(); ++i ) { delete *i; @@ -494,6 +534,17 @@ const Schema &Rule::schema() const { return schema_; } +void Rule::check(const Key& key) const { + for (const auto& pred : predicates_ ) { + auto k = key.find(pred->keyword()); + if (k != key.end()) { + const std::string& value = (*k).second; + const Type& type = registry_.lookupType(pred->keyword()); + ASSERT(value == type.tidy(pred->keyword(), value)); + } + } +} + std::ostream &operator<<(std::ostream &s, const Rule &x) { x.print(s); return s; diff --git a/src/fdb5/rules/Rule.h b/src/fdb5/rules/Rule.h index ab30b8c22..5986fbece 100644 --- a/src/fdb5/rules/Rule.h +++ b/src/fdb5/rules/Rule.h @@ -19,9 +19,10 @@ #include #include -#include "eckit/memory/NonCopyable.h" +#include "eckit/serialisation/Streamable.h" #include "eckit/types/Types.h" #include "fdb5/types/TypesRegistry.h" +#include "eckit/serialisation/Reanimator.h" namespace metkit { namespace mars { @@ -39,7 +40,7 @@ class Key; //---------------------------------------------------------------------------------------------------------------------- -class Rule : public eckit::NonCopyable { +class Rule : public eckit::Streamable { public: // methods @@ -50,6 +51,8 @@ class Rule : public eckit::NonCopyable { std::vector &rules, const std::map &types ); + Rule(eckit::Stream& s); + Rule(const Schema &schema, eckit::Stream& s); ~Rule(); @@ -83,6 +86,11 @@ class Rule : public eckit::NonCopyable { const Schema &schema() const; const TypesRegistry ®istry() const; + const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } + static const eckit::ClassSpec& classSpec() { return classSpec_; } + + void check(const Key& key) const; + private: // methods void expand(const metkit::mars::MarsRequest &request, @@ -114,11 +122,15 @@ class Rule : public eckit::NonCopyable { friend std::ostream &operator<<(std::ostream &s, const Rule &x); - void print( std::ostream &out ) const; + void encode(eckit::Stream& s) const override; + void print( std::ostream &out ) const; private: // members + static eckit::ClassSpec classSpec_; + static eckit::Reanimator reanimator_; + const Schema& schema_; const Rule* parent_; diff --git a/src/fdb5/rules/Schema.cc b/src/fdb5/rules/Schema.cc index 16ac2bf92..f66d073d2 100644 --- a/src/fdb5/rules/Schema.cc +++ b/src/fdb5/rules/Schema.cc @@ -21,6 +21,10 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- +eckit::ClassSpec Schema::classSpec_ = { &eckit::Streamable::classSpec(), "Schema", }; + +eckit::Reanimator Schema::reanimator_; + Schema::Schema() { } @@ -31,6 +35,27 @@ Schema::Schema(const eckit::PathName &path) { Schema::Schema(std::istream& s) { load(s); } +Schema::Schema(eckit::Stream& s) : + registry_(s) { + + size_t numRules; + s >> path_; + s >> numRules; + for (size_t i=0; i < numRules; i++) { + rules_.push_back(new Rule(*this, s)); + } + + check(); +} + +void Schema::encode(eckit::Stream& s) const { + registry_.encode(s); + s << path_; + s << rules_.size(); + for (const Rule* rule : rules_) { + rule->encode(s); + } +} Schema::~Schema() { clear(); @@ -41,8 +66,8 @@ const Rule* Schema::ruleFor(const Key& dbKey, const Key& idxKey) const { keys.push_back(dbKey); keys.push_back(idxKey); - for (std::vector::const_iterator i = rules_.begin(); i != rules_.end(); ++i ) { - const Rule* r = (*i)->ruleFor(keys , 0); + for (const Rule* rule : rules_) { + const Rule* r = rule->ruleFor(keys , 0); if (r) { return r; } @@ -54,10 +79,10 @@ void Schema::expand(const metkit::mars::MarsRequest &request, ReadVisitor &visit Key full; std::vector keys(3); - for (std::vector::const_iterator i = rules_.begin(); i != rules_.end(); ++i ) { + for (const Rule* rule : rules_) { // eckit::Log::info() << "Rule " << **i << std::endl; - // (*i)->dump(eckit::Log::info()); - (*i)->expand(request, visitor, 0, keys, full); + // rule->dump(eckit::Log::info()); + rule->expand(request, visitor, 0, keys, full); } } @@ -67,17 +92,17 @@ void Schema::expand(const Key &field, WriteVisitor &visitor) const { visitor.rule(0); // reset to no rule so we verify that we pick at least one - for (std::vector::const_iterator i = rules_.begin(); i != rules_.end(); ++i ) { - (*i)->expand(field, visitor, 0, keys, full); + for (const Rule* rule : rules_) { + rule->expand(field, visitor, 0, keys, full); } } void Schema::expandSecond(const metkit::mars::MarsRequest& request, ReadVisitor& visitor, const Key& dbKey) const { const Rule* dbRule = nullptr; - for (const Rule* r : rules_) { - if (r->match(dbKey)) { - dbRule = r; + for (const Rule* rule : rules_) { + if (rule->match(dbKey)) { + dbRule = rule; break; } } @@ -87,17 +112,17 @@ void Schema::expandSecond(const metkit::mars::MarsRequest& request, ReadVisitor& std::vector keys(3); keys[0] = dbKey; - for (std::vector:: const_iterator i = dbRule->rules_.begin(); i != dbRule->rules_.end(); ++i) { - (*i)->expand(request, visitor, 1, keys, full); + for (const Rule* rule : dbRule->rules_) { + rule->expand(request, visitor, 1, keys, full); } } void Schema::expandSecond(const Key& field, WriteVisitor& visitor, const Key& dbKey) const { const Rule* dbRule = nullptr; - for (const Rule* r : rules_) { - if (r->match(dbKey)) { - dbRule = r; + for (const Rule* rule : rules_) { + if (rule->match(dbKey)) { + dbRule = rule; break; } } @@ -107,15 +132,16 @@ void Schema::expandSecond(const Key& field, WriteVisitor& visitor, const Key& db std::vector keys(3); keys[0] = dbKey; - for (std::vector:: const_iterator i = dbRule->rules_.begin(); i != dbRule->rules_.end(); ++i) { - (*i)->expand(field, visitor, 1, keys, full); + for (const Rule* rule : dbRule->rules_) { + rule->expand(field, visitor, 1, keys, full); } } bool Schema::expandFirstLevel(const Key &dbKey, Key &result) const { bool found = false; - for (std::vector::const_iterator i = rules_.begin(); i != rules_.end() && !found; ++i ) { - (*i)->expandFirstLevel(dbKey, result, found); + for (const Rule* rule : rules_) { + rule->expandFirstLevel(dbKey, result, found); + if (found) break; } return found; } @@ -130,14 +156,14 @@ bool Schema::expandFirstLevel(const metkit::mars::MarsRequest& request, Key &res } void Schema::matchFirstLevel(const Key &dbKey, std::set &result, const char* missing) const { - for (std::vector::const_iterator i = rules_.begin(); i != rules_.end(); ++i ) { - (*i)->matchFirstLevel(dbKey, result, missing); + for (const Rule* rule : rules_) { + rule->matchFirstLevel(dbKey, result, missing); } } void Schema::matchFirstLevel(const metkit::mars::MarsRequest& request, std::set& result, const char* missing) const { - for (std::vector::const_iterator i = rules_.begin(); i != rules_.end(); ++i ) { - (*i)->matchFirstLevel(request, result, missing); + for (const Rule* rule : rules_) { + rule->matchFirstLevel(request, result, missing); } } @@ -177,18 +203,18 @@ void Schema::clear() { void Schema::dump(std::ostream &s) const { registry_.dump(s); - for (std::vector::const_iterator i = rules_.begin(); i != rules_.end(); ++i ) { - (*i)->dump(s); + for (const Rule* rule : rules_) { + rule->dump(s); s << std::endl; } } void Schema::check() { - for (std::vector::iterator i = rules_.begin(); i != rules_.end(); ++i ) { + for (Rule* rule : rules_) { /// @todo print offending rule in meaningful message - ASSERT((*i)->depth() == 3); - (*i)->registry_.updateParent(®istry_); - (*i)->updateParent(0); + ASSERT(rule->depth() == 3); + rule->registry_.updateParent(®istry_); + rule->updateParent(nullptr); } } diff --git a/src/fdb5/rules/Schema.h b/src/fdb5/rules/Schema.h index 43d565d97..035d67118 100644 --- a/src/fdb5/rules/Schema.h +++ b/src/fdb5/rules/Schema.h @@ -22,7 +22,8 @@ #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "eckit/io/DataHandle.h" -#include "eckit/memory/NonCopyable.h" +#include "eckit/serialisation/Streamable.h" +#include "eckit/serialisation/Reanimator.h" #include "fdb5/types/TypesRegistry.h" @@ -38,13 +39,14 @@ class Schema; //---------------------------------------------------------------------------------------------------------------------- -class Schema : private eckit::NonCopyable { +class Schema : public eckit::Streamable { public: // methods Schema(); Schema(const eckit::PathName &path); Schema(std::istream& s); + Schema(eckit::Stream& s); ~Schema(); @@ -77,6 +79,8 @@ class Schema : private eckit::NonCopyable { const TypesRegistry& registry() const; + const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } + static const eckit::ClassSpec& classSpec() { return classSpec_; } private: // methods @@ -85,10 +89,15 @@ class Schema : private eckit::NonCopyable { friend std::ostream &operator<<(std::ostream &s, const Schema &x); + void encode(eckit::Stream& s) const override; + void print( std::ostream &out ) const; private: // members + static eckit::ClassSpec classSpec_; + static eckit::Reanimator reanimator_; + TypesRegistry registry_; std::vector rules_; std::string path_; diff --git a/src/fdb5/types/TypesRegistry.cc b/src/fdb5/types/TypesRegistry.cc index 7f2e9ec74..cf6b29c3d 100644 --- a/src/fdb5/types/TypesRegistry.cc +++ b/src/fdb5/types/TypesRegistry.cc @@ -18,10 +18,39 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- +eckit::ClassSpec TypesRegistry::classSpec_ = { &eckit::Streamable::classSpec(), "TypesRegistry", }; + +eckit::Reanimator TypesRegistry::reanimator_; + TypesRegistry::TypesRegistry(): - parent_(0) { + parent_(nullptr) { +} + +TypesRegistry::TypesRegistry(eckit::Stream& s): + parent_(nullptr) { + + size_t numTypes; + std::string name; + std::string type; + + s >> numTypes; + for (size_t i=0; i> name; + s >> type; + types_[name] = type; + } +} + +void TypesRegistry::encode(eckit::Stream& s) const { + + s << types_.size(); + for (auto t: types_) { + s << t.first; + s << t.second; + } } + TypesRegistry::~TypesRegistry() { for (TypeMap::iterator i = cache_.begin(); i != cache_.end(); ++i) { delete (*i).second; diff --git a/src/fdb5/types/TypesRegistry.h b/src/fdb5/types/TypesRegistry.h index 9fc082318..84b244ce4 100644 --- a/src/fdb5/types/TypesRegistry.h +++ b/src/fdb5/types/TypesRegistry.h @@ -19,7 +19,7 @@ #include #include -#include "eckit/memory/NonCopyable.h" +#include "eckit/serialisation/Streamable.h" namespace fdb5 { @@ -27,11 +27,12 @@ class Type; //---------------------------------------------------------------------------------------------------------------------- -class TypesRegistry : private eckit::NonCopyable { +class TypesRegistry : public eckit::Streamable { public: // methods TypesRegistry(); + TypesRegistry(eckit::Stream& s); ~TypesRegistry(); @@ -42,9 +43,15 @@ class TypesRegistry : private eckit::NonCopyable { void dump( std::ostream &out ) const; void dump( std::ostream &out, const std::string &keyword ) const; + const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } + static const eckit::ClassSpec& classSpec() { return classSpec_; } + void encode(eckit::Stream& s) const override; private: // members + static eckit::ClassSpec classSpec_; + static eckit::Reanimator reanimator_; + typedef std::map TypeMap; mutable TypeMap cache_; @@ -54,6 +61,7 @@ class TypesRegistry : private eckit::NonCopyable { friend std::ostream &operator<<(std::ostream &s, const TypesRegistry &x); + void print( std::ostream &out ) const; }; diff --git a/tests/fdb/test_fdb5_service.cc b/tests/fdb/test_fdb5_service.cc index a0becae8b..eebad8fbc 100644 --- a/tests/fdb/test_fdb5_service.cc +++ b/tests/fdb/test_fdb5_service.cc @@ -24,6 +24,9 @@ #include "eckit/runtime/Main.h" #include "eckit/types/Types.h" #include "eckit/utils/Translator.h" +#include "eckit/serialisation/MemoryStream.h" +#include "eckit/serialisation/FileStream.h" +#include "eckit/io/AutoCloser.h" #include "metkit/mars/MarsRequest.h" #include "metkit/mars/TypeAny.h" @@ -455,6 +458,42 @@ CASE ( "test_fdb_service_subtoc" ) { } } +CASE( "schemaSerialisation" ) { + + PathName filename = PathName::unique("data"); + std::string filepath = filename.asString(); + + std::string original; + + { + eckit::FileStream sout(filepath.c_str(), "w"); + auto c = eckit::closer(sout); + + fdb5::Config config; + config = config.expandConfig(); + + std::stringstream ss; + config.schema().dump(ss); + original = ss.str(); + + sout << config.schema(); + } + { + eckit::FileStream sin(filepath.c_str(), "r"); + auto c = eckit::closer(sin); + + fdb5::Schema* clone = eckit::Reanimator::reanimate(sin); + + std::stringstream ss; + clone->dump(ss); + + EXPECT(original == ss.str()); + } + if (filename.exists()) { + filename.unlink(); + } +} + //----------------------------------------------------------------------------- } // namespace test From 1d65ce7645437b1a9f5a4f23c38c8f025bc64546 Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Tue, 10 Oct 2023 07:54:11 +0100 Subject: [PATCH 003/186] wip --- src/fdb5/CMakeLists.txt | 6 +- src/fdb5/api/FDB.h | 2 +- src/fdb5/api/local/ControlVisitor.h | 2 +- src/fdb5/api/local/DumpVisitor.h | 2 +- src/fdb5/api/local/ListVisitor.h | 7 +- src/fdb5/api/local/MoveVisitor.h | 2 +- src/fdb5/api/local/PurgeVisitor.cc | 2 +- src/fdb5/api/local/PurgeVisitor.h | 2 +- src/fdb5/api/local/StatsVisitor.cc | 2 +- src/fdb5/api/local/StatsVisitor.h | 2 +- src/fdb5/api/local/StatusVisitor.h | 2 +- src/fdb5/api/local/WipeVisitor.h | 2 +- src/fdb5/config/Config.cc | 3 +- src/fdb5/database/ArchiveVisitor.cc | 7 +- src/fdb5/database/ArchiveVisitor.h | 4 +- src/fdb5/database/Archiver.cc | 15 +- src/fdb5/database/Archiver.h | 2 +- src/fdb5/database/BaseArchiveVisitor.cc | 24 +-- src/fdb5/database/BaseArchiveVisitor.h | 8 +- src/fdb5/database/Catalogue.cc | 6 +- src/fdb5/database/Catalogue.h | 9 +- src/fdb5/database/DB.cc | 26 +-- src/fdb5/database/DB.h | 4 +- src/fdb5/database/Engine.h | 4 +- src/fdb5/database/EntryVisitMechanism.cc | 16 +- src/fdb5/database/EntryVisitMechanism.h | 4 +- src/fdb5/database/Index.cc | 11 +- src/fdb5/database/Index.h | 25 ++- src/fdb5/database/IndexAxis.cc | 7 +- src/fdb5/database/IndexAxis.h | 6 +- src/fdb5/database/InspectionKey.cc | 233 ++++++++++++++++++++++ src/fdb5/database/InspectionKey.h | 86 ++++++++ src/fdb5/database/Inspector.cc | 4 +- src/fdb5/database/Inspector.h | 4 +- src/fdb5/database/Key.cc | 143 +++---------- src/fdb5/database/Key.h | 38 ++-- src/fdb5/database/Manager.cc | 26 +-- src/fdb5/database/Manager.h | 4 +- src/fdb5/database/MoveVisitor.h | 2 +- src/fdb5/database/MultiRetrieveVisitor.cc | 46 ++--- src/fdb5/database/MultiRetrieveVisitor.h | 8 +- src/fdb5/database/ReadVisitor.cc | 10 + src/fdb5/database/ReadVisitor.h | 13 +- src/fdb5/database/RetrieveVisitor.cc | 55 +++-- src/fdb5/database/RetrieveVisitor.h | 9 +- src/fdb5/database/WipeVisitor.h | 2 +- src/fdb5/database/WriteVisitor.h | 2 +- src/fdb5/pmem/PMemDB.cc | 4 +- src/fdb5/pmem/PMemDB.h | 2 +- src/fdb5/pmem/PMemDBWriter.cc | 2 +- src/fdb5/pmem/PMemDBWriter.h | 2 +- src/fdb5/pmem/PMemStats.h | 2 +- src/fdb5/remote/CatalogueHandler.cc | 2 +- src/fdb5/remote/RemoteStore.cc | 12 -- src/fdb5/remote/StoreHandler.cc | 12 +- src/fdb5/remote/StoreHandler.h | 2 +- src/fdb5/rules/Rule.cc | 44 ++-- src/fdb5/rules/Rule.h | 25 +-- src/fdb5/rules/Schema.cc | 24 +-- src/fdb5/rules/Schema.h | 8 +- src/fdb5/toc/AdoptVisitor.cc | 2 +- src/fdb5/toc/AdoptVisitor.h | 2 +- src/fdb5/toc/RootManager.cc | 15 +- src/fdb5/toc/RootManager.h | 6 +- src/fdb5/toc/TocCatalogue.cc | 6 +- src/fdb5/toc/TocCatalogue.h | 4 +- src/fdb5/toc/TocCatalogueReader.cc | 16 +- src/fdb5/toc/TocCatalogueReader.h | 6 +- src/fdb5/toc/TocCatalogueWriter.cc | 32 +-- src/fdb5/toc/TocCatalogueWriter.h | 8 +- src/fdb5/toc/TocEngine.cc | 24 +-- src/fdb5/toc/TocEngine.h | 6 +- src/fdb5/toc/TocHandler.cc | 5 +- src/fdb5/toc/TocHandler.h | 2 +- src/fdb5/toc/TocIndex.cc | 4 +- src/fdb5/toc/TocIndex.h | 4 +- src/fdb5/toc/TocStats.h | 2 +- src/fdb5/toc/TocStore.cc | 11 +- src/fdb5/tools/fdb-hide.cc | 2 +- src/fdb5/tools/fdb-overlay.cc | 27 ++- src/fdb5/tools/fdb-reconsolidate-toc.cc | 8 +- src/fdb5/tools/fdb-root.cc | 13 +- src/fdb5/tools/fdb-schema.cc | 6 +- src/fdb5/tools/fdb-stats-new.cc | 6 +- src/fdb5/types/Type.cc | 2 +- src/fdb5/types/Type.h | 4 +- src/fdb5/types/TypeAbbreviation.cc | 2 +- src/fdb5/types/TypeAbbreviation.h | 2 +- src/fdb5/types/TypeClimateDaily.cc | 2 +- src/fdb5/types/TypeClimateDaily.h | 2 +- src/fdb5/types/TypeClimateMonthly.cc | 2 +- src/fdb5/types/TypeClimateMonthly.h | 2 +- src/fdb5/types/TypeDate.cc | 2 +- src/fdb5/types/TypeDate.h | 2 +- src/fdb5/types/TypeDouble.cc | 2 +- src/fdb5/types/TypeDouble.h | 2 +- src/fdb5/types/TypeGrid.cc | 2 +- src/fdb5/types/TypeGrid.h | 2 +- src/fdb5/types/TypeIgnore.cc | 2 +- src/fdb5/types/TypeIgnore.h | 2 +- src/fdb5/types/TypeInteger.cc | 2 +- src/fdb5/types/TypeInteger.h | 2 +- src/fdb5/types/TypeMonth.cc | 2 +- src/fdb5/types/TypeMonth.h | 2 +- src/fdb5/types/TypeParam.cc | 10 +- src/fdb5/types/TypeParam.h | 2 +- src/fdb5/types/TypeStep.cc | 10 +- src/fdb5/types/TypeStep.h | 2 +- tests/fdb/type/test_toKey.cc | 22 +- 109 files changed, 794 insertions(+), 551 deletions(-) create mode 100644 src/fdb5/database/InspectionKey.cc create mode 100644 src/fdb5/database/InspectionKey.h diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt index 64fa282e4..17abb8ab0 100644 --- a/src/fdb5/CMakeLists.txt +++ b/src/fdb5/CMakeLists.txt @@ -75,8 +75,8 @@ list( APPEND fdb5_srcs database/BaseArchiveVisitor.h database/Catalogue.cc database/Catalogue.h - database/DB.cc - database/DB.h + # database/DB.cc + # database/DB.h database/DataStats.cc database/DataStats.h database/DbStats.cc @@ -119,6 +119,8 @@ list( APPEND fdb5_srcs database/IndexFactory.h database/Key.cc database/Key.h + database/InspectionKey.cc + database/InspectionKey.h database/ReadVisitor.cc database/ReadVisitor.h database/Report.cc diff --git a/src/fdb5/api/FDB.h b/src/fdb5/api/FDB.h index cb9879363..97f629321 100644 --- a/src/fdb5/api/FDB.h +++ b/src/fdb5/api/FDB.h @@ -48,7 +48,7 @@ namespace fdb5 { class FDBBase; class FDBToolRequest; -class Key; +class InspectionKey; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/api/local/ControlVisitor.h b/src/fdb5/api/local/ControlVisitor.h index 2273b0efb..87d2dc391 100644 --- a/src/fdb5/api/local/ControlVisitor.h +++ b/src/fdb5/api/local/ControlVisitor.h @@ -39,7 +39,7 @@ class ControlVisitor : public QueryVisitor { bool visitDatabase(const Catalogue& catalogue, const Store& store) override; bool visitIndex(const Index&) override { NOTIMP; } - void visitDatum(const Field&, const Key&) override { NOTIMP; } + void visitDatum(const Field&, const InspectionKey&) override { NOTIMP; } private: // members diff --git a/src/fdb5/api/local/DumpVisitor.h b/src/fdb5/api/local/DumpVisitor.h index ed0a5cde9..3b7bdc9a9 100644 --- a/src/fdb5/api/local/DumpVisitor.h +++ b/src/fdb5/api/local/DumpVisitor.h @@ -49,7 +49,7 @@ class DumpVisitor : public QueryVisitor { return true; } bool visitIndex(const Index&) override { NOTIMP; } - void visitDatum(const Field&, const Key&) override { NOTIMP; } + void visitDatum(const Field&, const InspectionKey&) override { NOTIMP; } void visitDatum(const Field& field, const std::string& keyFingerprint) override { EntryVisitor::visitDatum(field, keyFingerprint); diff --git a/src/fdb5/api/local/ListVisitor.h b/src/fdb5/api/local/ListVisitor.h index 938eeec5e..7956af637 100644 --- a/src/fdb5/api/local/ListVisitor.h +++ b/src/fdb5/api/local/ListVisitor.h @@ -21,6 +21,7 @@ #include "fdb5/database/DB.h" #include "fdb5/database/Index.h" +#include "fdb5/database/InspectionKey.h" #include "fdb5/api/local/QueryVisitor.h" #include "fdb5/api/helpers/ListIterator.h" @@ -79,12 +80,12 @@ struct ListVisitor : public QueryVisitor { } /// Test if entry matches the current request. If so, add to the output queue. - void visitDatum(const Field& field, const Key& key) override { + void visitDatum(const Field& field, const InspectionKey& datumKey) override { ASSERT(currentCatalogue_); ASSERT(currentIndex_); - if (key.match(datumRequest_)) { - queue_.emplace(ListElement({currentCatalogue_->key(), currentIndex_->key(), key}, field.stableLocation(), field.timestamp())); + if (datumKey.match(datumRequest_)) { + queue_.emplace(ListElement({currentCatalogue_->key(), currentIndex_->key(), datumKey.canonical()}, field.stableLocation(), field.timestamp())); } } diff --git a/src/fdb5/api/local/MoveVisitor.h b/src/fdb5/api/local/MoveVisitor.h index c6c79fcc1..ec1e834ec 100644 --- a/src/fdb5/api/local/MoveVisitor.h +++ b/src/fdb5/api/local/MoveVisitor.h @@ -44,7 +44,7 @@ class MoveVisitor : public QueryVisitor { bool visitDatabase(const Catalogue& catalogue, const Store& store) override; bool visitIndex(const Index&) override { NOTIMP; } - void visitDatum(const Field&, const Key&) override { NOTIMP; } + void visitDatum(const Field&, const InspectionKey&) override { NOTIMP; } void visitDatum(const Field& field, const std::string& keyFingerprint) override { NOTIMP; } private: // members diff --git a/src/fdb5/api/local/PurgeVisitor.cc b/src/fdb5/api/local/PurgeVisitor.cc index 212eb0947..e608894cd 100644 --- a/src/fdb5/api/local/PurgeVisitor.cc +++ b/src/fdb5/api/local/PurgeVisitor.cc @@ -82,7 +82,7 @@ void PurgeVisitor::visitDatum(const Field& field, const std::string& keyFingerpr internalVisitor_->visitDatum(field, keyFingerprint); } -void PurgeVisitor::visitDatum(const Field&, const Key&) { NOTIMP; } +void PurgeVisitor::visitDatum(const Field&, const InspectionKey&) { NOTIMP; } void PurgeVisitor::catalogueComplete(const Catalogue& catalogue) { internalVisitor_->catalogueComplete(catalogue); diff --git a/src/fdb5/api/local/PurgeVisitor.h b/src/fdb5/api/local/PurgeVisitor.h index 9c441bac4..43d9b7b8b 100644 --- a/src/fdb5/api/local/PurgeVisitor.h +++ b/src/fdb5/api/local/PurgeVisitor.h @@ -48,7 +48,7 @@ class PurgeVisitor : public QueryVisitor { bool visitIndex(const Index& index) override; void catalogueComplete(const Catalogue& catalogue) override; void visitDatum(const Field& field, const std::string& keyFingerprint) override; - void visitDatum(const Field&, const Key&) override; + void visitDatum(const Field&, const InspectionKey&) override; private: // members diff --git a/src/fdb5/api/local/StatsVisitor.cc b/src/fdb5/api/local/StatsVisitor.cc index 1ebc30cfb..b681a1eac 100644 --- a/src/fdb5/api/local/StatsVisitor.cc +++ b/src/fdb5/api/local/StatsVisitor.cc @@ -47,7 +47,7 @@ void StatsVisitor::visitDatum(const Field& field, const std::string& keyFingerpr internalVisitor_->visitDatum(field, keyFingerprint); } -void StatsVisitor::visitDatum(const Field&, const Key&) { NOTIMP; } +void StatsVisitor::visitDatum(const Field&, const InspectionKey&) { NOTIMP; } void StatsVisitor::catalogueComplete(const Catalogue& catalogue) { internalVisitor_->catalogueComplete(catalogue); diff --git a/src/fdb5/api/local/StatsVisitor.h b/src/fdb5/api/local/StatsVisitor.h index 1712e5ada..6ef95117c 100644 --- a/src/fdb5/api/local/StatsVisitor.h +++ b/src/fdb5/api/local/StatsVisitor.h @@ -42,7 +42,7 @@ class StatsVisitor : public QueryVisitor { bool visitIndex(const Index& index) override; void catalogueComplete(const Catalogue& catalogue) override; void visitDatum(const Field& field, const std::string& keyFingerprint) override; - void visitDatum(const Field&, const Key&) override; + void visitDatum(const Field&, const InspectionKey&) override; private: // members diff --git a/src/fdb5/api/local/StatusVisitor.h b/src/fdb5/api/local/StatusVisitor.h index 8f4a542ce..e28431f56 100644 --- a/src/fdb5/api/local/StatusVisitor.h +++ b/src/fdb5/api/local/StatusVisitor.h @@ -38,7 +38,7 @@ class StatusVisitor : public QueryVisitor { bool visitEntries() override { return false; } bool visitDatabase(const Catalogue& catalogue, const Store& store) override { queue_.emplace(catalogue); return true; } bool visitIndex(const Index&) override { NOTIMP; } - void visitDatum(const Field&, const Key&) override { NOTIMP; } + void visitDatum(const Field&, const InspectionKey&) override { NOTIMP; } void visitDatum(const Field& field, const std::string& keyFingerprint) override { EntryVisitor::visitDatum(field, keyFingerprint); diff --git a/src/fdb5/api/local/WipeVisitor.h b/src/fdb5/api/local/WipeVisitor.h index 6cd6db2a4..c40fc424e 100644 --- a/src/fdb5/api/local/WipeVisitor.h +++ b/src/fdb5/api/local/WipeVisitor.h @@ -50,7 +50,7 @@ class WipeVisitor : public QueryVisitor { bool visitDatabase(const Catalogue& catalogue, const Store& store) override; bool visitIndex(const Index& index) override; void catalogueComplete(const Catalogue& catalogue) override; - void visitDatum(const Field&, const Key&) override { NOTIMP; } + void visitDatum(const Field&, const InspectionKey&) override { NOTIMP; } void visitDatum(const Field& field, const std::string& keyFingerprint) override { NOTIMP; } private: // members diff --git a/src/fdb5/config/Config.cc b/src/fdb5/config/Config.cc index f59df7fae..17c311072 100644 --- a/src/fdb5/config/Config.cc +++ b/src/fdb5/config/Config.cc @@ -170,7 +170,7 @@ PathName Config::expandPath(const std::string& path) const { } const PathName& Config::schemaPath() const { - if (schemaPath_.path().empty()) { + if (schemaPath_.path().empty() || !schemaPathInitialised_) { initializeSchemaPath(); } return schemaPath_; @@ -196,6 +196,7 @@ void Config::initializeSchemaPath() const { schemaPath_ = expandPath(fdbSchemaFile); } + schemaPathInitialised_ = true; eckit::Log::debug() << "Using FDB schema: " << schemaPath_ << std::endl; } diff --git a/src/fdb5/database/ArchiveVisitor.cc b/src/fdb5/database/ArchiveVisitor.cc index 76595627f..1c9b2f0d3 100644 --- a/src/fdb5/database/ArchiveVisitor.cc +++ b/src/fdb5/database/ArchiveVisitor.cc @@ -16,13 +16,13 @@ namespace fdb5 { -ArchiveVisitor::ArchiveVisitor(Archiver &owner, const Key &field, const void *data, size_t size) : - BaseArchiveVisitor(owner, field), +ArchiveVisitor::ArchiveVisitor(Archiver &owner, const Key &dataKey, const void *data, size_t size) : + BaseArchiveVisitor(owner, dataKey), data_(data), size_(size) { } -bool ArchiveVisitor::selectDatum(const Key &key, const Key &full) { +bool ArchiveVisitor::selectDatum(const InspectionKey &key, const Key &full) { // eckit::Log::info() << "selectDatum " << key << ", " << full << " " << size_ << std::endl; checkMissingKeys(full); @@ -33,7 +33,6 @@ bool ArchiveVisitor::selectDatum(const Key &key, const Key &full) { // here we could create a queue... and keep accepting archival request until the queue is full auto futureLocation = store()->archive(idx.key(), data_, size_); writeCatalogue->archive(key, futureLocation.get()); -// writeCatalogue->archive(catalogue->key(), idx.key(), key, futureLocation.get()); return true; } diff --git a/src/fdb5/database/ArchiveVisitor.h b/src/fdb5/database/ArchiveVisitor.h index f6213465f..6ae5fd74f 100644 --- a/src/fdb5/database/ArchiveVisitor.h +++ b/src/fdb5/database/ArchiveVisitor.h @@ -30,11 +30,11 @@ class ArchiveVisitor : public BaseArchiveVisitor { public: // methods - ArchiveVisitor(Archiver &owner, const Key &field, const void *data, size_t size); + ArchiveVisitor(Archiver &owner, const Key &dataKey, const void *data, size_t size); protected: // methods - virtual bool selectDatum(const Key &key, const Key &full) override; + virtual bool selectDatum(const InspectionKey &key, const Key &full) override; virtual void print( std::ostream &out ) const override; diff --git a/src/fdb5/database/Archiver.cc b/src/fdb5/database/Archiver.cc index 9538f202f..faaf68807 100644 --- a/src/fdb5/database/Archiver.cc +++ b/src/fdb5/database/Archiver.cc @@ -26,7 +26,7 @@ namespace fdb5 { Archiver::Archiver(const Config& dbConfig) : dbConfig_(dbConfig), - current_(nullptr) { + catalogue_(nullptr) { } Archiver::~Archiver() { @@ -64,13 +64,12 @@ void Archiver::flush() { } } -void Archiver::selectDatabase(const Key &key) { +void Archiver::selectDatabase(const Key &dbKey) { - store_t::iterator i = databases_.find(key); + store_t::iterator i = databases_.find(dbKey); if (i != databases_.end() ) { - std::cout << "found key: " << key << std::endl; - current_ = i->second.second.first.get(); + catalogue_ = i->second.second.first.get(); store_ = i->second.second.second.get(); i->second.first = ::time(0); return; @@ -96,7 +95,7 @@ void Archiver::selectDatabase(const Key &key) { } } - std::unique_ptr cat = CatalogueFactory::instance().build(key, dbConfig_, false); + std::unique_ptr cat = CatalogueFactory::instance().build(dbKey, dbConfig_, false); ASSERT(cat); // If this database is locked for writing then this is an error @@ -107,10 +106,10 @@ void Archiver::selectDatabase(const Key &key) { } std::unique_ptr str = cat->buildStore(); - current_ = cat.get(); + catalogue_ = cat.get(); store_ = str.get(); - databases_[key] = std::make_pair(::time(0), std::make_pair(std::move(cat), std::move(str))); + databases_[dbKey] = std::make_pair(::time(0), std::make_pair(std::move(cat), std::move(str))); } void Archiver::print(std::ostream &out) const { diff --git a/src/fdb5/database/Archiver.h b/src/fdb5/database/Archiver.h index c3d72bc35..3abe0e53b 100644 --- a/src/fdb5/database/Archiver.h +++ b/src/fdb5/database/Archiver.h @@ -76,7 +76,7 @@ class Archiver : public eckit::NonCopyable { std::vector prev_; - Catalogue* current_; + Catalogue* catalogue_; Store* store_; }; diff --git a/src/fdb5/database/BaseArchiveVisitor.cc b/src/fdb5/database/BaseArchiveVisitor.cc index 40dde3c2a..b60ab40a3 100644 --- a/src/fdb5/database/BaseArchiveVisitor.cc +++ b/src/fdb5/database/BaseArchiveVisitor.cc @@ -17,31 +17,31 @@ namespace fdb5 { -BaseArchiveVisitor::BaseArchiveVisitor(Archiver &owner, const Key &field) : +BaseArchiveVisitor::BaseArchiveVisitor(Archiver &owner, const Key &dataKey) : WriteVisitor(owner.prev_), owner_(owner), - field_(field) { + dataKey_(dataKey) { checkMissingKeysOnWrite_ = eckit::Resource("checkMissingKeysOnWrite", true); } -bool BaseArchiveVisitor::selectDatabase(const Key &key, const Key&) { - eckit::Log::debug() << "selectDatabase " << key << std::endl; - owner_.selectDatabase(key); - ASSERT(owner_.current_); - owner_.current_->deselectIndex(); +bool BaseArchiveVisitor::selectDatabase(const Key &dbKey, const Key&) { + eckit::Log::debug() << "selectDatabase " << dbKey << std::endl; + owner_.selectDatabase(dbKey); + ASSERT(owner_.catalogue_); + owner_.catalogue_->deselectIndex(); return true; } -bool BaseArchiveVisitor::selectIndex(const Key &key, const Key&) { +bool BaseArchiveVisitor::selectIndex(const Key &idxKey, const Key&) { // eckit::Log::info() << "selectIndex " << key << std::endl; - ASSERT(owner_.current_); - return owner_.current_->selectIndex(key); + ASSERT(owner_.catalogue_); + return owner_.catalogue_->selectIndex(idxKey); } void BaseArchiveVisitor::checkMissingKeys(const Key &full) { if (checkMissingKeysOnWrite_) { - field_.validateKeysOf(full); + dataKey_.validateKeysOf(full); } } @@ -51,7 +51,7 @@ const Schema& BaseArchiveVisitor::databaseSchema() const { } Catalogue* BaseArchiveVisitor::current() const { - return owner_.current_; + return owner_.catalogue_; } Store* BaseArchiveVisitor::store() const { diff --git a/src/fdb5/database/BaseArchiveVisitor.h b/src/fdb5/database/BaseArchiveVisitor.h index 29733b7fa..b4bd41398 100644 --- a/src/fdb5/database/BaseArchiveVisitor.h +++ b/src/fdb5/database/BaseArchiveVisitor.h @@ -32,13 +32,13 @@ class BaseArchiveVisitor : public WriteVisitor { public: // methods - BaseArchiveVisitor(Archiver &owner, const Key &field); + BaseArchiveVisitor(Archiver &owner, const Key &dataKey); protected: // methods - virtual bool selectDatabase(const Key &key, const Key &full); + virtual bool selectDatabase(const Key &dbKey, const Key &full); - virtual bool selectIndex(const Key &key, const Key &full); + virtual bool selectIndex(const Key &idxKey, const Key &full); virtual void checkMissingKeys(const Key &full); @@ -51,7 +51,7 @@ class BaseArchiveVisitor : public WriteVisitor { Archiver &owner_; - const Key &field_; + const Key &dataKey_; bool checkMissingKeysOnWrite_; }; diff --git a/src/fdb5/database/Catalogue.cc b/src/fdb5/database/Catalogue.cc index 317ad75fe..f7a337a65 100644 --- a/src/fdb5/database/Catalogue.cc +++ b/src/fdb5/database/Catalogue.cc @@ -85,8 +85,8 @@ void CatalogueFactory::list(std::ostream& out) { } } -std::unique_ptr CatalogueFactory::build(const Key& key, const Config& config, bool read) { - std::string name = Manager(config).engine(key); +std::unique_ptr CatalogueFactory::build(const Key& dbKey, const Config& config, bool read) { + std::string name = Manager(config).engine(dbKey); std::string nameLowercase = eckit::StringTools::lower(name); nameLowercase += read ? ".reader" : ".writer"; @@ -104,7 +104,7 @@ std::unique_ptr CatalogueFactory::build(const Key& key, const Config& throw eckit::SeriousBug(std::string("No CatalogueBuilder called ") + nameLowercase); } - return j->second->make(key, config); + return j->second->make(dbKey, config); } std::unique_ptr CatalogueFactory::build(const eckit::URI& uri, const fdb5::Config& config, bool read) { diff --git a/src/fdb5/database/Catalogue.h b/src/fdb5/database/Catalogue.h index 733abb5d5..e03987dbf 100644 --- a/src/fdb5/database/Catalogue.h +++ b/src/fdb5/database/Catalogue.h @@ -33,6 +33,7 @@ #include "fdb5/database/WipeVisitor.h" #include "fdb5/database/MoveVisitor.h" #include "fdb5/rules/Schema.h" +#include "fdb5/database/InspectionKey.h" namespace fdb5 { @@ -55,7 +56,7 @@ class Catalogue { std::unique_ptr buildStore(); virtual const Schema& schema() const = 0; - virtual bool selectIndex(const Key& key) = 0; + virtual bool selectIndex(const Key& idxKey) = 0; virtual void deselectIndex() = 0; virtual std::vector metadataPaths() const = 0; @@ -118,16 +119,16 @@ class CatalogueReader { public: virtual DbStats stats() const = 0; virtual bool axis(const std::string& keyword, eckit::StringSet& s) const = 0; - virtual bool retrieve(const Key& key, Field& field) const = 0; + virtual bool retrieve(const InspectionKey& key, Field& field) const = 0; }; class CatalogueWriter { public: virtual const Index& currentIndex() = 0; - virtual void archive(const Key& key, std::unique_ptr fieldLocation) = 0; + virtual void archive(const InspectionKey& key, std::unique_ptr fieldLocation) = 0; virtual void overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) = 0; - virtual void index(const Key& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) = 0; + virtual void index(const InspectionKey& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) = 0; virtual void reconsolidate() = 0; }; diff --git a/src/fdb5/database/DB.cc b/src/fdb5/database/DB.cc index aa30b9413..8f3f20a2a 100644 --- a/src/fdb5/database/DB.cc +++ b/src/fdb5/database/DB.cc @@ -83,25 +83,25 @@ bool DB::axis(const std::string &keyword, eckit::StringSet &s) const { return cat->axis(keyword, s); } -bool DB::inspect(const Key& key, Field& field) { +// bool DB::inspect(const Key& key, Field& field) { - eckit::Log::debug() << "Trying to retrieve key " << key << std::endl; +// eckit::Log::debug() << "Trying to retrieve key " << key << std::endl; - CatalogueReader* cat = dynamic_cast(catalogue_.get()); - ASSERT(cat); +// CatalogueReader* cat = dynamic_cast(catalogue_.get()); +// ASSERT(cat); - return cat->retrieve(key, field); -} +// return cat->retrieve(key, field); +// } -eckit::DataHandle *DB::retrieve(const Key& key) { +// eckit::DataHandle *DB::retrieve(const Key& key) { - Field field; - if (inspect(key, field)) { - return store().retrieve(field); - } +// Field field; +// if (inspect(key, field)) { +// return store().retrieve(field); +// } - return nullptr; -} +// return nullptr; +// } // void DB::archive(const Key& key, const void* data, eckit::Length length) { diff --git a/src/fdb5/database/DB.h b/src/fdb5/database/DB.h index fd593c31f..a6de6e53a 100644 --- a/src/fdb5/database/DB.h +++ b/src/fdb5/database/DB.h @@ -54,8 +54,8 @@ class DB final : public eckit::OwnedLock { const Schema& schema() const; bool axis(const std::string &keyword, eckit::StringSet &s) const; - bool inspect(const Key& key, Field& field); - eckit::DataHandle *retrieve(const Key &key); + // bool inspect(const Key& key, Field& field); + // eckit::DataHandle *retrieve(const Key &key); // void archive(const Key &key, const void *data, eckit::Length length); bool open(); diff --git a/src/fdb5/database/Engine.h b/src/fdb5/database/Engine.h index b9560d50e..d49c66c1d 100644 --- a/src/fdb5/database/Engine.h +++ b/src/fdb5/database/Engine.h @@ -57,11 +57,11 @@ class Engine : private eckit::NonCopyable { virtual std::vector allLocations(const Key& key, const Config& config) const = 0; /// Lists the roots that can be visited given a DB key - virtual std::vector visitableLocations(const Key& key, const Config& config) const = 0; + virtual std::vector visitableLocations(const Config& config) const = 0; virtual std::vector visitableLocations(const metkit::mars::MarsRequest& rq, const Config& config) const = 0; /// Lists the roots where a DB key would be able to be written - virtual std::vector writableLocations(const Key& key, const Config& config) const = 0; +// virtual std::vector writableLocations(const InspectionKey& key, const Config& config) const = 0; friend std::ostream &operator<<(std::ostream &s, const Engine& x); diff --git a/src/fdb5/database/EntryVisitMechanism.cc b/src/fdb5/database/EntryVisitMechanism.cc index adac0fa9e..a97b4f9bd 100644 --- a/src/fdb5/database/EntryVisitMechanism.cc +++ b/src/fdb5/database/EntryVisitMechanism.cc @@ -14,6 +14,7 @@ #include "fdb5/api/helpers/FDBToolRequest.h" #include "fdb5/database/Manager.h" +#include "fdb5/database/InspectionKey.h" #include "fdb5/LibFdb5.h" #include "fdb5/rules/Schema.h" @@ -59,7 +60,7 @@ void EntryVisitor::visitDatum(const Field& field, const std::string& keyFingerpr ASSERT(currentCatalogue_); ASSERT(currentIndex_); - Key key(keyFingerprint, currentCatalogue_->schema().ruleFor(currentCatalogue_->key(), currentIndex_->key())); + InspectionKey key(keyFingerprint, currentCatalogue_->schema().ruleFor(currentCatalogue_->key(), currentIndex_->key())); visitDatum(field, key); } @@ -106,11 +107,16 @@ void EntryVisitMechanism::visit(const FDBToolRequest& request, EntryVisitor& vis Log::debug() << "FDB processing Path " << path << std::endl; - std::unique_ptr db = DB::buildReader(eckit::URI(uri.scheme(), path), dbConfig_); - ASSERT(db->open()); - eckit::AutoCloser closer(*db); + std::unique_ptr catalogue = CatalogueFactory::instance().build(eckit::URI(uri.scheme(), path), dbConfig_, true); + ASSERT(catalogue->open()); - db->visitEntries(visitor, false); + std::unique_ptr store = catalogue->buildStore(); + ASSERT(store->open()); + + eckit::AutoCloser closer(*catalogue); + eckit::AutoCloser storeCloser(*store); + + catalogue->visitEntries(visitor, *store, false); } } diff --git a/src/fdb5/database/EntryVisitMechanism.h b/src/fdb5/database/EntryVisitMechanism.h index 441818ba8..2ba78a5c4 100644 --- a/src/fdb5/database/EntryVisitMechanism.h +++ b/src/fdb5/database/EntryVisitMechanism.h @@ -25,7 +25,7 @@ class Catalogue; class Store; class FDBToolRequest; class Index; -class Key; +class InspectionKey; //---------------------------------------------------------------------------------------------------------------------- @@ -49,7 +49,7 @@ class EntryVisitor : public eckit::NonCopyable { private: // methods - virtual void visitDatum(const Field& field, const Key& key) = 0; + virtual void visitDatum(const Field& field, const InspectionKey& key) = 0; protected: // members diff --git a/src/fdb5/database/Index.cc b/src/fdb5/database/Index.cc index ea7ee32d1..488b0332d 100755 --- a/src/fdb5/database/Index.cc +++ b/src/fdb5/database/Index.cc @@ -79,6 +79,7 @@ void IndexBase::decodeLegacy(eckit::Stream& s, const int version) { // decoding axes_.decode(s, version); + std::string dummy; s >> key_; s >> dummy; ///< legacy entry, no longer used but stays here so we can read existing indexes @@ -121,11 +122,11 @@ void IndexBase::encodeLegacy(eckit::Stream& s, const int version) const { axes_.encode(s, version); s << key_; - s << key_.valuesToString(); // we no longer write this field, required in the previous index format + s << ""; // key_.valuesToString(); we no longer write this field, required in the previous index format s << type_; } -void IndexBase::put(const Key &key, const Field &field) { +void IndexBase::put(const InspectionKey &key, const Field &field) { eckit::Log::debug() << "FDB Index " << indexer_ << " " << key << " -> " << field << std::endl; @@ -142,7 +143,7 @@ bool IndexBase::partialMatch(const metkit::mars::MarsRequest& request) const { return true; } -bool IndexBase::mayContain(const Key &key) const { +bool IndexBase::mayContain(const InspectionKey &key) const { return axes_.contains(key); } @@ -201,8 +202,8 @@ class NullIndex : public IndexBase { virtual void visit(IndexLocationVisitor&) const override { NOTIMP; } - virtual bool get( const Key&, const Key&, Field&) const override { NOTIMP; } - virtual void add( const Key&, const Field&) override { NOTIMP; } + virtual bool get( const InspectionKey&, const Key&, Field&) const override { NOTIMP; } + virtual void add( const InspectionKey&, const Field&) override { NOTIMP; } virtual void flush() override { NOTIMP; } virtual void encode(eckit::Stream&, const int version) const override { NOTIMP; } virtual void entries(EntryVisitor&) const override { NOTIMP; } diff --git a/src/fdb5/database/Index.h b/src/fdb5/database/Index.h index 5c9a53c57..05542c4ba 100755 --- a/src/fdb5/database/Index.h +++ b/src/fdb5/database/Index.h @@ -30,7 +30,7 @@ #include "fdb5/database/IndexAxis.h" #include "fdb5/database/IndexLocation.h" #include "fdb5/database/Indexer.h" -#include "fdb5/database/Key.h" +#include "fdb5/database/InspectionKey.h" namespace eckit { class Stream; @@ -38,7 +38,6 @@ class Stream; namespace fdb5 { -class Key; class Index; class IndexLocationVisitor; class Schema; @@ -80,15 +79,15 @@ class IndexBase : public eckit::Counted { time_t timestamp() const { return timestamp_; } - virtual bool get(const Key &key, const Key &remapKey, Field &field) const = 0; - virtual void put(const Key &key, const Field &field); + virtual bool get(const InspectionKey &key, const Key &remapKey, Field &field) const = 0; + virtual void put(const InspectionKey &key, const Field &field); virtual void encode(eckit::Stream& s, const int version) const; virtual void entries(EntryVisitor& visitor) const = 0; virtual void dump(std::ostream& out, const char* indent, bool simple = false, bool dumpFields = false) const = 0; virtual bool partialMatch(const metkit::mars::MarsRequest& request) const; - virtual bool mayContain(const Key& key) const; + virtual bool mayContain(const InspectionKey& key) const; virtual IndexStats statistics() const = 0; @@ -108,18 +107,18 @@ class IndexBase : public eckit::Counted { void decodeCurrent(eckit::Stream& s, const int version); void decodeLegacy(eckit::Stream& s, const int version); - virtual void add(const Key &key, const Field &field) = 0; + virtual void add(const InspectionKey &key, const Field &field) = 0; protected: // members std::string type_; /// @note Order of members is important here ... - IndexAxis axes_; ///< This Index spans along these axis - Key key_; ///< key that selected this index - time_t timestamp_; ///< timestamp when this Index was flushed + IndexAxis axes_; ///< This Index spans along these axis + Key key_; ///< key that selected this index + time_t timestamp_; ///< timestamp when this Index was flushed - Indexer indexer_; + Indexer indexer_; friend std::ostream& operator<<(std::ostream& s, const IndexBase& o) { o.print(s); return s; @@ -160,8 +159,8 @@ class Index { time_t timestamp() const { return content_->timestamp(); } - bool get(const Key& key, const Key& remapKey, Field& field) const { return content_->get(key, remapKey, field); } - void put(const Key& key, const Field& field) { content_->put(key, field); } + bool get(const InspectionKey& key, const Key& remapKey, Field& field) const { return content_->get(key, remapKey, field); } + void put(const InspectionKey& key, const Field& field) { content_->put(key, field); } void encode(eckit::Stream& s, const int version) const { content_->encode(s, version); } void entries(EntryVisitor& v) const { content_->entries(v); } @@ -175,7 +174,7 @@ class Index { const IndexBase* content() const { return content_; } bool partialMatch(const metkit::mars::MarsRequest& request) const { return content_->partialMatch(request); } - bool mayContain(const Key& key) const { return content_->mayContain(key); } + bool mayContain(const InspectionKey& key) const { return content_->mayContain(key); } bool null() const { return null_; } diff --git a/src/fdb5/database/IndexAxis.cc b/src/fdb5/database/IndexAxis.cc index 912b49be7..f535bd9eb 100755 --- a/src/fdb5/database/IndexAxis.cc +++ b/src/fdb5/database/IndexAxis.cc @@ -16,7 +16,7 @@ #include "fdb5/database/AxisRegistry.h" #include "fdb5/database/IndexAxis.h" -#include "fdb5/database/Key.h" +#include "fdb5/database/InspectionKey.h" #include "fdb5/types/TypesRegistry.h" #include "fdb5/types/Type.h" @@ -150,6 +150,7 @@ void IndexAxis::decodeCurrent(eckit::Stream &s, const int version) { } void IndexAxis::decodeLegacy(eckit::Stream& s, const int version) { + ASSERT(version <= 2); size_t n; @@ -220,7 +221,7 @@ bool IndexAxis::partialMatch(const metkit::mars::MarsRequest& request) const { return true; } -bool IndexAxis::contains(const Key &key) const { +bool IndexAxis::contains(const InspectionKey &key) const { for (AxisMap::const_iterator i = axis_.begin(); i != axis_.end(); ++i) { if (!key.match(i->first, *(i->second))) { @@ -230,7 +231,7 @@ bool IndexAxis::contains(const Key &key) const { return true; } -void IndexAxis::insert(const Key &key) { +void IndexAxis::insert(const InspectionKey &key) { ASSERT(!readOnly_); for (Key::const_iterator i = key.begin(); i != key.end(); ++i) { diff --git a/src/fdb5/database/IndexAxis.h b/src/fdb5/database/IndexAxis.h index a83b92b6d..72fe85a88 100644 --- a/src/fdb5/database/IndexAxis.h +++ b/src/fdb5/database/IndexAxis.h @@ -35,7 +35,7 @@ class MarsRequest; namespace fdb5 { -class Key; +class InspectionKey; //---------------------------------------------------------------------------------------------------------------------- @@ -48,7 +48,7 @@ class IndexAxis : private eckit::NonCopyable { ~IndexAxis(); - void insert(const Key &key); + void insert(const InspectionKey &key); void encode(eckit::Stream &s, const int version) const; // Decode can be used for two-stage initialisation (IndexAxis a; a.decode(s);) @@ -60,7 +60,7 @@ class IndexAxis : private eckit::NonCopyable { void dump(std::ostream &out, const char* indent) const; bool partialMatch(const metkit::mars::MarsRequest& request) const; - bool contains(const Key& key) const; + bool contains(const InspectionKey& key) const; /// Provide a means to test if the index has changed since it was last written out, and to /// mark that it has been written out. diff --git a/src/fdb5/database/InspectionKey.cc b/src/fdb5/database/InspectionKey.cc new file mode 100644 index 000000000..b36952c93 --- /dev/null +++ b/src/fdb5/database/InspectionKey.cc @@ -0,0 +1,233 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include + +#include "eckit/container/DenseSet.h" +#include "eckit/utils/Tokenizer.h" + +#include "metkit/mars/MarsRequest.h" + +#include "fdb5/config/Config.h" +#include "fdb5/database/InspectionKey.h" +#include "fdb5/LibFdb5.h" +#include "fdb5/rules/Rule.h" +#include "fdb5/rules/Schema.h" +#include "fdb5/types/Type.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +InspectionKey::InspectionKey() : + Key(), rule_(nullptr) {} + +InspectionKey::InspectionKey(const Key& other) { + for (const auto& name : other.names_) { + names_.push_back(name); + } + for (const auto& k : other.keys_) { + keys_[k.first] = k.second; + } +} + +InspectionKey::InspectionKey(const std::string &s, const Rule *rule) : + Key(), rule_(nullptr) { + eckit::Tokenizer parse(":", true); + eckit::StringList values; + parse(s, values); + + ASSERT(rule); + rule->fill(*this, values); +} + +InspectionKey::InspectionKey(const std::string &s) : + Key(), + rule_(nullptr) { + + const TypesRegistry ®istry = this->registry(); + + eckit::Tokenizer parse1(","); + eckit::StringList v; + + parse1(s, v); + + eckit::Tokenizer parse2("="); + for (eckit::StringList::const_iterator i = v.begin(); i != v.end(); ++i) { + eckit::StringList kv; + parse2(*i, kv); + ASSERT(kv.size() == 2); + + const Type &t = registry.lookupType(kv[0]); + + std::string v = t.tidy(kv[0], kv[1]); + + if (find(kv[0]) == end()) { + push(kv[0], v); + } else { + set(kv[0], v); + } + } + +} + +InspectionKey::InspectionKey(const eckit::StringDict &keys) : + Key(keys), rule_(nullptr) { + + eckit::StringDict::const_iterator it = keys.begin(); + eckit::StringDict::const_iterator end = keys.end(); + for (; it != end; ++it) { + names_.emplace_back(it->first); + } +} + +InspectionKey::InspectionKey(eckit::Stream& s) : + rule_(nullptr) { + decode(s); +} + +Key InspectionKey::canonical() const { + Key key; + + for (const auto& name: names_) { + auto m = keys_.find(name); + ASSERT(m != keys_.end()); + + key.set(name, canonicalise(name, m->second)); + } + + return key; +} + +// void InspectionKey::decode(eckit::Stream& s) { + +// ASSERT(rule_ == nullptr); + +// keys_.clear(); +// names_.clear(); + + +// size_t n; + +// s >> n; +// std::string k; +// std::string v; +// for (size_t i = 0; i < n; ++i) { +// s >> k; +// s >> v; +// keys_[k] = v; +// } + +// s >> n; +// for (size_t i = 0; i < n; ++i) { +// s >> k; +// s >> v; // this is the type (ignoring FTM) +// names_.push_back(k); +// } +// s >> canonical_; +// } + +// void InspectionKey::encode(eckit::Stream& s) const { +// const TypesRegistry& registry = this->registry(); + +// s << keys_.size(); +// for (eckit::StringDict::const_iterator i = keys_.begin(); i != keys_.end(); ++i) { +// const Type &t = registry.lookupType(i->first); +// s << i->first << canonicalise(i->first, i->second); +// } + +// s << names_.size(); +// for (eckit::StringList::const_iterator i = names_.begin(); i != names_.end(); ++i) { +// const Type &t = registry.lookupType(*i); +// s << (*i); +// s << t.type(); +// } + +// s << true; +// } + +void InspectionKey::rule(const Rule *rule) { + rule_ = rule; +} + +// const Rule *InspectionKey::rule() const { +// return rule_; +// } + +const TypesRegistry& InspectionKey::registry() const { + if(rule_) { + return rule_->registry(); + } + else { + return LibFdb5::instance().defaultConfig().schema().registry(); + } +} + +std::string InspectionKey::canonicalise(const std::string& keyword, const std::string& value) const { + if (value.empty()) { + return value; + } else { + return this->registry().lookupType(keyword).toKey(keyword, value); + } +} + + +fdb5::InspectionKey::operator eckit::StringDict() const +{ + eckit::StringDict res; + + ASSERT(names_.size() == keys_.size()); + + const TypesRegistry ®istry = this->registry(); + + for (eckit::StringList::const_iterator j = names_.begin(); j != names_.end(); ++j) { + + eckit::StringDict::const_iterator i = keys_.find(*j); + + ASSERT(i != keys_.end()); + ASSERT(!(*i).second.empty()); + + res[*j] = registry.lookupType(*j).tidy(*j, (*i).second); + } + + return res; +} + +// void InspectionKey::print(std::ostream &out) const { +// if (names_.size() == keys_.size()) { +// out << "{" << toString() << "}"; +// if (rule_) { +// out << " (" << *rule_ << ")"; +// } +// } else { +// out << keys_; +// if (rule_) { +// out << " (" << *rule_ << ")"; +// } +// } +// } + +// std::string Key::toString() const { +// std::string res; +// const char *sep = ""; +// for (eckit::StringList::const_iterator j = names_.begin(); j != names_.end(); ++j) { +// eckit::StringDict::const_iterator i = keys_.find(*j); +// ASSERT(i != keys_.end()); +// if (!i->second.empty()) { +// res += sep + *j + '=' + i->second; +// sep = ","; +// } +// } +// return res; +// } + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/database/InspectionKey.h b/src/fdb5/database/InspectionKey.h new file mode 100644 index 000000000..4e6587ad7 --- /dev/null +++ b/src/fdb5/database/InspectionKey.h @@ -0,0 +1,86 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @file Key.h +/// @author Baudouin Raoult +/// @author Tiago Quintino +/// @date Mar 2016 + +#pragma once + +#include "fdb5/database/Key.h" +#include "eckit/types/Types.h" + + +namespace fdb5 { + +class TypesRegistry; +class Rule; + +//---------------------------------------------------------------------------------------------------------------------- + +class InspectionKey : public Key { + +public: // methods + + InspectionKey(); + + explicit InspectionKey(const Key& other); + explicit InspectionKey(eckit::Stream &); + explicit InspectionKey(const std::string &request); + explicit InspectionKey(const std::string &keys, const Rule* rule); + + explicit InspectionKey(const eckit::StringDict &keys); + + ~InspectionKey() override {} + + Key canonical() const; + + void rule(const Rule *rule); +// const Rule *rule() const; + const TypesRegistry& registry() const; + + friend eckit::Stream& operator>>(eckit::Stream& s, InspectionKey& x) { + x = InspectionKey(s); + return s; + } + friend eckit::Stream& operator<<(eckit::Stream &s, const InspectionKey &x) { + x.encode(s); + return s; + } + + friend std::ostream& operator<<(std::ostream &s, const InspectionKey& x) { + x.print(s); + return s; + } + +private: // methods + + //TODO add unit test for each type + std::string canonicalise(const std::string& keyword, const std::string& value) const override; + + operator eckit::StringDict() const override; + +// void validateKeysOf(const Key& other, bool checkAlsoValues = false) const; + + // void print( std::ostream &out ) const; + // void decode(eckit::Stream& s); + // void encode(eckit::Stream &s) const; + + // std::string toString() const; + +private: // members + + const Rule *rule_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/database/Inspector.cc b/src/fdb5/database/Inspector.cc index 838c104d4..c5f5f200e 100644 --- a/src/fdb5/database/Inspector.cc +++ b/src/fdb5/database/Inspector.cc @@ -48,13 +48,13 @@ bool InspectIterator::next(ListElement& elem) { //---------------------------------------------------------------------------------------------------------------------- -static void purgeDB(Key& key, DB*& db) { +static void purgeCatalogue(Key& key, Catalogue*& db) { Log::debug() << "Purging DB with key " << key << std::endl; delete db; } Inspector::Inspector(const Config& dbConfig) : - databases_(Resource("fdbMaxOpenDatabases", 16), &purgeDB), + databases_(Resource("fdbMaxOpenDatabases", 16), &purgeCatalogue), dbConfig_(dbConfig) {} Inspector::~Inspector() { diff --git a/src/fdb5/database/Inspector.h b/src/fdb5/database/Inspector.h index 05f91b411..58d39b5ff 100644 --- a/src/fdb5/database/Inspector.h +++ b/src/fdb5/database/Inspector.h @@ -40,7 +40,7 @@ namespace fdb5 { class Key; class Op; -class DB; +class Catalogue; class Schema; class Notifier; class FDBToolRequest; @@ -98,7 +98,7 @@ class Inspector : public eckit::NonCopyable { private: // data - mutable eckit::CacheLRU databases_; + mutable eckit::CacheLRU databases_; Config dbConfig_; }; diff --git a/src/fdb5/database/Key.cc b/src/fdb5/database/Key.cc index 76a3103a2..956ba5ebc 100644 --- a/src/fdb5/database/Key.cc +++ b/src/fdb5/database/Key.cc @@ -27,25 +27,10 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- Key::Key() : - keys_(), - rule_(0) {} - -Key::Key(const std::string &s, const Rule *rule) : - keys_(), - rule_(0) { - eckit::Tokenizer parse(":", true); - eckit::StringList values; - parse(s, values); - - ASSERT(rule); - rule->fill(*this, values); -} + keys_(), names_() {} Key::Key(const std::string &s) : - keys_(), - rule_(0) { - - const TypesRegistry ®istry = this->registry(); + keys_(), names_() { eckit::Tokenizer parse1(","); eckit::StringList v; @@ -58,22 +43,17 @@ Key::Key(const std::string &s) : parse2(*i, kv); ASSERT(kv.size() == 2); - const Type &t = registry.lookupType(kv[0]); - - std::string v = t.tidy(kv[0], kv[1]); - if (find(kv[0]) == end()) { - push(kv[0], v); + push(kv[0], kv[1]); } else { - set(kv[0], v); + set(kv[0], kv[1]); } } } Key::Key(const eckit::StringDict &keys) : - keys_(keys), - rule_(0) { + keys_(keys) { eckit::StringDict::const_iterator it = keys.begin(); eckit::StringDict::const_iterator end = keys.end(); @@ -82,19 +62,15 @@ Key::Key(const eckit::StringDict &keys) : } } -Key::Key(eckit::Stream& s) : - rule_(nullptr) { +Key::Key(eckit::Stream& s) { decode(s); } void Key::decode(eckit::Stream& s) { - ASSERT(rule_ == nullptr); - keys_.clear(); names_.clear(); - size_t n; s >> n; @@ -115,20 +91,17 @@ void Key::decode(eckit::Stream& s) { } void Key::encode(eckit::Stream& s) const { - const TypesRegistry& registry = this->registry(); s << keys_.size(); for (eckit::StringDict::const_iterator i = keys_.begin(); i != keys_.end(); ++i) { - const Type &t = registry.lookupType(i->first); - s << i->first << canonicalise(i->first, i->second); - + s << i->first; + s << i->second; } s << names_.size(); - for (eckit::StringList::const_iterator i = names_.begin(); i != names_.end(); ++i) { - const Type &t = registry.lookupType(*i); - s << (*i); - s << t.type(); + for (const auto& name : names_) { + s << name; + s << ""; } } @@ -143,15 +116,6 @@ std::set Key::keys() const { return k; } - -void Key::rule(const Rule *rule) { - rule_ = rule; -} - -const Rule *Key::rule() const { - return rule_; -} - void Key::clear() { keys_.clear(); names_.clear(); @@ -291,32 +255,14 @@ bool Key::partialMatch(const metkit::mars::MarsRequest& request) const { } -const TypesRegistry& Key::registry() const { - if(rule_) { - return rule_->registry(); - } - else { - return LibFdb5::instance().defaultConfig().schema().registry(); - } -} - -std::string Key::canonicalise(const std::string& keyword, const std::string& value) const { - if (value.empty()) { - return value; - } else { - return this->registry().lookupType(keyword).toKey(keyword, value); - } -} - std::string Key::canonicalValue(const std::string& keyword) const { eckit::StringDict::const_iterator it = keys_.find(keyword); ASSERT(it != keys_.end()); - return canonicalise(keyword, it->second); } -std::string Key::valuesToString(bool ruleOrDefault) const { +std::string Key::valuesToString() const { ASSERT(names_.size() == keys_.size()); @@ -328,12 +274,7 @@ std::string Key::valuesToString(bool ruleOrDefault) const { ASSERT(i != keys_.end()); oss << sep; - if (rule_ || ruleOrDefault) { - oss << canonicalise(*j, i->second); - } - else { - oss << i->second; - } + oss << canonicalise(*j, i->second); sep = ":"; } @@ -341,23 +282,9 @@ std::string Key::valuesToString(bool ruleOrDefault) const { } -const eckit::StringList& Key::names() const { - return names_; -} - -std::string Key::value(const std::string& key) const { - - eckit::StringDict::const_iterator it = keys_.find(key); - ASSERT(it != keys_.end()); - return it->second; -} - -void Key::validateKeysOf(const Key& other, bool checkAlsoValues) const +void Key::validateKeysOf(const Key& other) const { eckit::StringSet missing; - eckit::StringSet mismatch; - - const TypesRegistry& registry = this->registry(); for (Key::const_iterator j = begin(); j != end(); ++j) { const std::string& keyword = (*j).first; @@ -365,34 +292,32 @@ void Key::validateKeysOf(const Key& other, bool checkAlsoValues) const if (k == other.end()) { missing.insert(keyword); } - else { - if(checkAlsoValues && !registry.lookupType(keyword).match(keyword, k->second, j->second) ) { - mismatch.insert((*j).first + "=" + j->second + " and " + k->second); - } - } } - if (missing.size() || mismatch.size()) { + if (missing.size()) { std::ostringstream oss; if(missing.size()) { oss << "Keywords not used: " << missing << " "; } - - if(mismatch.size()) { - oss << "Values mismatch: " << mismatch << " "; - } - oss << "for key " << *this << " validating " << other; - if (rule()) { - oss << " " << *rule(); - } - throw eckit::SeriousBug(oss.str()); } } + +const eckit::StringList& Key::names() const { + return names_; +} + +std::string Key::value(const std::string& key) const { + + eckit::StringDict::const_iterator it = keys_.find(key); + ASSERT(it != keys_.end()); + return it->second; +} + const eckit::StringDict &Key::keyDict() const { return keys_; } @@ -413,14 +338,16 @@ fdb5::Key::operator std::string() const { return toString(); } +std::string Key::canonicalise(const std::string&, const std::string& value) const { + return value; +} + fdb5::Key::operator eckit::StringDict() const { eckit::StringDict res; ASSERT(names_.size() == keys_.size()); - const TypesRegistry ®istry = this->registry(); - for (eckit::StringList::const_iterator j = names_.begin(); j != names_.end(); ++j) { eckit::StringDict::const_iterator i = keys_.find(*j); @@ -428,7 +355,7 @@ fdb5::Key::operator eckit::StringDict() const ASSERT(i != keys_.end()); ASSERT(!(*i).second.empty()); - res[*j] = registry.lookupType(*j).tidy(*j, (*i).second); + res[*j] = (*i).second; } return res; @@ -437,14 +364,8 @@ fdb5::Key::operator eckit::StringDict() const void Key::print(std::ostream &out) const { if (names_.size() == keys_.size()) { out << "{" << toString() << "}"; - if (rule_) { - out << " (" << *rule_ << ")"; - } } else { out << keys_; - if (rule_) { - out << " (" << *rule_ << ")"; - } } } diff --git a/src/fdb5/database/Key.h b/src/fdb5/database/Key.h index 27b6d24b5..f17eecf3e 100644 --- a/src/fdb5/database/Key.h +++ b/src/fdb5/database/Key.h @@ -21,6 +21,7 @@ #include #include +#include "eckit/serialisation/Stream.h" #include "eckit/types/Types.h" namespace eckit { @@ -36,20 +37,19 @@ namespace mars { namespace fdb5 { -class TypesRegistry; -class Rule; - //---------------------------------------------------------------------------------------------------------------------- +class InspectionKey; + class Key { public: // methods Key(); + virtual ~Key() {} explicit Key(eckit::Stream &); explicit Key(const std::string &request); - explicit Key(const std::string &keys, const Rule* rule); explicit Key(const eckit::StringDict &keys); @@ -89,26 +89,21 @@ class Key { return keys_ == other.keys_; } - friend std::ostream& operator<<(std::ostream &s, const Key &x) { - x.print(s); + friend eckit::Stream& operator>>(eckit::Stream& s, Key& x) { + x = Key(s); return s; } - friend eckit::Stream& operator<<(eckit::Stream &s, const Key &x) { x.encode(s); return s; } - friend eckit::Stream& operator>>(eckit::Stream& s, Key& x) { - x = Key(s); + friend std::ostream& operator<<(std::ostream &s, const Key &x) { + x.print(s); return s; } - void rule(const Rule *rule); - const Rule *rule() const; - const TypesRegistry& registry() const; - - std::string valuesToString(bool ruleOrDefault = true) const; + std::string valuesToString() const; const eckit::StringList& names() const; @@ -131,7 +126,7 @@ class Key { bool empty() const { return keys_.empty(); } /// @throws When "other" doesn't contain all the keys of "this" - void validateKeysOf(const Key& other, bool checkAlsoValues = false) const; + void validateKeysOf(const Key& other) const; const eckit::StringDict& keyDict() const; @@ -139,24 +134,27 @@ class Key { operator std::string() const; - operator eckit::StringDict() const; + virtual operator eckit::StringDict() const; -private: // members +protected: // methods //TODO add unit test for each type - std::string canonicalise(const std::string& keyword, const std::string& value) const; + virtual std::string canonicalise(const std::string& keyword, const std::string& value) const; void print( std::ostream &out ) const; void decode(eckit::Stream& s); void encode(eckit::Stream &s) const; +private: // methods + std::string toString() const; +protected: // members + eckit::StringDict keys_; eckit::StringList names_; - const Rule *rule_; - + friend InspectionKey; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/Manager.cc b/src/fdb5/database/Manager.cc index 8724c2fa8..f431b6e49 100644 --- a/src/fdb5/database/Manager.cc +++ b/src/fdb5/database/Manager.cc @@ -222,7 +222,7 @@ std::set Manager::engines(const metkit::mars::MarsRequest& rq, bool } else { // Match all possible expansions of the first level according to the schema - std::set keys; + std::set keys; config_.schema().matchFirstLevel(rq, keys, ""); std::set expandedKeys; @@ -306,7 +306,7 @@ std::vector Manager::visitableLocations(const metkit::mars::MarsRequ Log::debug() << "Selected FDB engine " << *i << std::endl; std::vector p; if (all) { - p = Engine::backend(*i).visitableLocations(Key(), config_); + p = Engine::backend(*i).visitableLocations(config_); } else { p = Engine::backend(*i).visitableLocations(rq, config_); } @@ -317,22 +317,22 @@ std::vector Manager::visitableLocations(const metkit::mars::MarsRequ } -std::vector Manager::writableLocations(const Key& key) { +// std::vector Manager::writableLocations(const Key& key) { - std::set engines = Manager::engines(key); +// std::set engines = Manager::engines(key); - Log::debug() << "Matching engines for key " << key << " -> " << engines << std::endl; +// Log::debug() << "Matching engines for key " << key << " -> " << engines << std::endl; - std::vector r; // union of all locations +// std::vector r; // union of all locations - for(std::set::const_iterator i = engines.begin(); i != engines.end(); ++i) { - Log::debug() << "Selected FDB engine " << *i << std::endl; - std::vector p = Engine::backend(*i).writableLocations(key, config_); - r.insert(r.end(), p.begin(), p.end()); - } +// for(std::set::const_iterator i = engines.begin(); i != engines.end(); ++i) { +// Log::debug() << "Selected FDB engine " << *i << std::endl; +// std::vector p = Engine::backend(*i).writableLocations(key, config_); +// r.insert(r.end(), p.begin(), p.end()); +// } - return r; -} +// return r; +// } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/Manager.h b/src/fdb5/database/Manager.h index 238e0e196..41e859e16 100644 --- a/src/fdb5/database/Manager.h +++ b/src/fdb5/database/Manager.h @@ -56,8 +56,8 @@ class Manager { /// Lists the roots that can be visited given a DB key std::vector visitableLocations(const metkit::mars::MarsRequest& request, bool all); - /// Lists the roots where a DB key would be able to be written - std::vector writableLocations(const Key& key); + // /// Lists the roots where a DB key would be able to be written + // std::vector writableLocations(const Key& key); private: // members diff --git a/src/fdb5/database/MoveVisitor.h b/src/fdb5/database/MoveVisitor.h index e6efd693d..c6ec2d865 100644 --- a/src/fdb5/database/MoveVisitor.h +++ b/src/fdb5/database/MoveVisitor.h @@ -38,7 +38,7 @@ class MoveVisitor : public EntryVisitor { bool visitEntries() override { return false; } bool visitIndex(const Index&) override { NOTIMP; } - void visitDatum(const Field&, const Key&) override { NOTIMP; } + void visitDatum(const Field&, const InspectionKey&) override { NOTIMP; } void visitDatum(const Field& /*field*/, const std::string& /*keyFingerprint*/) override { NOTIMP; } protected: // members diff --git a/src/fdb5/database/MultiRetrieveVisitor.cc b/src/fdb5/database/MultiRetrieveVisitor.cc index dbee32764..70812bd09 100644 --- a/src/fdb5/database/MultiRetrieveVisitor.cc +++ b/src/fdb5/database/MultiRetrieveVisitor.cc @@ -29,9 +29,8 @@ namespace fdb5 { MultiRetrieveVisitor::MultiRetrieveVisitor(const Notifier& wind, InspectIterator& iterator, - eckit::CacheLRU& databases, + eckit::CacheLRU& databases, const Config& config) : - db_(nullptr), wind_(wind), databases_(databases), iterator_(iterator), @@ -49,8 +48,8 @@ bool MultiRetrieveVisitor::selectDatabase(const Key& key, const Key&) { /* is it the current DB ? */ - if(db_) { - if(key == db_->key()) { + if(catalogue_) { + if(key == catalogue_->key()) { eckit::Log::info() << "This is the current db" << std::endl; return true; } @@ -60,47 +59,46 @@ bool MultiRetrieveVisitor::selectDatabase(const Key& key, const Key&) { if(databases_.exists(key)) { eckit::Log::debug() << "FDB5 Reusing database " << key << std::endl; - db_ = databases_.access(key); + catalogue_ = databases_.access(key); return true; } /* DB not yet open */ - //std::unique_ptr newDB( DBFactory::buildReader(key, config_) ); - std::unique_ptr newDB = DB::buildReader(key, config_); + std::unique_ptr newCatalogue = CatalogueFactory::instance().build(key, config_, true); // If this database is locked for retrieval then it "does not exist" - if (!newDB->enabled(ControlIdentifier::Retrieve)) { + if (!newCatalogue->enabled(ControlIdentifier::Retrieve)) { std::ostringstream ss; - ss << "Database " << *newDB << " is LOCKED for retrieval"; + ss << "Catalogue " << *newCatalogue << " is LOCKED for retrieval"; eckit::Log::warning() << ss.str() << std::endl; return false; } - eckit::Log::debug() << "selectDatabase opening database " << key << " (type=" << newDB->dbType() << ")" << std::endl; + eckit::Log::debug() << "selectDatabase opening database " << key << " (type=" << newCatalogue->type() << ")" << std::endl; - if (!newDB->open()) { + if (!newCatalogue->open()) { eckit::Log::debug() << "Database does not exist " << key << std::endl; return false; } else { - db_ = newDB.release(); - databases_.insert(key, db_); + catalogue_ = newCatalogue.release(); + databases_.insert(key, catalogue_); return true; } } bool MultiRetrieveVisitor::selectIndex(const Key& key, const Key&) { - ASSERT(db_); + ASSERT(catalogue_); eckit::Log::debug() << "selectIndex " << key << std::endl; - return db_->selectIndex(key); + return catalogue_->selectIndex(key); } -bool MultiRetrieveVisitor::selectDatum(const Key& key, const Key& full) { - ASSERT(db_); +bool MultiRetrieveVisitor::selectDatum(const InspectionKey& key, const Key& full) { + ASSERT(catalogue_); eckit::Log::debug() << "selectDatum " << key << ", " << full << std::endl; Field field; - if (db_->inspect(key, field)) { + if (reader()->retrieve(key, field)) { Key simplifiedKey; for (auto k = key.begin(); k != key.end(); k++) { @@ -108,7 +106,7 @@ bool MultiRetrieveVisitor::selectDatum(const Key& key, const Key& full) { simplifiedKey.set(k->first, k->second); } - iterator_.emplace(ListElement({db_->key(), db_->indexKey(), simplifiedKey}, field.stableLocation(), field.timestamp())); + iterator_.emplace(ListElement({catalogue_->key(), catalogue_->indexKey(), simplifiedKey}, field.stableLocation(), field.timestamp())); return true; } @@ -120,12 +118,12 @@ void MultiRetrieveVisitor::values(const metkit::mars::MarsRequest &request, const TypesRegistry ®istry, eckit::StringList &values) { eckit::StringList list; - registry.lookupType(keyword).getValues(request, keyword, list, wind_, db_); + registry.lookupType(keyword).getValues(request, keyword, list, wind_, reader()); eckit::StringSet filter; bool toFilter = false; - if (db_) { - toFilter = db_->axis(keyword, filter); + if (catalogue_) { + toFilter = reader()->axis(keyword, filter); } for(auto l: list) { @@ -141,8 +139,8 @@ void MultiRetrieveVisitor::print( std::ostream &out ) const { } const Schema& MultiRetrieveVisitor::databaseSchema() const { - ASSERT(db_); - return db_->schema(); + ASSERT(catalogue_); + return catalogue_->schema(); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/MultiRetrieveVisitor.h b/src/fdb5/database/MultiRetrieveVisitor.h index 9ee81f62b..788907755 100644 --- a/src/fdb5/database/MultiRetrieveVisitor.h +++ b/src/fdb5/database/MultiRetrieveVisitor.h @@ -41,7 +41,7 @@ class MultiRetrieveVisitor : public ReadVisitor { MultiRetrieveVisitor(const Notifier& wind, InspectIterator& queue, - eckit::CacheLRU& databases, + eckit::CacheLRU& databases, const Config& config); ~MultiRetrieveVisitor(); @@ -54,7 +54,7 @@ class MultiRetrieveVisitor : public ReadVisitor { virtual bool selectIndex(const Key &key, const Key &full) override; - virtual bool selectDatum(const Key &key, const Key &full) override; + virtual bool selectDatum(const InspectionKey &key, const Key &full) override; virtual void values(const metkit::mars::MarsRequest& request, const std::string& keyword, @@ -67,11 +67,9 @@ class MultiRetrieveVisitor : public ReadVisitor { private: - DB* db_; - const Notifier& wind_; - eckit::CacheLRU& databases_; + eckit::CacheLRU& databases_; InspectIterator& iterator_; diff --git a/src/fdb5/database/ReadVisitor.cc b/src/fdb5/database/ReadVisitor.cc index 8c95a00c5..a80a675f9 100644 --- a/src/fdb5/database/ReadVisitor.cc +++ b/src/fdb5/database/ReadVisitor.cc @@ -11,6 +11,7 @@ // #include "eckit/log/Log.h" #include "fdb5/database/ReadVisitor.h" +#include "fdb5/database/Store.h" namespace fdb5 { @@ -20,6 +21,15 @@ namespace fdb5 { ReadVisitor::~ReadVisitor() { } +CatalogueReader* ReadVisitor::reader() const { + if (catalogue_) { + CatalogueReader* catalogueReader = dynamic_cast(catalogue_); + ASSERT(catalogueReader); + return catalogueReader; + } + return nullptr; +} + //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 diff --git a/src/fdb5/database/ReadVisitor.h b/src/fdb5/database/ReadVisitor.h index 5f5d3259e..bfdd229d2 100644 --- a/src/fdb5/database/ReadVisitor.h +++ b/src/fdb5/database/ReadVisitor.h @@ -19,6 +19,7 @@ #include #include "eckit/memory/NonCopyable.h" +#include "fdb5/database/Catalogue.h" #include "eckit/types/Types.h" namespace metkit { @@ -30,7 +31,9 @@ namespace mars { namespace fdb5 { class Key; +class InspectionKey; class TypesRegistry; +class Store; class Schema; //---------------------------------------------------------------------------------------------------------------------- @@ -39,11 +42,13 @@ class ReadVisitor : public eckit::NonCopyable { public: // methods + ReadVisitor() : catalogue_(nullptr) {} + virtual ~ReadVisitor(); virtual bool selectDatabase(const Key &key, const Key &full) = 0; virtual bool selectIndex(const Key &key, const Key &full) = 0; - virtual bool selectDatum(const Key &key, const Key &full) = 0; + virtual bool selectDatum(const InspectionKey &key, const Key &full) = 0; // Once we have selected a database, return its schema. Used for further iteration. virtual const Schema& databaseSchema() const = 0; @@ -55,8 +60,14 @@ class ReadVisitor : public eckit::NonCopyable { protected: // methods + CatalogueReader* reader() const; + virtual void print( std::ostream &out ) const = 0; +protected: // members + + Catalogue* catalogue_; + private: // members friend std::ostream &operator<<(std::ostream &s, const ReadVisitor &x) { diff --git a/src/fdb5/database/RetrieveVisitor.cc b/src/fdb5/database/RetrieveVisitor.cc index 056f58620..9ccb86d8b 100644 --- a/src/fdb5/database/RetrieveVisitor.cc +++ b/src/fdb5/database/RetrieveVisitor.cc @@ -14,7 +14,6 @@ #include "fdb5/LibFdb5.h" #include "fdb5/database/Key.h" -#include "fdb5/database/DB.h" #include "fdb5/io/HandleGatherer.h" #include "fdb5/types/Type.h" #include "fdb5/types/TypesRegistry.h" @@ -25,9 +24,8 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -RetrieveVisitor::RetrieveVisitor(const Notifier &wind, HandleGatherer &gatherer): - wind_(wind), - gatherer_(gatherer) { +RetrieveVisitor::RetrieveVisitor(const Notifier &wind, HandleGatherer &gatherer) : + store_(nullptr), wind_(wind), gatherer_(gatherer) { } RetrieveVisitor::~RetrieveVisitor() { @@ -37,26 +35,25 @@ RetrieveVisitor::~RetrieveVisitor() { bool RetrieveVisitor::selectDatabase(const Key& key, const Key&) { - if(db_) { - if(key == db_->key()) { + if(catalogue_) { + if(key == catalogue_->key()) { return true; } } eckit::Log::debug() << "selectDatabase " << key << std::endl; -// db_.reset(DBFactory::buildReader(key)); - db_ = DB::buildReader(key); + catalogue_ = CatalogueFactory::instance().build(key, fdb5::Config(), true).get(); // If this database is locked for retrieval then it "does not exist" - if (!db_->enabled(ControlIdentifier::Retrieve)) { + if (!catalogue_->enabled(ControlIdentifier::Retrieve)) { std::ostringstream ss; - ss << "Database " << *db_ << " is LOCKED for retrieval"; + ss << "Database " << *catalogue_ << " is LOCKED for retrieval"; eckit::Log::warning() << ss.str() << std::endl; - db_.reset(); + catalogue_ = nullptr; return false; } - if (!db_->open()) { + if (!catalogue_->open()) { eckit::Log::info() << "Database does not exists " << key << std::endl; return false; } else { @@ -65,16 +62,20 @@ bool RetrieveVisitor::selectDatabase(const Key& key, const Key&) { } bool RetrieveVisitor::selectIndex(const Key& key, const Key&) { - ASSERT(db_); + ASSERT(catalogue_); // eckit::Log::info() << "selectIndex " << key << std::endl; - return db_->selectIndex(key); + return catalogue_->selectIndex(key); } -bool RetrieveVisitor::selectDatum(const Key& key, const Key&) { - ASSERT(db_); +bool RetrieveVisitor::selectDatum(const InspectionKey& key, const Key&) { + ASSERT(catalogue_); // eckit::Log::info() << "selectDatum " << key << ", " << full << std::endl; - eckit::DataHandle *dh = db_->retrieve(key); + Field field; + eckit::DataHandle *dh = nullptr; + if (reader()->retrieve(key, field)) { + dh = store().retrieve(field); + } if (dh) { gatherer_.add(dh); @@ -88,12 +89,12 @@ void RetrieveVisitor::values(const metkit::mars::MarsRequest &request, const TypesRegistry ®istry, eckit::StringList &values) { eckit::StringList list; - registry.lookupType(keyword).getValues(request, keyword, list, wind_, db_.get()); + registry.lookupType(keyword).getValues(request, keyword, list, wind_, reader()); eckit::StringSet filter; bool toFilter = false; - if (db_) { - toFilter = db_->axis(keyword, filter); + if (catalogue_) { + toFilter = reader()->axis(keyword, filter); } for(auto l: list) { @@ -104,13 +105,23 @@ void RetrieveVisitor::values(const metkit::mars::MarsRequest &request, } } + +Store& RetrieveVisitor::store() { + if (!store_) { + ASSERT(catalogue_); + store_ = catalogue_->buildStore(); + ASSERT(store_); + } + return *store_; +} + void RetrieveVisitor::print( std::ostream &out ) const { out << "RetrieveVisitor[]"; } const Schema& RetrieveVisitor::databaseSchema() const { - ASSERT(db_); - return db_->schema(); + ASSERT(catalogue_); + return catalogue_->schema(); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/RetrieveVisitor.h b/src/fdb5/database/RetrieveVisitor.h index 805d6a60b..dd1a2cf64 100644 --- a/src/fdb5/database/RetrieveVisitor.h +++ b/src/fdb5/database/RetrieveVisitor.h @@ -25,8 +25,6 @@ namespace fdb5 { class HandleGatherer; class Notifier; -class DB; - //---------------------------------------------------------------------------------------------------------------------- class RetrieveVisitor : public ReadVisitor { @@ -46,7 +44,7 @@ class RetrieveVisitor : public ReadVisitor { virtual bool selectIndex(const Key &key, const Key &full) override; - virtual bool selectDatum(const Key &key, const Key &full) override; + virtual bool selectDatum(const InspectionKey &key, const Key &full) override; virtual void values(const metkit::mars::MarsRequest& request, const std::string& keyword, @@ -55,13 +53,14 @@ class RetrieveVisitor : public ReadVisitor { virtual void print( std::ostream &out ) const override; + Store& store(); virtual const Schema& databaseSchema() const override; private: - const Notifier &wind_; + std::unique_ptr store_; - std::unique_ptr db_; + const Notifier &wind_; HandleGatherer &gatherer_; }; diff --git a/src/fdb5/database/WipeVisitor.h b/src/fdb5/database/WipeVisitor.h index 272e15931..e4bcb5582 100644 --- a/src/fdb5/database/WipeVisitor.h +++ b/src/fdb5/database/WipeVisitor.h @@ -37,7 +37,7 @@ class WipeVisitor : public EntryVisitor { ~WipeVisitor() override; bool visitEntries() override { return false; } - void visitDatum(const Field&, const Key&) override { NOTIMP; } + void visitDatum(const Field&, const InspectionKey&) override { NOTIMP; } void visitDatum(const Field& /*field*/, const std::string& /*keyFingerprint*/) override { NOTIMP; } protected: // members diff --git a/src/fdb5/database/WriteVisitor.h b/src/fdb5/database/WriteVisitor.h index a3798a905..12f551932 100644 --- a/src/fdb5/database/WriteVisitor.h +++ b/src/fdb5/database/WriteVisitor.h @@ -43,7 +43,7 @@ class WriteVisitor : public eckit::NonCopyable { virtual bool selectDatabase(const Key &key, const Key &full) = 0; virtual bool selectIndex(const Key &key, const Key &full) = 0; - virtual bool selectDatum(const Key &key, const Key &full) = 0; + virtual bool selectDatum(const InspectionKey &key, const Key &full) = 0; // Once we have selected a database, return its schema. Used for further iteration. virtual const Schema& databaseSchema() const = 0; diff --git a/src/fdb5/pmem/PMemDB.cc b/src/fdb5/pmem/PMemDB.cc index 1188c52b6..165edc7d7 100644 --- a/src/fdb5/pmem/PMemDB.cc +++ b/src/fdb5/pmem/PMemDB.cc @@ -111,7 +111,7 @@ bool PMemDB::exists() const { return Pool::exists(poolDir_); } -void PMemDB::archive(const Key &key, const void *data, Length length) { +void PMemDB::archive(const InspectionKey &key, const void *data, Length length) { Log::error() << "archive not implemented for " << *this << std::endl; NOTIMP; } @@ -188,7 +188,7 @@ void PMemDB::dump(std::ostream& out, bool simple) const { struct DumpVisitor : EntryVisitor { DumpVisitor(std::ostream& out) : out_(out) {} - void visitDatum(const Field& field, const Key& key) override { + void visitDatum(const Field& field, const InspectionKey& key) override { out_ << "ENTRY" << std::endl; out_ << " " << key << std::endl; field.location().dump(out_); diff --git a/src/fdb5/pmem/PMemDB.h b/src/fdb5/pmem/PMemDB.h index 8a4667a1c..f6d9dddae 100644 --- a/src/fdb5/pmem/PMemDB.h +++ b/src/fdb5/pmem/PMemDB.h @@ -78,7 +78,7 @@ class PMemDB : public DB { const Schema& schema() const override; virtual eckit::DataHandle *retrieve(const Key &key) const; - virtual void archive(const Key &key, const void *data, eckit::Length length); + virtual void archive(const InspectionKey &key, const void *data, eckit::Length length); virtual void axis(const std::string &keyword, eckit::StringSet &s) const; virtual StatsReportVisitor* statsReportVisitor() const override; diff --git a/src/fdb5/pmem/PMemDBWriter.cc b/src/fdb5/pmem/PMemDBWriter.cc index 1e1c8ed04..e55bb6d42 100644 --- a/src/fdb5/pmem/PMemDBWriter.cc +++ b/src/fdb5/pmem/PMemDBWriter.cc @@ -61,7 +61,7 @@ void PMemDBWriter::close() { indexes_.clear(); } -void PMemDBWriter::archive(const Key &key, const void *data, Length length) { +void PMemDBWriter::archive(const InspectionKey &key, const void *data, Length length) { // Get the key:value identifier associated with this key diff --git a/src/fdb5/pmem/PMemDBWriter.h b/src/fdb5/pmem/PMemDBWriter.h index 54c74ab24..19ee00e81 100644 --- a/src/fdb5/pmem/PMemDBWriter.h +++ b/src/fdb5/pmem/PMemDBWriter.h @@ -40,7 +40,7 @@ class PMemDBWriter : public PMemDB { virtual bool selectIndex(const Key &key); virtual void close(); - virtual void archive(const Key &key, const void *data, eckit::Length length); + virtual void archive(const InspectionKey &key, const void *data, eckit::Length length); private: // methods diff --git a/src/fdb5/pmem/PMemStats.h b/src/fdb5/pmem/PMemStats.h index d3bceaba5..018cf3f58 100644 --- a/src/fdb5/pmem/PMemStats.h +++ b/src/fdb5/pmem/PMemStats.h @@ -162,7 +162,7 @@ class PMemStatsReportVisitor : public virtual StatsReportVisitor { bool visitDatabase(const DB& db) override; void visitDatum(const Field& field, const std::string& keyFingerprint) override; - void visitDatum(const Field& field, const Key& key) override { NOTIMP; } + void visitDatum(const Field& field, const InspectionKey& key) override { NOTIMP; } // This visitor is only legit for one DB - so don't reset database void databaseComplete(const DB& db) override; diff --git a/src/fdb5/remote/CatalogueHandler.cc b/src/fdb5/remote/CatalogueHandler.cc index f466e1bad..2171f06f8 100644 --- a/src/fdb5/remote/CatalogueHandler.cc +++ b/src/fdb5/remote/CatalogueHandler.cc @@ -221,7 +221,7 @@ template void CatalogueHandler::forwardApiCall(const MessageHeader& hdr) { HelperClass helper; - std::cout << "CatalogueHandler::forwardApiCall" << std::endl; +// std::cout << "CatalogueHandler::forwardApiCall" << std::endl; Buffer payload(receivePayload(hdr, controlSocket_)); MemoryStream s(payload); diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc index c36a608ca..1590a462e 100644 --- a/src/fdb5/remote/RemoteStore.cc +++ b/src/fdb5/remote/RemoteStore.cc @@ -743,18 +743,6 @@ void RemoteStore::sendArchiveData(uint32_t id, const Key& key, const void* data, Buffer keyBuffer(4096); MemoryStream keyStream(keyBuffer); - if (!key_.rule()) { - std::cout << "RemoteStore::sendArchiveData - missing rule in " << key_ << std::endl; - } else { - std::cout << "Good so far... RemoteStore::sendArchiveData - existing rule in " << key_ << std::endl; - - } - if (!key.rule()) { - std::cout << "RemoteStore::sendArchiveData - missing rule in " << key << std::endl; - } else { - std::cout << "Good so far... RemoteStore::sendArchiveData - existing rule in " << key << std::endl; - } - keyStream << key_; keyStream << key; diff --git a/src/fdb5/remote/StoreHandler.cc b/src/fdb5/remote/StoreHandler.cc index 8fec55dbe..3a04e5198 100644 --- a/src/fdb5/remote/StoreHandler.cc +++ b/src/fdb5/remote/StoreHandler.cc @@ -202,7 +202,7 @@ void StoreHandler::writeToParent(const uint32_t requestID, std::unique_ptr(data); // To allow pointer arithmetic Log::status() << "Archiving data: " << ss_key.str() << std::endl; - std::cout << "StoreHandler::archiveBlobPayload - Archiving data: " << ss_key.str() << " - size: " << length << std::endl; Store& ss = store(dbKey); - std::cout << "StoreHandler::archiveBlobPayload - Created store " << dbKey << std::endl; auto futureLocation = ss.archive(idxKey, charData + s.position(), length - s.position()); std::cout << "StoreHandler::archiveBlobPayload - Archiving done " << std::endl; diff --git a/src/fdb5/remote/StoreHandler.h b/src/fdb5/remote/StoreHandler.h index 74681b215..bcfc20bb7 100644 --- a/src/fdb5/remote/StoreHandler.h +++ b/src/fdb5/remote/StoreHandler.h @@ -30,7 +30,7 @@ class StoreHandler : public DecoupledHandler { void readLocationThreadLoop(); void writeToParent(const uint32_t requestID, std::unique_ptr dh); - void store(const MessageHeader& hdr) override; + void archive(const MessageHeader& hdr) override; size_t archiveThreadLoop(uint32_t id); void archiveBlobPayload(uint32_t id, const void* data, size_t length); diff --git a/src/fdb5/rules/Rule.cc b/src/fdb5/rules/Rule.cc index 8a1f9aac1..c5d72c6dd 100644 --- a/src/fdb5/rules/Rule.cc +++ b/src/fdb5/rules/Rule.cc @@ -21,6 +21,7 @@ #include "fdb5/database/ReadVisitor.h" #include "fdb5/database/WriteVisitor.h" #include "fdb5/types/Type.h" +#include "fdb5/database/InspectionKey.h" namespace fdb5 { @@ -92,7 +93,7 @@ Rule::~Rule() { void Rule::expand( const metkit::mars::MarsRequest &request, std::vector::const_iterator cur, size_t depth, - std::vector &keys, + std::vector &keys, Key &full, ReadVisitor &visitor) const { @@ -170,15 +171,15 @@ void Rule::expand( const metkit::mars::MarsRequest &request, } -void Rule::expand(const metkit::mars::MarsRequest &request, ReadVisitor &visitor, size_t depth, std::vector &keys, Key &full) const { +void Rule::expand(const metkit::mars::MarsRequest &request, ReadVisitor &visitor, size_t depth, std::vector &keys, Key &full) const { ASSERT(keys.size() == 3); expand(request, predicates_.begin(), depth, keys, full, visitor); } -void Rule::expand( const Key &field, +void Rule::expand( const Key &dataKey, std::vector::const_iterator cur, size_t depth, - std::vector &keys, + std::vector &keys, Key &full, WriteVisitor &visitor) const { @@ -220,7 +221,7 @@ void Rule::expand( const Key &field, } // Here we recurse on the database's schema (rather than the master schema) - visitor.databaseSchema().expandSecond(field, visitor, keys[0]); + visitor.databaseSchema().expandSecond(dataKey, visitor, keys[0]); return; case 1: @@ -236,7 +237,7 @@ void Rule::expand( const Key &field, } for (std::vector::const_iterator i = rules_.begin(); i != rules_.end(); ++i ) { - (*i)->expand(field, visitor, depth + 1, keys, full); + (*i)->expand(dataKey, visitor, depth + 1, keys, full); } } return; @@ -246,7 +247,7 @@ void Rule::expand( const Key &field, ++next; const std::string &keyword = (*cur)->keyword(); - const std::string &value = (*cur)->value(field); + const std::string &value = (*cur)->value(dataKey); Key &k = keys[depth]; @@ -254,20 +255,19 @@ void Rule::expand( const Key &field, full.push(keyword, value); if ((*cur)->match(k)) { - expand(field, next, depth, keys, full, visitor); + expand(dataKey, next, depth, keys, full, visitor); } full.pop(keyword); k.pop(keyword); - - } -void Rule::expand(const Key &field, WriteVisitor &visitor, size_t depth, std::vector &keys, Key &full) const { + +void Rule::expand(const Key &dataKey, WriteVisitor &visitor, size_t depth, std::vector &keys, Key &full) const { ASSERT(keys.size() == 3); - expand(field, predicates_.begin(), depth, keys, full, visitor); + expand(dataKey, predicates_.begin(), depth, keys, full, visitor); } -void Rule::expandFirstLevel( const Key &dbKey, std::vector::const_iterator cur, Key &result, bool& found) const { +void Rule::expandFirstLevel( const Key &dbKey, std::vector::const_iterator cur, InspectionKey &result, bool& found) const { if (cur == predicates_.end()) { found = true; @@ -292,11 +292,11 @@ void Rule::expandFirstLevel( const Key &dbKey, std::vector::const_i } } -void Rule::expandFirstLevel(const Key &dbKey, Key &result, bool& found) const { +void Rule::expandFirstLevel(const Key &dbKey, InspectionKey &result, bool& found) const { expandFirstLevel(dbKey, predicates_.begin(), result, found); } -void Rule::expandFirstLevel(const metkit::mars::MarsRequest& rq, std::vector::const_iterator cur, Key& result, bool& found) const { +void Rule::expandFirstLevel(const metkit::mars::MarsRequest& rq, std::vector::const_iterator cur, InspectionKey& result, bool& found) const { if (cur == predicates_.end()) { found = true; @@ -329,12 +329,12 @@ void Rule::expandFirstLevel(const metkit::mars::MarsRequest& rq, std::vector
::const_iterator cur, Key& tmp, std::set& result, const char* missing) const {
+void Rule::matchFirstLevel( const Key &dbKey, std::vector::const_iterator cur, InspectionKey& tmp, std::set& result, const char* missing) const {
 
     if (cur == predicates_.end()) {
         if (tmp.match(dbKey)) {
@@ -365,13 +365,13 @@ void Rule::matchFirstLevel( const Key &dbKey, std::vector::const_it
 
 }
 
-void Rule::matchFirstLevel(const Key &dbKey,  std::set& result, const char* missing) const {
-    Key tmp;
+void Rule::matchFirstLevel(const Key &dbKey,  std::set& result, const char* missing) const {
+    InspectionKey tmp;
     matchFirstLevel(dbKey, predicates_.begin(), tmp, result, missing);
 }
 
 
-void Rule::matchFirstLevel(const metkit::mars::MarsRequest& request, std::vector::const_iterator cur, Key& tmp, std::set& result, const char* missing) const {
+void Rule::matchFirstLevel(const metkit::mars::MarsRequest& request, std::vector::const_iterator cur, InspectionKey& tmp, std::set& result, const char* missing) const {
 
     if (cur == predicates_.end()) {
 //        if (tmp.match(request)) {
@@ -403,8 +403,8 @@ void Rule::matchFirstLevel(const metkit::mars::MarsRequest& request, std::vector
     }
 }
 
-void Rule::matchFirstLevel(const metkit::mars::MarsRequest& request,  std::set& result, const char* missing) const {
-    Key tmp;
+void Rule::matchFirstLevel(const metkit::mars::MarsRequest& request,  std::set& result, const char* missing) const {
+    InspectionKey tmp;
     matchFirstLevel(request, predicates_.begin(), tmp, result, missing);
 }
 
diff --git a/src/fdb5/rules/Rule.h b/src/fdb5/rules/Rule.h
index 5986fbece..a729a223a 100644
--- a/src/fdb5/rules/Rule.h
+++ b/src/fdb5/rules/Rule.h
@@ -37,6 +37,7 @@ class Predicate;
 class ReadVisitor;
 class WriteVisitor;
 class Key;
+class InspectionKey;
 
 //----------------------------------------------------------------------------------------------------------------------
 
@@ -65,13 +66,13 @@ class Rule : public eckit::Streamable {
     void expand(const metkit::mars::MarsRequest &request,
                 ReadVisitor &Visitor,
                 size_t depth,
-                std::vector &keys,
+                std::vector &keys,
                 Key &full) const;
 
     void expand(const Key &field,
                 WriteVisitor &Visitor,
                 size_t depth,
-                std::vector &keys,
+                std::vector &keys,
                 Key &full) const;
 
     const Rule* ruleFor(const std::vector &keys, size_t depth) const;
@@ -96,26 +97,26 @@ class Rule : public eckit::Streamable {
     void expand(const metkit::mars::MarsRequest &request,
                 std::vector::const_iterator cur,
                 size_t depth,
-                std::vector &keys,
+                std::vector &keys,
                 Key &full,
                 ReadVisitor &Visitor) const;
 
     void expand(const Key &field,
                 std::vector::const_iterator cur,
                 size_t depth,
-                std::vector &keys,
+                std::vector &keys,
                 Key &full,
                 WriteVisitor &Visitor) const;
 
-    void expandFirstLevel(const Key &dbKey, std::vector::const_iterator cur, Key &result, bool& done) const;
-    void expandFirstLevel(const Key &dbKey,  Key &result, bool& done) const ;
-    void expandFirstLevel(const metkit::mars::MarsRequest& request, std::vector::const_iterator cur, Key& result, bool& done) const;
-    void expandFirstLevel(const metkit::mars::MarsRequest& request,  Key& result, bool& done) const;
+    void expandFirstLevel(const Key &dbKey, std::vector::const_iterator cur, InspectionKey &result, bool& done) const;
+    void expandFirstLevel(const Key &dbKey,  InspectionKey &result, bool& done) const ;
+    void expandFirstLevel(const metkit::mars::MarsRequest& request, std::vector::const_iterator cur, InspectionKey& result, bool& done) const;
+    void expandFirstLevel(const metkit::mars::MarsRequest& request,  InspectionKey& result, bool& done) const;
 
-    void matchFirstLevel(const Key &dbKey, std::vector::const_iterator cur, Key &tmp, std::set& result, const char* missing) const;
-    void matchFirstLevel(const Key &dbKey, std::set& result, const char* missing) const ;
-    void matchFirstLevel(const metkit::mars::MarsRequest& request, std::vector::const_iterator cur, Key &tmp, std::set& result, const char* missing) const;
-    void matchFirstLevel(const metkit::mars::MarsRequest& request, std::set& result, const char* missing) const ;
+    void matchFirstLevel(const Key &dbKey, std::vector::const_iterator cur, InspectionKey &tmp, std::set& result, const char* missing) const;
+    void matchFirstLevel(const Key &dbKey, std::set& result, const char* missing) const ;
+    void matchFirstLevel(const metkit::mars::MarsRequest& request, std::vector::const_iterator cur, InspectionKey &tmp, std::set& result, const char* missing) const;
+    void matchFirstLevel(const metkit::mars::MarsRequest& request, std::set& result, const char* missing) const ;
 
 
     void keys(size_t level, size_t depth, eckit::StringList &result, eckit::StringSet &seen) const;
diff --git a/src/fdb5/rules/Schema.cc b/src/fdb5/rules/Schema.cc
index f66d073d2..8954b9af1 100644
--- a/src/fdb5/rules/Schema.cc
+++ b/src/fdb5/rules/Schema.cc
@@ -77,7 +77,7 @@ const Rule*  Schema::ruleFor(const Key& dbKey, const Key& idxKey) const {
 
 void Schema::expand(const metkit::mars::MarsRequest &request, ReadVisitor &visitor) const {
     Key full;
-    std::vector keys(3);
+    std::vector keys(3);
 
     for (const Rule* rule : rules_) {
 		// eckit::Log::info() << "Rule " << **i <<  std::endl;
@@ -86,14 +86,14 @@ void Schema::expand(const metkit::mars::MarsRequest &request, ReadVisitor &visit
     }
 }
 
-void Schema::expand(const Key &field, WriteVisitor &visitor) const {
+void Schema::expand(const Key &dataKey, WriteVisitor &visitor) const {
     Key full;
-    std::vector keys(3);
+    std::vector keys(3);
 
     visitor.rule(0); // reset to no rule so we verify that we pick at least one
 
     for (const Rule* rule : rules_) {
-        rule->expand(field, visitor, 0, keys, full);
+        rule->expand(dataKey, visitor, 0, keys, full);
     }
 }
 
@@ -109,8 +109,8 @@ void Schema::expandSecond(const metkit::mars::MarsRequest& request, ReadVisitor&
     ASSERT(dbRule);
 
     Key full = dbKey;
-    std::vector keys(3);
-    keys[0] = dbKey;
+    std::vector keys(3);
+    keys[0] = InspectionKey(dbKey);
 
     for (const Rule* rule : dbRule->rules_) {
         rule->expand(request, visitor, 1, keys, full);
@@ -129,15 +129,15 @@ void Schema::expandSecond(const Key& field, WriteVisitor& visitor, const Key& db
     ASSERT(dbRule);
 
     Key full = dbKey;
-    std::vector keys(3);
-    keys[0] = dbKey;
+    std::vector keys(3);
+    keys[0] = InspectionKey(dbKey);
 
     for (const Rule* rule : dbRule->rules_) {
         rule->expand(field, visitor, 1, keys, full);
     }
 }
 
-bool Schema::expandFirstLevel(const Key &dbKey,  Key &result) const {
+bool Schema::expandFirstLevel(const Key &dbKey,  InspectionKey &result) const {
     bool found = false;
     for (const Rule* rule : rules_) {
         rule->expandFirstLevel(dbKey, result, found);
@@ -146,7 +146,7 @@ bool Schema::expandFirstLevel(const Key &dbKey,  Key &result) const {
     return found;
 }
 
-bool Schema::expandFirstLevel(const metkit::mars::MarsRequest& request, Key &result) const {
+bool Schema::expandFirstLevel(const metkit::mars::MarsRequest& request, InspectionKey &result) const {
     bool found = false;
     for (const Rule* rule : rules_) {
         rule->expandFirstLevel(request, result, found);
@@ -155,13 +155,13 @@ bool Schema::expandFirstLevel(const metkit::mars::MarsRequest& request, Key &res
     return found;
 }
 
-void Schema::matchFirstLevel(const Key &dbKey,  std::set &result, const char* missing) const {
+void Schema::matchFirstLevel(const Key &dbKey,  std::set &result, const char* missing) const {
     for (const Rule* rule : rules_) {
         rule->matchFirstLevel(dbKey, result, missing);
     }
 }
 
-void Schema::matchFirstLevel(const metkit::mars::MarsRequest& request,  std::set& result, const char* missing) const {
+void Schema::matchFirstLevel(const metkit::mars::MarsRequest& request,  std::set& result, const char* missing) const {
     for (const Rule* rule : rules_) {
         rule->matchFirstLevel(request, result, missing);
     }
diff --git a/src/fdb5/rules/Schema.h b/src/fdb5/rules/Schema.h
index 035d67118..94143de09 100644
--- a/src/fdb5/rules/Schema.h
+++ b/src/fdb5/rules/Schema.h
@@ -59,10 +59,10 @@ class Schema : public eckit::Streamable {
     void expandSecond(const Key& field, WriteVisitor &visitor, const Key& dbKey) const;
     void expandSecond(const metkit::mars::MarsRequest& request, ReadVisitor &visitor, const Key& dbKey) const;
 
-    bool expandFirstLevel(const Key &dbKey,  Key &result) const ;
-    bool expandFirstLevel(const metkit::mars::MarsRequest& request,  Key& result) const ;
-    void matchFirstLevel(const Key &dbKey,  std::set &result, const char* missing) const ;
-    void matchFirstLevel(const metkit::mars::MarsRequest& request,  std::set& result, const char* missing) const ;
+    bool expandFirstLevel(const Key &dbKey,  InspectionKey &result) const ;
+    bool expandFirstLevel(const metkit::mars::MarsRequest& request,  InspectionKey& result) const ;
+    void matchFirstLevel(const Key &dbKey,  std::set &result, const char* missing) const ;
+    void matchFirstLevel(const metkit::mars::MarsRequest& request,  std::set& result, const char* missing) const ;
 
     const Rule* ruleFor(const Key &dbKey, const Key& idxKey) const;
 
diff --git a/src/fdb5/toc/AdoptVisitor.cc b/src/fdb5/toc/AdoptVisitor.cc
index 3aff5340a..ad661f259 100644
--- a/src/fdb5/toc/AdoptVisitor.cc
+++ b/src/fdb5/toc/AdoptVisitor.cc
@@ -30,7 +30,7 @@ AdoptVisitor::AdoptVisitor(Archiver &owner, const Key &field, const PathName &pa
     ASSERT(length_ > Length(0));
 }
 
-bool AdoptVisitor::selectDatum(const Key &key, const Key &full) {
+bool AdoptVisitor::selectDatum(const InspectionKey &key, const Key &full) {
 
     // Log::info() << "selectDatum " << key << ", " << full << " " << length_ << std::endl;
     checkMissingKeys(full);
diff --git a/src/fdb5/toc/AdoptVisitor.h b/src/fdb5/toc/AdoptVisitor.h
index 65d8d05fa..78a4cb609 100644
--- a/src/fdb5/toc/AdoptVisitor.h
+++ b/src/fdb5/toc/AdoptVisitor.h
@@ -40,7 +40,7 @@ class AdoptVisitor : public BaseArchiveVisitor {
 
 protected: // methods
 
-    virtual bool selectDatum(const Key &key, const Key &full) override;
+    virtual bool selectDatum(const InspectionKey &key, const Key &full) override;
 
     virtual void print( std::ostream &out ) const override;
 
diff --git a/src/fdb5/toc/RootManager.cc b/src/fdb5/toc/RootManager.cc
index ae2408b0e..ca925d47f 100644
--- a/src/fdb5/toc/RootManager.cc
+++ b/src/fdb5/toc/RootManager.cc
@@ -575,18 +575,13 @@ std::string RootManager::dbPathName(const Key& key)
         }
     }
 
-    if (!key.rule()) {
-        std::cout << "RootManager::dbPathName - missing rule in " << key << std::endl;
-    }
-
-
     // default naming convention for DB's
     dbpath = key.valuesToString();
     eckit::Log::debug() << "Using default naming convention for key " << key << " -> " << dbpath <<  std::endl;
     return dbpath;
 }
 
-std::vector RootManager::possibleDbPathNames(const Key& key, const char* missing)
+std::vector RootManager::possibleDbPathNames(const InspectionKey& key, const char* missing)
 {
     std::vector result;
     for (DbPathNamerTable::const_iterator i = dbPathNamers_.begin(); i != dbPathNamers_.end() ; ++i) {
@@ -662,7 +657,7 @@ std::vector RootManager::allRoots(const Key& key)
     return std::vector(roots.begin(), roots.end());
 }
 
-std::vector RootManager::visitableRoots(const std::set& keys) {
+std::vector RootManager::visitableRoots(const std::set& keys) {
 
     eckit::StringSet roots;
 
@@ -693,15 +688,15 @@ std::vector RootManager::visitableRoots(const std::set& keys) {
 }
 
 
-std::vector RootManager::visitableRoots(const Key& key) {
-    return visitableRoots(std::set{ key });
+std::vector RootManager::visitableRoots(const InspectionKey& key) {
+    return visitableRoots(std::set{ key });
 }
 
 std::vector RootManager::visitableRoots(const metkit::mars::MarsRequest& request) {
 
 //    Key key;
 //    config_.schema().expandFirstLevel(request, key);
-    std::set keys;
+    std::set keys;
     config_.schema().matchFirstLevel(request, keys, "");
     return visitableRoots(keys);
 }
diff --git a/src/fdb5/toc/RootManager.h b/src/fdb5/toc/RootManager.h
index 4e2dab558..da3ce89f2 100644
--- a/src/fdb5/toc/RootManager.h
+++ b/src/fdb5/toc/RootManager.h
@@ -52,8 +52,8 @@ class RootManager  {
     std::vector allRoots(const Key& key);
 
     /// Lists the roots that can be visited given a DB key
-    std::vector visitableRoots(const Key& key);
-    std::vector visitableRoots(const std::set& keys);
+    std::vector visitableRoots(const InspectionKey& key);
+    std::vector visitableRoots(const std::set& keys);
     std::vector visitableRoots(const metkit::mars::MarsRequest& request);
 
     /// Lists the roots where a DB key would be able to be written
@@ -62,7 +62,7 @@ class RootManager  {
 
     std::string dbPathName(const Key& key);
 
-    std::vector possibleDbPathNames(const Key& key, const char* missing);
+    std::vector possibleDbPathNames(const InspectionKey& key, const char* missing);
 
 protected: // methods
 
diff --git a/src/fdb5/toc/TocCatalogue.cc b/src/fdb5/toc/TocCatalogue.cc
index 9241d1eb4..280d61fc9 100644
--- a/src/fdb5/toc/TocCatalogue.cc
+++ b/src/fdb5/toc/TocCatalogue.cc
@@ -26,11 +26,13 @@ namespace fdb5 {
 //----------------------------------------------------------------------------------------------------------------------
 
 TocCatalogue::TocCatalogue(const Key& key, const fdb5::Config& config) :
-    TocCatalogue(key, CatalogueRootManager(config).directory(key), config) {}
+    TocCatalogue(key, CatalogueRootManager(config).directory(key), config) {
+}
 
 TocCatalogue::TocCatalogue(const Key& key, const TocPath& tocPath, const fdb5::Config& config) :
     Catalogue(key, tocPath.controlIdentifiers_, config),
-    TocHandler(tocPath.directory_, config) {}
+    TocHandler(tocPath.directory_, config) {
+}
 
 TocCatalogue::TocCatalogue(const eckit::PathName& directory, const ControlIdentifiers& controlIdentifiers, const fdb5::Config& config) :
     Catalogue(Key(), controlIdentifiers, config),
diff --git a/src/fdb5/toc/TocCatalogue.h b/src/fdb5/toc/TocCatalogue.h
index 2ad22112c..d5e593267 100644
--- a/src/fdb5/toc/TocCatalogue.h
+++ b/src/fdb5/toc/TocCatalogue.h
@@ -33,7 +33,7 @@ class TocCatalogue : public Catalogue, public TocHandler {
 
 public: // methods
 
-    TocCatalogue(const Key& key, const fdb5::Config& config);
+    TocCatalogue(const Key& dbKey, const fdb5::Config& config);
     TocCatalogue(const eckit::PathName& directory, const ControlIdentifiers& controlIdentifiers, const fdb5::Config& config);
 
     ~TocCatalogue() override {}
@@ -52,7 +52,7 @@ class TocCatalogue : public Catalogue, public TocHandler {
 
 protected: // methods
 
-    TocCatalogue(const Key& key, const TocPath& tocPath, const fdb5::Config& config);
+    TocCatalogue(const Key& dbKey, const TocPath& tocPath, const fdb5::Config& config);
 
     std::string type() const override;
 
diff --git a/src/fdb5/toc/TocCatalogueReader.cc b/src/fdb5/toc/TocCatalogueReader.cc
index 0dd6b1119..fd04738f8 100644
--- a/src/fdb5/toc/TocCatalogueReader.cc
+++ b/src/fdb5/toc/TocCatalogueReader.cc
@@ -21,8 +21,8 @@ namespace fdb5 {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-TocCatalogueReader::TocCatalogueReader(const Key& key, const fdb5::Config& config) :
-    TocCatalogue(key, config) {
+TocCatalogueReader::TocCatalogueReader(const Key& dbKey, const fdb5::Config& config) :
+    TocCatalogue(dbKey, config) {
     loadIndexesAndRemap();
 }
 
@@ -46,22 +46,22 @@ void TocCatalogueReader::loadIndexesAndRemap() {
     }
 }
 
-bool TocCatalogueReader::selectIndex(const Key &key) {
+bool TocCatalogueReader::selectIndex(const Key &idxKey) {
 
-    if(currentIndexKey_ == key) {
+    if(currentIndexKey_ == idxKey) {
         return true;
     }
 
-    currentIndexKey_ = key;
+    currentIndexKey_ = idxKey;
     matching_.clear();
 
     for (auto idx = indexes_.begin(); idx != indexes_.end(); ++idx) {
-        if (idx->first.key() == key) {
+        if (idx->first.key() == idxKey) {
             matching_.push_back(&(*idx));
         }
     }
 
-    eckit::Log::debug() << "TocCatalogueReader::selectIndex " << key << ", found "
+    eckit::Log::debug() << "TocCatalogueReader::selectIndex " << idxKey << ", found "
                                 << matching_.size() << " matche(s)" << std::endl;
 
     return (matching_.size() != 0);
@@ -103,7 +103,7 @@ void TocCatalogueReader::close() {
     }
 }
 
-bool TocCatalogueReader::retrieve(const Key& key, Field& field) const {
+bool TocCatalogueReader::retrieve(const InspectionKey& key, Field& field) const {
     eckit::Log::debug() << "Trying to retrieve key " << key << std::endl;
     eckit::Log::debug() << "Scanning indexes " << matching_.size() << std::endl;
 
diff --git a/src/fdb5/toc/TocCatalogueReader.h b/src/fdb5/toc/TocCatalogueReader.h
index 6b7cd025e..d0eef481b 100644
--- a/src/fdb5/toc/TocCatalogueReader.h
+++ b/src/fdb5/toc/TocCatalogueReader.h
@@ -28,7 +28,7 @@ class TocCatalogueReader : public TocCatalogue, public CatalogueReader {
 
 public: // methods
 
-    TocCatalogueReader(const Key& key, const fdb5::Config& config);
+    TocCatalogueReader(const Key& dbKey, const fdb5::Config& config);
     TocCatalogueReader(const eckit::URI& uri, const fdb5::Config& config);
 
     ~TocCatalogueReader() override;
@@ -39,7 +39,7 @@ class TocCatalogueReader : public TocCatalogue, public CatalogueReader {
 private: // methods
 
     void loadIndexesAndRemap();
-    bool selectIndex(const Key &key) override;
+    bool selectIndex(const Key &idxKey) override;
     void deselectIndex() override;
 
     bool open() override;
@@ -49,7 +49,7 @@ class TocCatalogueReader : public TocCatalogue, public CatalogueReader {
     
     bool axis(const std::string &keyword, eckit::StringSet &s) const override;
 
-    bool retrieve(const Key& key, Field& field) const override;
+    bool retrieve(const InspectionKey& key, Field& field) const override;
 
     void print( std::ostream &out ) const override;
 
diff --git a/src/fdb5/toc/TocCatalogueWriter.cc b/src/fdb5/toc/TocCatalogueWriter.cc
index 71c97d128..e90c2f3c2 100644
--- a/src/fdb5/toc/TocCatalogueWriter.cc
+++ b/src/fdb5/toc/TocCatalogueWriter.cc
@@ -30,10 +30,10 @@ namespace fdb5 {
 //----------------------------------------------------------------------------------------------------------------------
 
 
-TocCatalogueWriter::TocCatalogueWriter(const Key &key, const fdb5::Config& config) :
-    TocCatalogue(key, config),
+TocCatalogueWriter::TocCatalogueWriter(const Key &dbKey, const fdb5::Config& config) :
+    TocCatalogue(dbKey, config),
     umask_(config.umask()) {
-    writeInitRecord(key);
+    writeInitRecord(dbKey);
     TocCatalogue::loadSchema();
     TocCatalogue::checkUID();
 }
@@ -51,21 +51,21 @@ TocCatalogueWriter::~TocCatalogueWriter() {
     close();
 }
 
-bool TocCatalogueWriter::selectIndex(const Key& key) {
-    currentIndexKey_ = key;
+bool TocCatalogueWriter::selectIndex(const Key& idxKey) {
+    currentIndexKey_ = idxKey;
 
-    if (indexes_.find(key) == indexes_.end()) {
-        PathName indexPath(generateIndexPath(key));
+    if (indexes_.find(idxKey) == indexes_.end()) {
+        PathName indexPath(generateIndexPath(idxKey));
 
         // Enforce lustre striping if requested
         if (stripeLustre()) {
             fdb5LustreapiFileCreate(indexPath.localPath(), stripeIndexLustreSettings());
         }
 
-        indexes_[key] = Index(new TocIndex(key, indexPath, 0, TocIndex::WRITE));
+        indexes_[idxKey] = Index(new TocIndex(idxKey, indexPath, 0, TocIndex::WRITE));
     }
 
-    current_ = indexes_[key];
+    current_ = indexes_[idxKey];
     current_.open();
     current_.flock();
 
@@ -74,20 +74,20 @@ bool TocCatalogueWriter::selectIndex(const Key& key) {
 
     if (useSubToc()) {
 
-        if (fullIndexes_.find(key) == fullIndexes_.end()) {
+        if (fullIndexes_.find(idxKey) == fullIndexes_.end()) {
 
             // TODO TODO TODO .master.index
-            PathName indexPath(generateIndexPath(key));
+            PathName indexPath(generateIndexPath(idxKey));
 
             // Enforce lustre striping if requested
             if (stripeLustre()) {
                 fdb5LustreapiFileCreate(indexPath.localPath(), stripeIndexLustreSettings());
             }
 
-            fullIndexes_[key] = Index(new TocIndex(key, indexPath, 0, TocIndex::WRITE));
+            fullIndexes_[idxKey] = Index(new TocIndex(idxKey, indexPath, 0, TocIndex::WRITE));
         }
 
-        currentFull_ = fullIndexes_[key];
+        currentFull_ = fullIndexes_[idxKey];
         currentFull_.open();
         currentFull_.flock();
     }
@@ -121,7 +121,7 @@ void TocCatalogueWriter::close() {
     closeIndexes();
 }
 
-void TocCatalogueWriter::index(const Key &key, const eckit::URI &uri, eckit::Offset offset, eckit::Length length) {
+void TocCatalogueWriter::index(const InspectionKey &key, const eckit::URI &uri, eckit::Offset offset, eckit::Length length) {
     dirty_ = true;
 
     if (current_.null()) {
@@ -151,7 +151,7 @@ void TocCatalogueWriter::reconsolidateIndexesAndTocs() {
             writer_(writer) {}
         ~ConsolidateIndexVisitor() override {}
     private:
-        void visitDatum(const Field& field, const Key& key) override {
+        void visitDatum(const Field& field, const InspectionKey& key) override {
             // TODO: Do a sneaky schema.expand() here, prepopulated with the current DB/index/Rule,
             //       to extract the full key, including optional values.
             const TocFieldLocation& location(static_cast(field.location()));
@@ -294,7 +294,7 @@ bool TocCatalogueWriter::enabled(const ControlIdentifier& controlIdentifier) con
     return TocCatalogue::enabled(controlIdentifier);
 }
 
-void TocCatalogueWriter::archive(const Key& key, std::unique_ptr fieldLocation) {
+void TocCatalogueWriter::archive(const InspectionKey& key, std::unique_ptr fieldLocation) {
     dirty_ = true;
 
     if (current_.null()) {
diff --git a/src/fdb5/toc/TocCatalogueWriter.h b/src/fdb5/toc/TocCatalogueWriter.h
index 2d2caa88f..d97570330 100644
--- a/src/fdb5/toc/TocCatalogueWriter.h
+++ b/src/fdb5/toc/TocCatalogueWriter.h
@@ -37,13 +37,13 @@ class TocCatalogueWriter : public TocCatalogue, public CatalogueWriter {
 
 public: // methods
 
-    TocCatalogueWriter(const Key &key, const fdb5::Config& config);
+    TocCatalogueWriter(const Key &dbKey, const fdb5::Config& config);
     TocCatalogueWriter(const eckit::URI& uri, const fdb5::Config& config);
 
     virtual ~TocCatalogueWriter() override;
 
     /// Used for adopting & indexing external data to the TOC dir
-    void index(const Key &key, const eckit::URI &uri, eckit::Offset offset, eckit::Length length) override;
+    void index(const InspectionKey &key, const eckit::URI &uri, eckit::Offset offset, eckit::Length length) override;
 
     void reconsolidate() override { reconsolidateIndexesAndTocs(); }
 
@@ -62,7 +62,7 @@ class TocCatalogueWriter : public TocCatalogue, public CatalogueWriter {
 
 protected: // methods
 
-    virtual bool selectIndex(const Key &key) override;
+    virtual bool selectIndex(const Key& idxKey) override;
     virtual void deselectIndex() override;
 
     bool open() override;
@@ -70,7 +70,7 @@ class TocCatalogueWriter : public TocCatalogue, public CatalogueWriter {
     void clean() override;
     void close() override;
 
-    void archive(const Key& key, std::unique_ptr fieldLocation) override;
+    void archive(const InspectionKey& key, std::unique_ptr fieldLocation) override;
     void reconsolidateIndexesAndTocs();
 
     virtual void print( std::ostream &out ) const override;
diff --git a/src/fdb5/toc/TocEngine.cc b/src/fdb5/toc/TocEngine.cc
index a4c1a445b..2e32a0b4c 100644
--- a/src/fdb5/toc/TocEngine.cc
+++ b/src/fdb5/toc/TocEngine.cc
@@ -131,13 +131,13 @@ bool TocEngine::canHandle(const eckit::URI& uri) const
     return path.isDir() && toc.exists();
 }
 
-static void matchKeyToDB(const Key& key, std::set& keys, const char* missing, const Config& config)
+static void matchKeyToDB(const Key& key, std::set& keys, const char* missing, const Config& config)
 {
     const Schema& schema = config.schema();
     schema.matchFirstLevel(key, keys, missing);
 }
 
-static void matchRequestToDB(const metkit::mars::MarsRequest& rq, std::set& keys, const char* missing, const Config& config)
+static void matchRequestToDB(const metkit::mars::MarsRequest& rq, std::set& keys, const char* missing, const Config& config)
 {
     const Schema& schema = config.schema();
     schema.matchFirstLevel(rq, keys, missing);
@@ -145,7 +145,7 @@ static void matchRequestToDB(const metkit::mars::MarsRequest& rq, std::set&
 
 static constexpr const char* regexForMissingValues = "[^:/]*";
 
-std::set TocEngine::databases(const std::set& keys,
+std::set TocEngine::databases(const std::set& keys,
                                                const std::vector& roots,
                                                const Config& config) const {
 
@@ -158,7 +158,7 @@ std::set TocEngine::databases(const std::set& keys,
         std::list dbs;
         scan_dbs(*j, dbs);
 
-        for (std::set::const_iterator i = keys.begin(); i != keys.end(); ++i) {
+        for (std::set::const_iterator i = keys.begin(); i != keys.end(); ++i) {
 
             std::vector dbpaths = CatalogueRootManager(config).possibleDbPathNames(*i, regexForMissingValues);
 
@@ -195,7 +195,7 @@ std::vector TocEngine::databases(const Key& key,
                                                   const std::vector& roots,
                                                   const Config& config) const {
 
-    std::set keys;
+    std::set keys;
 
     matchKeyToDB(key, keys, regexForMissingValues, config);
 
@@ -224,7 +224,7 @@ std::vector TocEngine::databases(const metkit::mars::MarsRequest& re
                                                   const std::vector& roots,
                                                   const Config& config) const {
 
-    std::set keys;
+    std::set keys;
 
 //    matchRequestToDB(request, keys, regexForMissingValues, config);
     matchRequestToDB(request, keys, "", config);
@@ -255,9 +255,9 @@ std::vector TocEngine::allLocations(const Key& key, const Config& co
     return databases(key, CatalogueRootManager(config).allRoots(key), config);
 }
 
-std::vector TocEngine::visitableLocations(const Key& key, const Config& config) const
+std::vector TocEngine::visitableLocations(const Config& config) const
 {
-    return databases(key, CatalogueRootManager(config).visitableRoots(key), config);
+    return databases(Key(), CatalogueRootManager(config).visitableRoots(InspectionKey()), config);
 }
 
 std::vector TocEngine::visitableLocations(const metkit::mars::MarsRequest& request, const Config& config) const
@@ -265,10 +265,10 @@ std::vector TocEngine::visitableLocations(const metkit::mars::MarsRequest&
     return databases(request, CatalogueRootManager(config).visitableRoots(request), config);
 }
 
-std::vector TocEngine::writableLocations(const Key& key, const Config& config) const
-{
-    return databases(key, CatalogueRootManager(config).canArchiveRoots(key), config);
-}
+// std::vector TocEngine::writableLocations(const Key& key, const Config& config) const
+// {
+//     return databases(key, CatalogueRootManager(config).canArchiveRoots(key), config);
+// }
 
 void TocEngine::print(std::ostream& out) const
 {
diff --git a/src/fdb5/toc/TocEngine.h b/src/fdb5/toc/TocEngine.h
index 8c5af1709..5467d1dc2 100644
--- a/src/fdb5/toc/TocEngine.h
+++ b/src/fdb5/toc/TocEngine.h
@@ -29,7 +29,7 @@ class TocEngine : public fdb5::Engine {
     static const char* typeName() { return "toc"; }
 
 private:  // methods
-    std::set databases(const std::set& keys, const std::vector& dirs,
+    std::set databases(const std::set& keys, const std::vector& dirs,
                                         const Config& config) const;
 
     std::vector databases(const Key& key, const std::vector& dirs, const Config& config) const;
@@ -51,10 +51,10 @@ class TocEngine : public fdb5::Engine {
 
     virtual std::vector allLocations(const Key& key, const Config& config) const override;
 
-    virtual std::vector visitableLocations(const Key& key, const Config& config) const override;
+    virtual std::vector visitableLocations(const Config& config) const override;
     virtual std::vector visitableLocations(const metkit::mars::MarsRequest& rq, const Config& config) const override;
 
-    virtual std::vector writableLocations(const Key& key, const Config& config) const override;
+//    virtual std::vector writableLocations(const InspectionKey& key, const Config& config) const override;
 
     virtual void print( std::ostream &out ) const override;
 
diff --git a/src/fdb5/toc/TocHandler.cc b/src/fdb5/toc/TocHandler.cc
index 55676791e..057b48a02 100644
--- a/src/fdb5/toc/TocHandler.cc
+++ b/src/fdb5/toc/TocHandler.cc
@@ -133,7 +133,6 @@ TocHandler::TocHandler(const eckit::PathName& directory, const Config& config) :
     enumeratedMaskedEntries_(false),
     writeMode_(false)
 {
-
     // An override to enable using sub tocs without configurations being passed in, for ease
     // of debugging
     const char* subTocOverride = ::getenv("FDB5_SUB_TOCS");
@@ -145,7 +144,7 @@ TocHandler::TocHandler(const eckit::PathName& directory, const Config& config) :
 TocHandler::TocHandler(const eckit::PathName& path, const Key& parentKey) :
     TocCommon(path.dirName()),
     parentKey_(parentKey),
-    schemaPath_(directory_ / "schema"),
+    schemaPath_(TocCommon::findRealPath(path) / "schema"),
     tocPath_(TocCommon::findRealPath(path)),
     serialisationVersion_(TocSerialisationVersion(dbConfig_)),
     useSubToc_(false),
@@ -157,7 +156,6 @@ TocHandler::TocHandler(const eckit::PathName& path, const Key& parentKey) :
     enumeratedMaskedEntries_(false),
     writeMode_(false)
 {
-
     /// Are we remapping a mounted DB?
     if (exists()) {
         Key key(databaseKey());
@@ -726,7 +724,6 @@ void TocHandler::writeInitRecord(const Key &key) {
         if (!isSubToc_) {
 
             /* Copy schema first */
-
             eckit::Log::debug() << "Copy schema from "
                                << dbConfig_.schemaPath()
                                << " to "
diff --git a/src/fdb5/toc/TocHandler.h b/src/fdb5/toc/TocHandler.h
index 222306c09..122f0241a 100644
--- a/src/fdb5/toc/TocHandler.h
+++ b/src/fdb5/toc/TocHandler.h
@@ -102,7 +102,7 @@ class TocHandler : public TocCommon, private eckit::NonCopyable {
 
 public: // methods
 
-    TocHandler( const Key &key, const Config& config);
+    //TocHandler( const Key &key, const Config& config);
 
     TocHandler( const eckit::PathName &dir, const Config& config);
 
diff --git a/src/fdb5/toc/TocIndex.cc b/src/fdb5/toc/TocIndex.cc
index d91df4596..72faf1580 100644
--- a/src/fdb5/toc/TocIndex.cc
+++ b/src/fdb5/toc/TocIndex.cc
@@ -78,7 +78,7 @@ void TocIndex::encode(eckit::Stream& s, const int version) const {
 }
 
 
-bool TocIndex::get(const Key &key, const Key &remapKey, Field &field) const {
+bool TocIndex::get(const InspectionKey &key, const Key &remapKey, Field &field) const {
     ASSERT(btree_);
     FieldRef ref;
 
@@ -124,7 +124,7 @@ void TocIndex::close() {
     }
 }
 
-void TocIndex::add(const Key &key, const Field &field) {
+void TocIndex::add(const InspectionKey &key, const Field &field) {
     ASSERT(btree_);
     ASSERT( mode_ == TocIndex::WRITE );
 
diff --git a/src/fdb5/toc/TocIndex.h b/src/fdb5/toc/TocIndex.h
index 6d5874954..9361c2da8 100644
--- a/src/fdb5/toc/TocIndex.h
+++ b/src/fdb5/toc/TocIndex.h
@@ -97,8 +97,8 @@ class TocIndex :
 
     void visit(IndexLocationVisitor& visitor) const override;
 
-    bool get( const Key &key, const Key &remapKey, Field &field ) const override;
-    void add( const Key &key, const Field &field ) override;
+    bool get( const InspectionKey &key, const Key &remapKey, Field &field ) const override;
+    void add( const InspectionKey &key, const Field &field ) override;
     void flush() override;
     void encode(eckit::Stream& s, const int version) const override;
     void entries(EntryVisitor& visitor) const override;
diff --git a/src/fdb5/toc/TocStats.h b/src/fdb5/toc/TocStats.h
index d258751c0..91ba2049e 100644
--- a/src/fdb5/toc/TocStats.h
+++ b/src/fdb5/toc/TocStats.h
@@ -161,7 +161,7 @@ class TocStatsReportVisitor : public virtual StatsReportVisitor {
 
     bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
     void visitDatum(const Field& field, const std::string& keyFingerprint) override;
-    void visitDatum(const Field& field, const Key& key) override { NOTIMP; }
+    void visitDatum(const Field& field, const InspectionKey& key) override { NOTIMP; }
 
     // This visitor is only legit for one DB - so don't reset database
     void catalogueComplete(const Catalogue& catalogue) override;
diff --git a/src/fdb5/toc/TocStore.cc b/src/fdb5/toc/TocStore.cc
index 8832cf267..8c0764947 100644
--- a/src/fdb5/toc/TocStore.cc
+++ b/src/fdb5/toc/TocStore.cc
@@ -183,10 +183,7 @@ eckit::DataHandle& TocStore::getDataHandle( const eckit::PathName &path ) {
 eckit::PathName TocStore::generateDataPath(const Key &key) const {
 
     eckit::PathName dpath ( directory_ );
-    if (!key.rule()) {
-        std::cout << "TocStore::generateDataPath - missing rule in " << key << std::endl;
-    }
-    dpath /=  key.valuesToString(false);
+    dpath /=  key.valuesToString();
     dpath = eckit::PathName::unique(dpath) + ".data";
     return dpath;
 }
@@ -265,11 +262,7 @@ void TocStore::moveTo(const Key& key, const Config& config, const eckit::URI& de
 
 void TocStore::remove(const Key& key) const {
 
-    if (!key.rule()) {
-        std::cout << "TocStore::remove - missing rule in " << key << std::endl;
-    }
-
-    eckit::PathName src_db = directory_ / key.valuesToString(false);
+    eckit::PathName src_db = directory_ / key.valuesToString();
         
     DIR* dirp = ::opendir(src_db.asString().c_str());
     struct dirent* dp;
diff --git a/src/fdb5/tools/fdb-hide.cc b/src/fdb5/tools/fdb-hide.cc
index 92b06a8b9..dc92554c2 100644
--- a/src/fdb5/tools/fdb-hide.cc
+++ b/src/fdb5/tools/fdb-hide.cc
@@ -83,7 +83,7 @@ void FdbHide::execute(const option::CmdArgs& args) {
 
     const Schema& schema = conf.schema();
 
-    Key dbkey;
+    InspectionKey dbkey;
     ASSERT(schema.expandFirstLevel(dbrequest.request(), dbkey));
 
     std::unique_ptr db = CatalogueFactory::instance().build(dbkey, conf, true);
diff --git a/src/fdb5/tools/fdb-overlay.cc b/src/fdb5/tools/fdb-overlay.cc
index d83b8431d..da580ec16 100644
--- a/src/fdb5/tools/fdb-overlay.cc
+++ b/src/fdb5/tools/fdb-overlay.cc
@@ -95,8 +95,8 @@ void FdbOverlay::execute(const option::CmdArgs& args) {
     Config conf = config(args);
     const Schema& schema = conf.schema();
 
-    Key source;
-    Key target;
+    InspectionKey source;
+    InspectionKey target;
     ASSERT(schema.expandFirstLevel(sourceRequest.request(), source));
     ASSERT(schema.expandFirstLevel(targetRequest.request(), target));
 
@@ -123,25 +123,25 @@ void FdbOverlay::execute(const option::CmdArgs& args) {
         }
     }
 
-    std::unique_ptr dbSource = DB::buildReader(source, conf);
+    std::unique_ptr dbSource = CatalogueFactory::instance().build(source, conf, true);
     if (!dbSource->exists()) {
         std::stringstream ss;
         ss << "Source database not found: " << source << std::endl;
         throw UserError(ss.str(), Here());
     }
 
-    if (dbSource->dbType() != TocEngine::typeName()) {
+    if (dbSource->type() != TocEngine::typeName()) {
         std::stringstream ss;
         ss << "Only TOC DBs currently supported" << std::endl;
         throw UserError(ss.str(), Here());
     }
 
-    std::unique_ptr dbTarget = DB::buildReader(target, conf);
+    std::unique_ptr dbTarget = CatalogueFactory::instance().build(target, conf, true);
 
     if (remove_) {
         if (!dbTarget->exists()) {
             std::stringstream ss;
-            ss << "Target database must already already exist: " << target << std::endl;
+            ss << "Target database must already exist: " << target << std::endl;
             throw UserError(ss.str(), Here());
         }
     } else {
@@ -156,16 +156,13 @@ void FdbOverlay::execute(const option::CmdArgs& args) {
 
     ASSERT(dbTarget->uri() != dbSource->uri());
 
-    std::unique_ptr newDB = DB::buildWriter(target, conf);
+    std::unique_ptr newCatalogue = CatalogueFactory::instance().build(target, conf, false);
+    if (newCatalogue->type() == TocEngine::typeName() && dbSource->type() == TocEngine::typeName())  {
+        CatalogueWriter* cat = dynamic_cast(newCatalogue.get());
+        ASSERT(cat);
 
-    // This only works for tocDBs
-
-/*    TocDBReader* tocSourceDB = dynamic_cast(dbSource.get());
-    TocDBWriter* tocTargetDB = dynamic_cast(newDB.get());
-    ASSERT(tocSourceDB);
-    ASSERT(tocTargetDB);*/
-
-    newDB->overlayDB(*dbSource, vkeys, remove_);
+        cat->overlayDB(*dbSource, vkeys, remove_);
+    }
 }
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/tools/fdb-reconsolidate-toc.cc b/src/fdb5/tools/fdb-reconsolidate-toc.cc
index d5b1b60e1..3688e9cfc 100644
--- a/src/fdb5/tools/fdb-reconsolidate-toc.cc
+++ b/src/fdb5/tools/fdb-reconsolidate-toc.cc
@@ -12,6 +12,7 @@
 
 #include "eckit/option/CmdArgs.h"
 #include "eckit/config/LocalConfiguration.h"
+#include "fdb5/database/Catalogue.h"
 
 using namespace eckit;
 
@@ -53,8 +54,11 @@ void FDBReconsolidateToc::execute(const eckit::option::CmdArgs& args) {
     }
 
     // TODO: In updated version, grab default Config() here;
-    std::unique_ptr db = fdb5::DB::buildWriter(eckit::URI("toc", dbPath), eckit::LocalConfiguration());
-    db->reconsolidate();
+    std::unique_ptr catalogue = fdb5::CatalogueFactory::instance().build(eckit::URI("toc", dbPath), eckit::LocalConfiguration(), false);
+    fdb5::CatalogueWriter* cat = dynamic_cast(catalogue.get());
+    ASSERT(cat);
+
+    cat->reconsolidate();
 }
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/tools/fdb-root.cc b/src/fdb5/tools/fdb-root.cc
index eb2236875..3f521f001 100644
--- a/src/fdb5/tools/fdb-root.cc
+++ b/src/fdb5/tools/fdb-root.cc
@@ -65,20 +65,21 @@ void FdbRoot::execute(const eckit::option::CmdArgs& args) {
 
             Config conf = config(args);
             const Schema& schema = conf.schema();
-            Key result;
+            InspectionKey result;
             ASSERT( schema.expandFirstLevel(request.request(), result) );
 
             eckit::Log::info() << result << std::endl;
 
             // 'Touch' the database (which will create it if it doesn't exist)
-            std::unique_ptr db = DB::buildReader(result, conf);
 
-            if (!db->exists() && create_db) {
-                db = DB::buildWriter(result, conf);
+            std::unique_ptr cat = CatalogueFactory::instance().build(result, conf, true);
+
+            if (!cat->exists() && create_db) {
+                cat = CatalogueFactory::instance().build(result, conf, false);
             }
 
-            if (db->exists()) {
-                eckit::Log::info() << (*db) << std::endl;
+            if (cat->exists()) {
+                eckit::Log::info() << (*cat) << std::endl;
             }
         }
     }
diff --git a/src/fdb5/tools/fdb-schema.cc b/src/fdb5/tools/fdb-schema.cc
index 8b7ac9ddd..6b9a9dcfe 100644
--- a/src/fdb5/tools/fdb-schema.cc
+++ b/src/fdb5/tools/fdb-schema.cc
@@ -56,9 +56,9 @@ void FdbSchema:: execute(const eckit::option::CmdArgs &args) {
             eckit::PathName path(args(i));
 
             if (path.isDir()) {
-                std::unique_ptr db = DB::buildReader(eckit::URI("toc", path));
-                ASSERT(db->open());
-                db->schema().dump(Log::info());
+                std::unique_ptr cat = CatalogueFactory::instance().build(eckit::URI("toc", path), fdb5::Config(), true);
+                ASSERT(cat->open());
+                cat->schema().dump(Log::info());
             } else {
                 Schema schema;
                 schema.load(args(i));
diff --git a/src/fdb5/tools/fdb-stats-new.cc b/src/fdb5/tools/fdb-stats-new.cc
index 202d7f043..25d1d7492 100644
--- a/src/fdb5/tools/fdb-stats-new.cc
+++ b/src/fdb5/tools/fdb-stats-new.cc
@@ -61,11 +61,11 @@ void FDBStats::process(const eckit::PathName& path, const eckit::option::CmdArgs
 
     eckit::Log::info() << "Scanning " << path << std::endl;
 
-    std::unique_ptr db(fdb5::DBFactory::buildReader(path));
-    ASSERT(db->open());
+    std::unique_ptr cat = fdb5::CatalogueFactory::instance().build(path, fdb5::Config(), true);
+    ASSERT(cat->open());
 
     fdb5::ReportVisitor visitor(*db);
-    db->visitEntries(visitor);
+    cat->visitEntries(visitor);
 
     if (details_) {
         eckit::Log::info() << std::endl;
diff --git a/src/fdb5/types/Type.cc b/src/fdb5/types/Type.cc
index ac8fa7c7c..5b755122d 100644
--- a/src/fdb5/types/Type.cc
+++ b/src/fdb5/types/Type.cc
@@ -27,7 +27,7 @@ void Type::getValues(const metkit::mars::MarsRequest &request,
                      const std::string &keyword,
                      eckit::StringList &values,
                      const Notifier&,
-                     const DB*) const {
+                     const CatalogueReader*) const {
     request.getValues(keyword, values, true);
 }
 
diff --git a/src/fdb5/types/Type.h b/src/fdb5/types/Type.h
index 398079200..9f619fe08 100644
--- a/src/fdb5/types/Type.h
+++ b/src/fdb5/types/Type.h
@@ -31,7 +31,7 @@ namespace mars {
 
 namespace fdb5 {
 
-class DB;
+class CatalogueReader;
 class Notifier;
 
 //----------------------------------------------------------------------------------------------------------------------
@@ -54,7 +54,7 @@ class Type : private eckit::NonCopyable {
                            const std::string &keyword,
                            eckit::StringList &values,
                            const Notifier &wind,
-                           const DB *db) const;
+                           const CatalogueReader* cat) const;
 
     virtual bool match(const std::string& keyword, const std::string& value1, const std::string& value2) const;
 
diff --git a/src/fdb5/types/TypeAbbreviation.cc b/src/fdb5/types/TypeAbbreviation.cc
index 7ac104a35..efe8563bb 100644
--- a/src/fdb5/types/TypeAbbreviation.cc
+++ b/src/fdb5/types/TypeAbbreviation.cc
@@ -36,7 +36,7 @@ void TypeAbbreviation::getValues(const metkit::mars::MarsRequest &request,
                                  const std::string &keyword,
                                  eckit::StringList &values,
                                  const Notifier&,
-                                 const DB*) const {
+                                 const CatalogueReader*) const {
     std::vector vals;
 
     request.getValues(keyword, vals, true);
diff --git a/src/fdb5/types/TypeAbbreviation.h b/src/fdb5/types/TypeAbbreviation.h
index 29657d8fc..ac3a44a91 100644
--- a/src/fdb5/types/TypeAbbreviation.h
+++ b/src/fdb5/types/TypeAbbreviation.h
@@ -37,7 +37,7 @@ class TypeAbbreviation : public Type {
                            const std::string &keyword,
                            eckit::StringList &values,
                            const Notifier &wind,
-                           const DB *db) const override;
+                           const CatalogueReader* cat) const override;
 
 private: // methods
 
diff --git a/src/fdb5/types/TypeClimateDaily.cc b/src/fdb5/types/TypeClimateDaily.cc
index a02ec0251..279c50394 100644
--- a/src/fdb5/types/TypeClimateDaily.cc
+++ b/src/fdb5/types/TypeClimateDaily.cc
@@ -73,7 +73,7 @@ void TypeClimateDaily::getValues(const metkit::mars::MarsRequest &request,
                                  const std::string &keyword,
                                  eckit::StringList &values,
                                  const Notifier&,
-                                 const DB*) const {
+                                 const CatalogueReader*) const {
   std::vector dates;
 
   request.getValues(keyword, dates, true);
diff --git a/src/fdb5/types/TypeClimateDaily.h b/src/fdb5/types/TypeClimateDaily.h
index 7c569e238..7a77dd45a 100644
--- a/src/fdb5/types/TypeClimateDaily.h
+++ b/src/fdb5/types/TypeClimateDaily.h
@@ -37,7 +37,7 @@ class TypeClimateDaily : public Type {
                            const std::string &keyword,
                            eckit::StringList &values,
                            const Notifier &wind,
-                           const DB *db) const override;
+                           const CatalogueReader* cat) const override;
 
     virtual bool match(const std::string&,
                        const std::string& value1,
diff --git a/src/fdb5/types/TypeClimateMonthly.cc b/src/fdb5/types/TypeClimateMonthly.cc
index e4c2b568b..624fd2dd5 100644
--- a/src/fdb5/types/TypeClimateMonthly.cc
+++ b/src/fdb5/types/TypeClimateMonthly.cc
@@ -58,7 +58,7 @@ void TypeClimateMonthly::getValues(const metkit::mars::MarsRequest &request,
                                    const std::string &keyword,
                                    eckit::StringList &values,
                                    const Notifier&,
-                                   const DB*) const {
+                                   const CatalogueReader*) const {
     std::vector dates;
 
     request.getValues(keyword, dates, true);
diff --git a/src/fdb5/types/TypeClimateMonthly.h b/src/fdb5/types/TypeClimateMonthly.h
index ab588baa4..16deb5ae7 100644
--- a/src/fdb5/types/TypeClimateMonthly.h
+++ b/src/fdb5/types/TypeClimateMonthly.h
@@ -37,7 +37,7 @@ class TypeClimateMonthly : public Type {
                            const std::string &keyword,
                            eckit::StringList &values,
                            const Notifier &wind,
-                           const DB *db) const override;
+                           const CatalogueReader* cat) const override;
 
 private: // methods
 
diff --git a/src/fdb5/types/TypeDate.cc b/src/fdb5/types/TypeDate.cc
index 3e8119b3d..541feb9e6 100644
--- a/src/fdb5/types/TypeDate.cc
+++ b/src/fdb5/types/TypeDate.cc
@@ -47,7 +47,7 @@ void TypeDate::getValues(const metkit::mars::MarsRequest & request,
                          const std::string & keyword,
                          eckit::StringList & values,
                          const Notifier&,
-                         const DB*) const {
+                         const CatalogueReader*) const {
     std::vector dates;
 
     request.getValues(keyword, dates, true);
diff --git a/src/fdb5/types/TypeDate.h b/src/fdb5/types/TypeDate.h
index 405c2b0c7..d4eefb623 100644
--- a/src/fdb5/types/TypeDate.h
+++ b/src/fdb5/types/TypeDate.h
@@ -37,7 +37,7 @@ class TypeDate : public Type {
                            const std::string &keyword,
                            eckit::StringList &values,
                            const Notifier &wind,
-                           const DB *db) const override;
+                           const CatalogueReader* cat) const override;
 
 private: // methods
 
diff --git a/src/fdb5/types/TypeDouble.cc b/src/fdb5/types/TypeDouble.cc
index c87fed5b2..dc9417ead 100644
--- a/src/fdb5/types/TypeDouble.cc
+++ b/src/fdb5/types/TypeDouble.cc
@@ -42,7 +42,7 @@ void TypeDouble::getValues(const metkit::mars::MarsRequest& request,
                            const std::string& keyword,
                            eckit::StringList& values,
                            const Notifier&,
-                           const DB*) const {
+                           const CatalogueReader*) const {
   std::vector dblValues;
 
   request.getValues(keyword, dblValues, true);
diff --git a/src/fdb5/types/TypeDouble.h b/src/fdb5/types/TypeDouble.h
index 3de230d0b..85f7eb821 100644
--- a/src/fdb5/types/TypeDouble.h
+++ b/src/fdb5/types/TypeDouble.h
@@ -37,7 +37,7 @@ class TypeDouble : public Type {
                            const std::string &keyword,
                            eckit::StringList &values,
                            const Notifier &wind,
-                           const DB *db) const override;
+                           const CatalogueReader* cat) const override;
 
 private: // methods
 
diff --git a/src/fdb5/types/TypeGrid.cc b/src/fdb5/types/TypeGrid.cc
index 657260556..755321a62 100644
--- a/src/fdb5/types/TypeGrid.cc
+++ b/src/fdb5/types/TypeGrid.cc
@@ -39,7 +39,7 @@ void TypeGrid::getValues(const metkit::mars::MarsRequest& request,
                          const std::string& keyword,
                          eckit::StringList& values,
                          const Notifier&,
-                         const DB*) const {
+                         const CatalogueReader*) const {
     std::vector v;
     request.getValues(keyword, v, true);
     values.push_back(eckit::StringTools::join("+", v));
diff --git a/src/fdb5/types/TypeGrid.h b/src/fdb5/types/TypeGrid.h
index 61d44c464..0111a08ad 100644
--- a/src/fdb5/types/TypeGrid.h
+++ b/src/fdb5/types/TypeGrid.h
@@ -39,7 +39,7 @@ class TypeGrid : public Type {
                            const std::string &keyword,
                            eckit::StringList &values,
                            const Notifier &wind,
-                           const DB *db) const override;
+                           const CatalogueReader* cat) const override;
 
     virtual void print( std::ostream &out ) const override;
 
diff --git a/src/fdb5/types/TypeIgnore.cc b/src/fdb5/types/TypeIgnore.cc
index 8731b8dbc..5fed98b49 100644
--- a/src/fdb5/types/TypeIgnore.cc
+++ b/src/fdb5/types/TypeIgnore.cc
@@ -34,7 +34,7 @@ void TypeIgnore::getValues(const metkit::mars::MarsRequest&,
                            const std::string&,
                            eckit::StringList&,
                            const Notifier&,
-                           const DB*) const {
+                           const CatalogueReader*) const {
 }
 
 void TypeIgnore::print(std::ostream &out) const {
diff --git a/src/fdb5/types/TypeIgnore.h b/src/fdb5/types/TypeIgnore.h
index f90fe9d0f..1758422df 100644
--- a/src/fdb5/types/TypeIgnore.h
+++ b/src/fdb5/types/TypeIgnore.h
@@ -34,7 +34,7 @@ class TypeIgnore : public Type {
                            const std::string &keyword,
                            eckit::StringList &values,
                            const Notifier &wind,
-                           const DB *db) const override;
+                           const CatalogueReader* cat) const override;
 
     virtual std::string toKey(const std::string& keyword,
                               const std::string& value) const override;
diff --git a/src/fdb5/types/TypeInteger.cc b/src/fdb5/types/TypeInteger.cc
index 9f0dde42e..17801f217 100644
--- a/src/fdb5/types/TypeInteger.cc
+++ b/src/fdb5/types/TypeInteger.cc
@@ -30,7 +30,7 @@ void TypeInteger::getValues(const metkit::mars::MarsRequest& request,
                             const std::string& keyword,
                             eckit::StringList& values,
                             const Notifier&,
-                            const DB*) const {
+                            const CatalogueReader*) const {
     std::vector intValues;
 
     request.getValues(keyword, intValues, true);
diff --git a/src/fdb5/types/TypeInteger.h b/src/fdb5/types/TypeInteger.h
index 45a7165b5..a8c31c635 100644
--- a/src/fdb5/types/TypeInteger.h
+++ b/src/fdb5/types/TypeInteger.h
@@ -34,7 +34,7 @@ class TypeInteger : public Type {
                            const std::string &keyword,
                            eckit::StringList &values,
                            const Notifier &wind,
-                           const DB *db) const override;
+                           const CatalogueReader* cat) const override;
 
 private: // methods
 
diff --git a/src/fdb5/types/TypeMonth.cc b/src/fdb5/types/TypeMonth.cc
index c6080f820..ab1398561 100644
--- a/src/fdb5/types/TypeMonth.cc
+++ b/src/fdb5/types/TypeMonth.cc
@@ -40,7 +40,7 @@ void TypeMonth::getValues(const metkit::mars::MarsRequest& request,
                           const std::string& keyword,
                           eckit::StringList& values,
                           const Notifier&,
-                          const DB*) const {
+                          const CatalogueReader*) const {
     std::vector dates;
 
     request.getValues(keyword, dates, true);
diff --git a/src/fdb5/types/TypeMonth.h b/src/fdb5/types/TypeMonth.h
index 85fc242e4..5840ae432 100644
--- a/src/fdb5/types/TypeMonth.h
+++ b/src/fdb5/types/TypeMonth.h
@@ -37,7 +37,7 @@ class TypeMonth : public Type {
                            const std::string &keyword,
                            eckit::StringList &values,
                            const Notifier &wind,
-                           const DB *db) const override;
+                           const CatalogueReader* cat) const override;
 
 private: // methods
 
diff --git a/src/fdb5/types/TypeParam.cc b/src/fdb5/types/TypeParam.cc
index e081149cf..ea87bae71 100644
--- a/src/fdb5/types/TypeParam.cc
+++ b/src/fdb5/types/TypeParam.cc
@@ -13,7 +13,7 @@
 #include "metkit/mars/ParamID.h"
 #include "metkit/mars/Param.h"
 
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
 #include "fdb5/database/Notifier.h"
 #include "fdb5/types/TypeParam.h"
 #include "fdb5/types/TypesFactory.h"
@@ -35,16 +35,16 @@ void TypeParam::getValues(const metkit::mars::MarsRequest &request,
                           const std::string &keyword,
                           eckit::StringList &values,
                           const Notifier &wind,
-                          const DB *db) const {
-    ASSERT(db);
+                          const CatalogueReader* cat) const {
+    ASSERT(cat);
 
     eckit::StringSet ax;
 
-    db->axis(keyword, ax);
+    cat->axis(keyword, ax);
 
     eckit::StringList us;
 
-    Type::getValues(request, keyword, us, wind, db);
+    Type::getValues(request, keyword, us, wind, cat);
 
     std::vector user;
     std::copy(us.begin(), us.end(), std::back_inserter(user));
diff --git a/src/fdb5/types/TypeParam.h b/src/fdb5/types/TypeParam.h
index 22b705ac0..95705b5a9 100644
--- a/src/fdb5/types/TypeParam.h
+++ b/src/fdb5/types/TypeParam.h
@@ -34,7 +34,7 @@ class TypeParam : public Type {
                            const std::string &keyword,
                            eckit::StringList &values,
                            const Notifier &wind,
-                           const DB *db) const override;
+                           const CatalogueReader* cat) const override;
 
     virtual bool match(const std::string& keyword,
                        const std::string& value1,
diff --git a/src/fdb5/types/TypeStep.cc b/src/fdb5/types/TypeStep.cc
index afaece730..1e7066a1f 100644
--- a/src/fdb5/types/TypeStep.cc
+++ b/src/fdb5/types/TypeStep.cc
@@ -16,7 +16,7 @@
 
 #include "fdb5/types/TypesFactory.h"
 #include "fdb5/types/TypeStep.h"
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
 
 using metkit::mars::StepRange;
 using metkit::mars::StepRangeNormalise;
@@ -56,7 +56,7 @@ void TypeStep::getValues(const metkit::mars::MarsRequest& request,
                          const std::string& keyword,
                          eckit::StringList& values,
                          const Notifier&,
-                         const DB *db) const {
+                         const CatalogueReader* cat) const {
 
     // Get the steps / step ranges from the request
 
@@ -66,14 +66,14 @@ void TypeStep::getValues(const metkit::mars::MarsRequest& request,
     std::vector ranges;
     std::copy(steps.begin(), steps.end(), std::back_inserter(ranges));
 
-    // If this is before knowing the DB, we are constrained on what we can do.
+    // If this is before knowing the Catalogue, we are constrained on what we can do.
 
-    if (db) {
+    if (cat) {
 
         // Get the axis
 
         eckit::StringSet ax;
-        db->axis("step", ax);
+        cat->axis("step", ax);
 
         std::vector axis;
         for (auto step: ax) {
diff --git a/src/fdb5/types/TypeStep.h b/src/fdb5/types/TypeStep.h
index 46c752923..8d9ccd2ba 100644
--- a/src/fdb5/types/TypeStep.h
+++ b/src/fdb5/types/TypeStep.h
@@ -34,7 +34,7 @@ class TypeStep : public Type {
                            const std::string &keyword,
                            eckit::StringList &values,
                            const Notifier &wind,
-                           const DB *db) const override;
+                           const CatalogueReader* cat) const override;
 
     virtual std::string toKey(const std::string& keyword,
                               const std::string& value) const override;
diff --git a/tests/fdb/type/test_toKey.cc b/tests/fdb/type/test_toKey.cc
index 85c600299..e9816701a 100644
--- a/tests/fdb/type/test_toKey.cc
+++ b/tests/fdb/type/test_toKey.cc
@@ -16,7 +16,7 @@
 #include "fdb5/config/Config.h"
 #include "fdb5/database/Archiver.h"
 #include "fdb5/database/ArchiveVisitor.h"
-#include "fdb5/database/Key.h"
+#include "fdb5/database/InspectionKey.h"
 #include "fdb5/rules/Rule.h"
 
 using namespace eckit::testing;
@@ -31,7 +31,7 @@ char data[4];
 
 CASE( "ClimateDaily - no expansion" ) {
 
-    fdb5::Key key;
+    fdb5::InspectionKey key;
     EXPECT(key.valuesToString() == "");
     EXPECT_THROWS(key.canonicalValue("date"));
 
@@ -48,7 +48,7 @@ CASE( "ClimateDaily - no expansion" ) {
 
 CASE( "Step & ClimateDaily - expansion" ) {
 
-    fdb5::Key key;
+    fdb5::InspectionKey key;
     EXPECT(key.valuesToString() == "");
     EXPECT_THROWS(key.canonicalValue("date"));
 
@@ -130,9 +130,10 @@ CASE( "Step & ClimateDaily - expansion" ) {
     key.set("levelist", "50");
     key.set("param", "129.128");
 
-    fdb5::Archiver archiver;
+    fdb5::Config conf = config.expandConfig();
+    fdb5::Archiver archiver(conf);
     fdb5::ArchiveVisitor visitor(archiver, key, data, 4);
-    config.schema().expand(key, visitor);
+    conf.schema().expand(key, visitor);
     key.rule(visitor.rule());
 
     EXPECT(key.canonicalValue("date") == "0427");
@@ -151,7 +152,7 @@ CASE( "Levelist" ) {
     values.insert("0.333333");
     values.sort();
 
-    fdb5::Key key;
+    fdb5::InspectionKey key;
     EXPECT(key.valuesToString() == "");
     EXPECT_THROWS(key.canonicalValue("levelist"));
 
@@ -199,14 +200,13 @@ CASE( "Levelist" ) {
     // this works (probably shouldn't), simply becasue to_string uses the same precision as printf %f (default 6)
     /// @note don't use to_string when canonicalising Keys
     key.set("levelist", std::to_string(double(1./3.)));
-//    std::cout << key.canonicalValue("levelist") << std::endl;
     EXPECT(key.canonicalValue("levelist") == "0.333333");
     EXPECT(key.match("levelist", values));
 }
 
 CASE( "Expver, Time & ClimateDaily - string ctor - expansion" ) {
 
-    fdb5::Key key("class=ei,expver=1,stream=dacl,domain=g,type=pb,levtype=pl,date=20210427,time=6,step=0,quantile=99:100,levelist=50,param=129.128");
+    fdb5::InspectionKey key("class=ei,expver=1,stream=dacl,domain=g,type=pb,levtype=pl,date=20210427,time=6,step=0,quantile=99:100,levelist=50,param=129.128");
 
     EXPECT(key.canonicalValue("date") == "20210427");
     EXPECT(key.canonicalValue("time") == "0600");
@@ -224,7 +224,7 @@ CASE( "Expver, Time & ClimateDaily - string ctor - expansion" ) {
 
 CASE( "ClimateMonthly - string ctor - expansion" ) {
 
-    fdb5::Key key("class=op,expver=1,stream=mnth,domain=g,type=cl,levtype=pl,date=20210427,time=0000,levelist=50,param=129.128");
+    fdb5::InspectionKey key("class=op,expver=1,stream=mnth,domain=g,type=cl,levtype=pl,date=20210427,time=0000,levelist=50,param=129.128");
 
     EXPECT(key.canonicalValue("date") == "20210427");
     EXPECT(key.valuesToString() == "op:0001:mnth:g:cl:pl:20210427:0000:50:129.128");
@@ -240,10 +240,10 @@ CASE( "ClimateMonthly - string ctor - expansion" ) {
 
 }
 
-// do we need to keep this behaviour? should we rely on metkit for date expansion and remove it from Key?
+// do we need to keep this behaviour? should we rely on metkit for date expansion and remove it from InspectionKey?
 CASE( "Date - string ctor - expansion" ) {
 
-    fdb5::Key key("class=od,expver=1,stream=oper,type=ofb,date=-2,time=0000,obsgroup=MHS,reportype=3001");
+    fdb5::InspectionKey key("class=od,expver=1,stream=oper,type=ofb,date=-2,time=0000,obsgroup=MHS,reportype=3001");
 
     eckit::Date now(-2);
     eckit::Translator t;

From b6691d5fca4466d2e25ae189b0b8a5cd477675e3 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 10 Oct 2023 07:54:29 +0100
Subject: [PATCH 004/186] wip

---
 src/fdb5/database/DB.cc | 208 ----------------------------------------
 src/fdb5/database/DB.h  | 117 ----------------------
 2 files changed, 325 deletions(-)
 delete mode 100644 src/fdb5/database/DB.cc
 delete mode 100644 src/fdb5/database/DB.h

diff --git a/src/fdb5/database/DB.cc b/src/fdb5/database/DB.cc
deleted file mode 100644
index 8f3f20a2a..000000000
--- a/src/fdb5/database/DB.cc
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-#include "eckit/utils/StringTools.h"
-
-#include "fdb5/LibFdb5.h"
-#include "fdb5/database/DB.h"
-#include "fdb5/database/Field.h"
-#include "fdb5/toc/TocEngine.h"
-
-using eckit::Log;
-
-namespace fdb5 {
-
-//----------------------------------------------------------------------------------------------------------------------
-
-std::unique_ptr DB::buildReader(const Key &key, const fdb5::Config& config) {
-    return std::unique_ptr(new DB(key, config, true));
-}
-std::unique_ptr DB::buildWriter(const Key &key, const fdb5::Config& config) {
-    return std::unique_ptr(new DB(key, config, false));
-}
-std::unique_ptr DB::buildReader(const eckit::URI& uri, const fdb5::Config& config) {
-    return std::unique_ptr(new DB(uri, config, true));
-}
-std::unique_ptr DB::buildWriter(const eckit::URI& uri, const fdb5::Config& config) {
-    return std::unique_ptr(new DB(uri, config, false));
-}
-
-DB::DB(const Key& key, const fdb5::Config& config, bool read) {
-    catalogue_ = CatalogueFactory::instance().build(key, config.expandConfig(), read);
-}
-
-DB::DB(const eckit::URI& uri, const fdb5::Config& config, bool read) {
-    catalogue_ = CatalogueFactory::instance().build(uri, config.expandConfig(), read);
-}
-
-Store& DB::store() const {
-    if (store_ == nullptr) {
-        store_ = catalogue_->buildStore();
-    }
-
-    return *store_;
-}
-
-std::string DB::dbType() const {
-    return catalogue_->type();// + ":" + store_->type();
-}
-
-const Key& DB::key() const {
-    return catalogue_->key();
-}
-const Key& DB::indexKey() const {
-    return catalogue_->indexKey();
-}
-const Schema& DB::schema() const {
-    return catalogue_->schema();
-}
-
-bool DB::selectIndex(const Key &key) {
-    return catalogue_->selectIndex(key);
-}
-
-void DB::deselectIndex() {
-    return catalogue_->deselectIndex();
-}
-
-void DB::visitEntries(EntryVisitor& visitor, bool sorted) {
-    catalogue_->visitEntries(visitor, store(), sorted);
-}
-
-
-bool DB::axis(const std::string &keyword, eckit::StringSet &s) const {
-    CatalogueReader* cat = dynamic_cast(catalogue_.get());
-    ASSERT(cat);
-    return cat->axis(keyword, s);
-}
-
-// bool DB::inspect(const Key& key, Field& field) {
-
-//     eckit::Log::debug() << "Trying to retrieve key " << key << std::endl;
-
-//     CatalogueReader* cat = dynamic_cast(catalogue_.get());
-//     ASSERT(cat);
-
-//     return cat->retrieve(key, field);
-// }
-
-// eckit::DataHandle *DB::retrieve(const Key& key) {
-
-//     Field field;
-//     if (inspect(key, field)) {
-//         return store().retrieve(field);
-//     }
-
-//     return nullptr;
-// }
-
-// void DB::archive(const Key& key, const void* data, eckit::Length length) {
-
-//     CatalogueWriter* cat = dynamic_cast(catalogue_.get());
-//     ASSERT(cat);
-
-//     const Index& idx = cat->currentIndex();
-//     cat->archive(key, store().archive(idx.key(), data, length));
-// }
-
-bool DB::open() {
-    bool ret = catalogue_->open();
-    if (!ret)
-            return ret;
-
-    return store().open();
-}
-
-void DB::flush() {
-    if (store_ != nullptr)
-        store_->flush();
-    catalogue_->flush();
-}
-
-void DB::close() {
-    flush();
-    catalogue_->clean();
-    if (store_ != nullptr)
-        store_->close();
-    catalogue_->close();
-}
-
-bool DB::exists() const {
-    return (catalogue_->exists()/* && store_->exists()*/);
-}
-
-void DB::hideContents() {
-    if (catalogue_->type() == TocEngine::typeName()) {
-        catalogue_->hideContents();
-    }
-}
-
-eckit::URI DB::uri() const {
-    return catalogue_->uri();
-}
-
-void DB::overlayDB(const DB& otherDB, const std::set& variableKeys, bool unmount) {
-    if (catalogue_->type() == TocEngine::typeName() && otherDB.catalogue_->type() == TocEngine::typeName())  {
-        CatalogueWriter* cat = dynamic_cast(catalogue_.get());
-        ASSERT(cat);
-
-        cat->overlayDB(*(otherDB.catalogue_), variableKeys, unmount);
-    }
-}
-
-void DB::reconsolidate() {
-    CatalogueWriter* cat = dynamic_cast(catalogue_.get());
-    ASSERT(cat);
-
-    cat->reconsolidate();
-}
-
-// void DB::index(const Key &key, const eckit::PathName &path, eckit::Offset offset, eckit::Length length) {
-//     if (catalogue_->type() == TocEngine::typeName()) {
-//         CatalogueWriter* cat = dynamic_cast(catalogue_.get());
-//         ASSERT(cat);
-
-//         cat->index(key, eckit::URI("file", path), offset, length);
-//     }
-// }
-
-void DB::dump(std::ostream& out, bool simple, const eckit::Configuration& conf) const {
-    catalogue_->dump(out, simple, conf);
-}
-
-DbStats DB::stats() const {
-    CatalogueReader* cat = dynamic_cast(catalogue_.get());
-    ASSERT(cat);
-
-    return cat->stats();
-}
-
-void DB::control(const ControlAction& action, const ControlIdentifiers& identifiers) const {
-    catalogue_->control(action, identifiers);
-}
-bool DB::enabled(const ControlIdentifier& controlIdentifier) const {
-    return catalogue_->enabled(controlIdentifier);
-};
-
-void DB::print( std::ostream &out ) const {
-    catalogue_->print(out);
-}
-
-std::ostream &operator<<(std::ostream &s, const DB &x) {
-    x.print(s);
-    return s;
-}
-
-DBVisitor::~DBVisitor() {
-}
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace fdb5
diff --git a/src/fdb5/database/DB.h b/src/fdb5/database/DB.h
deleted file mode 100644
index a6de6e53a..000000000
--- a/src/fdb5/database/DB.h
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/// @file   DB.h
-/// @author Baudouin Raoult
-/// @author Tiago Quintino
-/// @date   Mar 2016
-
-#ifndef fdb5_DB_H
-#define fdb5_DB_H
-
-#include "eckit/types/Types.h"
-
-#include "fdb5/config/Config.h"
-#include "fdb5/database/Catalogue.h"
-#include "fdb5/database/EntryVisitMechanism.h"
-#include "fdb5/database/Key.h"
-#include "fdb5/database/Store.h"
-
-namespace eckit {
-class DataHandle;
-}
-
-namespace fdb5 {
-
-class Schema;
-class DbStats;
-
-enum class ControlAction : uint16_t;
-class ControlIdentifiers;
-
-//----------------------------------------------------------------------------------------------------------------------
-
-class DB final : public eckit::OwnedLock {
-
-public: // methods
-
-    static std::unique_ptr buildReader(const Key &key, const fdb5::Config& config = fdb5::Config());
-    static std::unique_ptr buildWriter(const Key &key, const fdb5::Config& config = fdb5::Config());
-    static std::unique_ptr buildReader(const eckit::URI& uri, const fdb5::Config& config = fdb5::Config());
-    static std::unique_ptr buildWriter(const eckit::URI& uri, const fdb5::Config& config = fdb5::Config());
-
-    std::string dbType() const;
-
-    const Key& key() const;
-    const Key& indexKey() const;
-    const Schema& schema() const;
-
-    bool axis(const std::string &keyword, eckit::StringSet &s) const;
-    // bool inspect(const Key& key, Field& field);
-    // eckit::DataHandle *retrieve(const Key &key);
-//    void archive(const Key &key, const void *data, eckit::Length length);
-
-    bool open();
-    void flush();
-    void close();
-
-    bool exists() const;
-
-    void dump(std::ostream& out, bool simple=false, const eckit::Configuration& conf = eckit::LocalConfiguration()) const;
-
-    bool selectIndex(const Key &key);
-    void deselectIndex();
-
-    DbStats stats() const;
-    void reconsolidate();
-
-    // for ToC tools
-    void hideContents();
-    eckit::URI uri() const;
-    void overlayDB(const DB& otherDB, const std::set& variableKeys, bool unmount);
-
-    void visitEntries(EntryVisitor& visitor, bool sorted = false);
-    /// Used for adopting & indexing external data to the TOC dir
-//    void index(const Key &key, const eckit::PathName &path, eckit::Offset offset, eckit::Length length);
-
-    // Control access properties of the DB
-    void control(const ControlAction& action, const ControlIdentifiers& identifiers) const;
-    bool enabled(const ControlIdentifier& controlIdentifier) const;
-
-protected: // methods
-
-    friend std::ostream &operator<<(std::ostream &s, const DB &x);
-    void print( std::ostream &out ) const;
-
-private: // members
-
-    DB(const Key &key, const fdb5::Config& config, bool read);
-    DB(const eckit::URI &uri, const fdb5::Config& config, bool read);
-
-    Store& store() const;
-
-    std::unique_ptr catalogue_;
-    mutable std::unique_ptr store_ = nullptr;
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-class DBVisitor : private eckit::NonCopyable {
-public:
-
-    virtual ~DBVisitor();
-    virtual void operator() (DB& db) = 0;
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace fdb5
-
-#endif

From 337cc514988f6366be30bce604f361d9c908e2b0 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 10 Oct 2023 08:46:45 +0100
Subject: [PATCH 005/186] cleanup

---
 src/fdb5/CMakeLists.txt                   |  2 --
 src/fdb5/LibFdb5.h                        |  2 +-
 src/fdb5/api/FDBFactory.h                 |  2 +-
 src/fdb5/api/LocalFDB.cc                  |  2 +-
 src/fdb5/api/local/DumpVisitor.h          |  2 +-
 src/fdb5/api/local/ListVisitor.h          |  2 +-
 src/fdb5/api/local/MoveVisitor.cc         |  2 +-
 src/fdb5/api/local/PurgeVisitor.cc        |  2 +-
 src/fdb5/api/local/StatsVisitor.cc        |  2 +-
 src/fdb5/api/local/WipeVisitor.cc         |  2 +-
 src/fdb5/database/ArchiveVisitor.cc       |  4 +--
 src/fdb5/database/Archiver.cc             |  1 +
 src/fdb5/database/Archiver.h              |  2 +-
 src/fdb5/database/BaseArchiveVisitor.cc   |  1 +
 src/fdb5/database/BaseArchiveVisitor.h    |  3 +-
 src/fdb5/database/Catalogue.cc            |  4 +--
 src/fdb5/database/Catalogue.h             |  2 +-
 src/fdb5/database/EntryVisitMechanism.cc  |  1 +
 src/fdb5/database/InspectionKey.cc        | 41 +++--------------------
 src/fdb5/database/InspectionKey.h         | 18 +++++-----
 src/fdb5/database/MultiRetrieveVisitor.cc |  2 +-
 src/fdb5/database/MultiRetrieveVisitor.h  |  2 --
 src/fdb5/database/RetrieveVisitor.cc      |  1 +
 src/fdb5/database/Store.h                 |  2 +-
 src/fdb5/rados/RadosStore.h               |  2 +-
 src/fdb5/remote/RemoteStore.h             |  2 +-
 src/fdb5/remote/StoreHandler.cc           |  7 ++--
 src/fdb5/toc/AdoptVisitor.cc              |  2 +-
 src/fdb5/toc/TocCatalogue.h               |  2 +-
 src/fdb5/toc/TocHandler.h                 |  2 +-
 src/fdb5/toc/TocMoveVisitor.cc            |  3 +-
 src/fdb5/toc/TocStore.h                   |  2 +-
 src/fdb5/toc/TocWipeVisitor.cc            | 14 ++++----
 src/fdb5/tools/FDBTool.h                  |  2 +-
 src/fdb5/tools/fdb-info.cc                |  2 +-
 src/fdb5/tools/fdb-list.cc                |  4 +--
 36 files changed, 58 insertions(+), 90 deletions(-)

diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt
index 17abb8ab0..8cdae8fb8 100644
--- a/src/fdb5/CMakeLists.txt
+++ b/src/fdb5/CMakeLists.txt
@@ -75,8 +75,6 @@ list( APPEND fdb5_srcs
     database/BaseArchiveVisitor.h
     database/Catalogue.cc
     database/Catalogue.h
-    # database/DB.cc
-    # database/DB.h
     database/DataStats.cc
     database/DataStats.h
     database/DbStats.cc
diff --git a/src/fdb5/LibFdb5.h b/src/fdb5/LibFdb5.h
index 23500f452..e7297ee8a 100644
--- a/src/fdb5/LibFdb5.h
+++ b/src/fdb5/LibFdb5.h
@@ -19,7 +19,7 @@
 
 #include "eckit/system/Library.h"
 
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
 #include "fdb5/types/TypesRegistry.h"
 
 namespace fdb5 {
diff --git a/src/fdb5/api/FDBFactory.h b/src/fdb5/api/FDBFactory.h
index 1d9970d5f..9c18f9ffe 100644
--- a/src/fdb5/api/FDBFactory.h
+++ b/src/fdb5/api/FDBFactory.h
@@ -25,7 +25,7 @@
 #include "eckit/utils/Regex.h"
 #include "eckit/memory/NonCopyable.h"
 
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
 #include "fdb5/config/Config.h"
 #include "fdb5/api/FDBStats.h"
 #include "fdb5/api/helpers/ListIterator.h"
diff --git a/src/fdb5/api/LocalFDB.cc b/src/fdb5/api/LocalFDB.cc
index b2ea1a7ea..a9d161d76 100644
--- a/src/fdb5/api/LocalFDB.cc
+++ b/src/fdb5/api/LocalFDB.cc
@@ -21,7 +21,7 @@
 #include "fdb5/api/helpers/FDBToolRequest.h"
 #include "fdb5/api/LocalFDB.h"
 #include "fdb5/database/Archiver.h"
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
 #include "fdb5/database/EntryVisitMechanism.h"
 #include "fdb5/database/Index.h"
 #include "fdb5/database/Inspector.h"
diff --git a/src/fdb5/api/local/DumpVisitor.h b/src/fdb5/api/local/DumpVisitor.h
index 3b7bdc9a9..e943d53e3 100644
--- a/src/fdb5/api/local/DumpVisitor.h
+++ b/src/fdb5/api/local/DumpVisitor.h
@@ -22,7 +22,7 @@
 #include "fdb5/api/local/QueryVisitor.h"
 #include "fdb5/api/local/QueueStringLogTarget.h"
 #include "fdb5/api/helpers/DumpIterator.h"
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
 
 namespace fdb5 {
 namespace api {
diff --git a/src/fdb5/api/local/ListVisitor.h b/src/fdb5/api/local/ListVisitor.h
index 7956af637..b65f4d25e 100644
--- a/src/fdb5/api/local/ListVisitor.h
+++ b/src/fdb5/api/local/ListVisitor.h
@@ -19,7 +19,7 @@
 #ifndef fdb5_api_local_ListVisitor_H
 #define fdb5_api_local_ListVisitor_H
 
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
 #include "fdb5/database/Index.h"
 #include "fdb5/database/InspectionKey.h"
 #include "fdb5/api/local/QueryVisitor.h"
diff --git a/src/fdb5/api/local/MoveVisitor.cc b/src/fdb5/api/local/MoveVisitor.cc
index a3d249a4a..ccf6a6046 100644
--- a/src/fdb5/api/local/MoveVisitor.cc
+++ b/src/fdb5/api/local/MoveVisitor.cc
@@ -17,7 +17,7 @@
 #include "fdb5/api/local/MoveVisitor.h"
 
 #include "fdb5/api/local/QueueStringLogTarget.h"
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
 #include "fdb5/database/Index.h"
 #include "fdb5/LibFdb5.h"
 
diff --git a/src/fdb5/api/local/PurgeVisitor.cc b/src/fdb5/api/local/PurgeVisitor.cc
index e608894cd..bc3e3a972 100644
--- a/src/fdb5/api/local/PurgeVisitor.cc
+++ b/src/fdb5/api/local/PurgeVisitor.cc
@@ -19,7 +19,7 @@
 #include "eckit/exception/Exceptions.h"
 
 #include "fdb5/api/local/QueueStringLogTarget.h"
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
 #include "fdb5/database/PurgeVisitor.h"
 #include "fdb5/LibFdb5.h"
 
diff --git a/src/fdb5/api/local/StatsVisitor.cc b/src/fdb5/api/local/StatsVisitor.cc
index b681a1eac..c520d8cc8 100644
--- a/src/fdb5/api/local/StatsVisitor.cc
+++ b/src/fdb5/api/local/StatsVisitor.cc
@@ -16,7 +16,7 @@
 
 #include "fdb5/api/local/StatsVisitor.h"
 
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
 #include "fdb5/database/StatsReportVisitor.h"
 
 namespace fdb5 {
diff --git a/src/fdb5/api/local/WipeVisitor.cc b/src/fdb5/api/local/WipeVisitor.cc
index cb6420363..3661b2e07 100644
--- a/src/fdb5/api/local/WipeVisitor.cc
+++ b/src/fdb5/api/local/WipeVisitor.cc
@@ -17,7 +17,7 @@
 #include "fdb5/api/local/WipeVisitor.h"
 
 #include "fdb5/api/local/QueueStringLogTarget.h"
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
 #include "fdb5/database/Index.h"
 #include "fdb5/LibFdb5.h"
 
diff --git a/src/fdb5/database/ArchiveVisitor.cc b/src/fdb5/database/ArchiveVisitor.cc
index 1c9b2f0d3..465262828 100644
--- a/src/fdb5/database/ArchiveVisitor.cc
+++ b/src/fdb5/database/ArchiveVisitor.cc
@@ -10,9 +10,9 @@
 
 #include "eckit/exception/Exceptions.h"
 
-
-#include "fdb5/database/DB.h"
 #include "fdb5/database/ArchiveVisitor.h"
+#include "fdb5/database/Catalogue.h"
+#include "fdb5/database/Store.h"
 
 namespace fdb5 {
 
diff --git a/src/fdb5/database/Archiver.cc b/src/fdb5/database/Archiver.cc
index faaf68807..8c74dae18 100644
--- a/src/fdb5/database/Archiver.cc
+++ b/src/fdb5/database/Archiver.cc
@@ -18,6 +18,7 @@
 #include "fdb5/database/BaseArchiveVisitor.h"
 #include "fdb5/rules/Schema.h"
 #include "fdb5/rules/Rule.h"
+#include "fdb5/database/Store.h"
 
 namespace fdb5 {
 
diff --git a/src/fdb5/database/Archiver.h b/src/fdb5/database/Archiver.h
index 3abe0e53b..a4ba3700b 100644
--- a/src/fdb5/database/Archiver.h
+++ b/src/fdb5/database/Archiver.h
@@ -21,7 +21,7 @@
 
 #include "eckit/memory/NonCopyable.h"
 
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
 #include "fdb5/config/Config.h"
 
 namespace eckit   {
diff --git a/src/fdb5/database/BaseArchiveVisitor.cc b/src/fdb5/database/BaseArchiveVisitor.cc
index b60ab40a3..f3b752528 100644
--- a/src/fdb5/database/BaseArchiveVisitor.cc
+++ b/src/fdb5/database/BaseArchiveVisitor.cc
@@ -14,6 +14,7 @@
 #include "fdb5/database/Archiver.h"
 #include "fdb5/database/BaseArchiveVisitor.h"
 #include "fdb5/rules/Rule.h"
+#include "fdb5/database/Store.h"
 
 namespace fdb5 {
 
diff --git a/src/fdb5/database/BaseArchiveVisitor.h b/src/fdb5/database/BaseArchiveVisitor.h
index b4bd41398..507c60885 100644
--- a/src/fdb5/database/BaseArchiveVisitor.h
+++ b/src/fdb5/database/BaseArchiveVisitor.h
@@ -23,7 +23,8 @@ namespace metkit { class MarsRequest; }
 namespace fdb5 {
 
 class Archiver;
-class DB;
+class Catalogue;
+class Store;
 class Schema;
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/database/Catalogue.cc b/src/fdb5/database/Catalogue.cc
index f7a337a65..af560436b 100644
--- a/src/fdb5/database/Catalogue.cc
+++ b/src/fdb5/database/Catalogue.cc
@@ -11,8 +11,6 @@
 #include 
 #include 
 
-#include "fdb5/database/Catalogue.h"
-
 #include "eckit/config/Resource.h"
 #include "eckit/exception/Exceptions.h"
 #include "eckit/thread/AutoLock.h"
@@ -20,7 +18,9 @@
 #include "eckit/utils/StringTools.h"
 
 #include "fdb5/LibFdb5.h"
+#include "fdb5/database/Catalogue.h"
 #include "fdb5/database/Manager.h"
+#include "fdb5/database/Store.h"
 
 namespace fdb5 {
 
diff --git a/src/fdb5/database/Catalogue.h b/src/fdb5/database/Catalogue.h
index e03987dbf..e094aa208 100644
--- a/src/fdb5/database/Catalogue.h
+++ b/src/fdb5/database/Catalogue.h
@@ -22,7 +22,7 @@
 #include "fdb5/api/helpers/ControlIterator.h"
 #include "fdb5/api/helpers/MoveIterator.h"
 #include "fdb5/config/Config.h"
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
 #include "fdb5/database/Field.h"
 #include "fdb5/database/FieldLocation.h"
 #include "fdb5/database/Key.h"
diff --git a/src/fdb5/database/EntryVisitMechanism.cc b/src/fdb5/database/EntryVisitMechanism.cc
index a97b4f9bd..0b10460b6 100644
--- a/src/fdb5/database/EntryVisitMechanism.cc
+++ b/src/fdb5/database/EntryVisitMechanism.cc
@@ -17,6 +17,7 @@
 #include "fdb5/database/InspectionKey.h"
 #include "fdb5/LibFdb5.h"
 #include "fdb5/rules/Schema.h"
+#include "fdb5/database/Store.h"
 
 using namespace eckit;
 
diff --git a/src/fdb5/database/InspectionKey.cc b/src/fdb5/database/InspectionKey.cc
index b36952c93..a8c43ad32 100644
--- a/src/fdb5/database/InspectionKey.cc
+++ b/src/fdb5/database/InspectionKey.cc
@@ -88,10 +88,10 @@ InspectionKey::InspectionKey(const eckit::StringDict &keys) :
     }
 }
 
-InspectionKey::InspectionKey(eckit::Stream& s) :
-    rule_(nullptr) {
-    decode(s);
-}
+// InspectionKey::InspectionKey(eckit::Stream& s) :
+//     rule_(nullptr) {
+//     decode(s);
+// }
 
 Key InspectionKey::canonical() const {
     Key key;
@@ -157,10 +157,6 @@ void InspectionKey::rule(const Rule *rule) {
     rule_ = rule;
 }
 
-// const Rule *InspectionKey::rule() const {
-//     return rule_;
-// }
-
 const TypesRegistry& InspectionKey::registry() const {
     if(rule_) {
         return rule_->registry();
@@ -178,7 +174,6 @@ std::string InspectionKey::canonicalise(const std::string& keyword, const std::s
     }
 }
 
-
 fdb5::InspectionKey::operator eckit::StringDict() const
 {
     eckit::StringDict res;
@@ -200,34 +195,6 @@ fdb5::InspectionKey::operator eckit::StringDict() const
     return res;
 }
 
-// void InspectionKey::print(std::ostream &out) const {
-//     if (names_.size() == keys_.size()) {
-//         out << "{" << toString() << "}";
-//         if (rule_) {
-//             out << " (" << *rule_ << ")";
-//         }
-//     } else {
-//         out << keys_;
-//         if (rule_) {
-//             out << " (" << *rule_ << ")";
-//         }
-//     }
-// }
-
-// std::string Key::toString() const {
-//     std::string res;
-//     const char *sep = "";
-//     for (eckit::StringList::const_iterator j = names_.begin(); j != names_.end(); ++j) {
-//         eckit::StringDict::const_iterator i = keys_.find(*j);
-//         ASSERT(i != keys_.end());
-//         if (!i->second.empty()) {
-//             res += sep + *j + '=' + i->second;
-//             sep = ",";
-//         }
-//     }
-//     return res;
-// }
-
 //----------------------------------------------------------------------------------------------------------------------
 
 } // namespace fdb5
diff --git a/src/fdb5/database/InspectionKey.h b/src/fdb5/database/InspectionKey.h
index 4e6587ad7..53ce7e878 100644
--- a/src/fdb5/database/InspectionKey.h
+++ b/src/fdb5/database/InspectionKey.h
@@ -33,7 +33,7 @@ class InspectionKey : public Key {
     InspectionKey();
 
     explicit InspectionKey(const Key& other);
-    explicit InspectionKey(eckit::Stream &);
+    // explicit InspectionKey(eckit::Stream &);
     explicit InspectionKey(const std::string &request);
     explicit InspectionKey(const std::string &keys, const Rule* rule);
 
@@ -47,14 +47,14 @@ class InspectionKey : public Key {
 //    const Rule *rule() const;
     const TypesRegistry& registry() const;
 
-    friend eckit::Stream& operator>>(eckit::Stream& s, InspectionKey& x) {
-        x = InspectionKey(s);
-        return s;
-    }
-    friend eckit::Stream& operator<<(eckit::Stream &s, const InspectionKey &x) {
-        x.encode(s);
-        return s;
-    }
+    // friend eckit::Stream& operator>>(eckit::Stream& s, InspectionKey& x) {
+    //     x = InspectionKey(s);
+    //     return s;
+    // }
+    // friend eckit::Stream& operator<<(eckit::Stream &s, const InspectionKey &x) {
+    //     x.encode(s);
+    //     return s;
+    // }
 
     friend std::ostream& operator<<(std::ostream &s, const InspectionKey& x) {
         x.print(s);
diff --git a/src/fdb5/database/MultiRetrieveVisitor.cc b/src/fdb5/database/MultiRetrieveVisitor.cc
index 70812bd09..dc1271367 100644
--- a/src/fdb5/database/MultiRetrieveVisitor.cc
+++ b/src/fdb5/database/MultiRetrieveVisitor.cc
@@ -15,7 +15,7 @@
 #include "eckit/config/Resource.h"
 
 #include "fdb5/LibFdb5.h"
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
 #include "fdb5/database/Key.h"
 #include "fdb5/io/HandleGatherer.h"
 #include "fdb5/types/Type.h"
diff --git a/src/fdb5/database/MultiRetrieveVisitor.h b/src/fdb5/database/MultiRetrieveVisitor.h
index 788907755..43a2b8ea4 100644
--- a/src/fdb5/database/MultiRetrieveVisitor.h
+++ b/src/fdb5/database/MultiRetrieveVisitor.h
@@ -31,8 +31,6 @@ namespace fdb5 {
 class HandleGatherer;
 class Notifier;
 
-class DB;
-
 //----------------------------------------------------------------------------------------------------------------------
 
 class MultiRetrieveVisitor : public ReadVisitor {
diff --git a/src/fdb5/database/RetrieveVisitor.cc b/src/fdb5/database/RetrieveVisitor.cc
index 9ccb86d8b..a08c0a9cc 100644
--- a/src/fdb5/database/RetrieveVisitor.cc
+++ b/src/fdb5/database/RetrieveVisitor.cc
@@ -14,6 +14,7 @@
 
 #include "fdb5/LibFdb5.h"
 #include "fdb5/database/Key.h"
+#include "fdb5/database/Store.h"
 #include "fdb5/io/HandleGatherer.h"
 #include "fdb5/types/Type.h"
 #include "fdb5/types/TypesRegistry.h"
diff --git a/src/fdb5/database/Store.h b/src/fdb5/database/Store.h
index 2af38484c..97143af83 100644
--- a/src/fdb5/database/Store.h
+++ b/src/fdb5/database/Store.h
@@ -22,7 +22,7 @@
 
 #include "fdb5/api/helpers/MoveIterator.h"
 #include "fdb5/config/Config.h"
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
 #include "fdb5/database/Field.h"
 #include "fdb5/database/FieldLocation.h"
 #include "fdb5/database/Key.h"
diff --git a/src/fdb5/rados/RadosStore.h b/src/fdb5/rados/RadosStore.h
index 367e78825..502caf03e 100644
--- a/src/fdb5/rados/RadosStore.h
+++ b/src/fdb5/rados/RadosStore.h
@@ -15,7 +15,7 @@
 #ifndef fdb5_RadosStore_H
 #define fdb5_RadosStore_H
 
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
 #include "fdb5/database/Index.h"
 #include "fdb5/database/Store.h"
 #include "fdb5/rules/Schema.h"
diff --git a/src/fdb5/remote/RemoteStore.h b/src/fdb5/remote/RemoteStore.h
index d46d04d21..d6c1f3904 100644
--- a/src/fdb5/remote/RemoteStore.h
+++ b/src/fdb5/remote/RemoteStore.h
@@ -14,7 +14,7 @@
 
 #pragma once
 
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
 #include "fdb5/database/Index.h"
 #include "fdb5/database/Store.h"
 #include "fdb5/remote/ClientConnection.h"
diff --git a/src/fdb5/remote/StoreHandler.cc b/src/fdb5/remote/StoreHandler.cc
index 3a04e5198..1fb5b6bd7 100644
--- a/src/fdb5/remote/StoreHandler.cc
+++ b/src/fdb5/remote/StoreHandler.cc
@@ -8,11 +8,12 @@
  * does it submit to any jurisdiction.
  */
 
-#include "fdb5/remote/StoreHandler.h"
-#include "eckit/serialisation/MemoryStream.h"
-#include "fdb5/LibFdb5.h"
 #include "eckit/config/Resource.h"
+#include "eckit/serialisation/MemoryStream.h"
 
+#include "fdb5/LibFdb5.h"
+#include "fdb5/database/Store.h"
+#include "fdb5/remote/StoreHandler.h"
 
 using namespace eckit;
 using metkit::mars::MarsRequest;
diff --git a/src/fdb5/toc/AdoptVisitor.cc b/src/fdb5/toc/AdoptVisitor.cc
index ad661f259..c44a98333 100644
--- a/src/fdb5/toc/AdoptVisitor.cc
+++ b/src/fdb5/toc/AdoptVisitor.cc
@@ -11,7 +11,7 @@
 #include "eckit/exception/Exceptions.h"
 #include "eckit/log/Log.h"
 
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
 #include "fdb5/toc/AdoptVisitor.h"
 #include "fdb5/toc/TocEngine.h"
 
diff --git a/src/fdb5/toc/TocCatalogue.h b/src/fdb5/toc/TocCatalogue.h
index d5e593267..a77e0546f 100644
--- a/src/fdb5/toc/TocCatalogue.h
+++ b/src/fdb5/toc/TocCatalogue.h
@@ -16,7 +16,7 @@
 #ifndef fdb5_TocDB_H
 #define fdb5_TocDB_H
 
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
 #include "fdb5/database/Index.h"
 #include "fdb5/rules/Schema.h"
 #include "fdb5/toc/FileSpace.h"
diff --git a/src/fdb5/toc/TocHandler.h b/src/fdb5/toc/TocHandler.h
index 122f0241a..ec660d714 100644
--- a/src/fdb5/toc/TocHandler.h
+++ b/src/fdb5/toc/TocHandler.h
@@ -25,7 +25,7 @@
 
 #include "fdb5/config/Config.h"
 #include "fdb5/database/DbStats.h"
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
 #include "fdb5/toc/TocCommon.h"
 #include "fdb5/toc/TocRecord.h"
 #include "fdb5/toc/TocSerialisationVersion.h"
diff --git a/src/fdb5/toc/TocMoveVisitor.cc b/src/fdb5/toc/TocMoveVisitor.cc
index 5ba90d3ca..0cd9b1daf 100644
--- a/src/fdb5/toc/TocMoveVisitor.cc
+++ b/src/fdb5/toc/TocMoveVisitor.cc
@@ -17,7 +17,8 @@
 #include "eckit/os/Stat.h"
 
 #include "fdb5/api/helpers/ControlIterator.h"
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
+#include "fdb5/database/Store.h"
 #include "fdb5/toc/TocCatalogue.h"
 #include "fdb5/toc/TocMoveVisitor.h"
 #include "fdb5/toc/RootManager.h"
diff --git a/src/fdb5/toc/TocStore.h b/src/fdb5/toc/TocStore.h
index 3fc2c1b5f..4983e2a22 100644
--- a/src/fdb5/toc/TocStore.h
+++ b/src/fdb5/toc/TocStore.h
@@ -16,7 +16,7 @@
 #ifndef fdb5_TocStore_H
 #define fdb5_TocStore_H
 
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
 #include "fdb5/database/Index.h"
 #include "fdb5/database/Store.h"
 #include "fdb5/rules/Schema.h"
diff --git a/src/fdb5/toc/TocWipeVisitor.cc b/src/fdb5/toc/TocWipeVisitor.cc
index eeba7204d..4311cce61 100644
--- a/src/fdb5/toc/TocWipeVisitor.cc
+++ b/src/fdb5/toc/TocWipeVisitor.cc
@@ -10,20 +10,20 @@
 
 
 #include 
+#include 
+#include 
+#include 
+#include 
+#include 
 
 #include "eckit/os/Stat.h"
 
 #include "fdb5/api/helpers/ControlIterator.h"
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
+#include "fdb5/database/Store.h"
 #include "fdb5/toc/TocCatalogue.h"
 #include "fdb5/toc/TocWipeVisitor.h"
 
-#include 
-#include 
-#include 
-#include 
-#include 
-
 
 using namespace eckit;
 
diff --git a/src/fdb5/tools/FDBTool.h b/src/fdb5/tools/FDBTool.h
index 14bbc15d1..ada18b5f4 100644
--- a/src/fdb5/tools/FDBTool.h
+++ b/src/fdb5/tools/FDBTool.h
@@ -21,7 +21,7 @@
 #include "eckit/runtime/Tool.h"
 #include "eckit/filesystem/PathName.h"
 
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
 #include "eckit/option/SimpleOption.h"
 
 namespace eckit {
diff --git a/src/fdb5/tools/fdb-info.cc b/src/fdb5/tools/fdb-info.cc
index b32d21d03..396647fc8 100644
--- a/src/fdb5/tools/fdb-info.cc
+++ b/src/fdb5/tools/fdb-info.cc
@@ -12,7 +12,7 @@
 
 #include "fdb5/LibFdb5.h"
 #include "fdb5/config/Config.h"
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
 #include "fdb5/database/Index.h"
 #include "fdb5/tools/FDBInspect.h"
 #include "fdb5/io/LustreSettings.h"
diff --git a/src/fdb5/tools/fdb-list.cc b/src/fdb5/tools/fdb-list.cc
index de4271646..fb2c7a715 100644
--- a/src/fdb5/tools/fdb-list.cc
+++ b/src/fdb5/tools/fdb-list.cc
@@ -19,7 +19,7 @@
 
 #include "fdb5/api/FDB.h"
 #include "fdb5/api/helpers/FDBToolRequest.h"
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
 #include "fdb5/database/Index.h"
 #include "fdb5/rules/Schema.h"
 #include "fdb5/tools/FDBVisitTool.h"
@@ -100,7 +100,6 @@ void FDBList::execute(const CmdArgs& args) {
         // If --full is supplied, then include all entries including duplicates.
         auto listObject = fdb.list(request, !full_);
 
-        size_t count = 0;
         ListElement elem;
         while (listObject.next(elem)) {
 
@@ -109,7 +108,6 @@ void FDBList::execute(const CmdArgs& args) {
             } else {
                 elem.print(Log::info(), location_, !porcelain_);
                 Log::info() << std::endl;
-                count++;
             }
         }
 

From d5f9ae85a8fdfa92e5f7fb6d58566f61f12bb92c Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 19 Oct 2023 10:54:05 +0100
Subject: [PATCH 006/186] removed DB, added fdb remote connection multiplexer

---
 src/fdb5/CMakeLists.txt                       |   30 +-
 src/fdb5/api/FDBFactory.h                     |   14 +-
 src/fdb5/api/RemoteFDB.cc                     | 1124 -----------------
 src/fdb5/api/RemoteFDB.h                      |  178 ---
 src/fdb5/api/local/ControlVisitor.cc          |    4 +-
 src/fdb5/api/local/ControlVisitor.h           |    3 +-
 src/fdb5/api/local/DumpVisitor.h              |    3 +-
 src/fdb5/api/local/ListVisitor.h              |    6 +-
 src/fdb5/api/local/MoveVisitor.cc             |   12 +-
 src/fdb5/api/local/MoveVisitor.h              |    3 +-
 src/fdb5/api/local/PurgeVisitor.cc            |    9 +-
 src/fdb5/api/local/PurgeVisitor.h             |    4 +-
 src/fdb5/api/local/StatsVisitor.cc            |    7 +-
 src/fdb5/api/local/StatsVisitor.h             |    3 +-
 src/fdb5/api/local/StatusVisitor.h            |    3 +-
 src/fdb5/api/local/WipeVisitor.cc             |   11 +-
 src/fdb5/api/local/WipeVisitor.h              |    3 +-
 src/fdb5/database/ArchiveVisitor.cc           |    5 +-
 src/fdb5/database/Archiver.cc                 |    2 +-
 src/fdb5/database/Archiver.h                  |    4 +-
 src/fdb5/database/BaseArchiveVisitor.cc       |    2 +-
 src/fdb5/database/BaseArchiveVisitor.h        |    4 +-
 src/fdb5/database/Catalogue.cc                |  154 ++-
 src/fdb5/database/Catalogue.h                 |  141 ++-
 src/fdb5/database/EntryVisitMechanism.cc      |   27 +-
 src/fdb5/database/EntryVisitMechanism.h       |    9 +-
 src/fdb5/database/Inspector.cc                |    2 +-
 src/fdb5/database/Inspector.h                 |    4 +-
 src/fdb5/database/MultiRetrieveVisitor.cc     |   10 +-
 src/fdb5/database/MultiRetrieveVisitor.h      |    4 +-
 src/fdb5/database/ReadVisitor.cc              |   20 +-
 src/fdb5/database/ReadVisitor.h               |    6 +-
 src/fdb5/database/RetrieveVisitor.cc          |    8 +-
 src/fdb5/rados/RadosStore.cc                  |    5 +
 src/fdb5/rados/RadosStore.h                   |    1 +
 src/fdb5/remote/CatalogueHandler.cc           |    6 +-
 src/fdb5/remote/FdbServer.cc                  |   14 +-
 src/fdb5/remote/Messages.h                    |   14 +-
 src/fdb5/remote/RemoteFieldLocation.cc        |   30 +-
 src/fdb5/remote/RemoteFieldLocation.h         |   13 +-
 src/fdb5/remote/RemoteStore.cc                |  656 ++++------
 src/fdb5/remote/RemoteStore.h                 |   33 +-
 src/fdb5/remote/client/Client.cc              |   47 +
 src/fdb5/remote/client/Client.h               |   73 ++
 .../remote/{ => client}/ClientConnection.cc   |  250 ++--
 .../remote/{ => client}/ClientConnection.h    |  105 +-
 .../remote/client/ClientConnectionRouter.cc   |  210 +++
 .../remote/client/ClientConnectionRouter.h    |  111 ++
 src/fdb5/remote/server/DataStoreStrategies.h  |  409 ++++++
 src/fdb5/remote/server/ServerConnection.cc    |  356 ++++++
 src/fdb5/remote/server/ServerConnection.h     |  112 ++
 src/fdb5/remote/{ => server}/StoreHandler.cc  |   68 +-
 src/fdb5/remote/{ => server}/StoreHandler.h   |   15 +-
 src/fdb5/rules/SchemaParser.cc                |   14 +-
 src/fdb5/rules/SchemaParser.h                 |    2 +-
 src/fdb5/toc/AdoptVisitor.cc                  |    7 +-
 src/fdb5/toc/FieldRef.cc                      |   20 +-
 src/fdb5/toc/TocCatalogue.cc                  |   10 +-
 src/fdb5/toc/TocCatalogue.h                   |    4 +-
 src/fdb5/toc/TocCatalogueReader.cc            |    5 +-
 src/fdb5/toc/TocCatalogueWriter.cc            |    3 +-
 src/fdb5/toc/TocIndex.cc                      |    1 +
 src/fdb5/toc/TocMoveVisitor.cc                |    6 +-
 src/fdb5/toc/TocMoveVisitor.h                 |    3 +-
 src/fdb5/toc/TocPurgeVisitor.cc               |    7 +-
 src/fdb5/toc/TocPurgeVisitor.h                |    3 +-
 src/fdb5/toc/TocStats.cc                      |    9 +-
 src/fdb5/toc/TocStats.h                       |    3 +-
 src/fdb5/toc/TocStore.h                       |    1 +
 src/fdb5/toc/TocWipeVisitor.cc                |    6 +-
 src/fdb5/toc/TocWipeVisitor.h                 |    3 +-
 src/fdb5/tools/fdb-hide.cc                    |    5 +-
 src/fdb5/tools/fdb-overlay.cc                 |   11 +-
 src/fdb5/tools/fdb-reconsolidate-toc.cc       |    6 +-
 src/fdb5/tools/fdb-root.cc                    |    4 +-
 src/fdb5/tools/fdb-schema.cc                  |    2 +-
 src/fdb5/tools/fdb-stats-new.cc               |    2 +-
 77 files changed, 2332 insertions(+), 2152 deletions(-)
 delete mode 100644 src/fdb5/api/RemoteFDB.cc
 delete mode 100644 src/fdb5/api/RemoteFDB.h
 create mode 100644 src/fdb5/remote/client/Client.cc
 create mode 100644 src/fdb5/remote/client/Client.h
 rename src/fdb5/remote/{ => client}/ClientConnection.cc (78%)
 rename src/fdb5/remote/{ => client}/ClientConnection.h (63%)
 create mode 100644 src/fdb5/remote/client/ClientConnectionRouter.cc
 create mode 100644 src/fdb5/remote/client/ClientConnectionRouter.h
 create mode 100644 src/fdb5/remote/server/DataStoreStrategies.h
 create mode 100644 src/fdb5/remote/server/ServerConnection.cc
 create mode 100644 src/fdb5/remote/server/ServerConnection.h
 rename src/fdb5/remote/{ => server}/StoreHandler.cc (87%)
 rename src/fdb5/remote/{ => server}/StoreHandler.h (75%)

diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt
index 8cdae8fb8..500023ba6 100644
--- a/src/fdb5/CMakeLists.txt
+++ b/src/fdb5/CMakeLists.txt
@@ -119,7 +119,7 @@ list( APPEND fdb5_srcs
     database/Key.h
     database/InspectionKey.cc
     database/InspectionKey.h
-    database/ReadVisitor.cc
+    # database/ReadVisitor.cc
     database/ReadVisitor.h
     database/Report.cc
     database/Report.h
@@ -210,21 +210,29 @@ list( APPEND fdb5_srcs
 
 if(HAVE_FDB_REMOTE)
     list( APPEND fdb5_srcs 
-        api/RemoteFDB.cc
-        api/RemoteFDB.h
+        # api/RemoteFDB.cc
+        # api/RemoteFDB.h
         remote/RemoteStore.cc
         remote/RemoteStore.h
 
-        remote/ClientConnection.h
-        remote/ClientConnection.cc
+        remote/client/Client.h
+        remote/client/Client.cc
+        remote/client/ClientConnection.h
+        remote/client/ClientConnection.cc
+        remote/client/ClientConnectionRouter.h
+        remote/client/ClientConnectionRouter.cc
+        remote/server/StoreHandler.h
+        remote/server/StoreHandler.cc
+        remote/server/ServerConnection.h
+        remote/server/ServerConnection.cc
         remote/RemoteConfiguration.h
         remote/RemoteConfiguration.cc
         remote/RemoteFieldLocation.h
         remote/RemoteFieldLocation.cc
         remote/Messages.h
         remote/Messages.cc
-        remote/Handler.h
-        remote/Handler.cc
+        # remote/Handler.h
+        # remote/Handler.cc
         remote/AvailablePortList.cc
         remote/AvailablePortList.h
         remote/FdbServer.h
@@ -232,10 +240,10 @@ if(HAVE_FDB_REMOTE)
 
         # remote/CatalogueHandler.h
         # remote/CatalogueHandler.cc
-        remote/StoreHandler.h
-        remote/StoreHandler.cc
-        remote/DecoupledHandler.h
-        remote/DecoupledHandler.cc
+        # remote/StoreHandler.h
+        # remote/StoreHandler.cc
+        # remote/DecoupledHandler.h
+        # remote/DecoupledHandler.cc
     )
 endif()
 
diff --git a/src/fdb5/api/FDBFactory.h b/src/fdb5/api/FDBFactory.h
index 9c18f9ffe..39119e102 100644
--- a/src/fdb5/api/FDBFactory.h
+++ b/src/fdb5/api/FDBFactory.h
@@ -38,13 +38,17 @@
 #include "fdb5/api/helpers/StatsIterator.h"
 #include "fdb5/api/helpers/StatusIterator.h"
 
-namespace eckit {
-namespace message {
+namespace eckit::message {
+
 class Message;
-}
-}  // namespace eckit
 
-namespace metkit { class MarsRequest; }
+}  // namespace eckit::message
+
+namespace metkit {
+
+class MarsRequest;
+
+} // namespace metkit
 
 namespace fdb5 {
 
diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
deleted file mode 100644
index db402846a..000000000
--- a/src/fdb5/api/RemoteFDB.cc
+++ /dev/null
@@ -1,1124 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-#include 
-#include 
-
-#include "fdb5/api/RemoteFDB.h"
-#include "fdb5/LibFdb5.h"
-#include "fdb5/io/HandleGatherer.h"
-#include "fdb5/remote/Messages.h"
-#include "fdb5/remote/RemoteFieldLocation.h"
-#include "fdb5/api/helpers/FDBToolRequest.h"
-#include "fdb5/database/Key.h"
-
-#include "eckit/config/LocalConfiguration.h"
-#include "eckit/io/Buffer.h"
-#include "eckit/log/Bytes.h"
-#include "eckit/log/Log.h"
-#include "eckit/message/Message.h"
-#include "eckit/distributed/Transport.h"
-#include "eckit/config/Resource.h"
-#include "eckit/serialisation/MemoryStream.h"
-#include "eckit/utils/Translator.h"
-#include "eckit/runtime/Main.h"
-#include "eckit/os/BackTrace.h"
-
-#include "metkit/mars/MarsRequest.h"
-
-using namespace eckit;
-using namespace eckit::net;
-using namespace fdb5::remote;
-
-
-namespace eckit {
-template<> struct Translator {
-    std::string operator()(const net::Endpoint& e) {
-        std::stringstream ss;
-        ss << e;
-        return ss.str();
-    }
-};
-}
-
-namespace fdb5 {
-
-//----------------------------------------------------------------------------------------------------------------------
-
-class ConnectionError : public eckit::Exception {
-public:
-    ConnectionError(const int);
-    ConnectionError(const int, const eckit::net::Endpoint&);
-
-    bool retryOnClient() const override { return true; } 
-};
-
-ConnectionError::ConnectionError(const int retries) {
-    std::ostringstream s;
-    s << "Unable to create a connection with the FDB server after " << retries << " retries";
-    reason(s.str());
-    Log::status() << what() << std::endl;
-}
-
-ConnectionError::ConnectionError(const int retries, const eckit::net::Endpoint& endpoint) {
-    std::ostringstream s;
-    s << "Unable to create a connection with the FDB endpoint " << endpoint << " after " << retries << " retries";
-    reason(s.str());
-    Log::status() << what() << std::endl;
-}
-
-//----------------------------------------------------------------------------------------------------------------------
-
-namespace {
-class TCPException : public Exception {
-public:
-    TCPException(const std::string& msg, const CodeLocation& here) :
-        Exception(std::string("TCPException: ") + msg, here) {
-
-        eckit::Log::error() << "TCP Exception; backtrace(): " << std::endl;
-        eckit::Log::error() << eckit::BackTrace::dump() << std::endl;
-    }
-};
-}
-
-class RemoteFDBException : public RemoteException {
-public:
-    RemoteFDBException(const std::string& msg, const net::Endpoint& endpoint):
-        RemoteException(msg, eckit::Translator()(endpoint)) {}
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-// n.b. if we get integer overflow, we reuse the IDs. This is not a
-//      big deal. The idea that we could be on the 2.1 billionth (successful)
-//      request, and still have an ongoing request 0 is ... laughable.
-
-static uint32_t generateRequestID() {
-
-    static std::mutex m;
-    static uint32_t id = 0;
-
-    std::lock_guard lock(m);
-    return ++id;
-}
-
-
-RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name) :
-    FDBBase(config, name),
-    controlEndpoint_(config.getString("host"), config.getInt("port")),
-    archiveID_(0),
-    maxArchiveQueueLength_(eckit::Resource("fdbRemoteArchiveQueueLength;$FDB_REMOTE_ARCHIVE_QUEUE_LENGTH", 200)),
-    maxArchiveBatchSize_(config.getInt("maxBatchSize", 1)),
-    retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
-    connected_(false) {}
-
-
-RemoteFDB::~RemoteFDB() {
-
-    // If we have launched a thread with an async and we manage to get here, this is
-    // an error. n.b. if we don't do something, we will block in the destructor
-    // of std::future.
-
-    if (archiveFuture_.valid()) {
-        Log::error() << "Attempting to destruct RemoteFDB with active archive thread" << std::endl;
-        eckit::Main::instance().terminate();
-    }
-
-    disconnect();
-}
-
-
-// Functions for management of the connection
-
-// Protocol negotiation:
-//
-// i) Connect to server. Send:
-//     - session identification
-//     - supported functionality for protocol negotiation
-//
-// ii) Server responds. Sends:
-//     - returns the client session id (for verification)
-//     - the server session id
-//     - endpoint for data connection
-//     - selected functionality for protocol negotiation
-//
-// iii) Open data connection, and write to server:
-//     - client session id
-//     - server session id
-//
-// This appears quite verbose in terms of protocol negotiation, but the repeated
-// sending of both session ids allows clients and servers to be sure that they are
-// connected to the correct endpoint in a multi-process, multi-host environment.
-//
-// This was an issue on the NextGenIO prototype machine, where TCP connections were
-// getting lost, and/or incorrectly reset, and as a result sessions were being
-// mispaired by the network stack!
-
-void RemoteFDB::connect() {
-
-    if (!connected_) {
-
-        static int fdbMaxConnectRetries = eckit::Resource("fdbMaxConnectRetries", 5);
-
-        try {
-            // Connect to server, and check that the server is happy on the response
-
-            Log::debug() << "Connecting to host: " << controlEndpoint_ << std::endl;
-            controlClient_.connect(controlEndpoint_, fdbMaxConnectRetries);
-            writeControlStartupMessage();
-            SessionID serverSession = verifyServerStartupResponse();
-
-            // Connect to the specified data port
-            Log::debug() << "Received data endpoint from host: " << dataEndpoint_ << std::endl;
-            dataClient_.connect(dataEndpoint_, fdbMaxConnectRetries);
-            writeDataStartupMessage(serverSession);
-
-            // And the connections are set up. Let everything start up!
-            listeningThread_ = std::thread([this] { listeningThreadLoop(); });
-            connected_ = true;
-        } catch(TooManyRetries& e) {
-            if (controlClient_.isConnected()) {
-                controlClient_.close();
-                throw ConnectionError(fdbMaxConnectRetries, dataEndpoint_);
-            } else {
-                throw ConnectionError(fdbMaxConnectRetries, controlEndpoint_);
-            }
-        }
-    }
-}
-
-void RemoteFDB::disconnect() {
-    if (connected_) {
-
-        // Send termination message
-        controlWrite(fdb5::remote::Message::Exit, generateRequestID());
-
-        listeningThread_.join();
-
-        // Close both the control and data connections
-        controlClient_.close();
-        dataClient_.close();
-        connected_ = false;
-    }
-}
-
-void RemoteFDB::writeControlStartupMessage() {
-
-    Buffer payload(4096);
-    MemoryStream s(payload);
-    s << sessionID_;
-    s << controlEndpoint_;
-    s << LibFdb5::instance().remoteProtocolVersion().used();
-
-    // TODO: Abstract this dictionary into a RemoteConfiguration object, which
-    //       understands how to do the negotiation, etc, but uses Value (i.e.
-    //       essentially JSON) over the wire for flexibility.
-    s << availableFunctionality().get();
-
-    controlWrite(fdb5::remote::Message::Startup, 0, payload.data(), s.position());
-}
-
-SessionID RemoteFDB::verifyServerStartupResponse() {
-
-    MessageHeader hdr;
-    controlRead(&hdr, sizeof(hdr));
-
-    ASSERT(hdr.marker == StartMarker);
-    ASSERT(hdr.version == CurrentVersion);
-    ASSERT(hdr.message == fdb5::remote::Message::Startup);
-    ASSERT(hdr.requestID == 0);
-
-    Buffer payload(hdr.payloadSize);
-    eckit::FixedString<4> tail;
-    controlRead(payload, hdr.payloadSize);
-    controlRead(&tail, sizeof(tail));
-    ASSERT(tail == EndMarker);
-
-    MemoryStream s(payload);
-    SessionID clientSession(s);
-    SessionID serverSession(s);
-    Endpoint dataEndpoint(s);
-    LocalConfiguration serverFunctionality(s);
-
-    dataEndpoint_ = dataEndpoint;
-
-    if (dataEndpoint_.hostname() != controlEndpoint_.hostname()) {
-        Log::warning() << "Data and control interface hostnames do not match. "
-                       << dataEndpoint_.hostname() << " /= "
-                       << controlEndpoint_.hostname() << std::endl;
-    }
-
-    if (clientSession != sessionID_) {
-        std::stringstream ss;
-        ss << "Session ID does not match session received from server: "
-           << sessionID_ << " != " << clientSession;
-        throw BadValue(ss.str(), Here());
-    }
-
-    return serverSession;
-}
-
-void RemoteFDB::writeDataStartupMessage(const eckit::SessionID& serverSession) {
-
-    Buffer payload(1024);
-    MemoryStream s(payload);
-
-    s << sessionID_;
-    s << serverSession;
-
-    dataWrite(fdb5::remote::Message::Startup, 0, payload.data(), s.position());
-}
-
-eckit::LocalConfiguration RemoteFDB::availableFunctionality() const {
-    eckit::LocalConfiguration conf;
-    std::vector remoteFieldLocationVersions = {1};
-    conf.set("RemoteFieldLocation", remoteFieldLocationVersions);
-    return conf;
-}
-
-void RemoteFDB::listeningThreadLoop() {
-
-    /// @note This routine retrieves BOTH normal API asynchronously returned data, AND
-    /// fields that are being returned by a retrieve() call. These need to go into different
-    /// queues
-    /// --> Test if the requestID is a known API request, otherwise push onto the retrieve queue
-
-
-    /// @note messageQueues_ is a map of requestID:MessageQueue. At the point that
-    /// a request is complete, errored or otherwise killed, it needs to be removed
-    /// from the map. The shared_ptr allows this removal to be asynchronous with
-    /// the actual task cleaning up and returning to the client.
-
-    try {
-
-    MessageHeader hdr;
-    eckit::FixedString<4> tail;
-
-    while (true) {
-
-        dataRead(&hdr, sizeof(hdr));
-
-        ASSERT(hdr.marker == StartMarker);
-        ASSERT(hdr.version == CurrentVersion);
-
-        switch (hdr.message) {
-
-        case fdb5::remote::Message::Exit:
-            return;
-
-        case fdb5::remote::Message::Blob: {
-            Buffer payload(hdr.payloadSize);
-            if (hdr.payloadSize > 0) dataRead(payload, hdr.payloadSize);
-
-            auto it = messageQueues_.find(hdr.requestID);
-            if (it != messageQueues_.end()) {
-                it->second->emplace(std::make_pair(hdr, std::move(payload)));
-            } else {
-                retrieveMessageQueue_.emplace(std::make_pair(hdr, std::move(payload)));
-            }
-            break;
-        }
-
-        case fdb5::remote::Message::Complete: {
-            auto it = messageQueues_.find(hdr.requestID);
-            if (it != messageQueues_.end()) {
-                it->second->close();
-
-                // Remove entry (shared_ptr --> message queue will be destroyed when it
-                // goes out of scope in the worker thread).
-                messageQueues_.erase(it);
-
-            } else {
-                retrieveMessageQueue_.emplace(std::make_pair(hdr, Buffer(0)));
-            }
-            break;
-        }
-
-        case fdb5::remote::Message::Error: {
-
-            auto it = messageQueues_.find(hdr.requestID);
-            if (it != messageQueues_.end()) {
-                std::string msg;
-                if (hdr.payloadSize > 0) {
-                    msg.resize(hdr.payloadSize, ' ');
-                    dataRead(&msg[0], hdr.payloadSize);
-                }
-                it->second->interrupt(std::make_exception_ptr(RemoteFDBException(msg, dataEndpoint_)));
-
-                // Remove entry (shared_ptr --> message queue will be destroyed when it
-                // goes out of scope in the worker thread).
-                messageQueues_.erase(it);
-
-            } else if (hdr.requestID == archiveID_) {
-
-                std::lock_guard lock(archiveQueuePtrMutex_);
-
-                if (archiveQueue_) {
-                    std::string msg;
-                    if (hdr.payloadSize > 0) {
-                        msg.resize(hdr.payloadSize, ' ');
-                        dataRead(&msg[0], hdr.payloadSize);
-                    }
-
-                    archiveQueue_->interrupt(std::make_exception_ptr(RemoteFDBException(msg, dataEndpoint_)));
-                }
-            } else {
-                Buffer payload(hdr.payloadSize);
-                if (hdr.payloadSize > 0) dataRead(payload, hdr.payloadSize);
-                retrieveMessageQueue_.emplace(std::make_pair(hdr, std::move(payload)));
-            }
-            break;
-        }
-
-        default: {
-            std::stringstream ss;
-            ss << "ERROR: Unexpected message recieved (" << static_cast(hdr.message) << "). ABORTING";
-            Log::status() << ss.str() << std::endl;
-            Log::error() << "Retrieving... " << ss.str() << std::endl;
-            throw SeriousBug(ss.str(), Here());
-        }
-        };
-
-        // Ensure we have consumed exactly the correct amount from the socket.
-
-        dataRead(&tail, sizeof(tail));
-        ASSERT(tail == EndMarker);
-    }
-
-    // We don't want to let exceptions escape inside a worker thread.
-
-    } catch (const std::exception& e) {
-        for (auto& it : messageQueues_) {
-            it.second->interrupt(std::make_exception_ptr(e));
-        }
-        messageQueues_.clear();
-        retrieveMessageQueue_.interrupt(std::make_exception_ptr(e));
-        {
-            std::lock_guard lock(archiveQueuePtrMutex_);
-            if (archiveQueue_) archiveQueue_->interrupt(std::make_exception_ptr(e));
-        }
-    } catch (...) {
-        for (auto& it : messageQueues_) {
-            it.second->interrupt(std::current_exception());
-        }
-        messageQueues_.clear();
-        retrieveMessageQueue_.interrupt(std::current_exception());
-        {
-            std::lock_guard lock(archiveQueuePtrMutex_);
-            if (archiveQueue_) archiveQueue_->interrupt(std::current_exception());
-        }
-    }
-}
-
-void RemoteFDB::controlWriteCheckResponse(fdb5::remote::Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
-
-    controlWrite(msg, requestID, payload, payloadLength);
-
-    // Wait for the receipt acknowledgement
-
-    MessageHeader response;
-    controlRead(&response, sizeof(MessageHeader));
-
-    handleError(response);
-
-    ASSERT(response.marker == StartMarker);
-    ASSERT(response.version == CurrentVersion);
-    ASSERT(response.message == fdb5::remote::Message::Received);
-
-    eckit::FixedString<4> tail;
-    controlRead(&tail, sizeof(tail));
-    ASSERT(tail == EndMarker);
-}
-
-void RemoteFDB::controlWrite(fdb5::remote::Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
-
-    ASSERT((payload == nullptr) == (payloadLength == 0));
-
-    MessageHeader message(msg, requestID, payloadLength);
-    controlWrite(&message, sizeof(message));
-    if (payload) {
-        controlWrite(payload, payloadLength);
-    }
-    controlWrite(&EndMarker, sizeof(EndMarker));
-}
-
-void RemoteFDB::controlWrite(const void* data, size_t length) {
-    size_t written = controlClient_.write(data, length);
-    if (length != written) {
-        std::stringstream ss;
-        ss << "Write error. Expected " << length << " bytes, wrote " << written;
-        throw TCPException(ss.str(), Here());
-    }
-}
-
-void RemoteFDB::controlRead(void* data, size_t length) {
-    size_t read = controlClient_.read(data, length);
-    if (length != read) {
-        std::stringstream ss;
-        ss << "Read error. Expected " << length << " bytes, read " << read;
-        throw TCPException(ss.str(), Here());
-    }
-}
-
-void RemoteFDB::dataWrite(fdb5::remote::Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
-
-    ASSERT((payload == nullptr) == (payloadLength == 0));
-
-    MessageHeader message(msg, requestID, payloadLength);
-    dataWrite(&message, sizeof(message));
-    if (payload) {
-        dataWrite(payload, payloadLength);
-    }
-    dataWrite(&EndMarker, sizeof(EndMarker));
-}
-
-void RemoteFDB::dataWrite(const void* data, size_t length) {
-    size_t written = dataClient_.write(data, length);
-    if (length != written) {
-        std::stringstream ss;
-        ss << "Write error. Expected " << length << " bytes, wrote " << written;
-        throw TCPException(ss.str(), Here());
-    }
-}
-
-void RemoteFDB::dataRead(void* data, size_t length) {
-    size_t read = dataClient_.read(data, length);
-    if (length != read) {
-        std::stringstream ss;
-        ss << "Read error. Expected " << length << " bytes, read " << read;
-        throw TCPException(ss.str(), Here());
-    }
-}
-
-void RemoteFDB::handleError(const MessageHeader& hdr) {
-
-    ASSERT(hdr.marker == StartMarker);
-    ASSERT(hdr.version == CurrentVersion);
-
-    if (hdr.message == fdb5::remote::Message::Error) {
-        ASSERT(hdr.payloadSize > 9);
-
-        std::string what(hdr.payloadSize, ' ');
-        controlRead(&what[0], hdr.payloadSize);
-        what[hdr.payloadSize] = 0; // Just in case
-
-        try {
-            eckit::FixedString<4> tail;
-            controlRead(&tail, sizeof(tail));
-        } catch (...) {}
-
-        throw RemoteFDBException(what, controlEndpoint_);
-    }
-}
-
-FDBStats RemoteFDB::stats() const {
-    return internalStats_;
-}
-
-
-// Implement the primary FDB API
-
-// -----------------------------------------------------------------------------------------------------
-// Helper classes describe the behaviour of the various API functions to be forwarded
-// -----------------------------------------------------------------------------------------------------
-
-namespace {
-
-template 
-struct BaseAPIHelper {
-
-    typedef T ValueType;
-
-    static size_t bufferSize() { return 4096; }
-    static size_t queueSize() { return 100; }
-    static fdb5::remote::Message message() { return msgID; }
-
-    void encodeExtra(eckit::Stream& s) const {}
-    static ValueType valueFromStream(eckit::Stream& s, RemoteFDB* fdb) { return ValueType(s); }
-};
-
-using ListHelper = BaseAPIHelper;
-
-struct InspectHelper : BaseAPIHelper {
-
-    static ListElement valueFromStream(eckit::Stream& s, RemoteFDB* fdb) {
-        ListElement elem(s);
-        return ListElement(elem.key(), RemoteFieldLocation(fdb, elem.location()).make_shared(), elem.timestamp());
-    }
-};
-
-using StatsHelper = BaseAPIHelper;
-
-struct StatusHelper : BaseAPIHelper {
-
-    static StatusElement valueFromStream(eckit::Stream& s, RemoteFDB* fdb) {
-        StatusElement elem(s);
-        elem.location.endpoint(fdb->controlEndpoint());
-        return elem;
-    }
-};
-
-struct DumpHelper : BaseAPIHelper {
-
-    DumpHelper(bool simple) : simple_(simple) {}
-    void encodeExtra(eckit::Stream& s) const { s << simple_; }
-    static DumpElement valueFromStream(eckit::Stream& s, RemoteFDB* fdb) {
-        DumpElement elem;
-        s >> elem;
-        return elem;
-    }
-
-private:
-    bool simple_;
-};
-
-struct PurgeHelper : BaseAPIHelper {
-
-    PurgeHelper(bool doit, bool porcelain) : doit_(doit), porcelain_(porcelain) {}
-    void encodeExtra(eckit::Stream& s) const {
-        s << doit_;
-        s << porcelain_;
-    }
-    static PurgeElement valueFromStream(eckit::Stream& s, RemoteFDB*) {
-        PurgeElement elem;
-        s >> elem;
-        return elem;
-    }
-
-private:
-    bool doit_;
-    bool porcelain_;
-};
-
-struct WipeHelper : BaseAPIHelper {
-
-    WipeHelper(bool doit, bool porcelain, bool unsafeWipeAll) :
-        doit_(doit), porcelain_(porcelain), unsafeWipeAll_(unsafeWipeAll) {}
-    void encodeExtra(eckit::Stream& s) const {
-        s << doit_;
-        s << porcelain_;
-        s << unsafeWipeAll_;
-    }
-    static WipeElement valueFromStream(eckit::Stream& s, RemoteFDB*) {
-        WipeElement elem;
-        s >> elem;
-        return elem;
-    }
-
-private:
-    bool doit_;
-    bool porcelain_;
-    bool unsafeWipeAll_;
-};
-
-struct MoveHelper : BaseAPIHelper {
-
-    
-    MoveHelper(const eckit::URI& dest) :
-        dest_(dest) {}
-
-    void encodeExtra(eckit::Stream& s) const {
-        s << dest_;
-    }
-    static MoveElement valueFromStream(eckit::Stream& s, RemoteFDB*) {
-        MoveElement elem(s);
-        return elem;
-    }
-
-private:
-    const eckit::URI& dest_;
-};
-
-struct ControlHelper : BaseAPIHelper {
-
-    ControlHelper(ControlAction action, ControlIdentifiers identifiers) :
-        action_(action),
-        identifiers_(identifiers) {}
-    void encodeExtra(eckit::Stream& s) const {
-        s << action_;
-        s << identifiers_;
-    }
-
-private:
-    ControlAction action_;
-    ControlIdentifiers identifiers_;
-};
-
-} // namespace
-
-// -----------------------------------------------------------------------------------------------------
-
-// forwardApiCall captures the asynchronous behaviour:
-//
-// i) Set up a Queue to receive the messages as they come in
-// ii) Encode the request+arguments and send them to the server
-// iii) Return an AsyncIterator that pulls messages off the queue, and returns them to the caller.
-
-
-template 
-auto RemoteFDB::forwardApiCall(const HelperClass& helper, const FDBToolRequest& request) -> APIIterator {
-
-    using ValueType = typename HelperClass::ValueType;
-    using IteratorType = APIIterator;
-    using AsyncIterator = APIAsyncIterator;
-
-    connect();
-
-    // Ensure we have an entry in the message queue before we trigger anything that
-    // will result in return messages
-
-    uint32_t id = generateRequestID();
-    auto entry = messageQueues_.emplace(id, std::make_shared(HelperClass::queueSize()));
-    ASSERT(entry.second);
-    std::shared_ptr messageQueue(entry.first->second);
-
-    // Encode the request and send it to the server
-
-    Buffer encodeBuffer(HelperClass::bufferSize());
-    MemoryStream s(encodeBuffer);
-    s << request;
-    helper.encodeExtra(s);
-
-    controlWriteCheckResponse(HelperClass::message(), id, encodeBuffer, s.position());
-
-    // Return an AsyncIterator to allow the messages to be retrieved in the API
-
-    RemoteFDB* remoteFDB = this;
-    return IteratorType(
-        // n.b. Don't worry about catching exceptions in lambda, as
-        // this is handled in the AsyncIterator.
-        new AsyncIterator([messageQueue, remoteFDB](eckit::Queue& queue) {
-            StoredMessage msg = std::make_pair(remote::MessageHeader{}, eckit::Buffer{0});
-                        while (true) {
-                            if (messageQueue->pop(msg) == -1) {
-                                break;
-                            } else {
-                                MemoryStream s(msg.second);
-                                queue.emplace(HelperClass::valueFromStream(s, remoteFDB));
-                            }
-                        }
-                        // messageQueue goes out of scope --> destructed
-                    }
-                )
-           );
-}
-
-ListIterator RemoteFDB::list(const FDBToolRequest& request) {
-    return forwardApiCall(ListHelper(), request);
-}
-
-ListIterator RemoteFDB::inspect(const metkit::mars::MarsRequest& request) {
-    return forwardApiCall(InspectHelper(), request);
-}
-
-
-DumpIterator RemoteFDB::dump(const FDBToolRequest& request, bool simple) {
-    return forwardApiCall(DumpHelper(simple), request);
-}
-
-StatusIterator RemoteFDB::status(const FDBToolRequest& request) {
-    return forwardApiCall(StatusHelper(), request);
-}
-
-WipeIterator RemoteFDB::wipe(const FDBToolRequest& request, bool doit, bool porcelain, bool unsafeWipeAll) {
-    return forwardApiCall(WipeHelper(doit, porcelain, unsafeWipeAll), request);
-}
-
-PurgeIterator RemoteFDB::purge(const FDBToolRequest& request, bool doit, bool porcelain) {
-    return forwardApiCall(PurgeHelper(doit, porcelain), request);
-}
-
-StatsIterator RemoteFDB::stats(const FDBToolRequest& request) {
-    return forwardApiCall(StatsHelper(), request);
-}
-
-ControlIterator RemoteFDB::control(const FDBToolRequest& request,
-                                   ControlAction action,
-                                   ControlIdentifiers identifiers) {
-    return forwardApiCall(ControlHelper(action, identifiers), request);
-};
-
-MoveIterator RemoteFDB::move(const FDBToolRequest& request, const eckit::URI& dest) {
-    return forwardApiCall(MoveHelper(dest), request);
-}
-
-// -----------------------------------------------------------------------------------------------------
-
-// Here we do archive/flush related stuff
-void RemoteFDB::archive(const Key& key, const void* data, size_t length) {
-
-    connect();
-
-    // if there is no archiving thread active, then start one.
-    // n.b. reset the archiveQueue_ after a potential flush() cycle.
-
-    if (!archiveFuture_.valid()) {
-
-        // Start the archival request on the remote side
-        ASSERT(archiveID_ == 0);
-        uint32_t id = generateRequestID();
-        controlWriteCheckResponse(fdb5::remote::Message::Archive, id);
-        archiveID_ = id;
-
-        // Reset the queue after previous done/errors
-        {
-            std::lock_guard lock(archiveQueuePtrMutex_);
-            ASSERT(!archiveQueue_);
-            archiveQueue_.reset(new ArchiveQueue(maxArchiveQueueLength_));
-        }
-
-        archiveFuture_ = std::async(std::launch::async, [this, id] { return archiveThreadLoop(id); });
-    }
-
-    ASSERT(archiveFuture_.valid());
-    ASSERT(archiveID_ != 0);
-    {
-        std::lock_guard lock(archiveQueuePtrMutex_);
-        ASSERT(archiveQueue_);
-        archiveQueue_->emplace(std::make_pair(key, Buffer(reinterpret_cast(data), length)));
-    }
-}
-
-
-void RemoteFDB::flush() {
-
-    Timer timer;
-
-    timer.start();
-
-    // Flush only does anything if there is an ongoing archive();
-    if (archiveFuture_.valid()) {
-
-        ASSERT(archiveID_ != 0);
-        {
-            ASSERT(archiveQueue_);
-            std::lock_guard lock(archiveQueuePtrMutex_);
-            archiveQueue_->close();
-        }
-        FDBStats stats = archiveFuture_.get();
-        ASSERT(!archiveQueue_);
-        archiveID_ = 0;
-
-        ASSERT(stats.numFlush() == 0);
-        size_t numArchive = stats.numArchive();
-
-        Buffer sendBuf(4096);
-        MemoryStream s(sendBuf);
-        s << numArchive;
-
-        // The flush call is blocking
-        controlWriteCheckResponse(fdb5::remote::Message::Flush, generateRequestID(), sendBuf, s.position());
-
-        internalStats_ += stats;
-    }
-
-    timer.stop();
-    internalStats_.addFlush(timer);
-}
-
-
-FDBStats RemoteFDB::archiveThreadLoop(uint32_t requestID) {
-
-    FDBStats localStats;
-    eckit::Timer timer;
-
-    // We can pop multiple elements off the archive queue simultaneously, if
-    // configured
-    std::vector> elements;
-    for (size_t i = 0; i < maxArchiveBatchSize_; ++i) {
-        elements.emplace_back(std::make_pair(Key{}, Buffer{0}));
-    }
-
-    try {
-
-        ASSERT(archiveQueue_);
-        ASSERT(archiveID_ != 0);
-
-        long popped;
-        while ((popped = archiveQueue_->pop(elements)) != -1) {
-
-            timer.start();
-            long dataSent = sendArchiveData(requestID, elements, popped);
-            timer.stop();
-            localStats.addArchive(dataSent, timer, popped);
-        }
-
-        // And note that we are done. (don't time this, as already being blocked
-        // on by the ::flush() routine)
-
-        MessageHeader hdr(fdb5::remote::Message::Flush, requestID);
-        dataWrite(&hdr, sizeof(hdr));
-        dataWrite(&EndMarker, sizeof(EndMarker));
-
-        archiveID_ = 0;
-        archiveQueue_.reset();
-
-    } catch (...) {
-        archiveQueue_->interrupt(std::current_exception());
-        throw;
-    }
-
-    return localStats;
-
-    // We are inside an async, so don't need to worry about exceptions escaping.
-    // They will be released when flush() is called.
-}
-
-long RemoteFDB::sendArchiveData(uint32_t id, const std::vector>& elements, size_t count) {
-
-    if (count == 1) {
-        sendArchiveData(id, elements[0].first, elements[0].second.data(), elements[0].second.size());
-        return elements[0].second.size();
-    }
-
-    // Serialise the keys
-
-    std::vector keyBuffers;
-    std::vector keySizes;
-    keyBuffers.reserve(count);
-    keySizes.reserve(count);
-
-    size_t containedSize = 0;
-
-    for (size_t i = 0; i < count; ++i) {
-        keyBuffers.emplace_back(Buffer {4096});
-        MemoryStream keyStream(keyBuffers.back());
-        keyStream << elements[i].first;
-        keySizes.push_back(keyStream.position());
-        containedSize += (keyStream.position() + elements[i].second.size() +
-                          sizeof(MessageHeader) + sizeof(EndMarker));
-    }
-
-    // Construct the containing message
-
-    MessageHeader message(fdb5::remote::Message::MultiBlob, id, containedSize);
-    dataWrite(&message, sizeof(message));
-
-    long dataSent = 0;
-
-    for (size_t i = 0; i < count; ++i) {
-        MessageHeader containedMessage(fdb5::remote::Message::Blob, id, elements[i].second.size() + keySizes[i]);
-        dataWrite(&containedMessage, sizeof(containedMessage));
-        dataWrite(keyBuffers[i], keySizes[i]);
-        dataWrite(elements[i].second.data(), elements[i].second.size());
-        dataWrite(&EndMarker, sizeof(EndMarker));
-        dataSent += elements[i].second.size();
-    }
-
-    dataWrite(&EndMarker, sizeof(EndMarker));
-    return dataSent;
-}
-
-
-void RemoteFDB::sendArchiveData(uint32_t id, const Key& key, const void* data, size_t length) {
-
-    ASSERT(data);
-    ASSERT(length != 0);
-
-    Buffer keyBuffer(4096);
-    MemoryStream keyStream(keyBuffer);
-    keyStream << key;
-
-    MessageHeader message(fdb5::remote::Message::Blob, id, length + keyStream.position());
-    dataWrite(&message, sizeof(message));
-    dataWrite(keyBuffer, keyStream.position());
-    dataWrite(data, length);
-    dataWrite(&EndMarker, sizeof(EndMarker));
-}
-
-// -----------------------------------------------------------------------------------------------------
-
-//
-/// @note The DataHandles returned by retrieve() MUST STRICTLY be read in order.
-///       We do not create multiple message queues, one for each requestID, even
-///       though that would be nice. This is because a commonly used retrieve
-///       pattern uses many retrieve() calls aggregated into a MultiHandle, and
-///       if we created lots of queues we would just run out of memory receiving
-///       from the stream. Further, if we curcumvented this by blocking, then we
-///       could get deadlocked if we try and read a message that is further back
-///       in the stream
-///
-/// --> Retrieve is a _streaming_ service.
-
-namespace {
-
-class FDBRemoteDataHandle : public DataHandle {
-
-public: // methods
-
-    FDBRemoteDataHandle(uint32_t requestID,
-                        RemoteFDB::MessageQueue& queue,
-                        const net::Endpoint& remoteEndpoint) :
-        requestID_(requestID),
-        queue_(queue),
-        remoteEndpoint_(remoteEndpoint),
-        pos_(0),
-        overallPosition_(0),
-        currentBuffer_(0),
-        complete_(false) {}
-    virtual bool canSeek() const override { return false; }
-
-private: // methods
-
-    void print(std::ostream& s) const override {
-        s << "FDBRemoteDataHandle(id=" << requestID_ << ")";
-    }
-
-    Length openForRead() override {
-        return estimate();
-    }
-    void openForWrite(const Length&) override { NOTIMP; }
-    void openForAppend(const Length&) override { NOTIMP; }
-    long write(const void*, long) override { NOTIMP; }
-    void close() override {}
-
-    long read(void* pos, long sz) override {
-
-        if (complete_) return 0;
-
-        if (currentBuffer_.size() != 0) return bufferRead(pos, sz);
-
-        // If we are in the DataHandle, then there MUST be data to read
-
-        RemoteFDB::StoredMessage msg = std::make_pair(remote::MessageHeader{}, eckit::Buffer{0});
-        ASSERT(queue_.pop(msg) != -1);
-
-        // TODO; Error handling in the retrieve pathway
-
-        const MessageHeader& hdr(msg.first);
-
-        ASSERT(hdr.marker == StartMarker);
-        ASSERT(hdr.version == CurrentVersion);
-
-        // Handle any remote errors communicated from the server
-
-        if (hdr.message == fdb5::remote::Message::Error) {
-            std::string errmsg(static_cast(msg.second), msg.second.size());
-            throw RemoteFDBException(errmsg, remoteEndpoint_);
-        }
-
-        // Are we now complete
-
-        if (hdr.message == fdb5::remote::Message::Complete) {
-            complete_ = 0;
-            return 0;
-        }
-
-        ASSERT(hdr.message == fdb5::remote::Message::Blob);
-
-        // Otherwise return the data!
-
-        std::swap(currentBuffer_, msg.second);
-
-        return bufferRead(pos, sz);
-    }
-
-    // A helper function that returns some, or all, of a buffer that has
-    // already been retrieved.
-
-    long bufferRead(void* pos, long sz) {
-
-        ASSERT(currentBuffer_.size() != 0);
-        ASSERT(pos_ < currentBuffer_.size());
-
-        long read = std::min(sz, long(currentBuffer_.size() - pos_));
-
-        ::memcpy(pos, ¤tBuffer_[pos_], read);
-        pos_ += read;
-        overallPosition_ += read;
-
-        // If we have exhausted this buffer, free it up.
-
-        if (pos_ >= currentBuffer_.size()) {
-            Buffer nullBuffer(0);
-            std::swap(currentBuffer_, nullBuffer);
-            pos_ = 0;
-            ASSERT(currentBuffer_.size() == 0);
-        }
-
-        return read;
-    }
-
-    Length estimate() override {
-        return 0;
-    }
-
-    Offset position() override {
-        return overallPosition_;
-    }
-
-private: // members
-
-    uint32_t requestID_;
-    RemoteFDB::MessageQueue& queue_;
-    net::Endpoint remoteEndpoint_;
-    size_t pos_;
-    Offset overallPosition_;
-    Buffer currentBuffer_;
-    bool complete_;
-};
-
-}
-
-// Here we do (asynchronous) retrieving related stuff
-
-//DataHandle* RemoteFDB::retrieve(const metkit::mars::MarsRequest& request) {
-
-//    connect();
-
-//    Buffer encodeBuffer(4096);
-//    MemoryStream s(encodeBuffer);
-//    s << request;
-
-//    uint32_t id = generateRequestID();
-
-//    controlWriteCheckResponse(fdb5::remote::Message::Retrieve, id, encodeBuffer, s.position());
-
-//    return new FDBRemoteDataHandle(id, retrieveMessageQueue_, controlEndpoint_);
-//}
-
-
-// Here we do (asynchronous) read related stuff
-
-
-eckit::DataHandle* RemoteFDB::dataHandle(const FieldLocation& fieldLocation) {
-    return dataHandle(fieldLocation, Key());
-}
-
-eckit::DataHandle* RemoteFDB::dataHandle(const FieldLocation& fieldLocation, const Key& remapKey) {
-
-    connect();
-
-    Buffer encodeBuffer(4096);
-    MemoryStream s(encodeBuffer);
-    s << fieldLocation;
-    s << remapKey;
-
-    uint32_t id = generateRequestID();
-
-    controlWriteCheckResponse(fdb5::remote::Message::Read, id, encodeBuffer, s.position());
-
-    return new FDBRemoteDataHandle(id, retrieveMessageQueue_, controlEndpoint_);
-}
-
-void RemoteFDB::print(std::ostream &s) const {
-    s << "RemoteFDB(host=" << controlEndpoint_ << ", data=" << dataEndpoint_ << ")";
-}
-
-static FDBBuilder remoteFdbBuilder("remote");
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace fdb5
diff --git a/src/fdb5/api/RemoteFDB.h b/src/fdb5/api/RemoteFDB.h
deleted file mode 100644
index 38f83e4f2..000000000
--- a/src/fdb5/api/RemoteFDB.h
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Simon Smart
-/// @date   Mar 2018
-
-#ifndef fdb5_remote_RemoteFDB_H
-#define fdb5_remote_RemoteFDB_H
-
-#include 
-#include 
-
-#include "eckit/container/Queue.h"
-#include "eckit/io/Buffer.h"
-#include "eckit/net/Endpoint.h"
-#include "eckit/net/TCPClient.h"
-#include "eckit/net/TCPStream.h"
-#include "eckit/runtime/SessionID.h"
-
-#include "fdb5/api/FDB.h"
-#include "fdb5/api/FDBFactory.h"
-#include "fdb5/remote/Messages.h"
-
-namespace fdb5 {
-
-class FDB;
-
-//----------------------------------------------------------------------------------------------------------------------
-
-class RemoteFDB : public FDBBase {
-
-public: // types
-
-    using StoredMessage = std::pair;
-    using MessageQueue = eckit::Queue;
-    using ArchiveQueue = eckit::Queue>;
-
-public: // method
-
-    using FDBBase::stats;
-
-    RemoteFDB(const eckit::Configuration& config, const std::string& name);
-    ~RemoteFDB() override;
-
-    /// Archive writes data into aggregation buffer
-    void archive(const Key& key, const void* data, size_t length) override;
-
-    eckit::DataHandle* dataHandle(const FieldLocation& fieldLocation);
-    eckit::DataHandle* dataHandle(const FieldLocation& fieldLocation, const Key& remapKey);
-
-    ListIterator inspect(const metkit::mars::MarsRequest& request) override;
-
-    ListIterator list(const FDBToolRequest& request) override;
-
-    DumpIterator dump(const FDBToolRequest& request, bool simple) override;
-
-    StatusIterator status(const FDBToolRequest& request) override;
-
-    WipeIterator wipe(const FDBToolRequest& request, bool doit, bool porcelain, bool unsafeWipeAll) override;
-
-    PurgeIterator purge(const FDBToolRequest& request, bool doit, bool porcelain) override;
-
-    StatsIterator stats(const FDBToolRequest& request) override;
-
-    ControlIterator control(const FDBToolRequest& request,
-                            ControlAction action,
-                            ControlIdentifiers identifiers) override;
-
-    MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) override;
-
-    void flush() override;
-
-    const eckit::net::Endpoint& controlEndpoint() const { return controlEndpoint_; }
-
-private: // methods
-
-    // Methods to control the connection
-
-    void connect();
-    void disconnect();
-
-    // Session negotiation with the server
-    void writeControlStartupMessage();
-    eckit::SessionID verifyServerStartupResponse();
-    void writeDataStartupMessage(const eckit::SessionID& serverSession);
-
-    // Construct dictionary for protocol negotiation
-
-    eckit::LocalConfiguration availableFunctionality() const;
-
-    // Listen to the dataClient for incoming messages, and push them onto
-    // appropriate queues.
-    void listeningThreadLoop();
-
-    // Handle data going in either direction on the wire
-    void controlWriteCheckResponse(remote::Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
-    void controlWrite(remote::Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
-    void controlWrite(const void* data, size_t length);
-    void controlRead(void* data, size_t length);
-    void dataWrite(remote::Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
-    void dataWrite(const void* data, size_t length);
-    void dataRead(void* data, size_t length);
-    void handleError(const remote::MessageHeader& hdr);
-
-    // Worker for the API functions
-
-    template 
-    auto forwardApiCall(const HelperClass& helper, const FDBToolRequest& request) -> APIIterator;
-
-    // Workers for archiving
-
-    FDBStats archiveThreadLoop(uint32_t requestID);
-
-    void sendArchiveData(uint32_t id, const Key& key, const void* data, size_t length);
-    long sendArchiveData(uint32_t id, const std::vector>& elements, size_t count);
-
-    virtual void print(std::ostream& s) const override;
-
-    virtual FDBStats stats() const override;
-
-private: // members
-
-    eckit::SessionID sessionID_;
-
-    eckit::net::Endpoint controlEndpoint_;
-    eckit::net::Endpoint dataEndpoint_;
-
-    eckit::net::TCPClient controlClient_;
-    eckit::net::TCPClient dataClient_;
-
-    FDBStats internalStats_;
-
-    // Listen on the dataClient for incoming messages.
-    std::thread listeningThread_;
-
-    // Where do we put received messages
-    // @note This is a map of requestID:MessageQueue. At the point that a request is
-    // complete, errored or otherwise killed, it needs to be removed from the map.
-    // The shared_ptr allows this removal to be asynchronous with the actual task
-    // cleaning up and returning to the client.
-
-    std::map> messageQueues_;
-
-    // Asynchronised helpers for archiving
-
-    friend class FDBRemoteDataHandle;
-
-    std::future archiveFuture_;
-
-    // Helpers for retrievals
-
-    uint32_t archiveID_;
-    size_t maxArchiveQueueLength_;
-    size_t maxArchiveBatchSize_;
-    std::mutex archiveQueuePtrMutex_;
-    std::unique_ptr archiveQueue_;
-    MessageQueue retrieveMessageQueue_;
-
-    bool connected_;
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace fdb5
-
-#endif // fdb5_remote_RemoteFDB_H
diff --git a/src/fdb5/api/local/ControlVisitor.cc b/src/fdb5/api/local/ControlVisitor.cc
index ea18e2f11..88b1d54e1 100644
--- a/src/fdb5/api/local/ControlVisitor.cc
+++ b/src/fdb5/api/local/ControlVisitor.cc
@@ -27,9 +27,9 @@ ControlVisitor::ControlVisitor(eckit::Queue& queue,
     identifiers_(identifiers) {}
 
 
-bool ControlVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
+bool ControlVisitor::visitDatabase(const Catalogue& catalogue) { //, const Store& store) {
 
-    EntryVisitor::visitDatabase(catalogue, store);
+    EntryVisitor::visitDatabase(catalogue); //, store);
 
     // Only lock/unlock things that match exactly.
 
diff --git a/src/fdb5/api/local/ControlVisitor.h b/src/fdb5/api/local/ControlVisitor.h
index 87d2dc391..9297c5dc0 100644
--- a/src/fdb5/api/local/ControlVisitor.h
+++ b/src/fdb5/api/local/ControlVisitor.h
@@ -37,7 +37,8 @@ class ControlVisitor : public QueryVisitor {
     bool visitIndexes() override { return false; }
     bool visitEntries() override { return false; }
 
-    bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
+    bool visitDatabase(const Catalogue& catalogue) override;
+//    bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
     bool visitIndex(const Index&) override { NOTIMP; }
     void visitDatum(const Field&, const InspectionKey&) override { NOTIMP; }
 
diff --git a/src/fdb5/api/local/DumpVisitor.h b/src/fdb5/api/local/DumpVisitor.h
index e943d53e3..374ccbe32 100644
--- a/src/fdb5/api/local/DumpVisitor.h
+++ b/src/fdb5/api/local/DumpVisitor.h
@@ -44,7 +44,8 @@ class DumpVisitor : public QueryVisitor {
     bool visitIndexes() override { return false; }
     bool visitEntries() override { return false; }
 
-    bool visitDatabase(const Catalogue& catalogue, const Store& store) override {
+//    bool visitDatabase(const Catalogue& catalogue, const Store& store) override {
+    bool visitDatabase(const Catalogue& catalogue) override {
         catalogue.dump(out_, simple_);
         return true;
     }
diff --git a/src/fdb5/api/local/ListVisitor.h b/src/fdb5/api/local/ListVisitor.h
index b65f4d25e..da234588b 100644
--- a/src/fdb5/api/local/ListVisitor.h
+++ b/src/fdb5/api/local/ListVisitor.h
@@ -40,14 +40,16 @@ struct ListVisitor : public QueryVisitor {
 
     /// Make a note of the current database. Subtract its key from the current
     /// request so we can test request is used in its entirety
-    bool visitDatabase(const Catalogue& catalogue, const Store& store) override {
+//    bool visitDatabase(const Catalogue& catalogue, const Store& store) override {
+    bool visitDatabase(const Catalogue& catalogue) override {
 
         // If the DB is locked for listing, then it "doesn't exist"
         if (!catalogue.enabled(ControlIdentifier::List)) {
             return false;
         }
 
-        bool ret = QueryVisitor::visitDatabase(catalogue, store);
+//        bool ret = QueryVisitor::visitDatabase(catalogue, store);
+        bool ret = QueryVisitor::visitDatabase(catalogue);
         ASSERT(catalogue.key().partialMatch(request_));
 
         // Subselect the parts of the request
diff --git a/src/fdb5/api/local/MoveVisitor.cc b/src/fdb5/api/local/MoveVisitor.cc
index ccf6a6046..f3121ea8c 100644
--- a/src/fdb5/api/local/MoveVisitor.cc
+++ b/src/fdb5/api/local/MoveVisitor.cc
@@ -41,7 +41,8 @@ MoveVisitor::MoveVisitor(eckit::Queue& queue,
     QueryVisitor(queue, request),
     dest_(dest) {}
 
-bool MoveVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
+//bool MoveVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
+bool MoveVisitor::visitDatabase(const Catalogue& catalogue) {
     if (catalogue.key().match(request_)) {
         catalogue.control(
             ControlAction::Disable,
@@ -52,11 +53,14 @@ bool MoveVisitor::visitDatabase(const Catalogue& catalogue, const Store& store)
         ASSERT(!catalogue.enabled(ControlIdentifier::Wipe));
         ASSERT(!catalogue.enabled(ControlIdentifier::UniqueRoot));
 
-        EntryVisitor::visitDatabase(catalogue, store);
+        EntryVisitor::visitDatabase(catalogue);
+//        EntryVisitor::visitDatabase(catalogue, store);
 
         ASSERT(!internalVisitor_);
-        internalVisitor_.reset(catalogue.moveVisitor(store, request_, dest_, queue_));
-        internalVisitor_->visitDatabase(catalogue, store);
+        // internalVisitor_.reset(catalogue.moveVisitor(store, request_, dest_, queue_));
+        // internalVisitor_->visitDatabase(catalogue, store);
+        internalVisitor_.reset(catalogue.moveVisitor(store(), request_, dest_, queue_));
+        internalVisitor_->visitDatabase(catalogue);
 
         return true;
     }
diff --git a/src/fdb5/api/local/MoveVisitor.h b/src/fdb5/api/local/MoveVisitor.h
index ec1e834ec..5c6794e2c 100644
--- a/src/fdb5/api/local/MoveVisitor.h
+++ b/src/fdb5/api/local/MoveVisitor.h
@@ -42,7 +42,8 @@ class MoveVisitor : public QueryVisitor {
     bool visitIndexes() override { return false; }
     bool visitEntries() override { return false; }
 
-    bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
+//    bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
+    bool visitDatabase(const Catalogue& catalogue) override;
     bool visitIndex(const Index&) override { NOTIMP; }
     void visitDatum(const Field&, const InspectionKey&) override { NOTIMP; }
     void visitDatum(const Field& field, const std::string& keyFingerprint) override { NOTIMP; }
diff --git a/src/fdb5/api/local/PurgeVisitor.cc b/src/fdb5/api/local/PurgeVisitor.cc
index bc3e3a972..e975ed02e 100644
--- a/src/fdb5/api/local/PurgeVisitor.cc
+++ b/src/fdb5/api/local/PurgeVisitor.cc
@@ -44,14 +44,15 @@ bool PurgeVisitor::visitCatalogue(const Catalogue& catalogue) {
     return false;
 }*/
 
-bool PurgeVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
+//bool PurgeVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
+bool PurgeVisitor::visitDatabase(const Catalogue& catalogue) {
 
     // If the DB is locked for wiping, then it "doesn't exist"
     if (!catalogue.enabled(ControlIdentifier::Wipe)) {
         return false;
     }
     
-    EntryVisitor::visitDatabase(catalogue, store);
+    EntryVisitor::visitDatabase(catalogue);
 
     // If the request is overspecified relative to the DB key, then we
     // bail out here.
@@ -65,9 +66,9 @@ bool PurgeVisitor::visitDatabase(const Catalogue& catalogue, const Store& store)
     }
 
     ASSERT(!internalVisitor_);
-    internalVisitor_.reset(catalogue.purgeVisitor(store));
+    internalVisitor_.reset(catalogue.purgeVisitor(store()));
 
-    internalVisitor_->visitDatabase(catalogue, store);
+    internalVisitor_->visitDatabase(catalogue);
 
     return true; // Explore contained indexes
 }
diff --git a/src/fdb5/api/local/PurgeVisitor.h b/src/fdb5/api/local/PurgeVisitor.h
index 43d9b7b8b..6e977e2d2 100644
--- a/src/fdb5/api/local/PurgeVisitor.h
+++ b/src/fdb5/api/local/PurgeVisitor.h
@@ -43,8 +43,8 @@ class PurgeVisitor : public QueryVisitor {
                  bool doit,
                  bool porcelain);
 
-    //bool visitCatalogue(const Catalogue& catalogue) override;
-    bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
+//    bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
+    bool visitDatabase(const Catalogue& catalogue) override;
     bool visitIndex(const Index& index) override;
     void catalogueComplete(const Catalogue& catalogue) override;
     void visitDatum(const Field& field, const std::string& keyFingerprint) override;
diff --git a/src/fdb5/api/local/StatsVisitor.cc b/src/fdb5/api/local/StatsVisitor.cc
index c520d8cc8..8d86e2ba8 100644
--- a/src/fdb5/api/local/StatsVisitor.cc
+++ b/src/fdb5/api/local/StatsVisitor.cc
@@ -25,14 +25,15 @@ namespace local {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-bool StatsVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
+// bool StatsVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
+bool StatsVisitor::visitDatabase(const Catalogue& catalogue) {
 
-    EntryVisitor::visitDatabase(catalogue, store);
+    EntryVisitor::visitDatabase(catalogue);
 
     ASSERT(!internalVisitor_);
     internalVisitor_.reset(catalogue.statsReportVisitor());
 
-    internalVisitor_->visitDatabase(catalogue, store);
+    internalVisitor_->visitDatabase(catalogue);
 
     return true; // Explore contained indexes
 }
diff --git a/src/fdb5/api/local/StatsVisitor.h b/src/fdb5/api/local/StatsVisitor.h
index 6ef95117c..79f580d84 100644
--- a/src/fdb5/api/local/StatsVisitor.h
+++ b/src/fdb5/api/local/StatsVisitor.h
@@ -38,7 +38,8 @@ class StatsVisitor : public QueryVisitor {
 
     using QueryVisitor::QueryVisitor;
 
-    bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
+//    bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
+    bool visitDatabase(const Catalogue& catalogue) override;
     bool visitIndex(const Index& index) override;
     void catalogueComplete(const Catalogue& catalogue) override;
     void visitDatum(const Field& field, const std::string& keyFingerprint) override;
diff --git a/src/fdb5/api/local/StatusVisitor.h b/src/fdb5/api/local/StatusVisitor.h
index e28431f56..bb2ff5a35 100644
--- a/src/fdb5/api/local/StatusVisitor.h
+++ b/src/fdb5/api/local/StatusVisitor.h
@@ -36,7 +36,8 @@ class StatusVisitor : public QueryVisitor {
     using QueryVisitor::QueryVisitor;
     bool visitIndexes() override { return false; }
     bool visitEntries() override { return false; }
-    bool visitDatabase(const Catalogue& catalogue, const Store& store) override { queue_.emplace(catalogue); return true; }
+    bool visitDatabase(const Catalogue& catalogue) override  { queue_.emplace(catalogue); return true; }
+//    bool visitDatabase(const Catalogue& catalogue, const Store& store) override { queue_.emplace(catalogue); return true; }
     bool visitIndex(const Index&) override { NOTIMP; }
     void visitDatum(const Field&, const InspectionKey&) override { NOTIMP; }
 
diff --git a/src/fdb5/api/local/WipeVisitor.cc b/src/fdb5/api/local/WipeVisitor.cc
index 3661b2e07..d290135e0 100644
--- a/src/fdb5/api/local/WipeVisitor.cc
+++ b/src/fdb5/api/local/WipeVisitor.cc
@@ -47,18 +47,21 @@ WipeVisitor::WipeVisitor(eckit::Queue& queue,
     unsafeWipeAll_(unsafeWipeAll) {}
 
 
-bool WipeVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
+//bool WipeVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
+bool WipeVisitor::visitDatabase(const Catalogue& catalogue) {
 
     // If the DB is locked for wiping, then it "doesn't exist"
     if (!catalogue.enabled(ControlIdentifier::Wipe)) {
         return false;
     }
 
-    EntryVisitor::visitDatabase(catalogue, store);
+    // EntryVisitor::visitDatabase(catalogue, store);
+    EntryVisitor::visitDatabase(catalogue);
 
     ASSERT(!internalVisitor_);
-    internalVisitor_.reset(catalogue.wipeVisitor(store, request_, out_, doit_, porcelain_, unsafeWipeAll_));
-    internalVisitor_->visitDatabase(catalogue, store);
+    internalVisitor_.reset(catalogue.wipeVisitor(store(), request_, out_, doit_, porcelain_, unsafeWipeAll_));
+    // internalVisitor_->visitDatabase(catalogue, store);
+    internalVisitor_->visitDatabase(catalogue);
 
     return true; // Explore contained indexes
 }
diff --git a/src/fdb5/api/local/WipeVisitor.h b/src/fdb5/api/local/WipeVisitor.h
index c40fc424e..098831a93 100644
--- a/src/fdb5/api/local/WipeVisitor.h
+++ b/src/fdb5/api/local/WipeVisitor.h
@@ -47,7 +47,8 @@ class WipeVisitor : public QueryVisitor {
     bool visitEntries() override { return false; }
     bool visitIndexes() override;
 
-    bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
+//    bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
+    bool visitDatabase(const Catalogue& catalogue) override;
     bool visitIndex(const Index& index) override;
     void catalogueComplete(const Catalogue& catalogue) override;
     void visitDatum(const Field&, const InspectionKey&) override { NOTIMP; }
diff --git a/src/fdb5/database/ArchiveVisitor.cc b/src/fdb5/database/ArchiveVisitor.cc
index 465262828..ce72d2dba 100644
--- a/src/fdb5/database/ArchiveVisitor.cc
+++ b/src/fdb5/database/ArchiveVisitor.cc
@@ -27,12 +27,11 @@ bool ArchiveVisitor::selectDatum(const InspectionKey &key, const Key &full) {
     // eckit::Log::info() << "selectDatum " << key << ", " << full << " " << size_ << std::endl;
     checkMissingKeys(full);
 
-    CatalogueWriter* writeCatalogue = dynamic_cast(current());
-    const Index& idx = writeCatalogue->currentIndex();
+    const Index& idx = current()->currentIndex();
 
     // here we could create a queue... and keep accepting archival request until the queue is full
     auto futureLocation = store()->archive(idx.key(), data_, size_);
-    writeCatalogue->archive(key, futureLocation.get());
+    current()->archive(key, futureLocation.get());
 
     return true;
 }
diff --git a/src/fdb5/database/Archiver.cc b/src/fdb5/database/Archiver.cc
index 8c74dae18..60f2465db 100644
--- a/src/fdb5/database/Archiver.cc
+++ b/src/fdb5/database/Archiver.cc
@@ -96,7 +96,7 @@ void Archiver::selectDatabase(const Key &dbKey) {
         }
     }
 
-    std::unique_ptr cat = CatalogueFactory::instance().build(dbKey, dbConfig_, false);
+    std::unique_ptr cat = CatalogueWriterFactory::instance().build(dbKey, dbConfig_);
     ASSERT(cat);
 
     // If this database is locked for writing then this is an error
diff --git a/src/fdb5/database/Archiver.h b/src/fdb5/database/Archiver.h
index a4ba3700b..a1415b57e 100644
--- a/src/fdb5/database/Archiver.h
+++ b/src/fdb5/database/Archiver.h
@@ -68,7 +68,7 @@ class Archiver : public eckit::NonCopyable {
 
     friend class BaseArchiveVisitor;
 
-    typedef std::map< Key, std::pair, std::unique_ptr > > > store_t;
+    typedef std::map< Key, std::pair, std::unique_ptr > > > store_t;
 
     Config dbConfig_;
 
@@ -76,7 +76,7 @@ class Archiver : public eckit::NonCopyable {
 
     std::vector prev_;
 
-    Catalogue* catalogue_;
+    CatalogueWriter* catalogue_;
     Store* store_;
 };
 
diff --git a/src/fdb5/database/BaseArchiveVisitor.cc b/src/fdb5/database/BaseArchiveVisitor.cc
index f3b752528..0ef41086f 100644
--- a/src/fdb5/database/BaseArchiveVisitor.cc
+++ b/src/fdb5/database/BaseArchiveVisitor.cc
@@ -51,7 +51,7 @@ const Schema& BaseArchiveVisitor::databaseSchema() const {
     return current()->schema();
 }
 
-Catalogue* BaseArchiveVisitor::current() const {
+CatalogueWriter* BaseArchiveVisitor::current() const {
     return owner_.catalogue_;
 }
 
diff --git a/src/fdb5/database/BaseArchiveVisitor.h b/src/fdb5/database/BaseArchiveVisitor.h
index 507c60885..cec173535 100644
--- a/src/fdb5/database/BaseArchiveVisitor.h
+++ b/src/fdb5/database/BaseArchiveVisitor.h
@@ -23,7 +23,7 @@ namespace metkit { class MarsRequest; }
 namespace fdb5 {
 
 class Archiver;
-class Catalogue;
+class CatalogueWriter;
 class Store;
 class Schema;
 
@@ -45,7 +45,7 @@ class BaseArchiveVisitor : public WriteVisitor {
 
     virtual const Schema& databaseSchema() const;
 
-    fdb5::Catalogue* current() const;
+    fdb5::CatalogueWriter* current() const;
     fdb5::Store* store() const;
 
 private: // members
diff --git a/src/fdb5/database/Catalogue.cc b/src/fdb5/database/Catalogue.cc
index af560436b..ad76f872c 100644
--- a/src/fdb5/database/Catalogue.cc
+++ b/src/fdb5/database/Catalogue.cc
@@ -24,7 +24,12 @@
 
 namespace fdb5 {
 
-std::unique_ptr Catalogue::buildStore() {
+std::ostream &operator<<(std::ostream &s, const Catalogue &x) {
+    x.print(s);
+    return s;
+}
+
+std::unique_ptr CatalogueImpl::buildStore() const {
     if (buildByKey_)
         return StoreFactory::instance().build(key(), config_);
     else {
@@ -34,96 +39,181 @@ std::unique_ptr Catalogue::buildStore() {
     }
 }
 
-bool Catalogue::enabled(const ControlIdentifier& controlIdentifier) const {
+bool CatalogueImpl::enabled(const ControlIdentifier& controlIdentifier) const {
     return controlIdentifiers_.enabled(controlIdentifier);
 }
 
-std::ostream &operator<<(std::ostream &s, const Catalogue &x) {
-    x.print(s);
-    return s;
+//----------------------------------------------------------------------------------------------------------------------
+
+CatalogueReaderFactory::CatalogueReaderFactory() {}
+
+CatalogueReaderFactory& CatalogueReaderFactory::instance() {
+    static CatalogueReaderFactory theOne;
+    return theOne;
+}
+
+void CatalogueReaderFactory::add(const std::string& name, CatalogueReaderBuilderBase* builder) {
+    std::string nameLowercase = eckit::StringTools::lower(name);
+
+    eckit::AutoLock lock(mutex_);
+    if (has(nameLowercase)) {
+        throw eckit::SeriousBug("Duplicate entry in CatalogueReaderFactory: " + nameLowercase, Here());
+    }
+    builders_[nameLowercase] = builder;
+}
+
+void CatalogueReaderFactory::remove(const std::string& name) {
+    std::string nameLowercase = eckit::StringTools::lower(name);
+
+    eckit::AutoLock lock(mutex_);
+    builders_.erase(nameLowercase);
+}
+
+bool CatalogueReaderFactory::has(const std::string& name) {
+    std::string nameLowercase = eckit::StringTools::lower(name);
+
+    eckit::AutoLock lock(mutex_);
+    return builders_.find(nameLowercase) != builders_.end();
+}
+
+void CatalogueReaderFactory::list(std::ostream& out) {
+    eckit::AutoLock lock(mutex_);
+    const char* sep = "";
+    for (std::map::const_iterator j = builders_.begin(); j != builders_.end(); ++j) {
+        out << sep << (*j).first;
+        sep = ", ";
+    }
+}
+
+std::unique_ptr CatalogueReaderFactory::build(const Key& dbKey, const Config& config) {
+    std::string name = Manager(config).engine(dbKey);
+    std::string nameLowercase = eckit::StringTools::lower(name);
+
+    eckit::AutoLock lock(mutex_);
+    auto j = builders_.find(nameLowercase);
+
+    eckit::Log::debug() << "Looking for CatalogueReaderBuilder [" << nameLowercase << "]" << std::endl;
+
+    if (j == builders_.end()) {
+        eckit::Log::error() << "No CatalogueReaderBuilder for [" << nameLowercase << "]" << std::endl;
+        eckit::Log::error() << "CatalogueReaderBuilders are:" << std::endl;
+        for (j = builders_.begin(); j != builders_.end(); ++j)
+            eckit::Log::error() << "   " << (*j).first << std::endl;
+        throw eckit::SeriousBug(std::string("No CatalogueReaderBuilder called ") + nameLowercase);
+    }
+
+    return j->second->make(dbKey, config);
+}
+
+std::unique_ptr CatalogueReaderFactory::build(const eckit::URI& uri, const fdb5::Config& config) {
+    std::string name = uri.scheme();
+    std::string nameLowercase = eckit::StringTools::lower(name);
+
+    eckit::AutoLock lock(mutex_);
+    auto j = builders_.find(nameLowercase);
+
+    eckit::Log::debug() << "Looking for CatalogueReaderBuilder [" << nameLowercase << "]" << std::endl;
+
+    if (j == builders_.end()) {
+        eckit::Log::error() << "No CatalogueReaderBuilder for [" << nameLowercase << "]" << std::endl;
+        eckit::Log::error() << "CatalogueReaderBuilders are:" << std::endl;
+        for (j = builders_.begin(); j != builders_.end(); ++j)
+            eckit::Log::error() << "   " << (*j).first << std::endl;
+        throw eckit::SeriousBug(std::string("No CatalogueReaderBuilder called ") + nameLowercase);
+    }
+
+    return j->second->make(uri, config);
 }
 
 //----------------------------------------------------------------------------------------------------------------------
 
-CatalogueFactory::CatalogueFactory() {}
+CatalogueReaderBuilderBase::CatalogueReaderBuilderBase(const std::string& name) : name_(name) {
+    CatalogueReaderFactory::instance().add(name_, this);
+}
 
-CatalogueFactory& CatalogueFactory::instance() {
-    static CatalogueFactory theOne;
+CatalogueReaderBuilderBase::~CatalogueReaderBuilderBase() {
+    if(LibFdb5::instance().dontDeregisterFactories()) return;
+    CatalogueReaderFactory::instance().remove(name_);
+}
+
+
+//----------------------------------------------------------------------------------------------------------------------
+
+CatalogueWriterFactory::CatalogueWriterFactory() {}
+
+CatalogueWriterFactory& CatalogueWriterFactory::instance() {
+    static CatalogueWriterFactory theOne;
     return theOne;
 }
 
-void CatalogueFactory::add(const std::string& name, CatalogueBuilderBase* builder) {
+void CatalogueWriterFactory::add(const std::string& name, CatalogueWriterBuilderBase* builder) {
     std::string nameLowercase = eckit::StringTools::lower(name);
 
     eckit::AutoLock lock(mutex_);
     if (has(nameLowercase)) {
-        throw eckit::SeriousBug("Duplicate entry in CatalogueFactory: " + nameLowercase, Here());
+        throw eckit::SeriousBug("Duplicate entry in CatalogueWriterFactory: " + nameLowercase, Here());
     }
     builders_[nameLowercase] = builder;
 }
 
-void CatalogueFactory::remove(const std::string& name) {
+void CatalogueWriterFactory::remove(const std::string& name) {
     std::string nameLowercase = eckit::StringTools::lower(name);
 
     eckit::AutoLock lock(mutex_);
     builders_.erase(nameLowercase);
 }
 
-bool CatalogueFactory::has(const std::string& name) {
+bool CatalogueWriterFactory::has(const std::string& name) {
     std::string nameLowercase = eckit::StringTools::lower(name);
 
     eckit::AutoLock lock(mutex_);
     return builders_.find(nameLowercase) != builders_.end();
 }
 
-void CatalogueFactory::list(std::ostream& out) {
+void CatalogueWriterFactory::list(std::ostream& out) {
     eckit::AutoLock lock(mutex_);
     const char* sep = "";
-    for (std::map::const_iterator j = builders_.begin(); j != builders_.end(); ++j) {
+    for (std::map::const_iterator j = builders_.begin(); j != builders_.end(); ++j) {
         out << sep << (*j).first;
         sep = ", ";
     }
 }
 
-std::unique_ptr CatalogueFactory::build(const Key& dbKey, const Config& config, bool read) {
+std::unique_ptr CatalogueWriterFactory::build(const Key& dbKey, const Config& config) {
     std::string name = Manager(config).engine(dbKey);
     std::string nameLowercase = eckit::StringTools::lower(name);
 
-    nameLowercase += read ? ".reader" : ".writer";
-
     eckit::AutoLock lock(mutex_);
     auto j = builders_.find(nameLowercase);
 
-    eckit::Log::debug() << "Looking for CatalogueBuilder [" << nameLowercase << "]" << std::endl;
+    eckit::Log::debug() << "Looking for CatalogueWriterBuilder [" << nameLowercase << "]" << std::endl;
 
     if (j == builders_.end()) {
-        eckit::Log::error() << "No CatalogueBuilder for [" << nameLowercase << "]" << std::endl;
-        eckit::Log::error() << "CatalogueBuilders are:" << std::endl;
+        eckit::Log::error() << "No CatalogueWriterBuilder for [" << nameLowercase << "]" << std::endl;
+        eckit::Log::error() << "CatalogueWriterBuilders are:" << std::endl;
         for (j = builders_.begin(); j != builders_.end(); ++j)
             eckit::Log::error() << "   " << (*j).first << std::endl;
-        throw eckit::SeriousBug(std::string("No CatalogueBuilder called ") + nameLowercase);
+        throw eckit::SeriousBug(std::string("No CatalogueWriterBuilder called ") + nameLowercase);
     }
 
     return j->second->make(dbKey, config);
 }
 
-std::unique_ptr CatalogueFactory::build(const eckit::URI& uri, const fdb5::Config& config, bool read) {
+std::unique_ptr CatalogueWriterFactory::build(const eckit::URI& uri, const fdb5::Config& config) {
     std::string name = uri.scheme();
     std::string nameLowercase = eckit::StringTools::lower(name);
 
-    nameLowercase += read ? ".reader" : ".writer";
-
     eckit::AutoLock lock(mutex_);
     auto j = builders_.find(nameLowercase);
 
-    eckit::Log::debug() << "Looking for CatalogueBuilder [" << nameLowercase << "]" << std::endl;
+    eckit::Log::debug() << "Looking for CatalogueWriterBuilder [" << nameLowercase << "]" << std::endl;
 
     if (j == builders_.end()) {
-        eckit::Log::error() << "No CatalogueBuilder for [" << nameLowercase << "]" << std::endl;
-        eckit::Log::error() << "CatalogueBuilders are:" << std::endl;
+        eckit::Log::error() << "No CatalogueWriterBuilder for [" << nameLowercase << "]" << std::endl;
+        eckit::Log::error() << "CatalogueWriterBuilders are:" << std::endl;
         for (j = builders_.begin(); j != builders_.end(); ++j)
             eckit::Log::error() << "   " << (*j).first << std::endl;
-        throw eckit::SeriousBug(std::string("No CatalogueBuilder called ") + nameLowercase);
+        throw eckit::SeriousBug(std::string("No CatalogueWriterBuilder called ") + nameLowercase);
     }
 
     return j->second->make(uri, config);
@@ -131,13 +221,13 @@ std::unique_ptr CatalogueFactory::build(const eckit::URI& uri, const
 
 //----------------------------------------------------------------------------------------------------------------------
 
-CatalogueBuilderBase::CatalogueBuilderBase(const std::string& name) : name_(name) {
-    CatalogueFactory::instance().add(name_, this);
+CatalogueWriterBuilderBase::CatalogueWriterBuilderBase(const std::string& name) : name_(name) {
+    CatalogueWriterFactory::instance().add(name_, this);
 }
 
-CatalogueBuilderBase::~CatalogueBuilderBase() {
+CatalogueWriterBuilderBase::~CatalogueWriterBuilderBase() {
     if(LibFdb5::instance().dontDeregisterFactories()) return;
-    CatalogueFactory::instance().remove(name_);
+    CatalogueWriterFactory::instance().remove(name_);
 }
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/database/Catalogue.h b/src/fdb5/database/Catalogue.h
index e094aa208..758baa439 100644
--- a/src/fdb5/database/Catalogue.h
+++ b/src/fdb5/database/Catalogue.h
@@ -44,16 +44,14 @@ typedef std::map IndexStore;
 class Catalogue {
 public:
 
-    Catalogue(const Key& key, ControlIdentifiers controlIdentifiers, const fdb5::Config& config)
-        : dbKey_(key), config_(config), controlIdentifiers_(controlIdentifiers), buildByKey_(!key.empty()) {}
-
+    Catalogue() {}
     virtual ~Catalogue() {}
 
-    const Key& key() const { return dbKey_; }
-    virtual const Key& indexKey() const { NOTIMP; }
-    const Config& config() const { return config_; }
+    virtual const Key& key() const = 0;
+    virtual const Key& indexKey() const = 0;
+    virtual const Config& config() const = 0;
 
-    std::unique_ptr buildStore();
+    virtual std::unique_ptr buildStore() const = 0;
     virtual const Schema& schema() const = 0;
 
     virtual bool selectIndex(const Key& idxKey) = 0;
@@ -61,9 +59,9 @@ class Catalogue {
 
     virtual std::vector metadataPaths() const = 0;
 
-    virtual void visitEntries(EntryVisitor& visitor, const Store& store, bool sorted = false) = 0;
+    virtual void visitEntries(EntryVisitor& visitor, /*const Store& store,*/ bool sorted = false) = 0;
 
-    virtual void hideContents() { NOTIMP; }
+    virtual void hideContents() = 0;
 
     virtual void dump(std::ostream& out, bool simple=false, const eckit::Configuration& conf = eckit::LocalConfiguration()) const = 0;
 
@@ -74,7 +72,7 @@ class Catalogue {
 
     virtual void control(const ControlAction& action, const ControlIdentifiers& identifiers) const = 0;
 
-    virtual bool enabled(const ControlIdentifier& controlIdentifier) const;
+    virtual bool enabled(const ControlIdentifier& controlIdentifier) const = 0;
 
     virtual std::vector indexes(bool sorted=false) const = 0;
 
@@ -103,28 +101,67 @@ class Catalogue {
 
     virtual void loadSchema() = 0;
 
+};
+
+class CatalogueImpl : virtual public Catalogue {
+public:
+
+    CatalogueImpl(const Key& key, ControlIdentifiers controlIdentifiers, const fdb5::Config& config)
+        : dbKey_(key), config_(config), controlIdentifiers_(controlIdentifiers), buildByKey_(!key.empty()) {}
+
+    virtual ~CatalogueImpl() {}
+
+    const Key& key() const override { return dbKey_; }
+    const Key& indexKey() const override { NOTIMP; }
+    const Config& config() const override { return config_; }
+
+    std::unique_ptr buildStore() const override;
+
+    void hideContents() override { NOTIMP; }
+
+    bool enabled(const ControlIdentifier& controlIdentifier) const override;
+
+protected: // methods
+
+    CatalogueImpl() : dbKey_(Key()), config_(Config()), controlIdentifiers_(ControlIdentifiers()) {}
+
 protected: // members
 
     Key dbKey_;
     Config config_;
     ControlIdentifiers controlIdentifiers_;
 
-
 private: // members
 
     bool buildByKey_ = false;
 };
 
-class CatalogueReader {
+class CatalogueReader : virtual public Catalogue {
+
 public:
+
+    // CatalogueReader(const Key& key, ControlIdentifiers controlIdentifiers, const fdb5::Config& config)
+    //     : Catalogue(key, controlIdentifiers, config) {}
+    CatalogueReader() {}
+    
+    virtual ~CatalogueReader() {}
+
     virtual DbStats stats() const = 0;
     virtual bool axis(const std::string& keyword, eckit::StringSet& s) const = 0;
     virtual bool retrieve(const InspectionKey& key, Field& field) const = 0;
 };
 
 
-class CatalogueWriter {
+class CatalogueWriter : virtual public Catalogue  {
+
 public:
+
+    CatalogueWriter() {}
+    // CatalogueWriter(const Key& key, ControlIdentifiers controlIdentifiers, const fdb5::Config& config)
+    //     : Catalogue(key, controlIdentifiers, config) {}
+    
+    virtual ~CatalogueWriter() {}
+
     virtual const Index& currentIndex() = 0;
     virtual void archive(const InspectionKey& key, std::unique_ptr fieldLocation) = 0;
     virtual void overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) = 0;
@@ -134,31 +171,75 @@ class CatalogueWriter {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-class CatalogueBuilderBase {
+class CatalogueReaderBuilderBase {
+    std::string name_;
+
+public:
+    CatalogueReaderBuilderBase(const std::string&);
+    virtual ~CatalogueReaderBuilderBase();
+    virtual std::unique_ptr make(const fdb5::Key& key, const fdb5::Config& config) = 0;
+    virtual std::unique_ptr make(const eckit::URI& uri, const fdb5::Config& config) = 0;
+};
+
+template 
+class CatalogueReaderBuilder : public CatalogueReaderBuilderBase {
+    virtual std::unique_ptr make(const fdb5::Key& key, const fdb5::Config& config) override { return std::unique_ptr(new T(key, config)); }
+    virtual std::unique_ptr make(const eckit::URI& uri, const fdb5::Config& config) override { return std::unique_ptr(new T(uri, config)); }
+
+public:
+    CatalogueReaderBuilder(const std::string& name) : CatalogueReaderBuilderBase(name) {}
+    virtual ~CatalogueReaderBuilder() = default;
+};
+
+class CatalogueReaderFactory {
+public:
+    static CatalogueReaderFactory& instance();
+
+    void add(const std::string& name, CatalogueReaderBuilderBase* builder);
+    void remove(const std::string& name);
+
+    bool has(const std::string& name);
+    void list(std::ostream&);
+
+    /// @param db        the db using the required catalogue
+    /// @returns         catalogue built by specified builder
+    std::unique_ptr build(const Key& key, const Config& config);
+    std::unique_ptr build(const eckit::URI& uri, const Config& config);
+
+private:
+    CatalogueReaderFactory();
+
+    std::map builders_;
+    eckit::Mutex mutex_;
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
+class CatalogueWriterBuilderBase {
     std::string name_;
 
 public:
-    CatalogueBuilderBase(const std::string&);
-    virtual ~CatalogueBuilderBase();
-    virtual std::unique_ptr make(const fdb5::Key& key, const fdb5::Config& config) = 0;
-    virtual std::unique_ptr make(const eckit::URI& uri, const fdb5::Config& config) = 0;
+    CatalogueWriterBuilderBase(const std::string&);
+    virtual ~CatalogueWriterBuilderBase();
+    virtual std::unique_ptr make(const fdb5::Key& key, const fdb5::Config& config) = 0;
+    virtual std::unique_ptr make(const eckit::URI& uri, const fdb5::Config& config) = 0;
 };
 
 template 
-class CatalogueBuilder : public CatalogueBuilderBase {
-    virtual std::unique_ptr make(const fdb5::Key& key, const fdb5::Config& config) override { return std::unique_ptr(new T(key, config)); }
-    virtual std::unique_ptr make(const eckit::URI& uri, const fdb5::Config& config) override { return std::unique_ptr(new T(uri, config)); }
+class CatalogueWriterBuilder : public CatalogueWriterBuilderBase {
+    virtual std::unique_ptr make(const fdb5::Key& key, const fdb5::Config& config) override { return std::unique_ptr(new T(key, config)); }
+    virtual std::unique_ptr make(const eckit::URI& uri, const fdb5::Config& config) override { return std::unique_ptr(new T(uri, config)); }
 
 public:
-    CatalogueBuilder(const std::string& name) : CatalogueBuilderBase(name) {}
-    virtual ~CatalogueBuilder() = default;
+    CatalogueWriterBuilder(const std::string& name) : CatalogueWriterBuilderBase(name) {}
+    virtual ~CatalogueWriterBuilder() = default;
 };
 
-class CatalogueFactory {
+class CatalogueWriterFactory {
 public:
-    static CatalogueFactory& instance();
+    static CatalogueWriterFactory& instance();
 
-    void add(const std::string& name, CatalogueBuilderBase* builder);
+    void add(const std::string& name, CatalogueWriterBuilderBase* builder);
     void remove(const std::string& name);
 
     bool has(const std::string& name);
@@ -166,13 +247,13 @@ class CatalogueFactory {
 
     /// @param db        the db using the required catalogue
     /// @returns         catalogue built by specified builder
-    std::unique_ptr build(const Key& key, const Config& config, bool read);
-    std::unique_ptr build(const eckit::URI& uri, const Config& config, bool read);
+    std::unique_ptr build(const Key& key, const Config& config);
+    std::unique_ptr build(const eckit::URI& uri, const Config& config);
 
 private:
-    CatalogueFactory();
+    CatalogueWriterFactory();
 
-    std::map builders_;
+    std::map builders_;
     eckit::Mutex mutex_;
 };
 
diff --git a/src/fdb5/database/EntryVisitMechanism.cc b/src/fdb5/database/EntryVisitMechanism.cc
index 0b10460b6..e243396b3 100644
--- a/src/fdb5/database/EntryVisitMechanism.cc
+++ b/src/fdb5/database/EntryVisitMechanism.cc
@@ -37,9 +37,20 @@ EntryVisitor::EntryVisitor() : currentCatalogue_(nullptr), currentIndex_(nullptr
 
 EntryVisitor::~EntryVisitor() {}
 
-bool EntryVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
+
+Store& EntryVisitor::store() {
+    if (!currentStore_) {
+        ASSERT(currentCatalogue_);
+        currentStore_ = currentCatalogue_->buildStore();
+        ASSERT(currentStore_);
+    }
+    return *currentStore_;
+}
+
+// bool EntryVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
+bool EntryVisitor::visitDatabase(const Catalogue& catalogue) {
     currentCatalogue_ = &catalogue;
-    currentStore_ = &store;
+    currentStore_.reset();
     return true;
 }
 
@@ -48,7 +59,7 @@ void EntryVisitor::catalogueComplete(const Catalogue& catalogue) {
         ASSERT(currentCatalogue_ == &catalogue);
     }
     currentCatalogue_ = nullptr;
-    currentStore_ = nullptr;
+    // currentStore_ = nullptr;
     currentIndex_ = nullptr;
 }
 
@@ -108,16 +119,16 @@ void EntryVisitMechanism::visit(const FDBToolRequest& request, EntryVisitor& vis
 
                 Log::debug() << "FDB processing Path " << path << std::endl;
 
-                std::unique_ptr catalogue = CatalogueFactory::instance().build(eckit::URI(uri.scheme(), path), dbConfig_, true);
+                std::unique_ptr catalogue = CatalogueReaderFactory::instance().build(eckit::URI(uri.scheme(), path), dbConfig_);
                 ASSERT(catalogue->open());
 
-                std::unique_ptr store = catalogue->buildStore();
-                ASSERT(store->open());
+                // std::unique_ptr store = catalogue->buildStore();
+                // ASSERT(store->open());
 
                 eckit::AutoCloser closer(*catalogue);
-                eckit::AutoCloser storeCloser(*store);
+                // eckit::AutoCloser storeCloser(*store);
 
-                catalogue->visitEntries(visitor, *store, false);
+                catalogue->visitEntries(visitor, /* *store, */ false);
             }
         }
 
diff --git a/src/fdb5/database/EntryVisitMechanism.h b/src/fdb5/database/EntryVisitMechanism.h
index 2ba78a5c4..6607c376a 100644
--- a/src/fdb5/database/EntryVisitMechanism.h
+++ b/src/fdb5/database/EntryVisitMechanism.h
@@ -40,13 +40,18 @@ class EntryVisitor : public eckit::NonCopyable {
     virtual bool visitIndexes() { return true; }
     virtual bool visitEntries() { return true; }
 
-    virtual bool visitDatabase(const Catalogue& catalogue, const Store& store);    // return true if Catalogue should be explored
+//    virtual bool visitDatabase(const Catalogue& catalogue, const Store& store);    // return true if Catalogue should be explored
+    virtual bool visitDatabase(const Catalogue& catalogue);    // return true if Catalogue should be explored
     virtual bool visitIndex(const Index& index); // return true if index should be explored
     virtual void catalogueComplete(const Catalogue& catalogue);
     virtual void visitDatum(const Field& field, const std::string& keyFingerprint);
 
     time_t indexTimestamp() const;
 
+protected:
+
+    Store& store();
+
 private: // methods
 
     virtual void visitDatum(const Field& field, const InspectionKey& key) = 0;
@@ -55,7 +60,7 @@ class EntryVisitor : public eckit::NonCopyable {
 
     // n.b. non-owning
     const Catalogue* currentCatalogue_;
-    const Store* currentStore_;
+    std::unique_ptr currentStore_;
     const Index* currentIndex_;
 };
 
diff --git a/src/fdb5/database/Inspector.cc b/src/fdb5/database/Inspector.cc
index c5f5f200e..73a5bb5ce 100644
--- a/src/fdb5/database/Inspector.cc
+++ b/src/fdb5/database/Inspector.cc
@@ -48,7 +48,7 @@ bool InspectIterator::next(ListElement& elem) {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-static void purgeCatalogue(Key& key, Catalogue*& db) {
+static void purgeCatalogue(Key& key, CatalogueReader*& db) {
     Log::debug() << "Purging DB with key " << key << std::endl;
     delete db;
 }
diff --git a/src/fdb5/database/Inspector.h b/src/fdb5/database/Inspector.h
index 58d39b5ff..7066425d3 100644
--- a/src/fdb5/database/Inspector.h
+++ b/src/fdb5/database/Inspector.h
@@ -40,7 +40,7 @@ namespace fdb5 {
 
 class Key;
 class Op;
-class Catalogue;
+class CatalogueReader;
 class Schema;
 class Notifier;
 class FDBToolRequest;
@@ -98,7 +98,7 @@ class Inspector : public eckit::NonCopyable {
 
 private: // data
 
-    mutable eckit::CacheLRU databases_;
+    mutable eckit::CacheLRU databases_;
 
     Config dbConfig_;
 };
diff --git a/src/fdb5/database/MultiRetrieveVisitor.cc b/src/fdb5/database/MultiRetrieveVisitor.cc
index dc1271367..07e3b76fa 100644
--- a/src/fdb5/database/MultiRetrieveVisitor.cc
+++ b/src/fdb5/database/MultiRetrieveVisitor.cc
@@ -29,7 +29,7 @@ namespace fdb5 {
 
 MultiRetrieveVisitor::MultiRetrieveVisitor(const Notifier& wind,
                                            InspectIterator& iterator,
-                                           eckit::CacheLRU& databases,
+                                           eckit::CacheLRU& databases,
                                            const Config& config) :
     wind_(wind),
     databases_(databases),
@@ -65,7 +65,7 @@ bool MultiRetrieveVisitor::selectDatabase(const Key& key, const Key&) {
 
     /* DB not yet open */
 
-    std::unique_ptr newCatalogue = CatalogueFactory::instance().build(key, config_, true);
+    std::unique_ptr newCatalogue = CatalogueReaderFactory::instance().build(key, config_);
 
     // If this database is locked for retrieval then it "does not exist"
     if (!newCatalogue->enabled(ControlIdentifier::Retrieve)) {
@@ -98,7 +98,7 @@ bool MultiRetrieveVisitor::selectDatum(const InspectionKey& key, const Key& full
     eckit::Log::debug() << "selectDatum " << key << ", " << full << std::endl;
 
     Field field;
-    if (reader()->retrieve(key, field)) {
+    if (catalogue_->retrieve(key, field)) {
 
         Key simplifiedKey;
         for (auto k = key.begin(); k != key.end(); k++) {
@@ -118,12 +118,12 @@ void MultiRetrieveVisitor::values(const metkit::mars::MarsRequest &request,
                              const TypesRegistry ®istry,
                              eckit::StringList &values) {
     eckit::StringList list;
-    registry.lookupType(keyword).getValues(request, keyword, list, wind_, reader());
+    registry.lookupType(keyword).getValues(request, keyword, list, wind_, catalogue_);
 
     eckit::StringSet filter;
     bool toFilter = false;
     if (catalogue_) {
-        toFilter = reader()->axis(keyword, filter);
+        toFilter = catalogue_->axis(keyword, filter);
     }
 
     for(auto l: list) {
diff --git a/src/fdb5/database/MultiRetrieveVisitor.h b/src/fdb5/database/MultiRetrieveVisitor.h
index 43a2b8ea4..baf9cc765 100644
--- a/src/fdb5/database/MultiRetrieveVisitor.h
+++ b/src/fdb5/database/MultiRetrieveVisitor.h
@@ -39,7 +39,7 @@ class MultiRetrieveVisitor : public ReadVisitor {
 
     MultiRetrieveVisitor(const Notifier& wind,
                          InspectIterator& queue,
-                         eckit::CacheLRU& databases,
+                         eckit::CacheLRU& databases,
                          const Config& config);
 
     ~MultiRetrieveVisitor();
@@ -67,7 +67,7 @@ class MultiRetrieveVisitor : public ReadVisitor {
 
     const Notifier& wind_;
 
-    eckit::CacheLRU& databases_;
+    eckit::CacheLRU& databases_;
 
     InspectIterator& iterator_;
 
diff --git a/src/fdb5/database/ReadVisitor.cc b/src/fdb5/database/ReadVisitor.cc
index a80a675f9..dfd4f4a4a 100644
--- a/src/fdb5/database/ReadVisitor.cc
+++ b/src/fdb5/database/ReadVisitor.cc
@@ -18,17 +18,15 @@ namespace fdb5 {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-ReadVisitor::~ReadVisitor() {
-}
-
-CatalogueReader* ReadVisitor::reader() const {
-    if (catalogue_) {
-        CatalogueReader* catalogueReader = dynamic_cast(catalogue_);
-        ASSERT(catalogueReader);
-        return catalogueReader;
-    }
-    return nullptr;
-}
+
+// CatalogueReader* ReadVisitor::reader() const {
+//     if (catalogue_) {
+//         CatalogueReader* catalogueReader = dynamic_cast(catalogue_);
+//         ASSERT(catalogueReader);
+//         return catalogueReader;
+//     }
+//     return nullptr;
+// }
 
 //----------------------------------------------------------------------------------------------------------------------
 
diff --git a/src/fdb5/database/ReadVisitor.h b/src/fdb5/database/ReadVisitor.h
index bfdd229d2..fb00f6e6f 100644
--- a/src/fdb5/database/ReadVisitor.h
+++ b/src/fdb5/database/ReadVisitor.h
@@ -44,7 +44,7 @@ class ReadVisitor : public eckit::NonCopyable {
 
     ReadVisitor() : catalogue_(nullptr) {}
 
-    virtual ~ReadVisitor();
+    virtual ~ReadVisitor() {}
 
     virtual bool selectDatabase(const Key &key, const Key &full) = 0;
     virtual bool selectIndex(const Key &key, const Key &full) = 0;
@@ -60,13 +60,11 @@ class ReadVisitor : public eckit::NonCopyable {
 
 protected: // methods
 
-    CatalogueReader* reader() const;
-
     virtual void print( std::ostream &out ) const = 0;
 
 protected: // members
 
-    Catalogue* catalogue_;
+    CatalogueReader* catalogue_;
 
 private: // members
 
diff --git a/src/fdb5/database/RetrieveVisitor.cc b/src/fdb5/database/RetrieveVisitor.cc
index a08c0a9cc..350fd587f 100644
--- a/src/fdb5/database/RetrieveVisitor.cc
+++ b/src/fdb5/database/RetrieveVisitor.cc
@@ -43,7 +43,7 @@ bool RetrieveVisitor::selectDatabase(const Key& key, const Key&) {
     }
 
     eckit::Log::debug() << "selectDatabase " << key << std::endl;
-    catalogue_ = CatalogueFactory::instance().build(key, fdb5::Config(), true).get();
+    catalogue_ = CatalogueReaderFactory::instance().build(key, fdb5::Config()).get();
 
     // If this database is locked for retrieval then it "does not exist"
     if (!catalogue_->enabled(ControlIdentifier::Retrieve)) {
@@ -74,7 +74,7 @@ bool RetrieveVisitor::selectDatum(const InspectionKey& key, const Key&) {
 
     Field field;
     eckit::DataHandle *dh = nullptr;
-    if (reader()->retrieve(key, field)) {
+    if (catalogue_->retrieve(key, field)) {
         dh = store().retrieve(field);
     }
 
@@ -90,12 +90,12 @@ void RetrieveVisitor::values(const metkit::mars::MarsRequest &request,
                              const TypesRegistry ®istry,
                              eckit::StringList &values) {
     eckit::StringList list;
-    registry.lookupType(keyword).getValues(request, keyword, list, wind_, reader());
+    registry.lookupType(keyword).getValues(request, keyword, list, wind_, catalogue_);
 
     eckit::StringSet filter;
     bool toFilter = false;
     if (catalogue_) {
-        toFilter = reader()->axis(keyword, filter);
+        toFilter = catalogue_->axis(keyword, filter);
     }
 
     for(auto l: list) {
diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc
index e27e89595..6842dc989 100644
--- a/src/fdb5/rados/RadosStore.cc
+++ b/src/fdb5/rados/RadosStore.cc
@@ -31,6 +31,11 @@ namespace fdb5 {
 RadosStore::RadosStore(const Key& key, const Config& config) :
     Store(), directory_("mars:"+key.valuesToString()) {}
 
+RadosStore(const Key& key, const Config& config, const eckit::net::Endpoint& controlEndpoint) :
+    Store(), directory_("mars:"+key.valuesToString()) {
+    NOTIMP;
+}
+
 RadosStore::RadosStore(const eckit::URI& uri, const Config& config) :
     Store(), directory_("mars:"+uri.path().dirName()) {}
 
diff --git a/src/fdb5/rados/RadosStore.h b/src/fdb5/rados/RadosStore.h
index 502caf03e..f05181ae8 100644
--- a/src/fdb5/rados/RadosStore.h
+++ b/src/fdb5/rados/RadosStore.h
@@ -31,6 +31,7 @@ class RadosStore : public Store {
 public: // methods
 
     RadosStore(const Key& key, const Config& config);
+    RadosStore(const Key& key, const Config& config, const eckit::net::Endpoint& controlEndpoint);
     RadosStore(const eckit::URI& uri, const Config& config);
 
     ~RadosStore() override {}
diff --git a/src/fdb5/remote/CatalogueHandler.cc b/src/fdb5/remote/CatalogueHandler.cc
index 2171f06f8..17a5cda0c 100644
--- a/src/fdb5/remote/CatalogueHandler.cc
+++ b/src/fdb5/remote/CatalogueHandler.cc
@@ -177,9 +177,9 @@ void CatalogueHandler::handle() {
                     archive(hdr);
                     break;
 
-                case Message::Store:
-                    store(hdr);
-                    break;
+                // case Message::Store:
+                //     store(hdr);
+                //     break;
 
                 default: {
                     std::stringstream ss;
diff --git a/src/fdb5/remote/FdbServer.cc b/src/fdb5/remote/FdbServer.cc
index 21194870f..5de9e8875 100644
--- a/src/fdb5/remote/FdbServer.cc
+++ b/src/fdb5/remote/FdbServer.cc
@@ -20,8 +20,9 @@
 
 #include "fdb5/remote/AvailablePortList.h"
 // #include "fdb5/remote/Handler.h"
-#include "fdb5/remote/CatalogueHandler.h"
-#include "fdb5/remote/StoreHandler.h"
+//#include "fdb5/remote/CatalogueHandler.h"
+#include "fdb5/remote/server/StoreHandler.h"
+#include "fdb5/remote/server/ServerConnection.h"
 #include "eckit/config/Resource.h"
 
 using namespace eckit;
@@ -49,6 +50,9 @@ void FDBForker::run() {
 
     eckit::Log::info() << "FDB forked pid " << ::getpid() << std::endl;
 
+    // ServerConnection handler(socket_, config_);
+    // handler.handle();
+    
     if (config_.getString("type", "local") == "catalogue" || (::getenv("FDB_IS_CAT") && ::getenv("FDB_IS_CAT")[0] == '1')) {
         eckit::Log::info() << "FDB using Catalogue Handler" << std::endl;
         // CatalogueHandler handler(socket_, config_);
@@ -95,6 +99,8 @@ void FDBServerThread::run() {
     std::cout << "FDBServerThread::run()" << std::endl;
     eckit::Log::info() << "FDB started handler thread" << std::endl;
 
+    // ServerConnection handler(socket_, config_);
+    // handler.handle();
 
     if (config_.getString("type", "local") == "catalogue" || (::getenv("FDB_IS_CAT") && ::getenv("FDB_IS_CAT")[0] == '1')) {
         eckit::Log::info() << "FDB using Catalogue Handler" << std::endl;
@@ -112,8 +118,8 @@ void FDBServerThread::run() {
     //     handler.handle();
     // }
 
-    // RemoteHandler handler(socket_, config_);
-    // handler.handle();
+    // // RemoteHandler handler(socket_, config_);
+    // // handler.handle();
 }
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/Messages.h b/src/fdb5/remote/Messages.h
index b7f927c15..d7176190b 100644
--- a/src/fdb5/remote/Messages.h
+++ b/src/fdb5/remote/Messages.h
@@ -37,7 +37,7 @@ namespace remote {
 const static eckit::FixedString<4> StartMarker {"SFDB"};
 const static eckit::FixedString<4> EndMarker {"EFDB"};
 
-constexpr uint16_t CurrentVersion = 9;
+constexpr uint16_t CurrentVersion = 10;
 
 
 enum class Message : uint16_t {
@@ -83,13 +83,15 @@ struct MessageHeader {
     MessageHeader() :
         version(CurrentVersion),
         message(Message::None),
+        remoteID(0),
         requestID(0),
         payloadSize(0) {}
 
-    MessageHeader(Message message, uint32_t requestID, uint32_t payloadSize=0) :
+    MessageHeader(Message message, uint32_t remoteID, uint32_t requestID, uint32_t payloadSize=0) :
         marker(StartMarker),
         version(CurrentVersion),
         message(message),
+        remoteID(remoteID),
         requestID(requestID),
         payloadSize(payloadSize) {}
 
@@ -99,11 +101,13 @@ struct MessageHeader {
 
     Message message;                // 2 bytes  --> 8
 
-    uint32_t requestID;             // 4 bytes  --> 12
+    uint32_t remoteID;              // 4 bytes  --> 12
 
-    uint32_t payloadSize;           // 4 bytes  --> 16
+    uint32_t requestID;             // 4 bytes  --> 16
 
-    eckit::FixedString<16> hash;    // 16 bytes --> 32
+    uint32_t payloadSize;           // 4 bytes  --> 20
+
+    eckit::FixedString<16> hash;    // 16 bytes --> 36
 };
 
 
diff --git a/src/fdb5/remote/RemoteFieldLocation.cc b/src/fdb5/remote/RemoteFieldLocation.cc
index a552f5474..2b89a8f4d 100644
--- a/src/fdb5/remote/RemoteFieldLocation.cc
+++ b/src/fdb5/remote/RemoteFieldLocation.cc
@@ -14,7 +14,8 @@
  */
 
 #include "fdb5/remote/RemoteFieldLocation.h"
-#include "fdb5/api/RemoteFDB.h"
+#include "fdb5/remote/RemoteStore.h"
+#include "fdb5/remote/client/ClientConnectionRouter.h"
 
 #include "eckit/exception/Exceptions.h"
 #include "eckit/filesystem/URIManager.h"
@@ -27,20 +28,20 @@ ::eckit::Reanimator RemoteFieldLocation::reanimator_;
 
 //----------------------------------------------------------------------------------------------------------------------
 
-RemoteFieldLocation::RemoteFieldLocation(RemoteFDB* remoteFDB, const FieldLocation& remoteLocation) :
-    FieldLocation(eckit::URI("fdb", remoteFDB->controlEndpoint().host(),  remoteFDB->controlEndpoint().port())),
-    remoteFDB_(remoteFDB),
+RemoteFieldLocation::RemoteFieldLocation(const eckit::net::Endpoint& endpoint, const FieldLocation& remoteLocation) :
+    FieldLocation(eckit::URI("fdbremote", remoteLocation.uri(), endpoint.host(),  endpoint.port())),
+//    remoteStore_(remoteStore),
     internal_(remoteLocation.make_shared()) {
-    ASSERT(remoteFDB);
-    ASSERT(remoteLocation.uri().scheme() != "fdb");
+//    ASSERT(remoteStore);
+    ASSERT(remoteLocation.uri().scheme() != "fdbremote");
 }
 
 RemoteFieldLocation::RemoteFieldLocation(const eckit::URI& uri) :
-    FieldLocation(eckit::URI("fdb://" + uri.hostport())),
+    FieldLocation(eckit::URI("fdbremote://" + uri.hostport())),
     internal_(std::shared_ptr(FieldLocationFactory::instance().build(uri.scheme(), uri))) {}
 
 RemoteFieldLocation::RemoteFieldLocation(const eckit::URI& uri, const eckit::Offset& offset, const eckit::Length& length, const Key& remapKey) :
-    FieldLocation(eckit::URI("fdb://" + uri.hostport())),
+    FieldLocation(eckit::URI("fdbremote://" + uri.hostport())),
     internal_(std::shared_ptr(FieldLocationFactory::instance().build(uri.scheme(), uri, offset, length, remapKey))) {}
 
 RemoteFieldLocation::RemoteFieldLocation(eckit::Stream& s) :
@@ -49,7 +50,7 @@ RemoteFieldLocation::RemoteFieldLocation(eckit::Stream& s) :
 
 RemoteFieldLocation::RemoteFieldLocation(const RemoteFieldLocation& rhs) :
     FieldLocation(rhs.uri_),
-    remoteFDB_(rhs.remoteFDB_),
+//    remoteStore_(rhs.remoteStore_),
     internal_(rhs.internal_) {}
 
 
@@ -58,8 +59,9 @@ std::shared_ptr RemoteFieldLocation::make_shared() const {
 }
 
 eckit::DataHandle* RemoteFieldLocation::dataHandle() const {
-    ASSERT(remoteFDB_);
-    return remoteFDB_->dataHandle(*internal_);
+//    ASSERT(remoteStore_);
+    //Connection conn = ClientConnectionRouter::instance().connectStore(uri_);
+    return nullptr; //conn.->dataHandle(*internal_);
 }
 
 void RemoteFieldLocation::visit(FieldLocationVisitor& visitor) const {
@@ -74,7 +76,7 @@ void RemoteFieldLocation::encode(eckit::Stream& s) const {
     FieldLocation::encode(s);
     s << *internal_;
     std::stringstream ss;
-    ss << remoteFDB_->config();
+//    ss << remoteStore_->config();
     s << ss.str();
 }
 
@@ -83,7 +85,7 @@ void RemoteFieldLocation::dump(std::ostream& out) const {
     internal_->dump(out);
 }
 
-static FieldLocationBuilder builder("fdb");
+static FieldLocationBuilder builder("fdbremote");
 
 //----------------------------------------------------------------------------------------------------------------------
 
@@ -115,7 +117,7 @@ class FdbURIManager : public eckit::URIManager {
     FdbURIManager(const std::string& name) : eckit::URIManager(name) {}
 };
 
-static FdbURIManager manager_fdb_file("fdb");
+static FdbURIManager manager_fdb_file("fdbremote");
 
 } // namespace remote
 } // namespace fdb5
diff --git a/src/fdb5/remote/RemoteFieldLocation.h b/src/fdb5/remote/RemoteFieldLocation.h
index c0947f559..2ee44bca7 100644
--- a/src/fdb5/remote/RemoteFieldLocation.h
+++ b/src/fdb5/remote/RemoteFieldLocation.h
@@ -21,18 +21,16 @@
 
 #include "fdb5/database/FieldLocation.h"
 
-namespace fdb5 {
+namespace fdb5::remote {
 
-class RemoteFDB;
-
-namespace remote {
+class RemoteStore;
 
 //----------------------------------------------------------------------------------------------------------------------
 
 class RemoteFieldLocation : public FieldLocation {
 public:
 
-    RemoteFieldLocation(RemoteFDB* remoteFDB, const FieldLocation& remoteLocation);
+    RemoteFieldLocation(const eckit::net::Endpoint& endpoint, const FieldLocation& remoteLocation);
     RemoteFieldLocation(const eckit::URI &uri);
     RemoteFieldLocation(const eckit::URI &uri, const eckit::Offset &offset, const eckit::Length &length, const Key& remapKey);
     RemoteFieldLocation(eckit::Stream&);
@@ -66,14 +64,13 @@ class RemoteFieldLocation : public FieldLocation {
 private: // members
 
     // not Owning
-    RemoteFDB* remoteFDB_;
+//    RemoteStore* remoteStore_;
     std::shared_ptr internal_;
 };
 
 
 //----------------------------------------------------------------------------------------------------------------------
 
-} // namespace remote
-} // namespace fdb5
+} // namespace fdb5::remote
 
 #endif // fdb5_RemoteFieldLocation_H
diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index 1590a462e..039d9b7d6 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -23,6 +23,8 @@
 #include "fdb5/rules/Rule.h"
 #include "fdb5/database/FieldLocation.h"
 #include "fdb5/remote/RemoteStore.h"
+#include "fdb5/remote/RemoteFieldLocation.h"
+#include "fdb5/remote/client/ClientConnectionRouter.h"
 #include "fdb5/io/FDBFileHandle.h"
 
 using namespace eckit;
@@ -45,27 +47,29 @@ namespace fdb5::remote {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-RemoteStore::RemoteStore(const Key& key, const Config& config) :
-    ClientConnection(eckit::net::Endpoint(config.getString("storeHost"), config.getInt("storePort")), config),
-    key_(key), config_(config), dirty_(false), archiveID_(0),
+RemoteStore::RemoteStore(const Key& dbKey, const Config& config) :
+    Client(std::vector{eckit::net::Endpoint("localhost", 7000)}),
+//    Client(std::vector{ClientConnectionRouter::instance().connectStore(dbKey_, std::vector{eckit::net::Endpoint("localhost", 7000)})}),
+//    ClientConnection(eckit::net::Endpoint(config.getString("storeHost"), config.getInt("storePort")), config),
+    dbKey_(dbKey), config_(config), dirty_(false), //archiveID_(0),
     maxArchiveQueueLength_(eckit::Resource("fdbRemoteArchiveQueueLength;$FDB_REMOTE_ARCHIVE_QUEUE_LENGTH", 200)),
     retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
-    maxArchiveBatchSize_(config.getInt("maxBatchSize", 1)) {}
-
-RemoteStore::RemoteStore(const Key& key, const Config& config, const eckit::net::Endpoint& controlEndpoint) :
-    ClientConnection(controlEndpoint, config),
-    key_(key), config_(config), dirty_(false), archiveID_(0),
-    maxArchiveQueueLength_(eckit::Resource("fdbRemoteArchiveQueueLength;$FDB_REMOTE_ARCHIVE_QUEUE_LENGTH", 200)),
-    retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
-    maxArchiveBatchSize_(config.getInt("maxBatchSize", 1)) {}
+    maxArchiveBatchSize_(config.getInt("maxBatchSize", 1)) {
+}
 
 RemoteStore::RemoteStore(const eckit::URI& uri, const Config& config) :
-    ClientConnection(eckit::net::Endpoint(uri.hostport()), config),
-    key_(Key()), config_(config), dirty_(false), archiveID_(0),
+    Client(std::vector{eckit::net::Endpoint("localhost", 7000)}),
+//    Client(std::vector{ClientConnectionRouter::instance().connectStore(Key(), std::vector{eckit::net::Endpoint("localhost", 7000)})}),
+    dbKey_(Key()), config_(config), dirty_(false), //archiveID_(0),
     maxArchiveQueueLength_(eckit::Resource("fdbRemoteArchiveQueueLength;$FDB_REMOTE_ARCHIVE_QUEUE_LENGTH", 200)),
     retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
     maxArchiveBatchSize_(config.getInt("maxBatchSize", 1)) {
-    ASSERT(uri.scheme() == "fdb");
+
+    std::cout << "RemoteStore ctor " << uri.asRawString() << std::endl;
+
+    ASSERT(uri.scheme() == "fdbremote");
+    // std::vector endpoints{eckit::net::Endpoint(uri.hostport())};
+    // connection_ = ClientConnectionRouter::instance().connectStore(dbKey_, endpoints);
 }
 
 RemoteStore::~RemoteStore() {
@@ -78,11 +82,11 @@ RemoteStore::~RemoteStore() {
     }
 
     ASSERT(!dirty_);
-    disconnect();
+//    disconnect();
 }
 
 eckit::URI RemoteStore::uri() const {
-    return URI("fdb", "");
+    return URI("fdbremote", "");
 }
 
 bool RemoteStore::exists() const {
@@ -98,18 +102,13 @@ std::future > RemoteStore::archive(const Key& key
     // dirty_ = true;
     // uint32_t id = generateRequestID();
 
-    connect();
+//    connect();
 
     // if there is no archiving thread active, then start one.
     // n.b. reset the archiveQueue_ after a potential flush() cycle.
-
     if (!archiveFuture_.valid()) {
 
         // Start the archival request on the remote side
-        ASSERT(archiveID_ == 0);
-        uint32_t id = generateRequestID();
-        archiveID_ = id;
-        controlWriteCheckResponse(fdb5::remote::Message::Archive, id);
 
         // Reset the queue after previous done/errors
         {
@@ -118,44 +117,65 @@ std::future > RemoteStore::archive(const Key& key
             archiveQueue_.reset(new ArchiveQueue(maxArchiveQueueLength_));
         }
 
-        archiveFuture_ = std::async(std::launch::async, [this, id] { return archiveThreadLoop(id); });
+        archiveFuture_ = std::async(std::launch::async, [this] { return archiveThreadLoop(); });
     }
 
     ASSERT(archiveFuture_.valid());
-    ASSERT(archiveID_ != 0);
-    {
-        std::lock_guard lock(archiveQueuePtrMutex_);
-        ASSERT(archiveQueue_);
-        // std::vector combinedKey = {key_, key};
-        // std::cout << "archiveID_: " << archiveID_ << "  Archiving " << combinedKey << std::endl;
-        archiveQueue_->emplace(std::make_pair(key, Buffer(reinterpret_cast(data), length)));
-    }
+
+    uint32_t id = controlWriteCheckResponse(Message::Archive);
+    std::lock_guard lock(archiveQueuePtrMutex_);
+    ASSERT(archiveQueue_);
+    // std::vector combinedKey = {dbKey_, key};
+    // std::cout << "archiveID_: " << archiveID_ << "  Archiving " << combinedKey << std::endl;
+    archiveQueue_->emplace(std::make_pair(id, std::make_pair(key, Buffer(reinterpret_cast(data), length))));
 
     std::promise > loc;
     auto futureLocation = loc.get_future();
-    locations_[archiveID_] = std::move(loc);
+    locations_[id] = std::move(loc);
     return futureLocation;
 }
 
 bool RemoteStore::open() {
-    connect();
+//    connect();
     return true;
 }
 
+
 void RemoteStore::flush() {
-//     if (!dirty_) {
-//         return;
-//     }
 
-//     // ensure consistent state before writing Toc entry
+    Timer timer;
+
+    timer.start();
+
+    // Flush only does anything if there is an ongoing archive();
+    if (archiveFuture_.valid()) {
+
+        ASSERT(archiveQueue_);
+        std::lock_guard lock(archiveQueuePtrMutex_);
+        archiveQueue_->close();
+
+        FDBStats stats = archiveFuture_.get();
+        ASSERT(!archiveQueue_);
+
+        ASSERT(stats.numFlush() == 0);
+        size_t numArchive = stats.numArchive();
+
+        Buffer sendBuf(4096);
+        MemoryStream s(sendBuf);
+        s << numArchive;
 
-//     flushDataHandles();
+        // The flush call is blocking
+        controlWriteCheckResponse(fdb5::remote::Message::Flush, sendBuf, s.position());
 
-//     dirty_ = false;
+//        internalStats_ += stats;
+    }
+
+    timer.stop();
+//    internalStats_.addFlush(timer);
 }
 
 void RemoteStore::close() {
-    disconnect();
+//    disconnect();
 }
 
 void RemoteStore::remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const {
@@ -195,14 +215,14 @@ void RemoteStore::remove(const Key& key) const {
 // }
 
 void RemoteStore::print(std::ostream &out) const {
-    out << "RemoteStore(host=" << controlEndpoint() << ", data=" << dataEndpoint() << ")";
+    out << "RemoteStore(host=" << /*controlEndpoint() << ", data=" << dataEndpoint() << */ ")";
 }
 
 
 
 
 
-// void RemoteFDB::flush() {
+// void RemoteStore::flush() {
 
 //     Timer timer;
 
@@ -240,39 +260,42 @@ void RemoteStore::print(std::ostream &out) const {
 
 
 
-FDBStats RemoteStore::archiveThreadLoop(uint32_t requestID) {
+FDBStats RemoteStore::archiveThreadLoop() {
     FDBStats localStats;
     eckit::Timer timer;
 
     // We can pop multiple elements off the archive queue simultaneously, if
     // configured
-    std::vector> elements;
-    for (size_t i = 0; i < maxArchiveBatchSize_; ++i) {
-        elements.emplace_back(std::make_pair(Key{}, Buffer{0}));
-    }
+    std::pair > element;
+    // for (size_t i = 0; i < maxArchiveBatchSize_; ++i) {
+    //     elements.emplace_back(std::make_pair(Key{}, Buffer{0}));
+    // }
 
     try {
 
         ASSERT(archiveQueue_);
-        ASSERT(archiveID_ != 0);
+//        ASSERT(archiveID_ != 0);
 
+//        std::cout << "RemoteStore::archiveThreadLoop \n";
         long popped;
-        while ((popped = archiveQueue_->pop(elements)) != -1) {
+        while ((popped = archiveQueue_->pop(element)) != -1) {
 
             timer.start();
-            long dataSent = sendArchiveData(requestID, elements, popped);
+//        std::cout << "RemoteStore::archiveThreadLoop - sendArchiveData " <<  element.second.second.size() << std::endl;
+            sendArchiveData(element.first, element.second.first, element.second.second.data(), element.second.second.size());
             timer.stop();
-            localStats.addArchive(dataSent, timer, popped);
+
+            localStats.addArchive(element.second.second.size(), timer, 1);
         }
 
         // And note that we are done. (don't time this, as already being blocked
         // on by the ::flush() routine)
 
-        MessageHeader hdr(Message::Flush, requestID);
-        dataWrite(&hdr, sizeof(hdr));
-        dataWrite(&EndMarker, sizeof(EndMarker));
+       //MessageHeader hdr(Message::Flush, 0, 0, 0);
+       dataWrite(Message::Flush, nullptr, 0);
+//       dataWrite(id, &EndMarker, sizeof(EndMarker));
 
-        archiveID_ = 0;
+//        archiveID_ = 0;
         archiveQueue_.reset();
 
     } catch (...) {
@@ -287,147 +310,40 @@ FDBStats RemoteStore::archiveThreadLoop(uint32_t requestID) {
 }
 
 
+bool RemoteStore::handle(Message message, uint32_t requestID) {
+//    std::cout << "RemoteStore::handle received message: " << ((uint) message) << " - requestID: " << requestID << std::endl;
+    return false;
 
-void RemoteStore::listeningThreadLoop() {
-
-    /// @note This routine retrieves BOTH normal API asynchronously returned data, AND
-    /// fields that are being returned by a retrieve() call. These need to go into different
-    /// queues
-    /// --> Test if the requestID is a known API request, otherwise push onto the retrieve queue
-
-
-    /// @note messageQueues_ is a map of requestID:MessageQueue. At the point that
-    /// a request is complete, errored or otherwise killed, it needs to be removed
-    /// from the map. The shared_ptr allows this removal to be asynchronous with
-    /// the actual task cleaning up and returning to the client.
-
-    try {
-
-    MessageHeader hdr;
-    eckit::FixedString<4> tail;
-
-    while (true) {
-
-        dataRead(&hdr, sizeof(hdr));
-
-        ASSERT(hdr.marker == StartMarker);
-        ASSERT(hdr.version == CurrentVersion);
-
-        switch (hdr.message) {
-
-        case Message::Exit:
-            return;
-
-        case Message::Blob: {
-            Buffer payload(hdr.payloadSize);
-            if (hdr.payloadSize > 0) dataRead(payload, hdr.payloadSize);
-
-            auto it = messageQueues_.find(hdr.requestID);
-            if (it != messageQueues_.end()) {
-                it->second->emplace(std::make_pair(hdr, std::move(payload)));
-            } else {
-                retrieveMessageQueue_.emplace(std::make_pair(hdr, std::move(payload)));
-            }
-            break;
-        }
-
-        case Message::Complete: {
-            auto it = messageQueues_.find(hdr.requestID);
-            if (it != messageQueues_.end()) {
-                it->second->close();
-
-                // Remove entry (shared_ptr --> message queue will be destroyed when it
-                // goes out of scope in the worker thread).
-                messageQueues_.erase(it);
-
-            } else {
-                retrieveMessageQueue_.emplace(std::make_pair(hdr, Buffer(0)));
-            }
-            break;
-        }
-
-        case Message::Error: {
-
-            auto it = messageQueues_.find(hdr.requestID);
-            if (it != messageQueues_.end()) {
-                std::string msg;
-                if (hdr.payloadSize > 0) {
-                    msg.resize(hdr.payloadSize, ' ');
-                    dataRead(&msg[0], hdr.payloadSize);
-                }
-                it->second->interrupt(std::make_exception_ptr(DecoupledFDBException(msg, dataEndpoint())));
-
-                // Remove entry (shared_ptr --> message queue will be destroyed when it
-                // goes out of scope in the worker thread).
-                messageQueues_.erase(it);
-
-            } else if (hdr.requestID == archiveID_) {
-
-                std::lock_guard lock(archiveQueuePtrMutex_);
-
-                if (archiveQueue_) {
-                    std::string msg;
-                    if (hdr.payloadSize > 0) {
-                        msg.resize(hdr.payloadSize, ' ');
-                        dataRead(&msg[0], hdr.payloadSize);
-                    }
-
-                    archiveQueue_->interrupt(std::make_exception_ptr(DecoupledFDBException(msg, dataEndpoint())));
-                }
-            } else {
-                Buffer payload(hdr.payloadSize);
-                if (hdr.payloadSize > 0) dataRead(payload, hdr.payloadSize);
-                retrieveMessageQueue_.emplace(std::make_pair(hdr, std::move(payload)));
-            }
-            break;
-        }
-
-        default: {
-            std::stringstream ss;
-            ss << "ERROR: Unexpected message recieved (" << static_cast(hdr.message) << "). ABORTING";
-            Log::status() << ss.str() << std::endl;
-            Log::error() << "Retrieving... " << ss.str() << std::endl;
-            throw SeriousBug(ss.str(), Here());
-        }
-        };
-
-        // Ensure we have consumed exactly the correct amount from the socket.
-
-        dataRead(&tail, sizeof(tail));
-        ASSERT(tail == EndMarker);
-    }
-
-    // We don't want to let exceptions escape inside a worker thread.
-
-    } catch (const std::exception& e) {
-        for (auto& it : messageQueues_) {
-            it.second->interrupt(std::make_exception_ptr(e));
-        }
-        messageQueues_.clear();
-        retrieveMessageQueue_.interrupt(std::make_exception_ptr(e));
-        {
-            std::lock_guard lock(archiveQueuePtrMutex_);
-            if (archiveQueue_) archiveQueue_->interrupt(std::make_exception_ptr(e));
-        }
-    } catch (...) {
-        for (auto& it : messageQueues_) {
-            it.second->interrupt(std::current_exception());
-        }
-        messageQueues_.clear();
-        retrieveMessageQueue_.interrupt(std::current_exception());
-        {
-            std::lock_guard lock(archiveQueuePtrMutex_);
-            if (archiveQueue_) archiveQueue_->interrupt(std::current_exception());
+}
+bool RemoteStore::handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) {
+
+//    std::cout << "RemoteStore::handle received message: " << ((uint) message) << " - requestID: " << requestID << std::endl;
+    if (message == Message::Archive) {
+        auto it = locations_.find(requestID);
+        if (it != locations_.end()) {
+            MemoryStream s(payload);
+            std::unique_ptr location(eckit::Reanimator::reanimate(s));
+            // std::cout <<  "RemoteStore::handle - " << location->uri().asRawString() << " " << location->length() << std::endl;
+            std::unique_ptr remoteLocation = std::unique_ptr(new RemoteFieldLocation(endpoint, *location));
+
+            it->second.set_value(std::move(remoteLocation));
         }
+        return true;
     }
+    return false;
+}
+void RemoteStore::handleException(std::exception_ptr e) {
+    std::cout << "RemoteStore::handleException" << std::endl;
 }
 
 
 void RemoteStore::flush(FDBStats& internalStats) {
+
+    std::cout << "###################################      RemoteStore::flush\n";
     // Flush only does anything if there is an ongoing archive();
     if (! archiveFuture_.valid()) return;
 
-    ASSERT(archiveID_ != 0);
+//    ASSERT(archiveID_ != 0);
     {
         ASSERT(archiveQueue_);
         std::lock_guard lock(archiveQueuePtrMutex_);
@@ -435,7 +351,7 @@ void RemoteStore::flush(FDBStats& internalStats) {
     }
     FDBStats stats = archiveFuture_.get();
     ASSERT(!archiveQueue_);
-    archiveID_ = 0;
+//    archiveID_ = 0;
 
     ASSERT(stats.numFlush() == 0);
     size_t numArchive = stats.numArchive();
@@ -445,211 +361,149 @@ void RemoteStore::flush(FDBStats& internalStats) {
     s << numArchive;
 
     // The flush call is blocking
-    controlWriteCheckResponse(Message::Flush, generateRequestID(), sendBuf, s.position());
+    controlWriteCheckResponse(Message::Flush, sendBuf, s.position());
 
     internalStats += stats;
 }
 
 
-// long RemoteFDB::sendArchiveData(uint32_t id, const std::vector>& elements, size_t count) {
-
-//     if (count == 1) {
-//         sendArchiveData(id, elements[0].first, elements[0].second.data(), elements[0].second.size());
-//         return elements[0].second.size();
-//     }
-
-//     // Serialise the keys
-
-//     std::vector keyBuffers;
-//     std::vector keySizes;
-//     keyBuffers.reserve(count);
-//     keySizes.reserve(count);
-
-//     size_t containedSize = 0;
-
-//     for (size_t i = 0; i < count; ++i) {
-//         keyBuffers.emplace_back(Buffer {4096});
-//         MemoryStream keyStream(keyBuffers.back());
-//         keyStream << elements[i].first;
-//         keySizes.push_back(keyStream.position());
-//         containedSize += (keyStream.position() + elements[i].second.size() +
-//                           sizeof(MessageHeader) + sizeof(EndMarker));
-//     }
-
-//     // Construct the containing message
-
-//     MessageHeader message(fdb5::remote::Message::MultiBlob, id, containedSize);
-//     dataWrite(&message, sizeof(message));
+// -----------------------------------------------------------------------------------------------------
 
-//     long dataSent = 0;
-
-//     for (size_t i = 0; i < count; ++i) {
-//         MessageHeader containedMessage(fdb5::remote::Message::Blob, id, elements[i].second.size() + keySizes[i]);
-//         dataWrite(&containedMessage, sizeof(containedMessage));
-//         dataWrite(keyBuffers[i], keySizes[i]);
-//         dataWrite(elements[i].second.data(), elements[i].second.size());
-//         dataWrite(&EndMarker, sizeof(EndMarker));
-//         dataSent += elements[i].second.size();
-//     }
-
-//     dataWrite(&EndMarker, sizeof(EndMarker));
-//     return dataSent;
-// }
+//
+/// @note The DataHandles returned by retrieve() MUST STRICTLY be read in order.
+///       We do not create multiple message queues, one for each requestID, even
+///       though that would be nice. This is because a commonly used retrieve
+///       pattern uses many retrieve() calls aggregated into a MultiHandle, and
+///       if we created lots of queues we would just run out of memory receiving
+///       from the stream. Further, if we curcumvented this by blocking, then we
+///       could get deadlocked if we try and read a message that is further back
+///       in the stream
+///
+/// --> Retrieve is a _streaming_ service.
 
+namespace {
 
-// void RemoteFDB::sendArchiveData(uint32_t id, const Key& key, const void* data, size_t length) {
+class FDBRemoteDataHandle : public DataHandle {
 
-//     ASSERT(data);
-//     ASSERT(length != 0);
+public: // methods
 
-//     Buffer keyBuffer(4096);
-//     MemoryStream keyStream(keyBuffer);
-//     keyStream << key;
+    FDBRemoteDataHandle(uint32_t requestID,
+                        RemoteStore::MessageQueue& queue,
+                        const net::Endpoint& remoteEndpoint) :
+        requestID_(requestID),
+        queue_(queue),
+        remoteEndpoint_(remoteEndpoint),
+        pos_(0),
+        overallPosition_(0),
+        currentBuffer_(0),
+        complete_(false) {}
+    virtual bool canSeek() const override { return false; }
 
-//     MessageHeader message(fdb5::remote::Message::Blob, id, length + keyStream.position());
-//     dataWrite(&message, sizeof(message));
-//     dataWrite(keyBuffer, keyStream.position());
-//     dataWrite(data, length);
-//     dataWrite(&EndMarker, sizeof(EndMarker));
-// }
+private: // methods
 
-// // -----------------------------------------------------------------------------------------------------
-
-// //
-// /// @note The DataHandles returned by retrieve() MUST STRICTLY be read in order.
-// ///       We do not create multiple message queues, one for each requestID, even
-// ///       though that would be nice. This is because a commonly used retrieve
-// ///       pattern uses many retrieve() calls aggregated into a MultiHandle, and
-// ///       if we created lots of queues we would just run out of memory receiving
-// ///       from the stream. Further, if we curcumvented this by blocking, then we
-// ///       could get deadlocked if we try and read a message that is further back
-// ///       in the stream
-// ///
-// /// --> Retrieve is a _streaming_ service.
-
-// namespace {
-
-// class FDBRemoteDataHandle : public DataHandle {
-
-// public: // methods
-
-//     FDBRemoteDataHandle(uint32_t requestID,
-//                         RemoteFDB::MessageQueue& queue,
-//                         const net::Endpoint& remoteEndpoint) :
-//         requestID_(requestID),
-//         queue_(queue),
-//         remoteEndpoint_(remoteEndpoint),
-//         pos_(0),
-//         overallPosition_(0),
-//         currentBuffer_(0),
-//         complete_(false) {}
-//     virtual bool canSeek() const override { return false; }
-
-// private: // methods
-
-//     void print(std::ostream& s) const override {
-//         s << "FDBRemoteDataHandle(id=" << requestID_ << ")";
-//     }
+    void print(std::ostream& s) const override {
+        s << "FDBRemoteDataHandle(id=" << requestID_ << ")";
+    }
 
-//     Length openForRead() override {
-//         return estimate();
-//     }
-//     void openForWrite(const Length&) override { NOTIMP; }
-//     void openForAppend(const Length&) override { NOTIMP; }
-//     long write(const void*, long) override { NOTIMP; }
-//     void close() override {}
+    Length openForRead() override {
+        return estimate();
+    }
+    void openForWrite(const Length&) override { NOTIMP; }
+    void openForAppend(const Length&) override { NOTIMP; }
+    long write(const void*, long) override { NOTIMP; }
+    void close() override {}
 
-//     long read(void* pos, long sz) override {
+    long read(void* pos, long sz) override {
 
-//         if (complete_) return 0;
+        if (complete_) return 0;
 
-//         if (currentBuffer_.size() != 0) return bufferRead(pos, sz);
+        if (currentBuffer_.size() != 0) return bufferRead(pos, sz);
 
-//         // If we are in the DataHandle, then there MUST be data to read
+        // If we are in the DataHandle, then there MUST be data to read
 
-//         RemoteFDB::StoredMessage msg = std::make_pair(remote::MessageHeader{}, eckit::Buffer{0});
-//         ASSERT(queue_.pop(msg) != -1);
+        RemoteStore::StoredMessage msg = std::make_pair(remote::MessageHeader{}, eckit::Buffer{0});
+        ASSERT(queue_.pop(msg) != -1);
 
-//         // TODO; Error handling in the retrieve pathway
+        // TODO; Error handling in the retrieve pathway
 
-//         const MessageHeader& hdr(msg.first);
+        const MessageHeader& hdr(msg.first);
 
-//         ASSERT(hdr.marker == StartMarker);
-//         ASSERT(hdr.version == CurrentVersion);
+        ASSERT(hdr.marker == StartMarker);
+        ASSERT(hdr.version == CurrentVersion);
 
-//         // Handle any remote errors communicated from the server
+        // Handle any remote errors communicated from the server
 
-//         if (hdr.message == fdb5::remote::Message::Error) {
-//             std::string errmsg(static_cast(msg.second), msg.second.size());
-//             throw RemoteFDBException(errmsg, remoteEndpoint_);
-//         }
+        if (hdr.message == fdb5::remote::Message::Error) {
+            std::string errmsg(static_cast(msg.second), msg.second.size());
+            throw DecoupledFDBException(errmsg, remoteEndpoint_);
+        }
 
-//         // Are we now complete
+        // Are we now complete
 
-//         if (hdr.message == fdb5::remote::Message::Complete) {
-//             complete_ = 0;
-//             return 0;
-//         }
+        if (hdr.message == fdb5::remote::Message::Complete) {
+            complete_ = 0;
+            return 0;
+        }
 
-//         ASSERT(hdr.message == fdb5::remote::Message::Blob);
+        ASSERT(hdr.message == fdb5::remote::Message::Blob);
 
-//         // Otherwise return the data!
+        // Otherwise return the data!
 
-//         std::swap(currentBuffer_, msg.second);
+        std::swap(currentBuffer_, msg.second);
 
-//         return bufferRead(pos, sz);
-//     }
+        return bufferRead(pos, sz);
+    }
 
-//     // A helper function that returns some, or all, of a buffer that has
-//     // already been retrieved.
+    // A helper function that returns some, or all, of a buffer that has
+    // already been retrieved.
 
-//     long bufferRead(void* pos, long sz) {
+    long bufferRead(void* pos, long sz) {
 
-//         ASSERT(currentBuffer_.size() != 0);
-//         ASSERT(pos_ < currentBuffer_.size());
+        ASSERT(currentBuffer_.size() != 0);
+        ASSERT(pos_ < currentBuffer_.size());
 
-//         long read = std::min(sz, long(currentBuffer_.size() - pos_));
+        long read = std::min(sz, long(currentBuffer_.size() - pos_));
 
-//         ::memcpy(pos, ¤tBuffer_[pos_], read);
-//         pos_ += read;
-//         overallPosition_ += read;
+        ::memcpy(pos, ¤tBuffer_[pos_], read);
+        pos_ += read;
+        overallPosition_ += read;
 
-//         // If we have exhausted this buffer, free it up.
+        // If we have exhausted this buffer, free it up.
 
-//         if (pos_ >= currentBuffer_.size()) {
-//             Buffer nullBuffer(0);
-//             std::swap(currentBuffer_, nullBuffer);
-//             pos_ = 0;
-//             ASSERT(currentBuffer_.size() == 0);
-//         }
+        if (pos_ >= currentBuffer_.size()) {
+            Buffer nullBuffer(0);
+            std::swap(currentBuffer_, nullBuffer);
+            pos_ = 0;
+            ASSERT(currentBuffer_.size() == 0);
+        }
 
-//         return read;
-//     }
+        return read;
+    }
 
-//     Length estimate() override {
-//         return 0;
-//     }
+    Length estimate() override {
+        return 0;
+    }
 
-//     Offset position() override {
-//         return overallPosition_;
-//     }
+    Offset position() override {
+        return overallPosition_;
+    }
 
-// private: // members
+private: // members
 
-//     uint32_t requestID_;
-//     RemoteFDB::MessageQueue& queue_;
-//     net::Endpoint remoteEndpoint_;
-//     size_t pos_;
-//     Offset overallPosition_;
-//     Buffer currentBuffer_;
-//     bool complete_;
-// };
+    uint32_t requestID_;
+    RemoteStore::MessageQueue& queue_;
+    net::Endpoint remoteEndpoint_;
+    size_t pos_;
+    Offset overallPosition_;
+    Buffer currentBuffer_;
+    bool complete_;
+};
 
-// }
+}
 
 // // Here we do (asynchronous) retrieving related stuff
 
-// //DataHandle* RemoteFDB::retrieve(const metkit::mars::MarsRequest& request) {
+// //DataHandle* RemoteStore::retrieve(const metkit::mars::MarsRequest& request) {
 
 // //    connect();
 
@@ -668,11 +522,11 @@ void RemoteStore::flush(FDBStats& internalStats) {
 // // Here we do (asynchronous) read related stuff
 
 
-// eckit::DataHandle* RemoteFDB::dataHandle(const FieldLocation& fieldLocation) {
+// eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation) {
 //     return dataHandle(fieldLocation, Key());
 // }
 
-// eckit::DataHandle* RemoteFDB::dataHandle(const FieldLocation& fieldLocation, const Key& remapKey) {
+// eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation, const Key& remapKey) {
 
 //     connect();
 
@@ -690,49 +544,49 @@ void RemoteStore::flush(FDBStats& internalStats) {
 
 
 
-long RemoteStore::sendArchiveData(uint32_t id, const std::vector>& elements, size_t count) {
-    if (count == 1) {
-        sendArchiveData(id, elements[0].first, elements[0].second.data(), elements[0].second.size());
-        return elements[0].second.size();
-    }
+// long RemoteStore::sendArchiveData(uint32_t id, const std::vector>& elements, size_t count) {
+//     if (count == 1) {
+//         sendArchiveData(id, elements[0].first, elements[0].second.data(), elements[0].second.size());
+//         return elements[0].second.size();
+//     }
 
-    // Serialise the keys
+//     // Serialise the keys
 
-    std::vector keyBuffers;
-    std::vector keySizes;
-    keyBuffers.reserve(count);
-    keySizes.reserve(count);
+//     std::vector keyBuffers;
+//     std::vector keySizes;
+//     keyBuffers.reserve(count);
+//     keySizes.reserve(count);
 
-    size_t containedSize = 0;
+//     size_t containedSize = 0;
 
-    for (size_t i = 0; i < count; ++i) {
-        keyBuffers.emplace_back(Buffer {4096});
-        MemoryStream keyStream(keyBuffers.back());
-        keyStream << elements[i].first;
-        keySizes.push_back(keyStream.position());
-        containedSize += (keyStream.position() + elements[i].second.size() +
-                          sizeof(MessageHeader) + sizeof(EndMarker));
-    }
+//     for (size_t i = 0; i < count; ++i) {
+//         keyBuffers.emplace_back(Buffer {4096});
+//         MemoryStream keyStream(keyBuffers.back());
+//         keyStream << elements[i].first;
+//         keySizes.push_back(keyStream.position());
+//         containedSize += (keyStream.position() + elements[i].second.size() +
+//                           sizeof(MessageHeader) + sizeof(EndMarker));
+//     }
 
-    // Construct the containing message
+//     // Construct the containing message
 
-    MessageHeader message(Message::MultiBlob, id, containedSize);
-    dataWrite(&message, sizeof(message));
+//     MessageHeader message(Message::MultiBlob, id, containedSize);
+//     dataWrite(&message, sizeof(message));
 
-    long dataSent = 0;
+//     long dataSent = 0;
 
-    for (size_t i = 0; i < count; ++i) {
-        MessageHeader containedMessage(Message::Blob, id, elements[i].second.size() + keySizes[i]);
-        dataWrite(&containedMessage, sizeof(containedMessage));
-        dataWrite(keyBuffers[i], keySizes[i]);
-        dataWrite(elements[i].second.data(), elements[i].second.size());
-        dataWrite(&EndMarker, sizeof(EndMarker));
-        dataSent += elements[i].second.size();
-    }
+//     for (size_t i = 0; i < count; ++i) {
+//         MessageHeader containedMessage(Message::Blob, id, elements[i].second.size() + keySizes[i]);
+//         dataWrite(&containedMessage, sizeof(containedMessage));
+//         dataWrite(keyBuffers[i], keySizes[i]);
+//         dataWrite(elements[i].second.data(), elements[i].second.size());
+//         dataWrite(&EndMarker, sizeof(EndMarker));
+//         dataSent += elements[i].second.size();
+//     }
 
-    dataWrite(&EndMarker, sizeof(EndMarker));
-    return dataSent;
-}
+//     dataWrite(&EndMarker, sizeof(EndMarker));
+//     return dataSent;
+// }
 
 
 void RemoteStore::sendArchiveData(uint32_t id, const Key& key, const void* data, size_t length) {
@@ -743,20 +597,42 @@ void RemoteStore::sendArchiveData(uint32_t id, const Key& key, const void* data,
     Buffer keyBuffer(4096);
     MemoryStream keyStream(keyBuffer);
 
-    keyStream << key_;
+    keyStream << dbKey_;
     keyStream << key;
 
-    MessageHeader message(Message::Blob, id, length + keyStream.position());
-    dataWrite(&message, sizeof(message));
-    dataWrite(keyBuffer, keyStream.position());
-    dataWrite(data, length);
-    dataWrite(&EndMarker, sizeof(EndMarker));
+    MessageHeader message(Message::Blob, 0, id, length + keyStream.position());
+    dataWrite(id, &message, sizeof(message));
+    dataWrite(id, keyBuffer, keyStream.position());
+    dataWrite(id, data, length);
+    dataWrite(id, &EndMarker, sizeof(EndMarker));
+}
+
+
+
+eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation) {
+    return dataHandle(fieldLocation, Key());
+}
+
+eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation, const Key& remapKey) {
+
+    //connect();
+
+    Buffer encodeBuffer(4096);
+    MemoryStream s(encodeBuffer);
+    s << fieldLocation;
+    s << remapKey;
+
+//    uint32_t id = generateRequestID();
+
+    uint32_t id = controlWriteCheckResponse(fdb5::remote::Message::Read, encodeBuffer, s.position());
+
+    return new FDBRemoteDataHandle(id, retrieveMessageQueue_, eckit::net::Endpoint("") /* controlEndpoint() */);
 }
 
 
 
 
-static StoreBuilder builder("remote");
+static StoreBuilder builder("fdbremote");
 
 //----------------------------------------------------------------------------------------------------------------------
 
diff --git a/src/fdb5/remote/RemoteStore.h b/src/fdb5/remote/RemoteStore.h
index d6c1f3904..9f31e5c5b 100644
--- a/src/fdb5/remote/RemoteStore.h
+++ b/src/fdb5/remote/RemoteStore.h
@@ -14,10 +14,11 @@
 
 #pragma once
 
+#include "fdb5/api/FDBStats.h"
 #include "fdb5/database/Catalogue.h"
 #include "fdb5/database/Index.h"
 #include "fdb5/database/Store.h"
-#include "fdb5/remote/ClientConnection.h"
+#include "fdb5/remote/client/Client.h"
 
 namespace fdb5::remote {
 
@@ -25,18 +26,18 @@ namespace fdb5::remote {
 
 /// Store that connects to a remote Store
 
-class RemoteStore : public Store, public ClientConnection {
+class RemoteStore : public Store, public Client {
 
 public: // types
 
-    using ArchiveQueue = eckit::Queue>;
+    using ArchiveQueue = eckit::Queue > >;
     using StoredMessage = std::pair;
     using MessageQueue = eckit::Queue;
 
 public: // methods
 
     RemoteStore(const Key& key, const Config& config);
-    RemoteStore(const Key& key, const Config& config, const eckit::net::Endpoint& controlEndpoint);
+//    RemoteStore(const Key& key, const Config& config, const eckit::net::Endpoint& controlEndpoint);
     RemoteStore(const eckit::URI& uri, const Config& config);
 
     ~RemoteStore() override;
@@ -49,13 +50,18 @@ class RemoteStore : public Store, public ClientConnection {
 
     void checkUID() const override { }
 
+    eckit::DataHandle* dataHandle(const FieldLocation& fieldLocation);
+    eckit::DataHandle* dataHandle(const FieldLocation& fieldLocation, const Key& remapKey);
+
     bool canMoveTo(const Key& key, const Config& config, const eckit::URI& dest) const override { return false; }
     void moveTo(const Key& key, const Config& config, const eckit::URI& dest, eckit::Queue& queue) const override { NOTIMP; }
     void remove(const Key& key) const override;
 
+   const Config& config() { return config_; }
+   
 protected: // methods
 
-    std::string type() const override { return "remote"; }
+    std::string type() const override { return "fdbremote"; }
 
     bool exists() const override;
 
@@ -68,11 +74,18 @@ class RemoteStore : public Store, public ClientConnection {
 
 private: // methods
 
-    FDBStats archiveThreadLoop(uint32_t requestID);
+    Key key() override { return dbKey_; }
+
+    // handlers for incoming messages - to be defined in the client class
+    bool handle(Message message, uint32_t requestID) override;
+    bool handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) override;
+    void handleException(std::exception_ptr e) override;
+
+    FDBStats archiveThreadLoop();
     // Listen to the dataClient for incoming messages, and push them onto
     // appropriate queues.
-    void listeningThreadLoop() override;
-    long sendArchiveData(uint32_t id, const std::vector>& elements, size_t count);
+//    void listeningThreadLoop() override;
+//    long sendArchiveData(uint32_t id, const std::vector>& elements, size_t count);
     void sendArchiveData(uint32_t id, const Key& key, const void* data, size_t length);
     void flush(FDBStats& stats);
 
@@ -81,12 +94,12 @@ class RemoteStore : public Store, public ClientConnection {
 
 private: // members
 
-    Key key_;
+    Key dbKey_;
     const Config& config_;
 
     bool dirty_;
     std::future archiveFuture_;
-    uint32_t archiveID_;
+//    uint32_t archiveID_;
     std::mutex archiveQueuePtrMutex_;
     std::unique_ptr archiveQueue_;
     size_t maxArchiveQueueLength_;
diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
new file mode 100644
index 000000000..6f45b4911
--- /dev/null
+++ b/src/fdb5/remote/client/Client.cc
@@ -0,0 +1,47 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+#include "fdb5/remote/client/Client.h"
+#include "fdb5/remote/client/ClientConnectionRouter.h"
+
+namespace fdb5::remote {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+Client::Client(eckit::net::Endpoint endpoint) :
+    endpoints_(std::vector{endpoint}) {}
+
+Client::Client(std::vector&& endpoints) :
+    endpoints_(std::move(endpoints)) {}
+
+uint32_t Client::controlWriteCheckResponse(Message msg, const void* payload, uint32_t payloadLength) {
+    return ClientConnectionRouter::instance().controlWriteCheckResponse(endpoints_, *this, msg, payload, payloadLength);
+}
+uint32_t Client::controlWrite(Message msg, const void* payload, uint32_t payloadLength) {
+    return ClientConnectionRouter::instance().controlWrite(endpoints_, *this, msg, payload, payloadLength);
+}
+// void Client::controlWrite(uint32_t requestId, const void* data, size_t length) {
+//     ClientConnectionRouter::instance().controlWrite(*this, requestId, data, length);
+// }
+// void Client::controlRead(uint32_t requestId, void* data, size_t length) {
+//     ClientConnectionRouter::instance().controlRead(*this, requestId, data, length);
+// }
+
+uint32_t Client::dataWrite(Message msg, const void* payload, uint32_t payloadLength) {
+    return ClientConnectionRouter::instance().dataWrite(endpoints_, *this, msg, payload, payloadLength);
+}
+void Client::dataWrite(uint32_t requestId, const void* data, size_t length){
+    ClientConnectionRouter::instance().dataWrite(*this, requestId, data, length);
+}
+// void Client::dataRead(uint32_t requestId, void* data, size_t length){
+//     ClientConnectionRouter::instance().dataRead(*this, requestId, data, length);
+// }
+
+}
\ No newline at end of file
diff --git a/src/fdb5/remote/client/Client.h b/src/fdb5/remote/client/Client.h
new file mode 100644
index 000000000..b998f4ee9
--- /dev/null
+++ b/src/fdb5/remote/client/Client.h
@@ -0,0 +1,73 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+#pragma once
+
+#include 
+
+#include "eckit/config/Configuration.h"
+#include "eckit/memory/NonCopyable.h"
+#include "eckit/net/Endpoint.h"
+#include "eckit/thread/Mutex.h"
+
+#include "fdb5/database/Store.h"
+#include "fdb5/remote/Messages.h"
+
+namespace fdb5::remote {
+
+class Connection;
+//----------------------------------------------------------------------------------------------------------------------
+
+// typedef eckit::net::Endpoint Connection;
+// typedef uint32_t ClientID;
+// typedef uint32_t DataLinkID;
+// typedef uint32_t HandlerID;
+// typedef uint32_t RequestID;
+
+// abstract class, gives ClientConnectionRouter access to RemoteCatalogue & RemoteStore handlers
+class Client : eckit::NonCopyable {
+public:
+    Client(eckit::net::Endpoint endpoint);
+    Client(std::vector&& endpoints);
+
+    uint32_t controlWriteCheckResponse(Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
+    uint32_t controlWrite(Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
+    // void controlWrite(uint32_t requestId, const void* data, size_t length);
+    // void controlRead(uint32_t requestId, void* data, size_t length);
+
+    uint32_t dataWrite(Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
+    void dataWrite(uint32_t requestId, const void* data, size_t length);
+    // void dataRead(uint32_t requestId, void* data, size_t length);
+
+    // handlers for incoming messages - to be defined in the client class
+    virtual bool handle(Message message, uint32_t requestID) = 0;
+    virtual bool handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) = 0;
+    virtual void handleException(std::exception_ptr e) = 0;
+
+    virtual Key key() = 0;
+private:
+
+    std::vector endpoints_;
+    // Client(DataLinkID dataLinkID, RemoteHandlerID remoteHandlerID);
+
+    // ClientConnectionProxy() {} ///< private constructor only used by singleton
+
+    // eckit::Mutex mutex_;
+
+    // // a Client is either a CatalogueProxy or a StoreProxy (local),
+    // // willing to communicate with a remote CatalogueHandler or StoreHandler at a given endpoint
+    // std::map > clientLinks_;
+
+    // std::map > connections_;
+
+    // std::map requests_;
+};
+
+}
\ No newline at end of file
diff --git a/src/fdb5/remote/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
similarity index 78%
rename from src/fdb5/remote/ClientConnection.cc
rename to src/fdb5/remote/client/ClientConnection.cc
index c64d86e55..0063dc60b 100644
--- a/src/fdb5/remote/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -3,7 +3,8 @@
 #include 
 #include 
 
-#include "fdb5/remote/ClientConnection.h"
+#include "fdb5/remote/client/ClientConnection.h"
+#include "fdb5/remote/client/ClientConnectionRouter.h"
 
 #include "fdb5/LibFdb5.h"
 #include "fdb5/io/HandleGatherer.h"
@@ -29,8 +30,7 @@ using namespace eckit;
 using namespace eckit::net;
 // using namespace fdb5::remote;
 
-namespace fdb5 {
-namespace remote {
+namespace fdb5::remote {
 
 //----------------------------------------------------------------------------------------------------------------------
 
@@ -118,74 +118,14 @@ void ClientConnection::connect() {
         }
     }
 }
-SessionID ClientConnection::verifyServerStartupResponse() {
-
-    MessageHeader hdr;
-    controlRead(&hdr, sizeof(hdr));
-
-    ASSERT(hdr.marker == StartMarker);
-    ASSERT(hdr.version == CurrentVersion);
-    ASSERT(hdr.message == Message::Startup);
-    ASSERT(hdr.requestID == 0);
-
-    Buffer payload(hdr.payloadSize);
-    eckit::FixedString<4> tail;
-    controlRead(payload, hdr.payloadSize);
-    controlRead(&tail, sizeof(tail));
-    ASSERT(tail == EndMarker);
-
-    MemoryStream s(payload);
-    SessionID clientSession(s);
-    SessionID serverSession(s);
-    Endpoint dataEndpoint(s);
-    LocalConfiguration serverFunctionality(s);
-
-    dataEndpoint_ = dataEndpoint;
-
-    if (dataEndpoint_.hostname() != controlEndpoint_.hostname()) {
-        Log::warning() << "Data and control interface hostnames do not match. "
-                       << dataEndpoint_.hostname() << " /= "
-                       << controlEndpoint_.hostname() << std::endl;
-    }
-
-    if (clientSession != sessionID_) {
-        std::stringstream ss;
-        ss << "Session ID does not match session received from server: "
-           << sessionID_ << " != " << clientSession;
-        throw BadValue(ss.str(), Here());
-    }
-
-    return serverSession;
-}
 
-void ClientConnection::writeControlStartupMessage() {
-
-    Buffer payload(4096);
-    MemoryStream s(payload);
-    s << sessionID_;
-    s << controlEndpoint_;
-    s << LibFdb5::instance().remoteProtocolVersion().used();
-
-    // TODO: Abstract this dictionary into a RemoteConfiguration object, which
-    //       understands how to do the negotiation, etc, but uses Value (i.e.
-    //       essentially JSON) over the wire for flexibility.
-    s << availableFunctionality().get();
 
-    controlWrite(Message::Startup, 0, payload.data(), s.position());
-}
-
-eckit::LocalConfiguration ClientConnection::availableFunctionality() const {
-    eckit::LocalConfiguration conf;
-    std::vector remoteFieldLocationVersions = {1};
-    conf.set("RemoteFieldLocation", remoteFieldLocationVersions);
-    return conf;
-}
 
 void ClientConnection::disconnect() {
     if (connected_) {
 
         // Send termination message
-        controlWrite(Message::Exit, generateRequestID()); //xxx why do we generate a request ID here? 
+        controlWrite(Message::Exit, 0); //xxx why do we generate a request ID here? 
 
         listeningThread_.join();
 
@@ -196,20 +136,25 @@ void ClientConnection::disconnect() {
     }
 }
 
-void ClientConnection::writeDataStartupMessage(const eckit::SessionID& serverSession) {
-
-    Buffer payload(1024);
-    MemoryStream s(payload);
 
-    s << sessionID_;
-    s << serverSession;
+const eckit::net::Endpoint& ClientConnection::controlEndpoint() const { 
+    return controlEndpoint_;
+} 
+const eckit::net::Endpoint& ClientConnection::dataEndpoint() const { 
+    return dataEndpoint_;
+} 
 
-    dataWrite(Message::Startup, 0, payload.data(), s.position());
+eckit::LocalConfiguration ClientConnection::availableFunctionality() const {
+    eckit::LocalConfiguration conf;
+    std::vector remoteFieldLocationVersions = {1};
+    conf.set("RemoteFieldLocation", remoteFieldLocationVersions);
+    return conf;
 }
 
 // -----------------------------------------------------------------------------------------------------
 
 void ClientConnection::controlWriteCheckResponse(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
+//    std::cout << "ClientConnection::controlWriteCheckResponse " << ((uint) msg) << "  " << requestID << "  " << payloadLength << std::endl;
 
     controlWrite(msg, requestID, payload, payloadLength);
 
@@ -230,10 +175,11 @@ void ClientConnection::controlWriteCheckResponse(Message msg, uint32_t requestID
 }
 
 void ClientConnection::controlWrite(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
+    //std::cout << "ClientConnection::controlWrite " << this << std::endl;
 
     ASSERT((payload == nullptr) == (payloadLength == 0));
 
-    MessageHeader message(msg, requestID, payloadLength);
+    MessageHeader message(msg, 0, requestID, payloadLength);
     controlWrite(&message, sizeof(message));
     if (payload) {
         controlWrite(payload, payloadLength);
@@ -251,6 +197,7 @@ void ClientConnection::controlWrite(const void* data, size_t length) {
 }
 
 void ClientConnection::controlRead(void* data, size_t length) {
+    //std::cout << "ClientConnection::controlRead " << this << std::endl;
     size_t read = controlClient_.read(data, length);
     if (length != read) {
         std::stringstream ss;
@@ -262,7 +209,7 @@ void ClientConnection::controlRead(void* data, size_t length) {
 void ClientConnection::dataWrite(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
 
     ASSERT((payload == nullptr) == (payloadLength == 0));
-    MessageHeader message(msg, requestID, payloadLength);
+    MessageHeader message(msg, 0, requestID, payloadLength);
     dataWrite(&message, sizeof(message));
     if (payload) {
         dataWrite(payload, payloadLength);
@@ -309,42 +256,125 @@ void ClientConnection::handleError(const MessageHeader& hdr) {
     }
 }
 
-void ClientConnection::index(const Key& key, const FieldLocation& location) {
-    NOTIMP;
+
+
+void ClientConnection::writeControlStartupMessage() {
+
+    Buffer payload(4096);
+    MemoryStream s(payload);
+    s << sessionID_;
+    s << controlEndpoint_;
+    s << LibFdb5::instance().remoteProtocolVersion().used();
+
+    // TODO: Abstract this dictionary into a RemoteConfiguration object, which
+    //       understands how to do the negotiation, etc, but uses Value (i.e.
+    //       essentially JSON) over the wire for flexibility.
+    s << availableFunctionality().get();
+
+    //std::cout << "writeControlStartupMessage" << std::endl;
+    controlWrite(Message::Startup, 0, payload.data(), s.position());
+}
+
+void ClientConnection::writeDataStartupMessage(const eckit::SessionID& serverSession) {
+
+    Buffer payload(1024);
+    MemoryStream s(payload);
+
+    s << sessionID_;
+    s << serverSession;
+
+    dataWrite(Message::Startup, 0, payload.data(), s.position());
+}
+
+SessionID ClientConnection::verifyServerStartupResponse() {
+
+    MessageHeader hdr;
+    controlRead(&hdr, sizeof(hdr));
+
+    ASSERT(hdr.marker == StartMarker);
+    ASSERT(hdr.version == CurrentVersion);
+    ASSERT(hdr.message == Message::Startup);
+    ASSERT(hdr.requestID == 0);
+
+    Buffer payload(hdr.payloadSize);
+    eckit::FixedString<4> tail;
+    controlRead(payload, hdr.payloadSize);
+    controlRead(&tail, sizeof(tail));
+    ASSERT(tail == EndMarker);
+
+    MemoryStream s(payload);
+    SessionID clientSession(s);
+    SessionID serverSession(s);
+    Endpoint dataEndpoint(s);
+    LocalConfiguration serverFunctionality(s);
+
+    dataEndpoint_ = dataEndpoint;
+
+    if (dataEndpoint_.hostname() != controlEndpoint_.hostname()) {
+        Log::warning() << "Data and control interface hostnames do not match. "
+                       << dataEndpoint_.hostname() << " /= "
+                       << controlEndpoint_.hostname() << std::endl;
+    }
+
+    if (clientSession != sessionID_) {
+        std::stringstream ss;
+        ss << "Session ID does not match session received from server: "
+           << sessionID_ << " != " << clientSession;
+        throw BadValue(ss.str(), Here());
+    }
+
+    return serverSession;
+}
+
+void ClientConnection::listeningThreadLoop() {
+
+    try {
+
+        MessageHeader hdr;
+        eckit::FixedString<4> tail;
+
+        while (true) {
+
+            dataRead(&hdr, sizeof(hdr));
+
+            ASSERT(hdr.marker == StartMarker);
+            ASSERT(hdr.version == CurrentVersion);
+
+            if (hdr.message == Message::Exit) {
+                return;
+            }
+
+            bool handled = false;
+
+            if (hdr.payloadSize == 0) {
+                handled = ClientConnectionRouter::instance().handle(hdr.message, hdr.requestID);
+            }
+            else {
+                Buffer payload(hdr.payloadSize);
+                dataRead(payload, hdr.payloadSize);
+
+                handled = ClientConnectionRouter::instance().handle(hdr.message, hdr.requestID, controlEndpoint_, std::move(payload));
+            }
+
+            if (!handled) {
+                std::stringstream ss;
+                ss << "ERROR: Unexpected message recieved (" << static_cast(hdr.message) << "). ABORTING";
+                Log::status() << ss.str() << std::endl;
+                Log::error() << "Retrieving... " << ss.str() << std::endl;
+                throw SeriousBug(ss.str(), Here());
+            }
+
+            // Ensure we have consumed exactly the correct amount from the socket.
+            dataRead(&tail, sizeof(tail));
+            ASSERT(tail == EndMarker);
+        }
+
+    // We don't want to let exceptions escape inside a worker thread.
+    } catch (const std::exception& e) {
+        ClientConnectionRouter::instance().handleException(std::make_exception_ptr(e));
+    } catch (...) {
+        ClientConnectionRouter::instance().handleException(std::current_exception());
+    }
 }
 
-// void ClientConnection::store(const Key& key, const void* data, size_t length) {
-      
-//     // if there is no archiving thread active, then start one.
-//     // n.b. reset the archiveQueue_ after a potential flush() cycle.
-
-//     if (!archiveFuture_.valid()) {
-
-//         // Start the archival request on the remote side
-//         ASSERT(archiveID_ == 0);
-//         uint32_t id = generateRequestID();
-//         controlWriteCheckResponse(Message::Archive, id);
-//         archiveID_ = id;
-
-//         // Reset the queue after previous done/errors
-//         {
-//             std::lock_guard lock(archiveQueuePtrMutex_);
-//             ASSERT(!archiveQueue_);
-//             archiveQueue_.reset(new ArchiveQueue(maxArchiveQueueLength_));
-//         }
-
-//         archiveFuture_ = std::async(std::launch::async, [this, id] { return archiveThreadLoop(id); });
-//     }
-
-//     ASSERT(archiveFuture_.valid());
-//     ASSERT(archiveID_ != 0);
-//     {
-//         std::lock_guard lock(archiveQueuePtrMutex_);
-//         ASSERT(archiveQueue_);
-//         archiveQueue_->emplace(std::make_pair(key, Buffer(reinterpret_cast(data), length)));
-//     }
-// }
-
-
-}  // namespace remote
-}  // namespace fdb5
\ No newline at end of file
+}  // namespace fdb5::remote
diff --git a/src/fdb5/remote/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
similarity index 63%
rename from src/fdb5/remote/ClientConnection.h
rename to src/fdb5/remote/client/ClientConnection.h
index 64debb695..25ce1a853 100644
--- a/src/fdb5/remote/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -10,7 +10,6 @@
 
 #pragma once
 
-#include 
 #include 
 
 #include "eckit/container/Queue.h"
@@ -27,6 +26,9 @@
 
 
 namespace eckit {
+
+class Buffer;
+
 // xxx can we give this code a better home?
 template<> struct Translator {
     std::string operator()(const net::Endpoint& e) {
@@ -35,13 +37,10 @@ template<> struct Translator {
         return ss.str();
     }
 };
-}
 
-namespace fdb5 {
-
-//class DecoupledFDB;
+}
 
-namespace remote {
+namespace fdb5::remote {
 
 //----------------------------------------------------------------------------------------------------------------------
 
@@ -49,79 +48,82 @@ class ClientConnection : eckit::NonCopyable {
 
 public: // types
 
-public:
+public: // methods
 
     ClientConnection(const eckit::net::Endpoint& controlEndpoint, const eckit::Configuration& config);
-    ~ClientConnection();
+    virtual ~ClientConnection();
+
+    void controlWriteCheckResponse(remote::Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
+    void controlWrite(remote::Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
+    void dataWrite(remote::Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
+
+    void controlRead(void* data, size_t length);
+    void controlWrite(const void* data, size_t length);
+    void dataWrite(const void* data, size_t length);
+    void dataRead(void* data, size_t length);
 
-    void setDataEndpoint(const eckit::net::Endpoint& dataEndpoint);
     void connect();
     void disconnect();
 
-    const eckit::net::Endpoint& controlEndpoint() const { 
-        return controlEndpoint_;
-    } 
-    const eckit::net::Endpoint& dataEndpoint() const { 
-        return dataEndpoint_;
-    } 
-    // const eckit::net::Endpoint& dataEndpoint() const { 
-    //     return dataEndpoint_;
-    // }
-protected:
+protected: // methods
+
+
+    const eckit::net::Endpoint& controlEndpoint() const;
+    const eckit::net::Endpoint& dataEndpoint() const;
+
+    // // handlers for incoming messages - to be defined in the client class
+    // virtual void handle(Message message, uint32_t requestID) = 0;
+    // virtual void handle(Message message, uint32_t requestID, eckit::Buffer&& payload) = 0;
+    // virtual void handleException(std::exception_ptr e) = 0;
+
+    // construct dictionary for protocol negotiation - to be defined in the client class
+    virtual eckit::LocalConfiguration availableFunctionality() const;
 
-    virtual void listeningThreadLoop() = 0;
+    
+private: // methods
 
-    // Construct dictionary for protocol negotiation
-    // im not sure if this belongs here, or in the above class
-    eckit::LocalConfiguration availableFunctionality() const;
 
     void writeControlStartupMessage();
     void writeDataStartupMessage(const eckit::SessionID& serverSession);
+
     eckit::SessionID verifyServerStartupResponse();
-    
-    void controlWriteCheckResponse(remote::Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
-    void controlWrite(remote::Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
-    void controlWrite(const void* data, size_t length);
-    void controlRead(void* data, size_t length);
-    void dataWrite(remote::Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
-    void dataWrite(const void* data, size_t length);
-    void dataRead(void* data, size_t length);
+
     void handleError(const remote::MessageHeader& hdr);
-    
-    void index(const Key& key, const FieldLocation& location);
-    // void store(const Key& key, const void* data, size_t length);
-    // void archive(const Key& key, const void* data, size_t length);
-    // long sendArchiveData(uint32_t id, const std::vector>& elements, size_t count);
-    // void sendArchiveData(uint32_t id, const Key& key, const void* data, size_t length);
-    // void flush(FDBStats& stats);
-
-private:
+
+    void listeningThreadLoop();
+
+private: // members
+
     eckit::SessionID sessionID_; 
 
     eckit::net::Endpoint controlEndpoint_;
     eckit::net::Endpoint dataEndpoint_;
+
     eckit::net::TCPClient controlClient_;
     eckit::net::TCPClient dataClient_;
+
     std::thread listeningThread_;
 
     bool connected_;
 
-//    friend class fdb5::DecoupledFDB;
+    eckit::Length freeSpace_;
+    float freeRatio_;
+
 };
 
 //----------------------------------------------------------------------------------------------------------------------
 
-// // n.b. if we get integer overflow, we reuse the IDs. This is not a
-// //      big deal. The idea that we could be on the 2.1 billionth (successful)
-// //      request, and still have an ongoing request 0 is ... laughable.
-static uint32_t generateRequestID() {
+// // // n.b. if we get integer overflow, we reuse the IDs. This is not a
+// // //      big deal. The idea that we could be on the 4.2 billionth (successful)
+// // //      request, and still have an ongoing request 0 is ... laughable.
+// static uint32_t generateRequestID() {
 
-    static std::mutex m;
-    static uint32_t id = 0;
+//     static std::mutex m;
+//     static uint32_t id = 0;
 
-    std::lock_guard lock(m);
-    return ++id;
-}
+//     std::lock_guard lock(m);
+//     return ++id;
+// }
 
 class DecoupledFDBException : public eckit::RemoteException {
 public:
@@ -129,5 +131,4 @@ class DecoupledFDBException : public eckit::RemoteException {
         eckit::RemoteException(msg, eckit::Translator()(endpoint)) {}
 };
 
-}  // namespace remote
-}  // namespace fdb5
\ No newline at end of file
+}  // namespace fdb5::remote
\ No newline at end of file
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.cc b/src/fdb5/remote/client/ClientConnectionRouter.cc
new file mode 100644
index 000000000..c6744c977
--- /dev/null
+++ b/src/fdb5/remote/client/ClientConnectionRouter.cc
@@ -0,0 +1,210 @@
+#include 
+#include 
+
+#include "fdb5/remote/client/ClientConnectionRouter.h"
+
+using namespace eckit;
+using namespace eckit::net;
+
+
+namespace {
+
+// static ClientID generateClientID() {
+
+//     static std::mutex m;
+//     static ClientID id = 0;
+
+//     std::lock_guard lock(m);
+//     return ++id;
+// }
+// static DataLinkID generateDataLinkID() {
+
+//     static std::mutex m;
+//     static DataLinkID id = 0;
+
+//     std::lock_guard lock(m);
+//     return ++id;
+// }
+// static HandlerID generateHandlerID() {
+
+//     static std::mutex m;
+//     static HandlerID id = 0;
+
+//     std::lock_guard lock(m);
+//     return ++id;
+// }
+static uint32_t generateRequestID() {
+
+    static std::mutex m;
+    static uint32_t id = 0;
+
+    std::lock_guard lock(m);
+    return ++id;
+}
+
+}
+
+
+namespace fdb5::remote {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+uint32_t ClientConnectionRouter::controlWriteCheckResponse(std::vector& endpoints, Client& client, Message msg, const void* payload, uint32_t payloadLength) {
+    //std::cout << "ClientConnectionRouter::controlWriteCheckResponse " << endpoints.size() << std::endl;
+    ClientConnection* conn;
+    uint32_t id = createConnection(endpoints, client, conn);
+
+    conn->controlWriteCheckResponse(msg, id, payload, payloadLength);
+
+    return id;
+}
+uint32_t ClientConnectionRouter::controlWrite(std::vector& endpoints, Client& client, Message msg, const void* payload, uint32_t payloadLength) {
+    //std::cout << "ClientConnectionRouter::controlWrite " << endpoints.size() << std::endl;
+    ClientConnection* conn;
+    uint32_t id = createConnection(endpoints, client, conn);
+
+    conn->controlWrite(msg, id, payload, payloadLength);
+
+    return id;
+}
+// void ClientConnectionRouter::controlWrite(Client& client, uint32_t requestId, const void* payload, size_t payloadLength) {
+//     auto it = requests_.find(requestId);
+//     ASSERT(it != requests_.end());
+//     ASSERT(it->second.second == &client); // check if the client owns the request
+
+//     it->second.first->controlWrite(payload, payloadLength);
+// }
+    // void controlRead(Client& client, uint32_t requestId, void* data, size_t length);
+
+    // void dataWrite(Client& client, uint32_t requestId, const void* data, size_t length);
+    // void dataRead(Client& client, uint32_t requestId, void* data, size_t length);
+
+void ClientConnectionRouter::controlRead(Client& client, uint32_t requestId, void* payload, size_t payloadLength) {
+    std::cout << "ClientConnectionRouter::controlRead " << requestId << std::endl;
+    auto it = requests_.find(requestId);
+    ASSERT(it != requests_.end());
+    ASSERT(it->second.second == &client); // check if the client owns the request
+
+    it->second.first->controlRead(payload, payloadLength);
+
+}
+
+// void ClientConnectionRouter::controlRead(Connection& connection, Client& client, void* data, size_t length) {
+
+// }
+
+
+uint32_t ClientConnectionRouter::createConnection(std::vector& endpoints, Client& client, ClientConnection*& conn) {
+
+    ASSERT(endpoints.size() > 0);
+
+    // pick the first endpoint - To be improved with DataStoreStrategies
+    eckit::net::Endpoint& endpoint = endpoints.at(0);
+    uint32_t id = generateRequestID();
+
+    auto it = connections_.find(endpoint.host());
+    if (it == connections_.end()) {
+        conn = (connections_[endpoint.host()] = std::unique_ptr(new ClientConnection(endpoint, LocalConfiguration()))).get();
+        conn->connect();
+        // client.key();
+        // connections_[endpoint] = std::make_pair, std::map >(std::move(conn))
+    } else {
+        conn = it->second.get();
+    }
+
+    ASSERT(conn);
+    requests_[id] = {conn, &client};
+
+    return id;
+}
+uint32_t ClientConnectionRouter::dataWrite(std::vector& endpoints, Client& client, Message msg, const void* payload, uint32_t payloadLength) {
+    ClientConnection* conn;
+    uint32_t id = createConnection(endpoints, client, conn);
+
+    conn->dataWrite(msg, id, payload, payloadLength);
+
+    return id;
+}
+void ClientConnectionRouter::dataWrite(Client& client, uint32_t requestId, const void* payload, size_t payloadLength) {
+    auto it = requests_.find(requestId);
+    ASSERT(it != requests_.end());
+    ASSERT(it->second.second == &client); // check if the client owns the request
+
+    it->second.first->dataWrite(payload, payloadLength);
+}
+// void ClientConnectionRouter::dataRead(Connection& connection, Client& client, void* data, size_t length) {
+
+// }
+
+
+bool ClientConnectionRouter::handle(Message message, uint32_t requestID) {
+    if (requestID != 0) {
+        auto it = requests_.find(requestID);
+        ASSERT(it != requests_.end());
+
+        return it->second.second->handle(message, requestID);
+    } else {
+        std::cout << "ClientConnectionRouter::handle [message=" << ((uint) message) << ",requestID=" << requestID << "]" << std::endl;
+        return true;
+    }
+}
+
+bool ClientConnectionRouter::handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) {
+    auto it = requests_.find(requestID);
+    ASSERT(it != requests_.end());
+
+    return it->second.second->handle(message, requestID, endpoint, std::move(payload));
+}
+void ClientConnectionRouter::handleException(std::exception_ptr e) {
+    // auto it = requests_.find(requestID);
+    // ASSERT(it != requests_.end());
+
+    // it->second->handle(message, requestID);
+}
+
+
+ClientConnectionRouter& ClientConnectionRouter::instance()
+{
+    static ClientConnectionRouter proxy;
+    return proxy;
+}
+
+// void ClientConnectionRouter::add(const std::string& name, const FDBBuilderBase* b)
+// {
+//     eckit::AutoLock lock(mutex_);
+
+//     ASSERT(registry_.find(name) == registry_.end());
+
+//     registry_[name] = b;
+// }
+
+// std::unique_ptr FDBFactory::build(const Config& config) {
+
+//     // Allow expanding of the config to make use of fdb_home supplied in a previous
+//     // configuration file, or to pick up the default configuration from ~fdb/etc/fdb/...
+
+//     Config actualConfig = config.expandConfig();
+
+//     /// We use "local" as a default type if not otherwise configured.
+
+//     std::string key = actualConfig.getString("type", "local");
+
+//     eckit::Log::debug() << "Selecting FDB implementation: " << key << std::endl;
+
+//     eckit::AutoLock lock(mutex_);
+
+//     auto it = registry_.find(key);
+
+//     if (it == registry_.end()) {
+//         std::stringstream ss;
+//         ss << "FDB factory \"" << key << "\" not found";
+//         throw eckit::SeriousBug(ss.str(), Here());
+//     }
+
+//     std::unique_ptr ret = it->second->make(actualConfig);
+//     eckit::Log::debug() << "Constructed FDB implementation: " << *ret << std::endl;
+//     return ret;
+// }
+
+
+}  // namespace fdb5::remote
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.h b/src/fdb5/remote/client/ClientConnectionRouter.h
new file mode 100644
index 000000000..1701c8bc3
--- /dev/null
+++ b/src/fdb5/remote/client/ClientConnectionRouter.h
@@ -0,0 +1,111 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+#pragma once
+
+#include 
+
+#include "eckit/config/Configuration.h"
+#include "eckit/memory/NonCopyable.h"
+#include "eckit/net/Endpoint.h"
+#include "eckit/thread/Mutex.h"
+
+#include "fdb5/database/Store.h"
+#include "fdb5/remote/Messages.h"
+#include "fdb5/remote/client/Client.h"
+#include "fdb5/remote/client/ClientConnection.h"
+
+namespace fdb5::remote {
+
+class Client;
+//----------------------------------------------------------------------------------------------------------------------
+
+class Connection {
+public:
+    Connection(ClientConnection* clientConnection, uint32_t remoteID) : clientConnection_(clientConnection), remoteID_(remoteID) {}
+    Connection(Connection& other) : clientConnection_(other.clientConnection_), remoteID_(other.remoteID_) {}
+    
+    ClientConnection* clientConnection_;
+    uint32_t remoteID_;
+};
+// typedef eckit::net::Endpoint Connection;
+// typedef uint32_t ClientID;
+// typedef uint32_t DataLinkID;
+// typedef uint32_t HandlerID;
+// typedef uint32_t RequestID;
+
+class ClientConnectionRouter : eckit::NonCopyable {
+public:
+
+    static ClientConnectionRouter& instance();
+    Connection connectCatalogue(Key dbKey, const eckit::Configuration& config);
+    Connection connectStore(Key dbKey, const std::vector& endpoints);
+//    Connection connectStore(eckit::URI uri);
+
+// protected:
+
+//     friend class Client;
+
+    uint32_t controlWriteCheckResponse(std::vector& endpoints, Client& client, Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
+    uint32_t controlWrite(std::vector& connections, Client& client, Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
+//    void controlWrite(Client& client, uint32_t requestId, const void* payload, size_t payloadLength);
+
+    // uint32_t controlWriteCheckResponse(Connection& connection, Client& client, Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
+    // uint32_t controlWrite(Connection& connection, Client& client, Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
+    // void controlWrite(Connection& connection, Client& client, const void* data, size_t length);
+    void controlRead(Client& client, uint32_t requestId, void* payload, size_t payloadLength);
+
+    uint32_t dataWrite(std::vector& endpoints, Client& client, Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
+    void dataWrite(Client& client, uint32_t requestId, const void* data, size_t length);
+    // void dataRead(Client& client, uint32_t requestId, void* data, size_t length);
+
+
+    // handlers for incoming messages - to be defined in the client class
+    bool handle(Message message, uint32_t requestID);
+    bool handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload);
+    void handleException(std::exception_ptr e);
+
+private:
+
+    ClientConnectionRouter() {} ///< private constructor only used by singleton
+
+    uint32_t createConnection(std::vector& endpoints, Client& client, ClientConnection*& conn);
+
+    eckit::Mutex mutex_;
+
+    // endpoint --> datalink
+    // remoteId --> datalink, remoteHandler
+    // a Client is either a CatalogueProxy or a StoreProxy (local),
+    // willing to communicate with a remote CatalogueHandler or StoreHandler at a given endpoint
+    // std::map > clientLinks_;
+
+    // std::map > connections_;
+
+    // std::map requests_;
+
+
+
+
+
+    // // key --> [endpoint1, endpoint2, ...]
+    // std::map > storeEndpoints_;
+
+    // requestID --> <, Client>
+    std::map > requests_;
+
+    // endpoint ->  remoteId>
+//    std::map, std::map > connections_;
+    std::map > connections_;
+    
+    // // not owning
+    // ClientConnection* catalogueConnection_;
+};
+
+}
\ No newline at end of file
diff --git a/src/fdb5/remote/server/DataStoreStrategies.h b/src/fdb5/remote/server/DataStoreStrategies.h
new file mode 100644
index 000000000..8e6d9c66e
--- /dev/null
+++ b/src/fdb5/remote/server/DataStoreStrategies.h
@@ -0,0 +1,409 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/// @file   DataStoreStrategies.h
+/// @date   Mar 1998
+/// @author Baudouin Raoult
+/// @author Tiago Quintino
+
+#pragma once
+
+#include "eckit/memory/NonCopyable.h"
+
+namespace fdb5 {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+template< class T >
+class DataStoreStrategies : private NonCopyable {
+public:
+    static const T& selectFileSystem(const std::vector& fileSystems, const std::string& s);
+
+    static const T& leastUsed(const std::vector& fileSystems);
+    static const T& leastUsedPercent(const std::vector& fileSystems);
+    static const T& roundRobin(const std::vector& fileSystems);
+    static const T& pureRandom(const std::vector& fileSystems);
+    static const T& weightedRandom(const std::vector& fileSystems);
+    static const T& weightedRandomPercent(const std::vector& fileSystems);
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
+struct DataStoreSize {
+    unsigned long long available;
+    unsigned long long total;
+    DataStoreSize() :
+        available(0), total(0) {}
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
+template< class T >
+struct Candidate {
+
+    const T* datastore_;
+
+    DataStoreSize size_;
+
+    double probability_;
+
+    Candidate(const T* datastore) :
+        datastore_(datastore) {}
+
+    void print(std::ostream& s) const {
+        s << "Candidate(datastore=" << datastore_->asString() << ",total=" << total() << ",available=" << available()
+          << ",percent=" << percent() << ",probability=" << probability_ << ")";
+    }
+
+    friend std::ostream& operator<<(std::ostream& s, const Candidate& v) {
+        v.print(s);
+        return s;
+    }
+
+    const T& datastore() const { return *datastore_; }
+
+    double probability() const { return probability_; }
+    void probability(double p) { probability_ = p; }
+
+    long percent() const { return long(100. * (double(size_.available) / size_.total)); }
+
+    unsigned long long total() const { return size_.total; }
+
+    unsigned long long available() const { return size_.available; }
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
+template< class T >
+const T& DataStoreStrategies::selectFileSystem(const std::vector& fileSystems, const std::string& s) {
+    Log::info() << "DataStoreStrategies::selectFileSystem is " << s << std::endl;
+
+    if (s == "roundRobin") {
+        return DataStoreStrategies::roundRobin(fileSystems);
+    }
+
+    if (s == "weightedRandom") {
+        return DataStoreStrategies::weightedRandom(fileSystems);
+    }
+
+    if (s == "pureRandom") {
+        return DataStoreStrategies::pureRandom(fileSystems);
+    }
+
+    if (s == "weightedRandomPercent") {
+        return DataStoreStrategies::weightedRandomPercent(fileSystems);
+    }
+
+    if (s == "leastUsedPercent") {
+        return DataStoreStrategies::leastUsedPercent(fileSystems);
+    }
+
+    return DataStoreStrategies::leastUsed(fileSystems);
+}
+
+template< class T >
+const T& DataStoreStrategies::leastUsed(const std::vector& fileSystems) {
+    unsigned long long free = 0;
+    Ordinal best            = 0;
+    Ordinal checked         = 0;
+
+    ASSERT(fileSystems.size() != 0);
+
+    for (Ordinal i = 0; i < fileSystems.size(); i++) {
+        // Log::info() << "leastUsed: " << fileSystems[i] << " " << fileSystems[i].available() << std::endl;
+        if (fileSystems[i].available()) {
+            DataStoreSize fs;
+
+            try {
+                fileSystems[i].fileSystemSize(fs);
+            }
+            catch (std::exception& e) {
+                Log::error() << "** " << e.what() << " Caught in " << Here() << std::endl;
+                Log::error() << "** Exception is ignored" << std::endl;
+                Log::error() << "Cannot stat " << fileSystems[i] << Log::syserr << std::endl;
+                continue;
+            }
+
+            if (fs.available >= free || checked == 0) {
+                free = fs.available;
+                best = i;
+                checked++;
+            }
+        }
+    }
+
+    if (!checked) {
+        throw Retry(std::string("No available filesystem (") + fileSystems[0] + ")");
+    }
+
+    Log::info() << "Filespace strategy leastUsed selected " << fileSystems[best] << " " << Bytes(free) << " available"
+                << std::endl;
+
+    return fileSystems[best];
+}
+
+template< class T >
+const T& DataStoreStrategies::leastUsedPercent(const std::vector& fileSystems) {
+    long percent = 0;
+    size_t best  = 0;
+
+    ASSERT(fileSystems.size() != 0);
+
+    for (size_t i = 0; i < fileSystems.size(); ++i) {
+        Candidate candidate(&fileSystems[i]);
+
+        Log::info() << "leastUsedPercent: " << fileSystems[i] << " " << fileSystems[i].available() << std::endl;
+        if (fileSystems[i].available()) {
+            DataStoreSize fs;
+
+            try {
+                fileSystems[i].fileSystemSize(candidate.size_);
+            }
+            catch (std::exception& e) {
+                Log::error() << "** " << e.what() << " Caught in " << Here() << std::endl;
+                Log::error() << "** Exception is ignored" << std::endl;
+                Log::error() << "Cannot stat " << fileSystems[i] << Log::syserr << std::endl;
+                continue;
+            }
+
+            if (candidate.percent() >= percent) {
+                percent = candidate.percent();
+                best    = i;
+            }
+        }
+    }
+
+    Log::info() << "Filespace strategy leastUsedPercent selected " << fileSystems[best] << " " << percent
+                << "% available" << std::endl;
+
+    return fileSystems[best];
+}
+
+typedef void (*compute_probability_t)(Candidate&);
+
+static void computePercent(Candidate& c) {
+    c.probability_ = double(c.percent());
+}
+
+static void computeAvailable(Candidate& c) {
+    c.probability_ = double(c.available());
+}
+
+static void computeIdentity(Candidate& c) {
+    c.probability_ = 1;
+}
+
+static void computeNull(Candidate& c) {
+    c.probability_ = 0;
+}
+
+static std::vector findCandidates(const std::vector& fileSystems,
+                                             compute_probability_t probability) {
+
+    ASSERT(fileSystems.size() != 0);
+
+    static Resource candidateFileSystemPercent("candidateFileSystem", 99);
+
+    std::vector result;
+
+    for (size_t i = 0; i < fileSystems.size(); ++i) {
+
+        Candidate candidate(&fileSystems[i]);
+
+        if (fileSystems[i].available()) {
+
+            try {
+                fileSystems[i].fileSystemSize(candidate.size_);
+            }
+            catch (std::exception& e) {
+                Log::error() << "** " << e.what() << " Caught in " << Here() << std::endl;
+                Log::error() << "** Exception is ignored" << std::endl;
+                Log::error() << "Cannot stat " << fileSystems[i] << Log::syserr << std::endl;
+                continue;
+            }
+
+            if (candidate.total() == 0) {
+                Log::warning() << "Cannot get total size of " << fileSystems[i] << std::endl;
+                return std::vector();
+            }
+
+            if (candidate.percent() <= candidateFileSystemPercent) {
+
+                probability(candidate);
+
+                //                Log::info() << candidate << std::endl;
+
+                result.push_back(candidate);
+            }
+        }
+    }
+
+    return result;
+}
+
+template< class T >
+const T& DataStoreStrategies::roundRobin(const std::vector& fileSystems) {
+    std::vector candidates = findCandidates(fileSystems, &computeNull);
+
+    if (candidates.empty()) {
+        return leastUsed(fileSystems);
+    }
+
+    static long value = -1;
+
+    if (value < 0) {
+        value = ::getpid();
+    }
+
+    value++;
+    value %= candidates.size();
+
+    Log::info() << "Filespace strategy roundRobin selected " << candidates[value].datastore() << " " << value << " out of "
+                << candidates.size() << std::endl;
+
+    return candidates[value].datastore();
+}
+
+template< class T >
+static void attenuateProbabilities(std::vector& candidates) {
+
+    ASSERT(!candidates.empty());
+
+    static double attenuation = Resource("attenuateFileSpacePeakProbability", 0.);
+
+    ASSERT(attenuation >= 0.);
+    ASSERT(attenuation <= 1.);
+
+    if (attenuation == 0.) {
+        return;
+    }
+
+    // compute mean
+
+    double mean = 0.;
+    for (std::vector::const_iterator i = candidates.begin(); i != candidates.end(); ++i) {
+        mean += i->probability();
+    }
+
+    mean /= candidates.size();
+
+    //    // compute variance
+
+    //    double variance = 0.;
+    //    for(std::vector::const_iterator i = candidates.begin(); i != candidates.end(); ++i) {
+    //        double diff = (i->probability() - mean);
+    //        variance += diff*diff;
+    //    }
+
+    //    variance /= candidates.size();
+
+    //    // compute stddev
+
+    //    double stddev = std::sqrt(variance);
+
+    //    // attenuate the peaks that exceed the stddev to the stddev value
+    //    double max = mean + attenuation * stddev;
+    //    for(std::vector::iterator i = candidates.begin(); i != candidates.end(); ++i) {
+    //        if(i->probability() > max) {
+    //            i->probability(max);
+    //        }
+    //    }
+
+
+    for (std::vector::iterator i = candidates.begin(); i != candidates.end(); ++i) {
+        double p    = i->probability();
+        double newp = attenuation * mean + (1. - attenuation) * p;
+        i->probability(newp);
+    }
+}
+
+
+template< class T >
+static const T& chooseByProbabylity(const char* strategy, const std::vector& candidates) {
+
+    double total = 0;
+    for (std::vector::const_iterator i = candidates.begin(); i != candidates.end(); ++i) {
+        //        Log::info() << "probability " << i->probability() << std::endl;
+        total += i->probability();
+    }
+
+    double choice = (double(random()) / double(RAND_MAX));
+
+    //    Log::info() << "choice " << choice << std::endl;
+
+    choice *= total;
+
+    std::vector::const_iterator select = candidates.begin();
+
+    double lower = 0;
+    double upper = 0;
+    for (std::vector::const_iterator i = candidates.begin(); i != candidates.end(); ++i) {
+
+        upper += i->probability();
+
+        //        Log::info() << "Choice " << choice << " total = " << total << " lower = " << lower << " upper = " <<
+        //        upper << std::endl;
+
+        if (choice >= lower && choice < upper) {
+            select = i;
+            break;
+        }
+
+        lower = upper;
+    }
+
+    Log::info() << "Filespace strategy " << strategy << " selected " << select->datastore() << " "
+                << Bytes(select->available()) << " available" << std::endl;
+
+    return select->datastore();
+}
+
+template< class T >
+const T& DataStoreStrategies::pureRandom(const std::vector& fileSystems) {
+    std::vector candidates = findCandidates(fileSystems, &computeIdentity);
+
+    if (candidates.empty()) {
+        return leastUsed(fileSystems);
+    }
+
+    attenuateProbabilities(candidates); /* has no effect */
+
+    return chooseByProbabylity("pureRandom", candidates);
+}
+
+template< class T >
+const T& DataStoreStrategies::weightedRandom(const std::vector& fileSystems) {
+    std::vector candidates = findCandidates(fileSystems, &computeAvailable);
+
+    if (candidates.empty()) {
+        return leastUsed(fileSystems);
+    }
+
+    attenuateProbabilities(candidates);
+
+    return chooseByProbabylity("weightedRandom", candidates);
+}
+
+template< class T >
+const T& DataStoreStrategies::weightedRandomPercent(const std::vector& fileSystems) {
+    std::vector candidates = findCandidates(fileSystems, &computePercent);
+
+    if (candidates.empty()) {
+        return leastUsed(fileSystems);
+    }
+
+    attenuateProbabilities(candidates);
+
+    return chooseByProbabylity("weightedRandomPercent", candidates);
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+
+}  // namespace eckit
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
new file mode 100644
index 000000000..11d9ce668
--- /dev/null
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -0,0 +1,356 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/*
+ * This software was developed as part of the EC H2020 funded project NextGenIO
+ * (Project ID: 671951) www.nextgenio.eu
+ */
+
+#include 
+
+#include "eckit/config/Resource.h"
+#include "eckit/maths/Functions.h"
+#include "eckit/net/Endpoint.h"
+#include "eckit/runtime/Main.h"
+#include "eckit/runtime/SessionID.h"
+#include "eckit/serialisation/MemoryStream.h"
+
+#include "metkit/mars/MarsRequest.h"
+
+#include "fdb5/LibFdb5.h"
+#include "fdb5/fdb5_version.h"
+#include "fdb5/api/helpers/FDBToolRequest.h"
+#include "fdb5/database/Key.h"
+#include "fdb5/remote/AvailablePortList.h"
+#include "fdb5/remote/Messages.h"
+#include "fdb5/remote/RemoteFieldLocation.h"
+#include "fdb5/api/FDB.h"
+
+#include "fdb5/remote/server/ServerConnection.h"
+
+using namespace eckit;
+using metkit::mars::MarsRequest;
+
+namespace fdb5 {
+namespace remote {
+
+// helpers
+namespace {
+
+class TCPException : public eckit::Exception {
+public:
+    TCPException(const std::string& msg, const eckit::CodeLocation& here) :
+        eckit::Exception(std::string("TCPException: ") + msg, here) {}
+};
+
+std::vector intersection(const LocalConfiguration& c1, const LocalConfiguration& c2, const std::string& field){
+
+    std::vector v1 = c1.getIntVector(field);
+    std::vector v2 = c2.getIntVector(field);
+    std::vector v3;
+
+    std::sort(v1.begin(), v1.end());
+    std::sort(v2.begin(), v2.end());
+
+    std::set_intersection(v1.begin(),v1.end(),
+                          v2.begin(),v2.end(),
+                          back_inserter(v3));
+    return v3;
+}
+} // namespace
+
+ServerConnection::ServerConnection(eckit::net::TCPSocket& socket, const Config& config) :
+        config_(config),
+        controlSocket_(socket),
+        dataSocket_(selectDataPort()),
+        dataListenHostname_(config.getString("dataListenHostname", "")),
+        // fdb_(config),
+        readLocationQueue_(eckit::Resource("fdbRetrieveQueueSize", 10000)) {
+            Log::debug() << "ServerConnection::ServerConnection initialized" << std::endl;
+    }
+
+ServerConnection::~ServerConnection() {
+    // We don't want to die before the worker threads are cleaned up
+    waitForWorkers();
+
+    // And notify the client that we are done.
+    Log::info() << "Sending exit message to client" << std::endl;
+    dataWrite(Message::Exit, 0);
+    Log::info() << "Done" << std::endl;
+}
+
+eckit::LocalConfiguration ServerConnection::availableFunctionality() const {
+    eckit::LocalConfiguration conf;
+//    Add to the configuration all the components that require to be versioned, as in the following example, with a vector of supported version numbers
+    std::vector remoteFieldLocationVersions = {1};
+    conf.set("RemoteFieldLocation", remoteFieldLocationVersions);
+    return conf;
+}
+
+void ServerConnection::initialiseConnections() {
+    // Read the startup message from the client. Check that it all checks out.
+
+    MessageHeader hdr;
+    socketRead(&hdr, sizeof(hdr), controlSocket_);
+
+
+    ASSERT(hdr.marker == StartMarker);
+    ASSERT(hdr.version == CurrentVersion);
+    ASSERT(hdr.message == Message::Startup);
+    ASSERT(hdr.remoteID == 0);
+    ASSERT(hdr.requestID == 0);
+
+    Buffer payload1 = receivePayload(hdr, controlSocket_);
+    eckit::FixedString<4> tail;
+    socketRead(&tail, sizeof(tail), controlSocket_);
+    ASSERT(tail == EndMarker);
+
+    MemoryStream s1(payload1);
+    SessionID clientSession(s1);
+    net::Endpoint endpointFromClient(s1);
+    unsigned int remoteProtocolVersion = 0;
+    std::string errorMsg;
+
+    try {
+        s1 >> remoteProtocolVersion;
+    } catch (...) {
+        errorMsg = "Error retrieving client protocol version";
+    }
+
+    if (errorMsg.empty() && !LibFdb5::instance().remoteProtocolVersion().check(remoteProtocolVersion, false)) {
+        std::stringstream ss;
+        ss << "FDB server version " << fdb5_version_str() << " - remote protocol version not supported:" << std::endl;
+        ss << "    versions supported by server: " << LibFdb5::instance().remoteProtocolVersion().supportedStr() << std::endl;
+        ss << "    version requested by client: " << remoteProtocolVersion << std::endl;
+        errorMsg = ss.str();
+    }
+
+    if (errorMsg.empty()) {
+        LocalConfiguration clientAvailableFunctionality(s1);
+        LocalConfiguration serverConf = availableFunctionality();
+        agreedConf_ = LocalConfiguration();
+
+        // agree on a common functionality by intersecting server and client version numbers
+         std::vector rflCommon = intersection(clientAvailableFunctionality, serverConf, "RemoteFieldLocation");
+         if (rflCommon.size() > 0) {
+             Log::debug() << "Protocol negotiation - RemoteFieldLocation version " << rflCommon.back() << std::endl;
+             agreedConf_.set("RemoteFieldLocation", rflCommon.back());
+         }
+         else {
+             std::stringstream ss;
+             ss << "FDB server version " << fdb5_version_str() << " - failed protocol negotiation with FDB client" << std::endl;
+             ss << "    server functionality: " << serverConf << std::endl;
+             ss << "    client functionality: " << clientAvailableFunctionality << std::endl;
+             errorMsg = ss.str();
+         }
+    }
+
+    // We want a data connection too. Send info to RemoteFDB, and wait for connection
+    // n.b. FDB-192: we use the host communicated from the client endpoint. This
+    //               ensures that if a specific interface has been selected and the
+    //               server has multiple, then we use that on, whilst retaining
+    //               the capacity in the protocol for the server to make a choice.
+
+    int dataport = dataSocket_.localPort();
+    net::Endpoint dataEndpoint(endpointFromClient.hostname(), dataport);
+
+    Log::info() << "Sending data endpoint to client: " << dataEndpoint << std::endl;
+    {
+        Buffer startupBuffer(1024);
+        MemoryStream s(startupBuffer);
+
+        s << clientSession;
+        s << sessionID_;
+        s << dataEndpoint;
+        s << agreedConf_.get();
+        // s << storeEndpoint; // xxx single-store case only: we cant do this with multiple stores // For now, dont send the store endpoint to the client 
+
+        Log::debug() << "Protocol negotiation - configuration: " << agreedConf_ <() << "ServerConnection::dataWrite [message="<< static_cast(msg) << ",requestID=" << requestID << ",payloadLength=" << payloadLength << "]" << std::endl;
+
+    ASSERT((payload == nullptr) == (payloadLength == 0));
+
+    MessageHeader message(msg, 0, requestID, payloadLength);
+
+    std::lock_guard lock(dataWriteMutex_);
+
+    dataWriteUnsafe(&message, sizeof(message));
+    if (payload) {
+        dataWriteUnsafe(payload, payloadLength);
+    }
+    dataWriteUnsafe(&EndMarker, sizeof(EndMarker));
+}
+
+void ServerConnection::dataWriteUnsafe(const void* data, size_t length) {
+    size_t written = dataSocket_.write(data, length);
+    if (length != written) {
+        std::stringstream ss;
+        ss << "Write error. Expected " << length << " bytes, wrote " << written;
+        throw TCPException(ss.str(), Here());
+    }
+}
+
+Buffer ServerConnection::receivePayload(const MessageHeader& hdr, net::TCPSocket& socket) {
+    Buffer payload(hdr.payloadSize);
+
+    ASSERT(hdr.payloadSize > 0);
+    socketRead(payload, hdr.payloadSize, socket);
+
+    return payload;
+}
+
+void ServerConnection::tidyWorkers() {
+    std::map>::iterator it = workerThreads_.begin();
+
+    for (; it != workerThreads_.end(); /* empty */) {
+        std::future_status stat = it->second.wait_for(std::chrono::milliseconds(0));
+
+        if (stat == std::future_status::ready) {
+            Log::info() << "Tidying up worker for request ID: " << it->first << std::endl;
+            workerThreads_.erase(it++);
+        }
+        else {
+            ++it;
+        }
+    }
+}
+
+void ServerConnection::waitForWorkers() {
+    readLocationQueue_.close();
+
+    tidyWorkers();
+
+    for (auto& it : workerThreads_) {
+        Log::error() << "Worker thread still alive for request ID: " << it.first << std::endl;
+        Log::error() << "Joining ..." << std::endl;
+        it.second.get();
+        Log::error() << "Thread complete" << std::endl;
+    }
+
+    if (readLocationWorker_.joinable()) {
+        readLocationWorker_.join();
+    }
+}
+
+
+// void ServerConnection::read(const MessageHeader& hdr) {
+//     // store only
+//     NOTIMP;
+// }
+
+// void ServerConnection::archive(const MessageHeader& hdr) {
+//     // catalogue only
+//     NOTIMP;
+// }
+
+// // void ServerConnection::store(const MessageHeader& hdr) {
+// //     // store only
+// //     NOTIMP;
+// // }
+
+// void ServerConnection::flush(const MessageHeader& hdr) {
+//     // store only
+//     NOTIMP;
+// }
+
+
+}  // namespace remote
+}  // namespace fdb5
diff --git a/src/fdb5/remote/server/ServerConnection.h b/src/fdb5/remote/server/ServerConnection.h
new file mode 100644
index 000000000..0a25c6182
--- /dev/null
+++ b/src/fdb5/remote/server/ServerConnection.h
@@ -0,0 +1,112 @@
+
+
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/*
+ * This software was developed as part of the EC H2020 funded project NextGenIO
+ * (Project ID: 671951) www.nextgenio.eu
+ */
+
+#pragma once
+
+#include 
+#include 
+
+#include "eckit/io/Buffer.h"
+#include "eckit/io/DataHandle.h"
+#include "eckit/net/TCPServer.h"
+#include "eckit/net/TCPSocket.h"
+#include "eckit/runtime/SessionID.h"
+
+#include "metkit/mars/MarsRequest.h"
+
+#include "fdb5/api/FDBFactory.h"
+#include "fdb5/config/Config.h"
+#include "fdb5/database/Key.h"
+#include "fdb5/remote/Messages.h"
+
+namespace fdb5 {
+
+class Config;
+
+namespace remote {
+
+struct MessageHeader;
+
+//----------------------------------------------------------------------------------------------------------------------
+// Base class for CatalogueHandler and StoreHandler
+
+class ServerConnection : private eckit::NonCopyable {
+public:  // methods
+    ServerConnection(eckit::net::TCPSocket& socket, const Config& config);
+    ~ServerConnection();
+
+    virtual void handle() { NOTIMP; }
+
+    std::string host() const { return controlSocket_.localHost(); }
+    int port() const { return controlSocket_.localPort(); }
+    const eckit::LocalConfiguration& agreedConf() const { return agreedConf_; }
+
+protected:
+
+    // socket methods
+    int selectDataPort();
+    virtual void initialiseConnections();
+    eckit::LocalConfiguration availableFunctionality() const;
+
+    void controlWrite(Message msg, uint32_t requestID, const void* payload = nullptr, uint32_t payloadLength = 0);
+    void controlWrite(const void* data, size_t length);
+    void socketRead(void* data, size_t length, eckit::net::TCPSocket& socket);
+    
+    // dataWrite is protected using a mutex, as we may have multiple workers.
+    void dataWrite(Message msg, uint32_t requestID, const void* payload = nullptr, uint32_t payloadLength = 0);
+    eckit::Buffer receivePayload(const MessageHeader& hdr, eckit::net::TCPSocket& socket);
+
+    // socket methods
+    void dataWriteUnsafe(const void* data, size_t length);
+
+    // Worker functionality
+    void tidyWorkers();
+    void waitForWorkers();
+
+
+    // virtual void read(const MessageHeader& hdr);
+    // virtual void archive(const MessageHeader& hdr);
+    // // virtual void store(const MessageHeader& hdr);
+    // virtual void flush(const MessageHeader& hdr);
+
+protected:
+
+    Config config_;
+    eckit::net::TCPSocket controlSocket_;
+    eckit::net::EphemeralTCPServer dataSocket_;
+    std::string dataListenHostname_;
+    // FDB fdb_;
+    eckit::Queue>> readLocationQueue_;
+
+    eckit::SessionID sessionID_;
+    eckit::LocalConfiguration agreedConf_;
+    std::mutex dataWriteMutex_;
+    std::map> workerThreads_;
+    std::future archiveFuture_;
+    std::thread readLocationWorker_;
+    
+
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
+}  // namespace remote
+}  // namespace fdb5
+
+
+//----------------------------------------------------------------------------------------------------------------------
+
diff --git a/src/fdb5/remote/StoreHandler.cc b/src/fdb5/remote/server/StoreHandler.cc
similarity index 87%
rename from src/fdb5/remote/StoreHandler.cc
rename to src/fdb5/remote/server/StoreHandler.cc
index 1fb5b6bd7..b15376d9f 100644
--- a/src/fdb5/remote/StoreHandler.cc
+++ b/src/fdb5/remote/server/StoreHandler.cc
@@ -13,7 +13,7 @@
 
 #include "fdb5/LibFdb5.h"
 #include "fdb5/database/Store.h"
-#include "fdb5/remote/StoreHandler.h"
+#include "fdb5/remote/server/StoreHandler.h"
 
 using namespace eckit;
 using metkit::mars::MarsRequest;
@@ -22,12 +22,12 @@ namespace fdb5 {
 namespace remote {
 
 StoreHandler::StoreHandler(eckit::net::TCPSocket& socket, const Config& config):
-    DecoupledHandler(socket, config) {}
+    ServerConnection(socket, config) {}
 
 StoreHandler::~StoreHandler() {}
 
 void StoreHandler::initialiseConnections() {
-    DecoupledHandler::initialiseConnections();
+    ServerConnection::initialiseConnections();
 
     Log::info() << "StoreHandler::initialiseConnections" << std::endl;
 }
@@ -46,6 +46,7 @@ void StoreHandler::handle() {
     while (true) {
         tidyWorkers();
 
+        std::cout << "StoreHandler::handle() - pre READ\n";
         socketRead(&hdr, sizeof(hdr), controlSocket_);
 
         ASSERT(hdr.marker == StartMarker);
@@ -209,12 +210,13 @@ void StoreHandler::archive(const MessageHeader& hdr) {
 
     // Ensure that we aren't already running a store()
 
-    ASSERT(!archiveFuture_.valid());
+    if(!archiveFuture_.valid()) {
 
-    // Start archive worker thread
+        // Start archive worker thread
 
-    uint32_t id    = hdr.requestID;
-    archiveFuture_ = std::async(std::launch::async, [this, id] { return archiveThreadLoop(id); });
+    //    uint32_t id    = hdr.requestID;
+        archiveFuture_ = std::async(std::launch::async, [this] { return archiveThreadLoop(); });
+    }
 }
 
 Store& StoreHandler::store(Key dbKey) {
@@ -242,32 +244,30 @@ void StoreHandler::archiveBlobPayload(uint32_t id, const void* data, size_t leng
     Store& ss = store(dbKey);
 
     auto futureLocation = ss.archive(idxKey, charData + s.position(), length - s.position());
-    std::cout << "StoreHandler::archiveBlobPayload - Archiving done " << std::endl;
     Log::status() << "Archiving done: " << ss_key.str() << std::endl;
     
     auto loc = futureLocation.get();
-    std::cout << "FieldLocation: " << (*loc) << std::endl;
 
-    ss.flush();
+//    ss.flush();
 
-    // eckit::Buffer locBuffer(2048);
-    // MemoryStream locStream(locBuffer);
-    // locStream << *(futureLocation.get());
-    // dataWrite(Message::Blob, id, locBuffer.data(), locStream.position());
+    eckit::Buffer locBuffer(16 * 1024);
+    MemoryStream locStream(locBuffer);
+    locStream << (*loc);
+    dataWrite(Message::Archive, id, locBuffer.data(), locStream.position());
 }
 
-size_t StoreHandler::archiveThreadLoop(uint32_t id) {
+size_t StoreHandler::archiveThreadLoop() {
     size_t totalArchived = 0;
 
     // Create a worker that will do the actual archiving
 
     static size_t queueSize(eckit::Resource("fdbServerMaxQueueSize", 32));
-    eckit::Queue> queue(queueSize);
+    eckit::Queue, bool>> queue(queueSize);
 
-    std::future worker = std::async(std::launch::async, [this, &queue, id] {
+    std::future worker = std::async(std::launch::async, [this, &queue] {
         size_t totalArchived = 0;
 
-        std::pair elem = std::make_pair(Buffer{0}, false);
+        std::pair, bool> elem = std::make_pair(std::make_pair(0, Buffer{0}), false);
 
         try {
             long queuelen;
@@ -275,14 +275,14 @@ size_t StoreHandler::archiveThreadLoop(uint32_t id) {
                 if (elem.second) {
                     // Handle MultiBlob
 
-                    const char* firstData = static_cast(elem.first.data());  // For pointer arithmetic
+                    const char* firstData = static_cast(elem.first.second.data());  // For pointer arithmetic
                     const char* charData = firstData;
-                    while (size_t(charData - firstData) < elem.first.size()) {
+                    while (size_t(charData - firstData) < elem.first.second.size()) {
                         const MessageHeader* hdr = static_cast(static_cast(charData));
                         ASSERT(hdr->marker == StartMarker);
                         ASSERT(hdr->version == CurrentVersion);
                         ASSERT(hdr->message == Message::Blob);
-                        ASSERT(hdr->requestID == id);
+                        ASSERT(hdr->requestID == elem.first.first);
                         charData += sizeof(MessageHeader);
 
                         const void* payloadData = charData;
@@ -292,18 +292,18 @@ size_t StoreHandler::archiveThreadLoop(uint32_t id) {
                         ASSERT(*e == EndMarker);
                         charData += sizeof(EndMarker);
 
-                        archiveBlobPayload(id, payloadData, hdr->payloadSize);
+                        archiveBlobPayload(elem.first.first, payloadData, hdr->payloadSize);
                         totalArchived += 1;
                     }
                 }
                 else {
                     // Handle single blob
-                    archiveBlobPayload(id, elem.first.data(), elem.first.size());
+                    archiveBlobPayload(elem.first.first, elem.first.second.data(), elem.first.second.size());
                     totalArchived += 1;
                 }
             }
             
-            dataWrite(Message::Complete, id);
+            // dataWrite(Message::Complete, 0);
         }
         catch (...) {
             // Ensure exception propagates across the queue back to the parent thread.
@@ -326,7 +326,7 @@ size_t StoreHandler::archiveThreadLoop(uint32_t id) {
 
             ASSERT(hdr.marker == StartMarker);
             ASSERT(hdr.version == CurrentVersion);
-            ASSERT(hdr.requestID == id);
+            //ASSERT(hdr.requestID == id);
 
             // Have we been told that we are done yet?
             if (hdr.message == Message::Flush)
@@ -346,7 +346,9 @@ size_t StoreHandler::archiveThreadLoop(uint32_t id) {
             size_t sz = payload.size();
             Log::debug() << "Queueing data: " << sz << std::endl;
             size_t queuelen = queue.emplace(
-                std::make_pair(std::move(payload), hdr.message == Message::MultiBlob));
+                std::make_pair(
+                    std::make_pair(hdr.requestID, std::move(payload)),
+                    hdr.message == Message::MultiBlob));
             Log::status() << "Queued data (" << queuelen << ", size=" << sz << ")" << std::endl;
             ;
             Log::debug() << "Queued data (" << queuelen << ", size=" << sz << ")"
@@ -371,13 +373,13 @@ size_t StoreHandler::archiveThreadLoop(uint32_t id) {
     catch (std::exception& e) {
         // n.b. more general than eckit::Exception
         std::string what(e.what());
-        dataWrite(Message::Error, id, what.c_str(), what.length());
+        dataWrite(Message::Error, 0, what.c_str(), what.length());
         queue.interrupt(std::current_exception());
         throw;
     }
     catch (...) {
         std::string what("Caught unexpected, unknown exception in retrieve worker");
-        dataWrite(Message::Error, id, what.c_str(), what.length());
+        dataWrite(Message::Error, 0, what.c_str(), what.length());
         queue.interrupt(std::current_exception());
         throw;
     }
@@ -389,7 +391,7 @@ void StoreHandler::flush(const MessageHeader& hdr) {
     Buffer payload(receivePayload(hdr, controlSocket_));
     MemoryStream s(payload);
 
-    fdb5::Key dbKey(s);
+    //fdb5::Key dbKey(s);
 
     size_t numArchived;
     s >> numArchived;
@@ -405,13 +407,13 @@ void StoreHandler::flush(const MessageHeader& hdr) {
         // Do the actual flush!
         Log::info() << "Flushing" << std::endl;
         Log::status() << "Flushing" << std::endl;
-        if (dbKey.empty()) {
+        //if (dbKey.empty()) {
             for (auto it = stores_.begin(); it != stores_.end(); it++) {
                 it->second->flush();
             }
-        } else {
-            store(dbKey).flush();
-        }
+        // } else {
+        //     store(dbKey).flush();
+        // }
         Log::info() << "Flush complete" << std::endl;
         Log::status() << "Flush complete" << std::endl;
     }
diff --git a/src/fdb5/remote/StoreHandler.h b/src/fdb5/remote/server/StoreHandler.h
similarity index 75%
rename from src/fdb5/remote/StoreHandler.h
rename to src/fdb5/remote/server/StoreHandler.h
index bcfc20bb7..02b5ac805 100644
--- a/src/fdb5/remote/StoreHandler.h
+++ b/src/fdb5/remote/server/StoreHandler.h
@@ -10,12 +10,12 @@
 
 #pragma once
 
-#include "fdb5/remote/DecoupledHandler.h"
+#include "fdb5/remote/server/ServerConnection.h"
 
 namespace fdb5::remote {
     
 //----------------------------------------------------------------------------------------------------------------------
-class StoreHandler : public DecoupledHandler {
+class StoreHandler : public ServerConnection {
 public:  // methods
     StoreHandler(eckit::net::TCPSocket& socket, const Config& config);
     ~StoreHandler();
@@ -24,21 +24,24 @@ class StoreHandler : public DecoupledHandler {
 
 private:  // methods
 
-    void read(const MessageHeader& hdr) override;
+    void read(const MessageHeader& hdr);
 
     void initialiseConnections() override;
     void readLocationThreadLoop();
     void writeToParent(const uint32_t requestID, std::unique_ptr dh);
 
-    void archive(const MessageHeader& hdr) override;
-    size_t archiveThreadLoop(uint32_t id);
+    void archive(const MessageHeader& hdr);
+    size_t archiveThreadLoop();
     void archiveBlobPayload(uint32_t id, const void* data, size_t length);
 
-    void flush(const MessageHeader& hdr) override;
+    void flush(const MessageHeader& hdr);
 
+  //  Catalogue& catalogue(Key dbKey);
     Store& store(Key dbKey);
+  //  Store& store(eckit::URI uri);
 
 private:  // members
+//    std::map> catalogues_;
     std::map> stores_;
 };
 
diff --git a/src/fdb5/rules/SchemaParser.cc b/src/fdb5/rules/SchemaParser.cc
index a4e62bb83..d87cbb308 100644
--- a/src/fdb5/rules/SchemaParser.cc
+++ b/src/fdb5/rules/SchemaParser.cc
@@ -28,7 +28,7 @@ namespace fdb5 {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-std::string SchemaParser::parseIdent(bool emptyOK) {
+std::string SchemaParser::parseIdent(bool key, bool emptyOK) {
     std::string s;
     for (;;) {
         char c = peek();
@@ -72,7 +72,7 @@ Predicate *SchemaParser::parsePredicate(std::map &type
 
     if (c == '?') {
         consume(c);
-        return new Predicate(k, new MatchOptional(parseIdent(true)));
+        return new Predicate(k, new MatchOptional(parseIdent(false, true)));
     }
 
     if (c == '-') {
@@ -81,17 +81,17 @@ Predicate *SchemaParser::parsePredicate(std::map &type
             // Register ignore type
             types[k] = "Ignore";
         }
-        return new Predicate(k, new MatchHidden(parseIdent(true)));
+        return new Predicate(k, new MatchHidden(parseIdent(false, true)));
     }
 
     if (c != ',' && c != '[' && c != ']') {
         consume("=");
 
-        values.insert(parseIdent());
+        values.insert(parseIdent(false));
 
         while ((c = peek()) == '/') {
             consume(c);
-            values.insert(parseIdent());
+            values.insert(parseIdent(false));
         }
     }
 
@@ -112,12 +112,12 @@ Predicate *SchemaParser::parsePredicate(std::map &type
 
 void SchemaParser::parseTypes(std::map &types) {
     for (;;) {
-        std::string name = parseIdent(true);
+        std::string name = parseIdent(true, true);
         if (name.empty()) {
             break;
         }
         consume(':');
-        std::string type = parseIdent();
+        std::string type = parseIdent(false);
         consume(';');
         ASSERT(types.find(name) == types.end());
         types[name] = type;
diff --git a/src/fdb5/rules/SchemaParser.h b/src/fdb5/rules/SchemaParser.h
index 2fc7ad323..a7e457803 100644
--- a/src/fdb5/rules/SchemaParser.h
+++ b/src/fdb5/rules/SchemaParser.h
@@ -36,7 +36,7 @@ class SchemaParser : public eckit::StreamParser {
 
 private: // methods
 
-    std::string parseIdent(bool emptyOK = false);
+    std::string parseIdent(bool key = true, bool emptyOK = false);
 
     Rule *parseRule(const Schema &owner);
 
diff --git a/src/fdb5/toc/AdoptVisitor.cc b/src/fdb5/toc/AdoptVisitor.cc
index c44a98333..254272ea8 100644
--- a/src/fdb5/toc/AdoptVisitor.cc
+++ b/src/fdb5/toc/AdoptVisitor.cc
@@ -35,14 +35,11 @@ bool AdoptVisitor::selectDatum(const InspectionKey &key, const Key &full) {
     // Log::info() << "selectDatum " << key << ", " << full << " " << length_ << std::endl;
     checkMissingKeys(full);
 
-    Catalogue* catalogue = current();
+    CatalogueWriter* catalogue = current();
     ASSERT(catalogue);
 
     if (catalogue->type() == TocEngine::typeName()) {
-        CatalogueWriter* cat = dynamic_cast(catalogue);
-        ASSERT(cat);
-
-        cat->index(key, eckit::URI("file", path_), offset_, length_);
+        catalogue->index(key, eckit::URI("file", path_), offset_, length_);
         return true;
     }
     return false;
diff --git a/src/fdb5/toc/FieldRef.cc b/src/fdb5/toc/FieldRef.cc
index ec5e0900c..4f8e1268e 100644
--- a/src/fdb5/toc/FieldRef.cc
+++ b/src/fdb5/toc/FieldRef.cc
@@ -33,14 +33,18 @@ FieldRefLocation::FieldRefLocation() {
 FieldRefLocation::FieldRefLocation(UriStore &store, const Field& field) {
 
     const FieldLocation& loc = field.location();
-    const TocFieldLocation* tocfloc = dynamic_cast(&loc);
-    if(!tocfloc) {
-        throw eckit::NotImplemented("Field location is not of TocFieldLocation type -- indexing other locations is not supported", Here());
-    }
-
-    uriId_ = store.insert(tocfloc->uri());
-    length_ = tocfloc->length();
-    offset_ = tocfloc->offset();
+    // const TocFieldLocation* tocfloc = dynamic_cast(&loc);
+    // if(!tocfloc) {
+    //     throw eckit::NotImplemented("Field location is not of TocFieldLocation type -- indexing other locations is not supported", Here());
+    // }
+
+    // uriId_ = store.insert(tocfloc->uri());
+    // length_ = tocfloc->length();
+    // offset_ = tocfloc->offset();
+
+    uriId_ = store.insert(loc.uri());
+    length_ = loc.length();
+    offset_ = loc.offset();
 }
 
 void FieldRefLocation::print(std::ostream &s) const {
diff --git a/src/fdb5/toc/TocCatalogue.cc b/src/fdb5/toc/TocCatalogue.cc
index 280d61fc9..dd13b7b28 100644
--- a/src/fdb5/toc/TocCatalogue.cc
+++ b/src/fdb5/toc/TocCatalogue.cc
@@ -30,12 +30,12 @@ TocCatalogue::TocCatalogue(const Key& key, const fdb5::Config& config) :
 }
 
 TocCatalogue::TocCatalogue(const Key& key, const TocPath& tocPath, const fdb5::Config& config) :
-    Catalogue(key, tocPath.controlIdentifiers_, config),
+    CatalogueImpl(key, tocPath.controlIdentifiers_, config),
     TocHandler(tocPath.directory_, config) {
 }
 
 TocCatalogue::TocCatalogue(const eckit::PathName& directory, const ControlIdentifiers& controlIdentifiers, const fdb5::Config& config) :
-    Catalogue(Key(), controlIdentifiers, config),
+    CatalogueImpl(Key(), controlIdentifiers, config),
     TocHandler(directory, config) {
     // Read the real DB key into the DB base object
     dbKey_ = databaseKey();
@@ -79,12 +79,12 @@ std::vector TocCatalogue::metadataPaths() const {
     return paths;
 }
 
-void TocCatalogue::visitEntries(EntryVisitor& visitor, const Store& store, bool sorted) {
+void TocCatalogue::visitEntries(EntryVisitor& visitor, /*const Store& store,*/ bool sorted) {
 
     std::vector all = indexes(sorted);
 
     // Allow the visitor to selectively reject this DB.
-    if (visitor.visitDatabase(*this, store)) {
+    if (visitor.visitDatabase(*this /*, store*/)) {
         if (visitor.visitIndexes()) {
             for (const Index& idx : all) {
                 if (visitor.visitEntries()) {
@@ -161,7 +161,7 @@ void TocCatalogue::control(const ControlAction& action, const ControlIdentifiers
 }
 
 bool TocCatalogue::enabled(const ControlIdentifier& controlIdentifier) const {
-    return Catalogue::enabled(controlIdentifier) && TocHandler::enabled(controlIdentifier);
+    return CatalogueImpl::enabled(controlIdentifier) && TocHandler::enabled(controlIdentifier);
 }
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/toc/TocCatalogue.h b/src/fdb5/toc/TocCatalogue.h
index a77e0546f..b4fd3e760 100644
--- a/src/fdb5/toc/TocCatalogue.h
+++ b/src/fdb5/toc/TocCatalogue.h
@@ -29,7 +29,7 @@ namespace fdb5 {
 
 /// DB that implements the FDB on POSIX filesystems
 
-class TocCatalogue : public Catalogue, public TocHandler {
+class TocCatalogue : public CatalogueImpl, public TocHandler {
 
 public: // methods
 
@@ -58,7 +58,7 @@ class TocCatalogue : public Catalogue, public TocHandler {
 
     void checkUID() const override;
     bool exists() const override;
-    void visitEntries(EntryVisitor& visitor, const Store& store, bool sorted) override;
+    void visitEntries(EntryVisitor& visitor, /*const Store& store,*/ bool sorted) override;
     void dump(std::ostream& out, bool simple, const eckit::Configuration& conf) const override;
     std::vector metadataPaths() const override;
     const Schema& schema() const override;
diff --git a/src/fdb5/toc/TocCatalogueReader.cc b/src/fdb5/toc/TocCatalogueReader.cc
index fd04738f8..adb5892fc 100644
--- a/src/fdb5/toc/TocCatalogueReader.cc
+++ b/src/fdb5/toc/TocCatalogueReader.cc
@@ -16,6 +16,7 @@
 #include "fdb5/toc/TocCatalogueReader.h"
 #include "fdb5/toc/TocIndex.h"
 #include "fdb5/toc/TocStats.h"
+#include "fdb5/toc/RootManager.h"
 
 namespace fdb5 {
 
@@ -32,7 +33,7 @@ TocCatalogueReader::TocCatalogueReader(const eckit::URI& uri, const fdb5::Config
 }
 
 TocCatalogueReader::~TocCatalogueReader() {
-    eckit::Log::debug() << "Closing DB " << *dynamic_cast(this) << std::endl;
+    eckit::Log::debug() << "Closing DB " << (*this) << std::endl;
 }
 
 void TocCatalogueReader::loadIndexesAndRemap() {
@@ -141,7 +142,7 @@ std::vector TocCatalogueReader::indexes(bool sorted) const {
     return returnedIndexes;
 }
 
-static CatalogueBuilder builder("toc.reader");
+static CatalogueReaderBuilder builder("toc");
 
 //----------------------------------------------------------------------------------------------------------------------
 
diff --git a/src/fdb5/toc/TocCatalogueWriter.cc b/src/fdb5/toc/TocCatalogueWriter.cc
index e90c2f3c2..9974c7464 100644
--- a/src/fdb5/toc/TocCatalogueWriter.cc
+++ b/src/fdb5/toc/TocCatalogueWriter.cc
@@ -21,6 +21,7 @@
 #include "fdb5/toc/TocCatalogueWriter.h"
 #include "fdb5/toc/TocFieldLocation.h"
 #include "fdb5/toc/TocIndex.h"
+#include "fdb5/toc/RootManager.h"
 #include "fdb5/io/LustreSettings.h"
 
 using namespace eckit;
@@ -403,7 +404,7 @@ void TocCatalogueWriter::print(std::ostream &out) const {
     out << "TocCatalogueWriter(" << directory() << ")";
 }
 
-static CatalogueBuilder builder("toc.writer");
+static CatalogueWriterBuilder builder("toc");
 
 //----------------------------------------------------------------------------------------------------------------------
 
diff --git a/src/fdb5/toc/TocIndex.cc b/src/fdb5/toc/TocIndex.cc
index 72faf1580..3f93527fc 100644
--- a/src/fdb5/toc/TocIndex.cc
+++ b/src/fdb5/toc/TocIndex.cc
@@ -86,6 +86,7 @@ bool TocIndex::get(const InspectionKey &key, const Key &remapKey, Field &field)
     if ( found ) {
         const eckit::URI& uri = files_.get(ref.uriId());
         FieldLocation* loc = FieldLocationFactory::instance().build(uri.scheme(), uri, ref.offset(), ref.length(), remapKey);
+        std::cout << "TocIndex::get " << *loc << std::endl;
         field = Field(std::move(*loc), timestamp_, ref.details());
         delete(loc);
     }
diff --git a/src/fdb5/toc/TocMoveVisitor.cc b/src/fdb5/toc/TocMoveVisitor.cc
index 0cd9b1daf..6fa9a82f0 100644
--- a/src/fdb5/toc/TocMoveVisitor.cc
+++ b/src/fdb5/toc/TocMoveVisitor.cc
@@ -50,12 +50,14 @@ TocMoveVisitor::TocMoveVisitor(const TocCatalogue& catalogue,
 
 TocMoveVisitor::~TocMoveVisitor() {}
 
-bool TocMoveVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
+//bool TocMoveVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
+bool TocMoveVisitor::visitDatabase(const Catalogue& catalogue) {
 
     // Overall checks
     ASSERT(&catalogue_ == &catalogue);
 
-    MoveVisitor::visitDatabase(catalogue_, store);
+//    MoveVisitor::visitDatabase(catalogue_, store);
+    MoveVisitor::visitDatabase(catalogue_);
 
     // TOC specific checks: index files not locked
     DIR* dirp = ::opendir(catalogue_.basePath().asString().c_str());
diff --git a/src/fdb5/toc/TocMoveVisitor.h b/src/fdb5/toc/TocMoveVisitor.h
index f0d64a702..3ff287c4b 100644
--- a/src/fdb5/toc/TocMoveVisitor.h
+++ b/src/fdb5/toc/TocMoveVisitor.h
@@ -35,7 +35,8 @@ class TocMoveVisitor : public MoveVisitor {
 
 private: // methods
 
-    bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
+    // bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
+    bool visitDatabase(const Catalogue& catalogue) override;
 
     void move();
 
diff --git a/src/fdb5/toc/TocPurgeVisitor.cc b/src/fdb5/toc/TocPurgeVisitor.cc
index 1d4104156..6a93d26db 100644
--- a/src/fdb5/toc/TocPurgeVisitor.cc
+++ b/src/fdb5/toc/TocPurgeVisitor.cc
@@ -29,7 +29,8 @@ TocPurgeVisitor::TocPurgeVisitor(const TocCatalogue& catalogue, const Store& sto
 
 TocPurgeVisitor::~TocPurgeVisitor() {}
 
-bool TocPurgeVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
+//bool TocPurgeVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
+bool TocPurgeVisitor::visitDatabase(const Catalogue& catalogue) {
 
     std::set> metadata;
     std::set data;
@@ -53,8 +54,8 @@ bool TocPurgeVisitor::visitDatabase(const Catalogue& catalogue, const Store& sto
 
 
 void TocPurgeVisitor::report(std::ostream& out) const {
-
-    const eckit::PathName& directory(((TocCatalogue*) currentCatalogue_)->basePath());
+    const TocCatalogue* cat = dynamic_cast(currentCatalogue_);
+    const eckit::PathName& directory(cat->basePath());
 
     out << std::endl;
     out << "Index Report:" << std::endl;
diff --git a/src/fdb5/toc/TocPurgeVisitor.h b/src/fdb5/toc/TocPurgeVisitor.h
index cda7b20b3..892b63067 100644
--- a/src/fdb5/toc/TocPurgeVisitor.h
+++ b/src/fdb5/toc/TocPurgeVisitor.h
@@ -32,7 +32,8 @@ class TocPurgeVisitor : public PurgeVisitor, public TocStatsReportVisitor {
     TocPurgeVisitor(const TocCatalogue& catalogue, const Store& store);
     ~TocPurgeVisitor() override;
 
-    bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
+    // bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
+    bool visitDatabase(const Catalogue& catalogue) override;
     void report(std::ostream& out) const override;
     void purge(std::ostream& out, bool porcelain, bool doit) const override;
 
diff --git a/src/fdb5/toc/TocStats.cc b/src/fdb5/toc/TocStats.cc
index ea556f5fc..19cda5208 100644
--- a/src/fdb5/toc/TocStats.cc
+++ b/src/fdb5/toc/TocStats.cc
@@ -218,7 +218,8 @@ TocStatsReportVisitor::TocStatsReportVisitor(const TocCatalogue& catalogue, bool
 
 TocStatsReportVisitor::~TocStatsReportVisitor() {}
 
-bool TocStatsReportVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
+//bool TocStatsReportVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
+bool TocStatsReportVisitor::visitDatabase(const Catalogue& catalogue) {
     ASSERT(&catalogue == currentCatalogue_);
     return true;
 }
@@ -232,8 +233,10 @@ void TocStatsReportVisitor::visitDatum(const Field& field, const std::string& fi
 
     // Exclude non-owned data if relevant
     if (!includeReferencedNonOwnedData_) {
-        if (!currentIndex_->location().uri().path().dirName().sameAs(((TocCatalogue*) currentCatalogue_)->basePath())) return;
-        if (!field.location().uri().path().dirName().sameAs(((TocCatalogue*) currentCatalogue_)->basePath())) return;
+        const TocCatalogue* cat = dynamic_cast(currentCatalogue_);
+
+        if (!currentIndex_->location().uri().path().dirName().sameAs(cat->basePath())) return;
+        if (!field.location().uri().path().dirName().sameAs(cat->basePath())) return;
     }
 
     // If this index is not yet in the map, then create an entry
diff --git a/src/fdb5/toc/TocStats.h b/src/fdb5/toc/TocStats.h
index 91ba2049e..c697aaf8b 100644
--- a/src/fdb5/toc/TocStats.h
+++ b/src/fdb5/toc/TocStats.h
@@ -159,7 +159,8 @@ class TocStatsReportVisitor : public virtual StatsReportVisitor {
 
 private: // methods
 
-    bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
+    // bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
+    bool visitDatabase(const Catalogue& catalogue) override;
     void visitDatum(const Field& field, const std::string& keyFingerprint) override;
     void visitDatum(const Field& field, const InspectionKey& key) override { NOTIMP; }
 
diff --git a/src/fdb5/toc/TocStore.h b/src/fdb5/toc/TocStore.h
index 4983e2a22..3391b6f95 100644
--- a/src/fdb5/toc/TocStore.h
+++ b/src/fdb5/toc/TocStore.h
@@ -34,6 +34,7 @@ class TocStore : public Store, public TocCommon {
 public: // methods
 
     TocStore(const Key& key, const Config& config);
+    TocStore(const Key& key, const Config& config, const eckit::net::Endpoint& controlEndpoint);
     TocStore(const eckit::URI& uri, const Config& config);
 
     ~TocStore() override {}
diff --git a/src/fdb5/toc/TocWipeVisitor.cc b/src/fdb5/toc/TocWipeVisitor.cc
index 4311cce61..bb10e7716 100644
--- a/src/fdb5/toc/TocWipeVisitor.cc
+++ b/src/fdb5/toc/TocWipeVisitor.cc
@@ -101,14 +101,16 @@ TocWipeVisitor::TocWipeVisitor(const TocCatalogue& catalogue,
 TocWipeVisitor::~TocWipeVisitor() {}
 
 
-bool TocWipeVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
+// bool TocWipeVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
+bool TocWipeVisitor::visitDatabase(const Catalogue& catalogue) {
 
     // Overall checks
 
     ASSERT(&catalogue_ == &catalogue);
 //    ASSERT(&store_ == &store);
     ASSERT(catalogue.enabled(ControlIdentifier::Wipe));
-    WipeVisitor::visitDatabase(catalogue, store);
+//    WipeVisitor::visitDatabase(catalogue, store);
+    WipeVisitor::visitDatabase(catalogue);
 
     // Check that we are in a clean state (i.e. we only visit one DB).
 
diff --git a/src/fdb5/toc/TocWipeVisitor.h b/src/fdb5/toc/TocWipeVisitor.h
index be1352396..518e1e78b 100644
--- a/src/fdb5/toc/TocWipeVisitor.h
+++ b/src/fdb5/toc/TocWipeVisitor.h
@@ -37,7 +37,8 @@ class TocWipeVisitor : public WipeVisitor {
 
 private: // methods
 
-    bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
+    // bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
+    bool visitDatabase(const Catalogue& catalogue) override;
     bool visitIndex(const Index& index) override;
     void catalogueComplete(const Catalogue& catalogue) override;
 
diff --git a/src/fdb5/tools/fdb-hide.cc b/src/fdb5/tools/fdb-hide.cc
index dc92554c2..ce1f6d3d5 100644
--- a/src/fdb5/tools/fdb-hide.cc
+++ b/src/fdb5/tools/fdb-hide.cc
@@ -86,7 +86,7 @@ void FdbHide::execute(const option::CmdArgs& args) {
     InspectionKey dbkey;
     ASSERT(schema.expandFirstLevel(dbrequest.request(), dbkey));
 
-    std::unique_ptr db = CatalogueFactory::instance().build(dbkey, conf, true);
+    std::unique_ptr db = CatalogueReaderFactory::instance().build(dbkey, conf);
     if (!db->exists()) {
         std::stringstream ss;
         ss << "Database not found: " << dbkey << std::endl;
@@ -101,8 +101,9 @@ void FdbHide::execute(const option::CmdArgs& args) {
 
     eckit::Log::info() << "Hide contents of DB: " << *db << std::endl;
     if (doit_) {
-        std::unique_ptr dbWriter = CatalogueFactory::instance().build(dbkey, conf, false);
+        std::unique_ptr dbWriter = CatalogueWriterFactory::instance().build(dbkey, conf);
         TocCatalogueWriter* tocDB = dynamic_cast(dbWriter.get());
+        ASSERT(tocDB);
         tocDB->hideContents();
     } else {
         eckit::Log::info() << "Run with --doit to make changes" << std::endl;
diff --git a/src/fdb5/tools/fdb-overlay.cc b/src/fdb5/tools/fdb-overlay.cc
index da580ec16..a2389b802 100644
--- a/src/fdb5/tools/fdb-overlay.cc
+++ b/src/fdb5/tools/fdb-overlay.cc
@@ -123,7 +123,7 @@ void FdbOverlay::execute(const option::CmdArgs& args) {
         }
     }
 
-    std::unique_ptr dbSource = CatalogueFactory::instance().build(source, conf, true);
+    std::unique_ptr dbSource = CatalogueReaderFactory::instance().build(source, conf);
     if (!dbSource->exists()) {
         std::stringstream ss;
         ss << "Source database not found: " << source << std::endl;
@@ -136,7 +136,7 @@ void FdbOverlay::execute(const option::CmdArgs& args) {
         throw UserError(ss.str(), Here());
     }
 
-    std::unique_ptr dbTarget = CatalogueFactory::instance().build(target, conf, true);
+    std::unique_ptr dbTarget = CatalogueReaderFactory::instance().build(target, conf);
 
     if (remove_) {
         if (!dbTarget->exists()) {
@@ -156,12 +156,9 @@ void FdbOverlay::execute(const option::CmdArgs& args) {
 
     ASSERT(dbTarget->uri() != dbSource->uri());
 
-    std::unique_ptr newCatalogue = CatalogueFactory::instance().build(target, conf, false);
+    std::unique_ptr newCatalogue = CatalogueWriterFactory::instance().build(target, conf);
     if (newCatalogue->type() == TocEngine::typeName() && dbSource->type() == TocEngine::typeName())  {
-        CatalogueWriter* cat = dynamic_cast(newCatalogue.get());
-        ASSERT(cat);
-
-        cat->overlayDB(*dbSource, vkeys, remove_);
+        newCatalogue->overlayDB(*dbSource, vkeys, remove_);
     }
 }
 
diff --git a/src/fdb5/tools/fdb-reconsolidate-toc.cc b/src/fdb5/tools/fdb-reconsolidate-toc.cc
index 3688e9cfc..26282dc3a 100644
--- a/src/fdb5/tools/fdb-reconsolidate-toc.cc
+++ b/src/fdb5/tools/fdb-reconsolidate-toc.cc
@@ -54,11 +54,9 @@ void FDBReconsolidateToc::execute(const eckit::option::CmdArgs& args) {
     }
 
     // TODO: In updated version, grab default Config() here;
-    std::unique_ptr catalogue = fdb5::CatalogueFactory::instance().build(eckit::URI("toc", dbPath), eckit::LocalConfiguration(), false);
-    fdb5::CatalogueWriter* cat = dynamic_cast(catalogue.get());
-    ASSERT(cat);
+    std::unique_ptr catalogue = fdb5::CatalogueWriterFactory::instance().build(eckit::URI("toc", dbPath), eckit::LocalConfiguration());
 
-    cat->reconsolidate();
+    catalogue->reconsolidate();
 }
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/tools/fdb-root.cc b/src/fdb5/tools/fdb-root.cc
index 3f521f001..f1951ce77 100644
--- a/src/fdb5/tools/fdb-root.cc
+++ b/src/fdb5/tools/fdb-root.cc
@@ -72,10 +72,10 @@ void FdbRoot::execute(const eckit::option::CmdArgs& args) {
 
             // 'Touch' the database (which will create it if it doesn't exist)
 
-            std::unique_ptr cat = CatalogueFactory::instance().build(result, conf, true);
+            std::unique_ptr cat = CatalogueReaderFactory::instance().build(result, conf);
 
             if (!cat->exists() && create_db) {
-                cat = CatalogueFactory::instance().build(result, conf, false);
+                cat = CatalogueWriterFactory::instance().build(result, conf);
             }
 
             if (cat->exists()) {
diff --git a/src/fdb5/tools/fdb-schema.cc b/src/fdb5/tools/fdb-schema.cc
index 6b9a9dcfe..929c53728 100644
--- a/src/fdb5/tools/fdb-schema.cc
+++ b/src/fdb5/tools/fdb-schema.cc
@@ -56,7 +56,7 @@ void FdbSchema:: execute(const eckit::option::CmdArgs &args) {
             eckit::PathName path(args(i));
 
             if (path.isDir()) {
-                std::unique_ptr cat = CatalogueFactory::instance().build(eckit::URI("toc", path), fdb5::Config(), true);
+                std::unique_ptr cat = CatalogueReaderFactory::instance().build(eckit::URI("toc", path), fdb5::Config());
                 ASSERT(cat->open());
                 cat->schema().dump(Log::info());
             } else {
diff --git a/src/fdb5/tools/fdb-stats-new.cc b/src/fdb5/tools/fdb-stats-new.cc
index 25d1d7492..9b926ac37 100644
--- a/src/fdb5/tools/fdb-stats-new.cc
+++ b/src/fdb5/tools/fdb-stats-new.cc
@@ -61,7 +61,7 @@ void FDBStats::process(const eckit::PathName& path, const eckit::option::CmdArgs
 
     eckit::Log::info() << "Scanning " << path << std::endl;
 
-    std::unique_ptr cat = fdb5::CatalogueFactory::instance().build(path, fdb5::Config(), true);
+    std::unique_ptr cat = fdb5::CatalogueReaderFactory::instance().build(path, fdb5::Config(), true);
     ASSERT(cat->open());
 
     fdb5::ReportVisitor visitor(*db);

From 563d1ad288ac9d6665269635a6b752b9a6f64fbb Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 24 Oct 2023 15:42:09 +0200
Subject: [PATCH 007/186] removed default Schema ctor

---
 src/fdb5/remote/RemoteFieldLocation.cc        | 64 +++++++++++--------
 src/fdb5/remote/RemoteFieldLocation.h         |  7 --
 src/fdb5/remote/RemoteStore.cc                | 13 ++--
 src/fdb5/remote/RemoteStore.h                 |  2 +-
 .../remote/client/ClientConnectionRouter.cc   |  6 +-
 .../remote/client/ClientConnectionRouter.h    | 21 +++---
 src/fdb5/remote/server/StoreHandler.cc        |  1 +
 src/fdb5/rules/Rule.cc                        |  3 +-
 src/fdb5/rules/Schema.cc                      |  3 -
 src/fdb5/rules/Schema.h                       |  1 -
 src/fdb5/toc/TocCatalogue.cc                  |  9 ++-
 src/fdb5/toc/TocCatalogue.h                   |  2 +-
 src/fdb5/toc/TocIndex.cc                      |  2 +-
 src/fdb5/tools/fdb-schema.cc                  |  6 +-
 14 files changed, 78 insertions(+), 62 deletions(-)

diff --git a/src/fdb5/remote/RemoteFieldLocation.cc b/src/fdb5/remote/RemoteFieldLocation.cc
index 2b89a8f4d..a98493335 100644
--- a/src/fdb5/remote/RemoteFieldLocation.cc
+++ b/src/fdb5/remote/RemoteFieldLocation.cc
@@ -29,29 +29,38 @@ ::eckit::Reanimator RemoteFieldLocation::reanimator_;
 //----------------------------------------------------------------------------------------------------------------------
 
 RemoteFieldLocation::RemoteFieldLocation(const eckit::net::Endpoint& endpoint, const FieldLocation& remoteLocation) :
-    FieldLocation(eckit::URI("fdbremote", remoteLocation.uri(), endpoint.host(),  endpoint.port())),
-//    remoteStore_(remoteStore),
-    internal_(remoteLocation.make_shared()) {
-//    ASSERT(remoteStore);
-    ASSERT(remoteLocation.uri().scheme() != "fdbremote");
+    FieldLocation(
+        eckit::URI("fdb", remoteLocation.uri(), endpoint.host(),  endpoint.port()),
+        remoteLocation.offset(),
+        remoteLocation.length(),
+        remoteLocation.remapKey()) {
+
+    ASSERT(remoteLocation.uri().scheme() != "fdb");
+    if (!remoteLocation.uri().scheme().empty()) {
+        uri_.query("internalScheme", remoteLocation.uri().scheme());
+    }
+    if (!remoteLocation.host().empty()) {
+        uri_.query("internalHost", remoteLocation.host());
+    }
 }
 
 RemoteFieldLocation::RemoteFieldLocation(const eckit::URI& uri) :
-    FieldLocation(eckit::URI("fdbremote://" + uri.hostport())),
-    internal_(std::shared_ptr(FieldLocationFactory::instance().build(uri.scheme(), uri))) {}
+    FieldLocation(eckit::URI("fdb", uri)) {
+    NOTIMP;
+}
 
 RemoteFieldLocation::RemoteFieldLocation(const eckit::URI& uri, const eckit::Offset& offset, const eckit::Length& length, const Key& remapKey) :
-    FieldLocation(eckit::URI("fdbremote://" + uri.hostport())),
-    internal_(std::shared_ptr(FieldLocationFactory::instance().build(uri.scheme(), uri, offset, length, remapKey))) {}
+    FieldLocation(uri, offset, length, remapKey) {
+
+    ASSERT(uri.scheme() == "fdb");
+}
 
 RemoteFieldLocation::RemoteFieldLocation(eckit::Stream& s) :
-    FieldLocation(s),
-    internal_(std::shared_ptr(eckit::Reanimator::reanimate(s))) {}
+    FieldLocation(s) {
+}
 
 RemoteFieldLocation::RemoteFieldLocation(const RemoteFieldLocation& rhs) :
-    FieldLocation(rhs.uri_),
-//    remoteStore_(rhs.remoteStore_),
-    internal_(rhs.internal_) {}
+    FieldLocation(rhs.uri_, rhs.offset_, rhs.length_, rhs.remapKey_) {}
 
 
 std::shared_ptr RemoteFieldLocation::make_shared() const {
@@ -59,8 +68,9 @@ std::shared_ptr RemoteFieldLocation::make_shared() const {
 }
 
 eckit::DataHandle* RemoteFieldLocation::dataHandle() const {
+    
 //    ASSERT(remoteStore_);
-    //Connection conn = ClientConnectionRouter::instance().connectStore(uri_);
+//    Connection conn = ClientConnectionRouter::instance().connectStore.connectStore(uri_);
     return nullptr; //conn.->dataHandle(*internal_);
 }
 
@@ -69,23 +79,25 @@ void RemoteFieldLocation::visit(FieldLocationVisitor& visitor) const {
 }
 
 void RemoteFieldLocation::print(std::ostream& out) const {
-    out << "RemoteFieldLocation[uri=" << uri_ << ",internal=" << *internal_ << "]";
+    out << "RemoteFieldLocation[uri=" << uri_ << "]";
 }
 
 void RemoteFieldLocation::encode(eckit::Stream& s) const {
     FieldLocation::encode(s);
-    s << *internal_;
-    std::stringstream ss;
-//    ss << remoteStore_->config();
-    s << ss.str();
+    // s << internalLocationScheme_;
+    // s << internalLocationHostPort_;
+//     std::stringstream ss;
+// //    ss << remoteStore_->config();
+//     s << ss.str();
 }
 
-void RemoteFieldLocation::dump(std::ostream& out) const {
-    out << "  uri: " << uri_ << std::endl;
-    internal_->dump(out);
-}
+// void RemoteFieldLocation::dump(std::ostream& out) const {
+//     out << "  uri: " << uri_ << std::endl;
+//     // out << "  internalScheme: " << internalLocationScheme_ << std::endl;
+//     // out << "  internalHost: " << internalLocationHostPort_ << std::endl;
+// }
 
-static FieldLocationBuilder builder("fdbremote");
+static FieldLocationBuilder builder("fdb");
 
 //----------------------------------------------------------------------------------------------------------------------
 
@@ -117,7 +129,7 @@ class FdbURIManager : public eckit::URIManager {
     FdbURIManager(const std::string& name) : eckit::URIManager(name) {}
 };
 
-static FdbURIManager manager_fdb_file("fdbremote");
+static FdbURIManager manager_fdb_file("fdb");
 
 } // namespace remote
 } // namespace fdb5
diff --git a/src/fdb5/remote/RemoteFieldLocation.h b/src/fdb5/remote/RemoteFieldLocation.h
index 2ee44bca7..91cf5b8b9 100644
--- a/src/fdb5/remote/RemoteFieldLocation.h
+++ b/src/fdb5/remote/RemoteFieldLocation.h
@@ -36,9 +36,6 @@ class RemoteFieldLocation : public FieldLocation {
     RemoteFieldLocation(eckit::Stream&);
     RemoteFieldLocation(const RemoteFieldLocation&);
 
-    eckit::Offset offset() const override { return internal_->offset(); }
-    eckit::Length length() const override { return internal_->length(); }
-
     virtual eckit::DataHandle *dataHandle() const override;
 
     virtual std::shared_ptr make_shared() const override;
@@ -58,14 +55,10 @@ class RemoteFieldLocation : public FieldLocation {
 
 private: // methods
 
-    virtual void dump(std::ostream &out) const override;
     virtual void print(std::ostream &out) const override;
 
 private: // members
 
-    // not Owning
-//    RemoteStore* remoteStore_;
-    std::shared_ptr internal_;
 };
 
 
diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index 039d9b7d6..ad80fad0b 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -65,9 +65,9 @@ RemoteStore::RemoteStore(const eckit::URI& uri, const Config& config) :
     retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
     maxArchiveBatchSize_(config.getInt("maxBatchSize", 1)) {
 
-    std::cout << "RemoteStore ctor " << uri.asRawString() << std::endl;
+    // std::cout << "RemoteStore ctor " << uri.asRawString() << std::endl;
 
-    ASSERT(uri.scheme() == "fdbremote");
+    ASSERT(uri.scheme() == "fdb");
     // std::vector endpoints{eckit::net::Endpoint(uri.hostport())};
     // connection_ = ClientConnectionRouter::instance().connectStore(dbKey_, endpoints);
 }
@@ -86,7 +86,7 @@ RemoteStore::~RemoteStore() {
 }
 
 eckit::URI RemoteStore::uri() const {
-    return URI("fdbremote", "");
+    return URI("fdb", "");
 }
 
 bool RemoteStore::exists() const {
@@ -339,7 +339,7 @@ void RemoteStore::handleException(std::exception_ptr e) {
 
 void RemoteStore::flush(FDBStats& internalStats) {
 
-    std::cout << "###################################      RemoteStore::flush\n";
+    // std::cout << "###################################      RemoteStore::flush\n";
     // Flush only does anything if there is an ongoing archive();
     if (! archiveFuture_.valid()) return;
 
@@ -591,6 +591,9 @@ class FDBRemoteDataHandle : public DataHandle {
 
 void RemoteStore::sendArchiveData(uint32_t id, const Key& key, const void* data, size_t length) {
     
+    ASSERT(!dbKey_.empty());
+    ASSERT(!key.empty());
+
     ASSERT(data);
     ASSERT(length != 0);
 
@@ -632,7 +635,7 @@ eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation, c
 
 
 
-static StoreBuilder builder("fdbremote");
+static StoreBuilder builder("remote");
 
 //----------------------------------------------------------------------------------------------------------------------
 
diff --git a/src/fdb5/remote/RemoteStore.h b/src/fdb5/remote/RemoteStore.h
index 9f31e5c5b..b6614403e 100644
--- a/src/fdb5/remote/RemoteStore.h
+++ b/src/fdb5/remote/RemoteStore.h
@@ -61,7 +61,7 @@ class RemoteStore : public Store, public Client {
    
 protected: // methods
 
-    std::string type() const override { return "fdbremote"; }
+    std::string type() const override { return "remote"; }
 
     bool exists() const override;
 
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.cc b/src/fdb5/remote/client/ClientConnectionRouter.cc
index c6744c977..1a7514b5a 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.cc
+++ b/src/fdb5/remote/client/ClientConnectionRouter.cc
@@ -80,7 +80,7 @@ uint32_t ClientConnectionRouter::controlWrite(std::vector&
     // void dataRead(Client& client, uint32_t requestId, void* data, size_t length);
 
 void ClientConnectionRouter::controlRead(Client& client, uint32_t requestId, void* payload, size_t payloadLength) {
-    std::cout << "ClientConnectionRouter::controlRead " << requestId << std::endl;
+    // std::cout << "ClientConnectionRouter::controlRead " << requestId << std::endl;
     auto it = requests_.find(requestId);
     ASSERT(it != requests_.end());
     ASSERT(it->second.second == &client); // check if the client owns the request
@@ -93,6 +93,10 @@ void ClientConnectionRouter::controlRead(Client& client, uint32_t requestId, voi
 
 // }
 
+// Client& ClientConnectionRouter::store(eckit::URI uri) {
+//     StoreFactory::instance().build()
+// }
+
 
 uint32_t ClientConnectionRouter::createConnection(std::vector& endpoints, Client& client, ClientConnection*& conn) {
 
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.h b/src/fdb5/remote/client/ClientConnectionRouter.h
index 1701c8bc3..902be73b1 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.h
+++ b/src/fdb5/remote/client/ClientConnectionRouter.h
@@ -27,14 +27,15 @@ namespace fdb5::remote {
 class Client;
 //----------------------------------------------------------------------------------------------------------------------
 
-class Connection {
-public:
-    Connection(ClientConnection* clientConnection, uint32_t remoteID) : clientConnection_(clientConnection), remoteID_(remoteID) {}
-    Connection(Connection& other) : clientConnection_(other.clientConnection_), remoteID_(other.remoteID_) {}
+// class Connection {
+// public:
+//     Connection(ClientConnection* clientConnection, uint32_t remoteID) : clientConnection_(clientConnection), remoteID_(remoteID) {}
+//     Connection(Connection& other) : clientConnection_(other.clientConnection_), remoteID_(other.remoteID_) {}
     
-    ClientConnection* clientConnection_;
-    uint32_t remoteID_;
-};
+//     ClientConnection* clientConnection_;
+//     uint32_t remoteID_;
+// };
+
 // typedef eckit::net::Endpoint Connection;
 // typedef uint32_t ClientID;
 // typedef uint32_t DataLinkID;
@@ -45,9 +46,9 @@ class ClientConnectionRouter : eckit::NonCopyable {
 public:
 
     static ClientConnectionRouter& instance();
-    Connection connectCatalogue(Key dbKey, const eckit::Configuration& config);
-    Connection connectStore(Key dbKey, const std::vector& endpoints);
-//    Connection connectStore(eckit::URI uri);
+//    Connection connectCatalogue(Key dbKey, const eckit::Configuration& config);
+//    Connection connectStore(Key dbKey, const std::vector& endpoints);
+//    RemoteStore& store(eckit::URI uri);
 
 // protected:
 
diff --git a/src/fdb5/remote/server/StoreHandler.cc b/src/fdb5/remote/server/StoreHandler.cc
index b15376d9f..2aee6d7cf 100644
--- a/src/fdb5/remote/server/StoreHandler.cc
+++ b/src/fdb5/remote/server/StoreHandler.cc
@@ -135,6 +135,7 @@ void StoreHandler::handle() {
 
 void StoreHandler::read(const MessageHeader& hdr) {
 
+    // std::cout << "readLocationThreadLoop\n";
     if (!readLocationWorker_.joinable()) {
         readLocationWorker_ = std::thread([this] { readLocationThreadLoop(); });
     }
diff --git a/src/fdb5/rules/Rule.cc b/src/fdb5/rules/Rule.cc
index c5d72c6dd..074b5f3dd 100644
--- a/src/fdb5/rules/Rule.cc
+++ b/src/fdb5/rules/Rule.cc
@@ -46,7 +46,8 @@ Rule::Rule(const Schema &schema,
 }
 
 Rule::Rule(eckit::Stream& s):
-    Rule(Schema(), s) {
+    Rule(Schema(""), s) {
+    NOTIMP;
 }
 
 Rule::Rule(const Schema &schema, eckit::Stream& s):
diff --git a/src/fdb5/rules/Schema.cc b/src/fdb5/rules/Schema.cc
index 8954b9af1..7d006ee7f 100644
--- a/src/fdb5/rules/Schema.cc
+++ b/src/fdb5/rules/Schema.cc
@@ -25,9 +25,6 @@ eckit::ClassSpec Schema::classSpec_ = { &eckit::Streamable::classSpec(), "Schema
 
 eckit::Reanimator Schema::reanimator_;
 
-Schema::Schema() {
-}
-
 Schema::Schema(const eckit::PathName &path) {
     load(path);
 }
diff --git a/src/fdb5/rules/Schema.h b/src/fdb5/rules/Schema.h
index 94143de09..21e4c956b 100644
--- a/src/fdb5/rules/Schema.h
+++ b/src/fdb5/rules/Schema.h
@@ -43,7 +43,6 @@ class Schema : public eckit::Streamable {
 
 public: // methods
 
-    Schema();
     Schema(const eckit::PathName &path);
     Schema(std::istream& s);
     Schema(eckit::Stream& s);
diff --git a/src/fdb5/toc/TocCatalogue.cc b/src/fdb5/toc/TocCatalogue.cc
index dd13b7b28..544626c12 100644
--- a/src/fdb5/toc/TocCatalogue.cc
+++ b/src/fdb5/toc/TocCatalogue.cc
@@ -59,7 +59,8 @@ eckit::URI TocCatalogue::uri() const {
 }
 
 const Schema& TocCatalogue::schema() const {
-    return schema_;
+    ASSERT(schema_);
+    return *schema_;
 }
 
 const eckit::PathName& TocCatalogue::basePath() const {
@@ -102,7 +103,11 @@ void TocCatalogue::visitEntries(EntryVisitor& visitor, /*const Store& store,*/ b
 
 void TocCatalogue::loadSchema() {
     Timer timer("TocCatalogue::loadSchema()", Log::debug());
-    schema_.load( schemaPath() );
+    if (schema_) {
+        schema_->load(schemaPath());
+    } else {
+        schema_.reset(new Schema(schemaPath()));
+    }
 }
 
 StatsReportVisitor* TocCatalogue::statsReportVisitor() const {
diff --git a/src/fdb5/toc/TocCatalogue.h b/src/fdb5/toc/TocCatalogue.h
index b4fd3e760..39985e34b 100644
--- a/src/fdb5/toc/TocCatalogue.h
+++ b/src/fdb5/toc/TocCatalogue.h
@@ -88,7 +88,7 @@ class TocCatalogue : public CatalogueImpl, public TocHandler {
     friend class TocWipeVisitor;
     friend class TocMoveVisitor;
 
-    Schema schema_;
+    std::unique_ptr schema_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/toc/TocIndex.cc b/src/fdb5/toc/TocIndex.cc
index 3f93527fc..2edccb4ca 100644
--- a/src/fdb5/toc/TocIndex.cc
+++ b/src/fdb5/toc/TocIndex.cc
@@ -86,7 +86,6 @@ bool TocIndex::get(const InspectionKey &key, const Key &remapKey, Field &field)
     if ( found ) {
         const eckit::URI& uri = files_.get(ref.uriId());
         FieldLocation* loc = FieldLocationFactory::instance().build(uri.scheme(), uri, ref.offset(), ref.length(), remapKey);
-        std::cout << "TocIndex::get " << *loc << std::endl;
         field = Field(std::move(*loc), timestamp_, ref.details());
         delete(loc);
     }
@@ -174,6 +173,7 @@ class TocIndexVisitor : public BTreeIndexVisitor {
         visitor_(visitor) {}
 
     void visit(const std::string& keyFingerprint, const FieldRef& ref) {
+
         Field field(TocFieldLocation(files_, ref), visitor_.indexTimestamp(), ref.details());
         visitor_.visitDatum(field, keyFingerprint);
     }
diff --git a/src/fdb5/tools/fdb-schema.cc b/src/fdb5/tools/fdb-schema.cc
index 929c53728..6892e361a 100644
--- a/src/fdb5/tools/fdb-schema.cc
+++ b/src/fdb5/tools/fdb-schema.cc
@@ -60,9 +60,9 @@ void FdbSchema:: execute(const eckit::option::CmdArgs &args) {
                 ASSERT(cat->open());
                 cat->schema().dump(Log::info());
             } else {
-                Schema schema;
-                schema.load(args(i));
-                schema.dump(Log::info());
+                Schema* schema = new Schema(args(i));
+                schema->dump(Log::info());
+                delete schema;
             }
         }
     }

From 020ce9ff65d12b92e7f2df5da56e8f06d0d36e7d Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 26 Oct 2023 11:10:16 +0100
Subject: [PATCH 008/186] fixed RemoteFieldLocation URI parsing + UriStore
 cleanup

---
 src/fdb5/database/UriStore.cc                 |  1 +
 src/fdb5/rados/RadosFieldLocation.cc          |  4 ++--
 src/fdb5/rados/RadosFieldLocation.h           |  4 ++--
 src/fdb5/remote/RemoteFieldLocation.cc        | 23 +++++++++++++++----
 src/fdb5/remote/RemoteStore.cc                |  7 +++---
 src/fdb5/remote/client/Client.h               |  3 ++-
 .../remote/client/ClientConnectionRouter.cc   | 13 ++++++++---
 .../remote/client/ClientConnectionRouter.h    |  6 +++--
 src/fdb5/toc/TocIndex.cc                      | 21 +++++++++--------
 src/fdb5/toc/TocIndex.h                       |  6 ++---
 10 files changed, 57 insertions(+), 31 deletions(-)

diff --git a/src/fdb5/database/UriStore.cc b/src/fdb5/database/UriStore.cc
index fbc78343f..7a50be58e 100644
--- a/src/fdb5/database/UriStore.cc
+++ b/src/fdb5/database/UriStore.cc
@@ -82,6 +82,7 @@ void UriStore::encode(eckit::Stream &s) const {
 UriStore::UriID UriStore::insert( const eckit::URI &path ) {
     ASSERT(!readOnly_);
 
+
     IdStore::iterator itr = ids_.find(path);
     if ( itr != ids_.end() )
         return itr->second;
diff --git a/src/fdb5/rados/RadosFieldLocation.cc b/src/fdb5/rados/RadosFieldLocation.cc
index 9c835769f..d61121341 100644
--- a/src/fdb5/rados/RadosFieldLocation.cc
+++ b/src/fdb5/rados/RadosFieldLocation.cc
@@ -35,8 +35,8 @@ RadosFieldLocation::RadosFieldLocation(const eckit::URI &uri, eckit::Offset offs
 RadosFieldLocation::RadosFieldLocation(const RadosFieldLocation& rhs) :
     FieldLocation(rhs.uri_) {}
 
-RadosFieldLocation::RadosFieldLocation(const FileStore &store, const FieldRef &ref) :
-    FieldLocation(store.get(ref.pathId()), ref.offset(), ref.length()) {}
+// RadosFieldLocation::RadosFieldLocation(const FileStore &store, const FieldRef &ref) :
+//     FieldLocation(store.get(ref.pathId()), ref.offset(), ref.length()) {}
 
 RadosFieldLocation::RadosFieldLocation(eckit::Stream& s) :
     FieldLocation(s) {}
diff --git a/src/fdb5/rados/RadosFieldLocation.h b/src/fdb5/rados/RadosFieldLocation.h
index b8f308549..3d03956de 100644
--- a/src/fdb5/rados/RadosFieldLocation.h
+++ b/src/fdb5/rados/RadosFieldLocation.h
@@ -20,7 +20,7 @@
 
 #include "fdb5/database/FieldLocation.h"
 #include "fdb5/database/FileStore.h"
-#include "fdb5/toc/FieldRef.h"
+//#include "fdb5/toc/FieldRef.h"
 
 namespace fdb5 {
 
@@ -34,7 +34,7 @@ class RadosFieldLocation : public FieldLocation {
     RadosFieldLocation(const eckit::PathName path, eckit::Offset offset, eckit::Length length);
     RadosFieldLocation(const eckit::URI &uri);
     RadosFieldLocation(const eckit::URI &uri, eckit::Offset offset, eckit::Length length);
-    RadosFieldLocation(const FileStore& store, const FieldRef& ref);
+//    RadosFieldLocation(const FileStore& store, const FieldRef& ref);
     RadosFieldLocation(eckit::Stream&);
 
 //    const eckit::PathName path() const { return uri_.name(); }
diff --git a/src/fdb5/remote/RemoteFieldLocation.cc b/src/fdb5/remote/RemoteFieldLocation.cc
index a98493335..35cf2979f 100644
--- a/src/fdb5/remote/RemoteFieldLocation.cc
+++ b/src/fdb5/remote/RemoteFieldLocation.cc
@@ -69,9 +69,23 @@ std::shared_ptr RemoteFieldLocation::make_shared() const {
 
 eckit::DataHandle* RemoteFieldLocation::dataHandle() const {
     
-//    ASSERT(remoteStore_);
-//    Connection conn = ClientConnectionRouter::instance().connectStore.connectStore(uri_);
-    return nullptr; //conn.->dataHandle(*internal_);
+    RemoteStore& store = ClientConnectionRouter::instance().store(uri_);
+    
+    const std::string scheme = uri_.query("internalScheme");
+    const std::string hostport = uri_.query("internalHost");
+    eckit::URI remote;
+    remote.query("internalScheme", "");
+    if (hostport.empty()) {
+        remote = eckit::URI(scheme, uri_, "",  -1);
+    }
+    else {
+        eckit::net::Endpoint endpoint{hostport};
+        remote = eckit::URI(scheme, uri_, endpoint.host(),  endpoint.port());
+        remote.query("internalHost", "");
+    }
+    FieldLocation* loc = FieldLocationFactory::instance().build(scheme, remote, offset_, length_, remapKey_);
+
+    return store.dataHandle(*loc);
 }
 
 void RemoteFieldLocation::visit(FieldLocationVisitor& visitor) const {
@@ -102,7 +116,8 @@ static FieldLocationBuilder builder("fdb");
 //----------------------------------------------------------------------------------------------------------------------
 
 class FdbURIManager : public eckit::URIManager {
-    virtual bool query() override { return false; }
+    bool authority() override { return true; }
+    virtual bool query() override { return true; }
     virtual bool fragment() override { return false; }
 
     virtual bool exists(const eckit::URI& f) override { return f.path().exists(); }
diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index ad80fad0b..aa307ddf0 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -58,7 +58,7 @@ RemoteStore::RemoteStore(const Key& dbKey, const Config& config) :
 }
 
 RemoteStore::RemoteStore(const eckit::URI& uri, const Config& config) :
-    Client(std::vector{eckit::net::Endpoint("localhost", 7000)}),
+    Client(std::vector{eckit::net::Endpoint(uri.hostport())}),
 //    Client(std::vector{ClientConnectionRouter::instance().connectStore(Key(), std::vector{eckit::net::Endpoint("localhost", 7000)})}),
     dbKey_(Key()), config_(config), dirty_(false), //archiveID_(0),
     maxArchiveQueueLength_(eckit::Resource("fdbRemoteArchiveQueueLength;$FDB_REMOTE_ARCHIVE_QUEUE_LENGTH", 200)),
@@ -618,6 +618,7 @@ eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation) {
 
 eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation, const Key& remapKey) {
 
+    ASSERT(endpoints_.size()==1);
     //connect();
 
     Buffer encodeBuffer(4096);
@@ -625,11 +626,9 @@ eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation, c
     s << fieldLocation;
     s << remapKey;
 
-//    uint32_t id = generateRequestID();
-
     uint32_t id = controlWriteCheckResponse(fdb5::remote::Message::Read, encodeBuffer, s.position());
 
-    return new FDBRemoteDataHandle(id, retrieveMessageQueue_, eckit::net::Endpoint("") /* controlEndpoint() */);
+    return new FDBRemoteDataHandle(id, retrieveMessageQueue_, endpoints_.at(0));
 }
 
 
diff --git a/src/fdb5/remote/client/Client.h b/src/fdb5/remote/client/Client.h
index b998f4ee9..55bb54ffa 100644
--- a/src/fdb5/remote/client/Client.h
+++ b/src/fdb5/remote/client/Client.h
@@ -52,7 +52,8 @@ class Client : eckit::NonCopyable {
     virtual void handleException(std::exception_ptr e) = 0;
 
     virtual Key key() = 0;
-private:
+
+protected:
 
     std::vector endpoints_;
     // Client(DataLinkID dataLinkID, RemoteHandlerID remoteHandlerID);
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.cc b/src/fdb5/remote/client/ClientConnectionRouter.cc
index 1a7514b5a..6925e0408 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.cc
+++ b/src/fdb5/remote/client/ClientConnectionRouter.cc
@@ -2,6 +2,7 @@
 #include 
 
 #include "fdb5/remote/client/ClientConnectionRouter.h"
+// #include "fdb5/remote/RemoteStore.h"
 
 using namespace eckit;
 using namespace eckit::net;
@@ -93,9 +94,15 @@ void ClientConnectionRouter::controlRead(Client& client, uint32_t requestId, voi
 
 // }
 
-// Client& ClientConnectionRouter::store(eckit::URI uri) {
-//     StoreFactory::instance().build()
-// }
+RemoteStore& ClientConnectionRouter::store(const eckit::URI& uri) {
+    const std::string& endpoint = uri.hostport();
+    auto it = readStores_.find(endpoint);
+    if (it != readStores_.end()) {
+        return *(it->second);
+    }
+
+    return *(readStores_[endpoint] = std::unique_ptr(new RemoteStore(uri, Config())));
+}
 
 
 uint32_t ClientConnectionRouter::createConnection(std::vector& endpoints, Client& client, ClientConnection*& conn) {
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.h b/src/fdb5/remote/client/ClientConnectionRouter.h
index 902be73b1..9fb77b9d6 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.h
+++ b/src/fdb5/remote/client/ClientConnectionRouter.h
@@ -21,10 +21,10 @@
 #include "fdb5/remote/Messages.h"
 #include "fdb5/remote/client/Client.h"
 #include "fdb5/remote/client/ClientConnection.h"
+#include "fdb5/remote/RemoteStore.h"
 
 namespace fdb5::remote {
 
-class Client;
 //----------------------------------------------------------------------------------------------------------------------
 
 // class Connection {
@@ -48,7 +48,7 @@ class ClientConnectionRouter : eckit::NonCopyable {
     static ClientConnectionRouter& instance();
 //    Connection connectCatalogue(Key dbKey, const eckit::Configuration& config);
 //    Connection connectStore(Key dbKey, const std::vector& endpoints);
-//    RemoteStore& store(eckit::URI uri);
+    RemoteStore& store(const eckit::URI& uri);
 
 // protected:
 
@@ -104,6 +104,8 @@ class ClientConnectionRouter : eckit::NonCopyable {
     // endpoint ->  remoteId>
 //    std::map, std::map > connections_;
     std::map > connections_;
+
+    std::map > readStores_;
     
     // // not owning
     // ClientConnection* catalogueConnection_;
diff --git a/src/fdb5/toc/TocIndex.cc b/src/fdb5/toc/TocIndex.cc
index 2edccb4ca..0c37ab70c 100644
--- a/src/fdb5/toc/TocIndex.cc
+++ b/src/fdb5/toc/TocIndex.cc
@@ -73,7 +73,7 @@ TocIndex::~TocIndex() {
 }
 
 void TocIndex::encode(eckit::Stream& s, const int version) const {
-    files_.encode(s);
+    uris_.encode(s);
     IndexBase::encode(s, version);
 }
 
@@ -84,7 +84,8 @@ bool TocIndex::get(const InspectionKey &key, const Key &remapKey, Field &field)
 
     bool found = btree_->get(key.valuesToString(), ref);
     if ( found ) {
-        const eckit::URI& uri = files_.get(ref.uriId());
+        const eckit::URI& uri = uris_.get(ref.uriId());
+        std::cout << "TocIndex::get " << uri << std::endl;
         FieldLocation* loc = FieldLocationFactory::instance().build(uri.scheme(), uri, ref.offset(), ref.length(), remapKey);
         field = Field(std::move(*loc), timestamp_, ref.details());
         delete(loc);
@@ -128,7 +129,7 @@ void TocIndex::add(const InspectionKey &key, const Field &field) {
     ASSERT(btree_);
     ASSERT( mode_ == TocIndex::WRITE );
 
-    FieldRef ref(files_, field);
+    FieldRef ref(uris_, field);
 
     //  bool replace =
     btree_->set(key.valuesToString(), ref); // returns true if replace, false if new insert
@@ -165,16 +166,16 @@ void TocIndex::funlock() const {
 }
 
 class TocIndexVisitor : public BTreeIndexVisitor {
-    const UriStore &files_;
+    const UriStore &uris_;
     EntryVisitor &visitor_;
 public:
-    TocIndexVisitor(const UriStore &files, EntryVisitor &visitor):
-        files_(files),
+    TocIndexVisitor(const UriStore &uris, EntryVisitor &visitor):
+        uris_(uris),
         visitor_(visitor) {}
 
     void visit(const std::string& keyFingerprint, const FieldRef& ref) {
 
-        Field field(TocFieldLocation(files_, ref), visitor_.indexTimestamp(), ref.details());
+        Field field(TocFieldLocation(uris_, ref), visitor_.indexTimestamp(), ref.details());
         visitor_.visitDatum(field, keyFingerprint);
     }
 };
@@ -186,7 +187,7 @@ void TocIndex::entries(EntryVisitor &visitor) const {
     // Allow the visitor to selectively decline to visit the entries in this index
     if (visitor.visitIndex(instantIndex)) {
         TocIndexCloser closer(*this);
-        TocIndexVisitor v(files_, visitor);
+        TocIndexVisitor v(uris_, visitor);
         btree_->visit(v);
     }
 }
@@ -201,7 +202,7 @@ std::string TocIndex::defaulType() {
 }
 
 const std::vector TocIndex::dataPaths() const {
-    return files_.paths();
+    return uris_.paths();
 }
 
 bool TocIndex::dirty() const {
@@ -229,7 +230,7 @@ void TocIndex::dump(std::ostream &out, const char* indent, bool simple, bool dum
 
     if(!simple) {
         out << std::endl;
-        files_.dump(out, indent);
+        uris_.dump(out, indent);
         axes_.dump(out, indent);
     }
 
diff --git a/src/fdb5/toc/TocIndex.h b/src/fdb5/toc/TocIndex.h
index 9361c2da8..4c2e5854e 100644
--- a/src/fdb5/toc/TocIndex.h
+++ b/src/fdb5/toc/TocIndex.h
@@ -42,10 +42,10 @@ class BTreeIndex;
 
 struct UriStoreWrapper {
 
-    UriStoreWrapper(const eckit::PathName& directory) : files_(directory) {}
-    UriStoreWrapper(const eckit::PathName& directory, eckit::Stream& s) : files_(directory, s) {}
+    UriStoreWrapper(const eckit::PathName& directory) : uris_(directory) {}
+    UriStoreWrapper(const eckit::PathName& directory, eckit::Stream& s) : uris_(directory, s) {}
 
-    UriStore files_;
+    UriStore uris_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------

From e9df4eeabeb1079225636717eecf3e38065c0a68 Mon Sep 17 00:00:00 2001
From: Chris Bradley 
Date: Thu, 26 Oct 2023 12:15:51 +0100
Subject: [PATCH 009/186] Remote catalogue write pathway wip

---
 src/fdb5/CMakeLists.txt                       |   6 +-
 src/fdb5/database/ArchiveVisitor.cc           |   5 +-
 src/fdb5/database/Catalogue.cc                |   9 +-
 src/fdb5/database/Catalogue.h                 |   1 +
 src/fdb5/database/Store.cc                    |   4 +-
 src/fdb5/remote/CatalogueHandler.cc           | 263 ------------
 src/fdb5/remote/FdbServer.cc                  |   5 +-
 src/fdb5/remote/Messages.h                    |   1 +
 src/fdb5/remote/RemoteCatalogue.cc            | 265 ++++++++++++
 src/fdb5/remote/RemoteCatalogue.h             | 101 +++++
 src/fdb5/remote/server/CatalogueHandler.cc    | 376 ++++++++++++++++++
 .../remote/{ => server}/CatalogueHandler.h    |  27 +-
 src/fdb5/toc/TocCatalogueWriter.cc            |   5 +
 src/fdb5/toc/TocCatalogueWriter.h             |   1 +
 14 files changed, 784 insertions(+), 285 deletions(-)
 delete mode 100644 src/fdb5/remote/CatalogueHandler.cc
 create mode 100644 src/fdb5/remote/RemoteCatalogue.cc
 create mode 100644 src/fdb5/remote/RemoteCatalogue.h
 create mode 100644 src/fdb5/remote/server/CatalogueHandler.cc
 rename src/fdb5/remote/{ => server}/CatalogueHandler.h (63%)

diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt
index 500023ba6..474b54fbb 100644
--- a/src/fdb5/CMakeLists.txt
+++ b/src/fdb5/CMakeLists.txt
@@ -214,6 +214,8 @@ if(HAVE_FDB_REMOTE)
         # api/RemoteFDB.h
         remote/RemoteStore.cc
         remote/RemoteStore.h
+        remote/RemoteCatalogue.h
+        remote/RemoteCatalogue.cc
 
         remote/client/Client.h
         remote/client/Client.cc
@@ -221,6 +223,8 @@ if(HAVE_FDB_REMOTE)
         remote/client/ClientConnection.cc
         remote/client/ClientConnectionRouter.h
         remote/client/ClientConnectionRouter.cc
+        remote/server/CatalogueHandler.h
+        remote/server/CatalogueHandler.cc
         remote/server/StoreHandler.h
         remote/server/StoreHandler.cc
         remote/server/ServerConnection.h
@@ -238,8 +242,6 @@ if(HAVE_FDB_REMOTE)
         remote/FdbServer.h
         remote/FdbServer.cc
 
-        # remote/CatalogueHandler.h
-        # remote/CatalogueHandler.cc
         # remote/StoreHandler.h
         # remote/StoreHandler.cc
         # remote/DecoupledHandler.h
diff --git a/src/fdb5/database/ArchiveVisitor.cc b/src/fdb5/database/ArchiveVisitor.cc
index ce72d2dba..e9802a77d 100644
--- a/src/fdb5/database/ArchiveVisitor.cc
+++ b/src/fdb5/database/ArchiveVisitor.cc
@@ -26,11 +26,10 @@ bool ArchiveVisitor::selectDatum(const InspectionKey &key, const Key &full) {
 
     // eckit::Log::info() << "selectDatum " << key << ", " << full << " " << size_ << std::endl;
     checkMissingKeys(full);
-
-    const Index& idx = current()->currentIndex();
+    const Key idxKey = current()->currentIndexKey();
 
     // here we could create a queue... and keep accepting archival request until the queue is full
-    auto futureLocation = store()->archive(idx.key(), data_, size_);
+    auto futureLocation = store()->archive(idxKey, data_, size_);
     current()->archive(key, futureLocation.get());
 
     return true;
diff --git a/src/fdb5/database/Catalogue.cc b/src/fdb5/database/Catalogue.cc
index ad76f872c..67dadf10f 100644
--- a/src/fdb5/database/Catalogue.cc
+++ b/src/fdb5/database/Catalogue.cc
@@ -30,11 +30,10 @@ std::ostream &operator<<(std::ostream &s, const Catalogue &x) {
 }
 
 std::unique_ptr CatalogueImpl::buildStore() const {
-    if (buildByKey_)
+    if (buildByKey_){
         return StoreFactory::instance().build(key(), config_);
-    else {
+    } else {
         std::string name = config_.getString("store", "file");
-
         return StoreFactory::instance().build(eckit::URI(name, uri()), config_);
     }
 }
@@ -186,7 +185,7 @@ std::unique_ptr CatalogueWriterFactory::build(const Key& dbKey,
     eckit::AutoLock lock(mutex_);
     auto j = builders_.find(nameLowercase);
 
-    eckit::Log::debug() << "Looking for CatalogueWriterBuilder [" << nameLowercase << "]" << std::endl;
+    eckit::Log::debug() << "Looking for CatalogueWriterBuilder [" << nameLowercase << "]" << std::endl;
 
     if (j == builders_.end()) {
         eckit::Log::error() << "No CatalogueWriterBuilder for [" << nameLowercase << "]" << std::endl;
@@ -206,7 +205,7 @@ std::unique_ptr CatalogueWriterFactory::build(const eckit::URI&
     eckit::AutoLock lock(mutex_);
     auto j = builders_.find(nameLowercase);
 
-    eckit::Log::debug() << "Looking for CatalogueWriterBuilder [" << nameLowercase << "]" << std::endl;
+    eckit::Log::debug() << "Looking for CatalogueWriterBuilder [" << nameLowercase << "]" << std::endl;
 
     if (j == builders_.end()) {
         eckit::Log::error() << "No CatalogueWriterBuilder for [" << nameLowercase << "]" << std::endl;
diff --git a/src/fdb5/database/Catalogue.h b/src/fdb5/database/Catalogue.h
index 758baa439..d8f050f74 100644
--- a/src/fdb5/database/Catalogue.h
+++ b/src/fdb5/database/Catalogue.h
@@ -163,6 +163,7 @@ class CatalogueWriter : virtual public Catalogue  {
     virtual ~CatalogueWriter() {}
 
     virtual const Index& currentIndex() = 0;
+    virtual const Key currentIndexKey() = 0;
     virtual void archive(const InspectionKey& key, std::unique_ptr fieldLocation) = 0;
     virtual void overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) = 0;
     virtual void index(const InspectionKey& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) = 0;
diff --git a/src/fdb5/database/Store.cc b/src/fdb5/database/Store.cc
index a87eb68ab..3c83951e3 100644
--- a/src/fdb5/database/Store.cc
+++ b/src/fdb5/database/Store.cc
@@ -79,7 +79,7 @@ std::unique_ptr StoreFactory::build(const Key& key, const Config& config)
     eckit::AutoLock lock(mutex_);
     auto j = builders_.find(nameLowercase);
 
-    eckit::Log::debug() << "Looking for StoreBuilder [" << nameLowercase << "]" << std::endl;
+    eckit::Log::debug() << "Looking for StoreBuilder [" << nameLowercase << "]" << std::endl;
 
     if (j == builders_.end()) {
         eckit::Log::error() << "No StoreBuilder for [" << nameLowercase << "]" << std::endl;
@@ -99,7 +99,7 @@ std::unique_ptr StoreFactory::build(const eckit::URI& uri, const Config&
     eckit::AutoLock lock(mutex_);
     auto j = builders_.find(nameLowercase);
 
-    eckit::Log::debug() << "Looking for StoreBuilder [" << nameLowercase << "]" << std::endl;
+    eckit::Log::debug() << "Looking for StoreBuilder [" << nameLowercase << "]" << std::endl;
 
     if (j == builders_.end()) {
         eckit::Log::error() << "No StoreBuilder for [" << nameLowercase << "]" << std::endl;
diff --git a/src/fdb5/remote/CatalogueHandler.cc b/src/fdb5/remote/CatalogueHandler.cc
deleted file mode 100644
index 17a5cda0c..000000000
--- a/src/fdb5/remote/CatalogueHandler.cc
+++ /dev/null
@@ -1,263 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-#include "fdb5/remote/CatalogueHandler.h"
-
-#include "eckit/serialisation/MemoryStream.h"
-
-#include "fdb5/LibFdb5.h"
-#include "fdb5/api/FDBFactory.h"
-#include "fdb5/api/helpers/FDBToolRequest.h"
-
-using namespace eckit;
-using metkit::mars::MarsRequest;
-
-namespace fdb5::remote {
-
-namespace {
-
-// ***************************************************************************************
-// All of the standard API functions behave in roughly the same manner. The Helper Classes
-// described here capture the manner in which their behaviours differ.
-//
-// See forwardApiCall() for how these helpers are used to forward an API call using a
-// worker thread.
-//
-// ***************************************************************************************
-//
-// Note that we use the "control" and "data" connections in a specific way, although these
-// may not be the optimal names for them. "control" is used for blocking requests,
-// and "data" is used for non-blocking activity.
-//
-// ***************************************************************************************
-
-template 
-struct BaseHelper {
-    virtual size_t encodeBufferSize(const ValueType&) const { return 4096; }
-    void extraDecode(eckit::Stream&) {}
-    ValueType apiCall(FDBCatalogueBase&, const FDBToolRequest&) const { NOTIMP; }
-
-    struct Encoded {
-        size_t position;
-        eckit::Buffer buf;
-    };
-
-    Encoded encode(const ValueType& elem, const DecoupledHandler&) const {
-        eckit::Buffer encodeBuffer(encodeBufferSize(elem));
-        MemoryStream s(encodeBuffer);
-        s << elem;
-        return {s.position(), std::move(encodeBuffer)};
-    }
-};
-
-struct ListHelper : public BaseHelper {
-    ListIterator apiCall(FDBCatalogueBase& fdb, const FDBToolRequest& request) const {
-        return fdb.list(request);
-    }
-};
-
-struct InspectHelper : public BaseHelper {
-    ListIterator apiCall(FDBCatalogueBase& fdb, const FDBToolRequest& request) const {
-        return fdb.inspect(request.request());
-    }
-};
-
-}
-
-
-CatalogueHandler::CatalogueHandler(eckit::net::TCPSocket& socket, const Config& config):
-    DecoupledHandler(socket, config) {
-
-}
-CatalogueHandler::~CatalogueHandler() {}
-
-void CatalogueHandler::initialiseConnections() {
-    DecoupledHandler::initialiseConnections();
-
-    std::string storeHost = ::getenv("FDB_STORE_HOST") ? ::getenv("FDB_STORE_HOST") : "localhost";
-    std::string storePort = ::getenv("FDB_STORE_PORT") ? ::getenv("FDB_STORE_PORT") : "7000";
-    net::Endpoint storeEndpoint(storeHost, std::stoi(storePort));
-
-
-    // Log::info() << "Sending store endpoint to client: " << storeEndpoint << std::endl;
-    // {
-    //     Buffer startupBuffer(1024);
-    //     MemoryStream s(startupBuffer);
-
-    //     s << clientSession;
-    //     s << sessionID_;
-    //     s << storeEndpoint;
-
-    //     // s << storeEndpoint; // xxx single-store case only: we cant do this with multiple stores // For now, dont send the store endpoint to the client 
-
-    //     Log::debug() << "Protocol negotiation - configuration: " << agreedConf_ < tail;
-
-    // listen loop
-    while (true) {
-        tidyWorkers();
-
-        socketRead(&hdr, sizeof(hdr), controlSocket_);
-
-        ASSERT(hdr.marker == StartMarker);
-        ASSERT(hdr.version == CurrentVersion);
-        Log::debug() << "Got message with request ID: " << hdr.requestID << std::endl;
-
-        try {
-            switch (hdr.message) {
-                case Message::Exit:
-                    Log::status() << "Exiting" << std::endl;
-                    Log::info() << "Exiting" << std::endl;
-                    return;
-
-                case Message::List:
-                    forwardApiCall(hdr);
-                    break;
-
-                case Message::Dump:
-                    NOTIMP;
-                    break;
-
-                case Message::Purge:
-                    NOTIMP;
-                    break;
-
-                case Message::Stats:
-                    NOTIMP;
-                    break;
-
-                case Message::Status:
-                    NOTIMP;
-                    break;
-
-                case Message::Wipe:
-                    NOTIMP;
-                    break;
-
-                case Message::Control:
-                    NOTIMP;
-                    break;
-
-                case Message::Inspect:
-                    forwardApiCall(hdr);
-                    break;
-
-                case Message::Read: 
-                    read(hdr);
-                    break;
-
-                case Message::Flush:
-                    flush(hdr);
-                    break;
-
-                case Message::Archive:
-                    archive(hdr);
-                    break;
-
-                // case Message::Store:
-                //     store(hdr);
-                //     break;
-
-                default: {
-                    std::stringstream ss;
-                    ss << "ERROR: Unexpected message recieved (" << static_cast(hdr.message)
-                       << "). ABORTING";
-                    Log::status() << ss.str() << std::endl;
-                    Log::error() << "Retrieving... " << ss.str() << std::endl;
-                    throw SeriousBug(ss.str(), Here());
-                }
-            }
-
-            // Ensure we have consumed exactly the correct amount from the socket.
-            socketRead(&tail, sizeof(tail), controlSocket_);
-            ASSERT(tail == EndMarker);
-
-            // Acknowledge receipt of command
-            controlWrite(Message::Received, hdr.requestID);
-        }
-        catch (std::exception& e) {
-            // n.b. more general than eckit::Exception
-            std::string what(e.what());
-            controlWrite(Message::Error, hdr.requestID, what.c_str(), what.length());
-        }
-        catch (...) {
-            std::string what("Caught unexpected and unknown error");
-            controlWrite(Message::Error, hdr.requestID, what.c_str(), what.length());
-        }
-    }
-}
-
-void CatalogueHandler::index(const MessageHeader& hdr) {
-    NOTIMP;
-}
-
-
-// API
-
-template 
-void CatalogueHandler::forwardApiCall(const MessageHeader& hdr) {
-    HelperClass helper;
-
-//    std::cout << "CatalogueHandler::forwardApiCall" << std::endl;
-
-    Buffer payload(receivePayload(hdr, controlSocket_));
-    MemoryStream s(payload);
-
-    FDBToolRequest request(s);
-    helper.extraDecode(s);
-
-    // Construct worker thread to feed responses back to client
-
-    ASSERT(catalogue_);
-    ASSERT(workerThreads_.find(hdr.requestID) == workerThreads_.end());
-
-    workerThreads_.emplace(
-        hdr.requestID, std::async(std::launch::async, [request, hdr, helper, this]() {
-            try {
-                auto iterator = helper.apiCall(*catalogue_, request);
-
-                typename decltype(iterator)::value_type elem;
-                while (iterator.next(elem)) {
-                    auto encoded(helper.encode(elem, *this));
-                    dataWrite(Message::Blob, hdr.requestID, encoded.buf, encoded.position);
-                }
-
-                dataWrite(Message::Complete, hdr.requestID);
-            }
-            catch (std::exception& e) {
-                // n.b. more general than eckit::Exception
-                std::string what(e.what());
-                dataWrite(Message::Error, hdr.requestID, what.c_str(), what.length());
-            }
-            catch (...) {
-                // We really don't want to std::terminate the thread
-                std::string what("Caught unexpected, unknown exception in worker");
-                dataWrite(Message::Error, hdr.requestID, what.c_str(), what.length());
-            }
-        }));
-}
-
-}  // namespace fdb5::remote
diff --git a/src/fdb5/remote/FdbServer.cc b/src/fdb5/remote/FdbServer.cc
index 5de9e8875..5bc2c27be 100644
--- a/src/fdb5/remote/FdbServer.cc
+++ b/src/fdb5/remote/FdbServer.cc
@@ -22,6 +22,7 @@
 // #include "fdb5/remote/Handler.h"
 //#include "fdb5/remote/CatalogueHandler.h"
 #include "fdb5/remote/server/StoreHandler.h"
+#include "fdb5/remote/server/CatalogueHandler.h"
 #include "fdb5/remote/server/ServerConnection.h"
 #include "eckit/config/Resource.h"
 
@@ -55,8 +56,8 @@ void FDBForker::run() {
     
     if (config_.getString("type", "local") == "catalogue" || (::getenv("FDB_IS_CAT") && ::getenv("FDB_IS_CAT")[0] == '1')) {
         eckit::Log::info() << "FDB using Catalogue Handler" << std::endl;
-        // CatalogueHandler handler(socket_, config_);
-        // handler.handle();
+        CatalogueHandler handler(socket_, config_);
+        handler.handle();
     } 
     else if (config_.getString("type", "local") == "store" || (::getenv("FDB_IS_STORE") && ::getenv("FDB_IS_STORE")[0] == '1')) {
         eckit::Log::info() << "FDB using Store Handler" << std::endl;
diff --git a/src/fdb5/remote/Messages.h b/src/fdb5/remote/Messages.h
index d7176190b..a4330e66c 100644
--- a/src/fdb5/remote/Messages.h
+++ b/src/fdb5/remote/Messages.h
@@ -63,6 +63,7 @@ enum class Message : uint16_t {
     Read,
     Move,
     Store,
+    Schema,
 
     // Responses
     Received = 200,
diff --git a/src/fdb5/remote/RemoteCatalogue.cc b/src/fdb5/remote/RemoteCatalogue.cc
new file mode 100644
index 000000000..4bcb0fc94
--- /dev/null
+++ b/src/fdb5/remote/RemoteCatalogue.cc
@@ -0,0 +1,265 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+#include "eckit/config/Resource.h"
+#include "eckit/log/Log.h"
+
+#include "fdb5/LibFdb5.h"
+#include "fdb5/remote/RemoteCatalogue.h"
+#include "eckit/serialisation/MemoryStream.h"
+
+#include  // xxx debug / development
+#include  // xxx debug / development
+
+
+using namespace eckit;
+namespace fdb5::remote {
+
+RemoteCatalogue::RemoteCatalogue(const Key& key, const Config& config):
+    CatalogueImpl(key, ControlIdentifiers(), config), // xxx what are control identifiers? Setting empty here...
+    Client(eckit::net::Endpoint("localhost", 7001)), // xxx hardcoded endpoint
+    dbKey_(key), config_(config), schema_(nullptr),
+    maxArchiveQueueLength_(eckit::Resource("fdbRemoteArchiveQueueLength;$FDB_REMOTE_ARCHIVE_QUEUE_LENGTH", 200))
+    {
+        loadSchema();
+    }
+
+RemoteCatalogue::RemoteCatalogue(const eckit::URI& uri, const Config& config):
+    Client(eckit::net::Endpoint("localhost", 7001)),  schema_(nullptr),
+    maxArchiveQueueLength_(eckit::Resource("fdbRemoteArchiveQueueLength;$FDB_REMOTE_ARCHIVE_QUEUE_LENGTH", 200))
+    {
+        NOTIMP;
+    }
+
+
+void RemoteCatalogue::sendArchiveData(uint32_t id, const Key& key, std::unique_ptr fieldLocation)
+{
+    ASSERT(!dbKey_.empty());
+    ASSERT(!currentIndexKey_.empty());
+    ASSERT(!key.empty());
+    ASSERT(fieldLocation);
+
+    Buffer keyBuffer(4096);
+    MemoryStream keyStream(keyBuffer);
+    keyStream << dbKey_;
+    keyStream << currentIndexKey_;
+    keyStream << key;
+
+    Buffer locBuffer(4096);
+    MemoryStream locStream(locBuffer);
+    locStream << *fieldLocation;
+
+    MessageHeader message(Message::Blob, 0, id, keyStream.position() + locStream.position());
+    dataWrite(id, &message, sizeof(message));
+    dataWrite(id, keyBuffer, keyStream.position());
+    dataWrite(id, locBuffer, locStream.position());
+    dataWrite(id, &EndMarker, sizeof(EndMarker));
+}
+
+FDBStats RemoteCatalogue::archiveThreadLoop(){
+    FDBStats localStats;
+    eckit::Timer timer;
+
+    std::pair > >  element;
+
+    try {
+
+        ASSERT(archiveQueue_);
+        long popped;
+        while ((popped = archiveQueue_->pop(element)) != -1) {
+            timer.start();
+            eckit::Log::debug() << " RemoteCatalogue::archiveThreadLoop() popped: " << popped << " id: " << element.first << 
+                " key: " << element.second.first << " fieldLocation: " << element.second.second->uri() << std::endl;
+            sendArchiveData(element.first, element.second.first, std::move(element.second.second));
+            timer.stop();
+
+            localStats.addArchive(0, timer, 1);
+
+        }
+
+        dataWrite(Message::Flush, nullptr, 0);
+        archiveQueue_.reset();
+
+    } catch (...) {
+        archiveQueue_->interrupt(std::current_exception());
+        throw;
+    }
+
+    return localStats;
+}
+
+void RemoteCatalogue::archive(const InspectionKey& key, std::unique_ptr fieldLocation) {
+    
+    // if there is no archiving thread active, then start one.
+    // n.b. reset the archiveQueue_ after a potential flush() cycle.
+    if (!archiveFuture_.valid()) {
+
+        // Start the archival request on the remote side
+
+        // Reset the queue after previous done/errors
+        {
+            std::lock_guard lock(archiveQueuePtrMutex_);
+            ASSERT(!archiveQueue_);
+            archiveQueue_.reset(new ArchiveQueue(maxArchiveQueueLength_));
+        }
+
+        archiveFuture_ = std::async(std::launch::async, [this] { return archiveThreadLoop(); });
+    }
+
+    ASSERT(archiveFuture_.valid());
+
+    // xxx I don't think we need to to this controlwrite everytime? On the remote side, the first call starts the archive thread loop.
+    // xxx repeated calls dont do anything? Its already listening for archive requests on the data connection after the first call.
+    // xxx so, try only doing this call on the first archive request.
+    uint32_t id = controlWriteCheckResponse(Message::Archive); 
+
+    std::lock_guard lock(archiveQueuePtrMutex_);
+    ASSERT(archiveQueue_);
+
+    eckit::Log::debug() << " RemoteCatalogue::archive() added to queue with id: " << id << 
+        " key: " << key << " fieldLocation: " << fieldLocation->uri() << std::endl;
+
+    archiveQueue_->emplace(std::make_pair(id, std::make_pair(key, std::move(fieldLocation))));
+
+}
+
+bool RemoteCatalogue::selectIndex(const Key& idxKey) {
+    currentIndexKey_ = idxKey;
+    return true; // xxx whats the return used for? TOC always returns true
+}
+
+const Index& RemoteCatalogue::currentIndex(){
+    return nullptr;
+}
+const Key RemoteCatalogue::currentIndexKey() {
+    return currentIndexKey_;
+}
+
+void RemoteCatalogue::deselectIndex() {
+    currentIndexKey_ = Key();
+}
+const Schema& RemoteCatalogue::schema() const {
+    ASSERT(schema_);
+    return *schema_;
+}
+
+void RemoteCatalogue::flush() {
+    
+    Timer timer;
+
+    timer.start();
+
+    // Flush only does anything if there is an ongoing archive();
+    if (archiveFuture_.valid()) {
+
+        ASSERT(archiveQueue_);
+        std::lock_guard lock(archiveQueuePtrMutex_);
+        archiveQueue_->close();
+
+        FDBStats stats = archiveFuture_.get();
+        ASSERT(!archiveQueue_);
+
+        ASSERT(stats.numFlush() == 0);
+        size_t numArchive = stats.numArchive();
+
+        Buffer sendBuf(4096);
+        MemoryStream s(sendBuf);
+        s << numArchive;
+
+        // The flush call is blocking
+        controlWriteCheckResponse(fdb5::remote::Message::Flush, sendBuf, s.position());
+
+//        internalStats_ += stats;
+    }
+
+    timer.stop();
+//    internalStats_.addFlush(timer);
+}
+
+void RemoteCatalogue::clean() {NOTIMP;}
+
+void RemoteCatalogue::close() {NOTIMP;}
+
+bool RemoteCatalogue::exists() const {NOTIMP;}
+
+void RemoteCatalogue::checkUID() const {NOTIMP;}
+
+// xxx what is this uri used for?
+eckit::URI RemoteCatalogue::uri() const {
+    // return eckit::URI(TocEngine::typeName(), basePath());
+    return eckit::URI("remotecat", ""); // todo
+}
+
+void RemoteCatalogue::loadSchema() {
+    // NB we're at the db level, so get the db schema. We will want to get the master schema beforehand.
+    // (outside of the catalogue) 
+
+    eckit::Log::debug() << "RemoteCatalogueWriter::loadSchema()" << std::endl;
+
+    // destroy previous schema if it exists
+    schema_.reset(nullptr);
+
+    // send dbkey to remote.
+    constexpr uint32_t bufferSize = 4096;
+    eckit::Buffer keyBuffer(bufferSize);
+    eckit::MemoryStream keyStream(keyBuffer);
+    keyStream << dbKey_;
+    
+    uint32_t id = controlWriteCheckResponse(Message::Schema, keyBuffer, keyStream.position());
+    eckit::Log::debug() << "RemoteCatalogueWriter::loadSchema() received id: " << id << std::endl;
+
+    // Should block - Use control connection.
+    // XXX This is a temporary work around while control read is being reimplemented.
+    while (!schema_) {
+        std::this_thread::sleep_for(std::chrono::milliseconds(10));
+    }
+
+}
+
+bool RemoteCatalogue::handle(Message message, uint32_t requestID) {
+    eckit::Log::debug() << "RemoteCatalogue::handle received message: " << ((uint) message) << " - requestID: " << requestID << std::endl;
+    NOTIMP;
+    return false;
+}
+bool RemoteCatalogue::handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) {
+    eckit::Log::debug()<< "RemoteCatalogue::handle received message: " << ((uint) message) << " - requestID: " << requestID << std::endl;
+    if (message == Message::Schema) {
+        eckit::Log::debug() << "RemoteCatalogue::handle received payload size: " << payload.size() << std::endl;
+        MemoryStream s(payload);
+        schema_ = std::unique_ptr(eckit::Reanimator::reanimate(s));
+        return true;
+    }
+    return false;
+}
+
+void RemoteCatalogue::handleException(std::exception_ptr e) {
+    NOTIMP;
+}
+
+void RemoteCatalogue::overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) {NOTIMP;}
+void RemoteCatalogue::index(const InspectionKey& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) {NOTIMP;}
+void RemoteCatalogue::reconsolidate(){NOTIMP;}
+std::vector RemoteCatalogue::metadataPaths() const {NOTIMP;}
+void RemoteCatalogue::visitEntries(EntryVisitor& visitor, /*const Store& store,*/ bool sorted) {NOTIMP;}
+void RemoteCatalogue::dump(std::ostream& out, bool simple, const eckit::Configuration& conf) const {NOTIMP;}
+StatsReportVisitor* RemoteCatalogue::statsReportVisitor() const {NOTIMP;}
+PurgeVisitor* RemoteCatalogue::purgeVisitor(const Store& store) const {NOTIMP;}
+WipeVisitor* RemoteCatalogue::wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, std::ostream& out, bool doit, bool porcelain, bool unsafeWipeAll) const {NOTIMP;}
+MoveVisitor* RemoteCatalogue::moveVisitor(const Store& store, const metkit::mars::MarsRequest& request, const eckit::URI& dest, eckit::Queue& queue) const {NOTIMP;}
+void RemoteCatalogue::control(const ControlAction& action, const ControlIdentifiers& identifiers) const {NOTIMP;}
+std::vector RemoteCatalogue::indexes(bool sorted) const {NOTIMP;}
+void RemoteCatalogue::maskIndexEntry(const Index& index) const {NOTIMP;}
+void RemoteCatalogue::allMasked(std::set>& metadata, std::set& data) const {NOTIMP;}
+void RemoteCatalogue::print( std::ostream &out ) const {NOTIMP;}
+std::string RemoteCatalogue::type() const {NOTIMP;}
+bool RemoteCatalogue::open() {NOTIMP;}
+
+static CatalogueWriterBuilder builder("remote");
+} // namespace fdb5::remote
\ No newline at end of file
diff --git a/src/fdb5/remote/RemoteCatalogue.h b/src/fdb5/remote/RemoteCatalogue.h
new file mode 100644
index 000000000..82dc10a7f
--- /dev/null
+++ b/src/fdb5/remote/RemoteCatalogue.h
@@ -0,0 +1,101 @@
+
+#pragma once
+
+#include "fdb5/api/FDBStats.h"
+#include "fdb5/database/Catalogue.h"
+#include "fdb5/database/Index.h"
+#include "fdb5/database/Store.h"
+#include "fdb5/remote/client/Client.h"
+
+namespace fdb5::remote {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+class RemoteCatalogue : public CatalogueWriter, public CatalogueImpl, public Client {
+public: //types
+    using ArchiveQueue = eckit::Queue > > >;
+
+public:
+
+    RemoteCatalogue(const Key& key, const Config& config);
+    
+    RemoteCatalogue(const eckit::URI& uri, const Config& config);
+
+    ~RemoteCatalogue() {}
+
+
+    // From CatalogueWriter
+    const Index& currentIndex() override;
+    void archive(const InspectionKey& key, std::unique_ptr fieldLocation) override;
+    void overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) override;
+    void index(const InspectionKey& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) override;
+    void reconsolidate() override;
+
+    // From Catalogue
+    bool selectIndex(const Key& idxKey) override;
+    const Key currentIndexKey() override;
+    void deselectIndex() override;
+    const Schema& schema() const override;
+
+    std::vector metadataPaths() const override;
+    void visitEntries(EntryVisitor& visitor, /*const Store& store,*/ bool sorted = false) override;
+    void dump(std::ostream& out, bool simple=false, const eckit::Configuration& conf = eckit::LocalConfiguration()) const override;
+    StatsReportVisitor* statsReportVisitor() const override;
+    PurgeVisitor* purgeVisitor(const Store& store) const override;
+    WipeVisitor* wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, std::ostream& out, bool doit, bool porcelain, bool unsafeWipeAll) const override;
+    MoveVisitor* moveVisitor(const Store& store, const metkit::mars::MarsRequest& request, const eckit::URI& dest, eckit::Queue& queue) const override;
+    void control(const ControlAction& action, const ControlIdentifiers& identifiers) const override;
+    std::vector indexes(bool sorted=false) const override;
+    void maskIndexEntry(const Index& index) const override;
+    void allMasked(std::set>& metadata, std::set& data) const override;
+    void print( std::ostream &out ) const override;
+    std::string type() const override;
+    bool open() override;
+    void flush() override;
+    void clean() override;
+    void close() override;
+    bool exists() const override;
+    void checkUID() const override;
+    eckit::URI uri() const override;
+
+
+protected:
+
+    Schema getSchema();
+    void loadSchema() override;
+
+private:
+    // From Client
+
+    // handlers for incoming messages - to be defined in the client class
+    bool handle(Message message, uint32_t requestID) override;
+    bool handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) override;
+    void handleException(std::exception_ptr e) override;
+    Key key() override { return dbKey_; } // xxx this is seperately declared by Client and CatalogueImpl
+
+
+    // note: in common with remotestore
+    FDBStats archiveThreadLoop();
+    void sendArchiveData(uint32_t id, const Key& key, std::unique_ptr fieldLocation);
+
+protected:
+
+    Key dbKey_;
+    Config config_;
+    ControlIdentifiers controlIdentifiers_;
+
+private:
+    Key currentIndexKey_;
+    std::unique_ptr schema_;
+
+    size_t maxArchiveQueueLength_;
+    std::mutex archiveQueuePtrMutex_;
+    std::future archiveFuture_;
+    std::unique_ptr archiveQueue_;
+
+
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
+} // namespace fdb5::remote
diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc
new file mode 100644
index 000000000..e6c16468f
--- /dev/null
+++ b/src/fdb5/remote/server/CatalogueHandler.cc
@@ -0,0 +1,376 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+#include "eckit/config/Resource.h"
+#include "eckit/serialisation/MemoryStream.h"
+
+#include "fdb5/LibFdb5.h"
+#include "fdb5/api/FDBFactory.h"
+#include "fdb5/api/helpers/FDBToolRequest.h"
+#include "fdb5/remote/server/CatalogueHandler.h"
+
+using namespace eckit;
+using metkit::mars::MarsRequest;
+
+namespace fdb5::remote {
+
+
+// ***************************************************************************************
+//
+// Note that we use the "control" and "data" connections in a specific way, although these
+// may not be the optimal names for them. "control" is used for blocking requests,
+// and "data" is used for non-blocking activity.
+//
+// ***************************************************************************************
+
+CatalogueHandler::CatalogueHandler(eckit::net::TCPSocket& socket, const Config& config):
+    ServerConnection(socket, config) {
+
+}
+CatalogueHandler::~CatalogueHandler() {}
+
+void CatalogueHandler::initialiseConnections() {
+    ServerConnection::initialiseConnections();
+
+    std::string storeHost = ::getenv("FDB_STORE_HOST") ? ::getenv("FDB_STORE_HOST") : "localhost";
+    std::string storePort = ::getenv("FDB_STORE_PORT") ? ::getenv("FDB_STORE_PORT") : "7000";
+    net::Endpoint storeEndpoint(storeHost, std::stoi(storePort));
+
+
+    // Log::info() << "Sending store endpoint to client: " << storeEndpoint << std::endl;
+    // {
+    //     Buffer startupBuffer(1024);
+    //     MemoryStream s(startupBuffer);
+
+    //     s << clientSession;
+    //     s << sessionID_;
+    //     s << storeEndpoint;
+
+    //     // s << storeEndpoint; // xxx single-store case only: we cant do this with multiple stores // For now, dont send the store endpoint to the client 
+
+    //     Log::debug() << "Protocol negotiation - configuration: " << agreedConf_ < tail;
+
+    // listen loop
+    while (true) {
+        tidyWorkers();
+
+        Log::debug() << "CatalogueHandler::handle() - pre READ" << std::endl;
+        socketRead(&hdr, sizeof(hdr), controlSocket_);
+
+        ASSERT(hdr.marker == StartMarker);
+        ASSERT(hdr.version == CurrentVersion);
+        Log::debug() << "Got message with request ID: " << hdr.requestID << std::endl;
+
+        try {
+            switch (hdr.message) {
+                case Message::Exit:
+                    Log::status() << "Exiting" << std::endl;
+                    Log::info() << "Exiting" << std::endl;
+                    return;
+
+                case Message::List:
+                    // forwardApiCall(hdr);
+                    list(hdr);
+                    break;
+
+                case Message::Dump:
+                    NOTIMP;
+                    break;
+
+                case Message::Purge:
+                    NOTIMP;
+                    break;
+
+                case Message::Stats:
+                    NOTIMP;
+                    break;
+
+                case Message::Status:
+                    NOTIMP;
+                    break;
+
+                case Message::Wipe:
+                    NOTIMP;
+                    break;
+
+                case Message::Control:
+                    NOTIMP;
+                    break;
+
+                case Message::Inspect:
+                    // forwardApiCall(hdr);
+                    inspect(hdr);
+                    break;
+
+                case Message::Read: 
+                    read(hdr);
+                    break;
+
+                case Message::Flush:
+                    flush(hdr);
+                    break;
+
+                case Message::Archive:
+                    archive(hdr);
+                    break;
+
+                case Message::Schema:
+                    schema(hdr);
+                    break;
+
+                // case Message::Store:
+                //     store(hdr);
+                //     break;
+
+                default: {
+                    std::stringstream ss;
+                    ss << "ERROR: Unexpected message recieved (" << static_cast(hdr.message)
+                       << "). ABORTING";
+                    Log::status() << ss.str() << std::endl;
+                    Log::error() << "Retrieving... " << ss.str() << std::endl;
+                    throw SeriousBug(ss.str(), Here());
+                }
+            }
+
+            // Ensure we have consumed exactly the correct amount from the socket.
+            socketRead(&tail, sizeof(tail), controlSocket_);
+            ASSERT(tail == EndMarker);
+
+            // Acknowledge receipt of command
+            controlWrite(Message::Received, hdr.requestID);
+        }
+        catch (std::exception& e) {
+            // n.b. more general than eckit::Exception
+            std::string what(e.what());
+            controlWrite(Message::Error, hdr.requestID, what.c_str(), what.length());
+        }
+        catch (...) {
+            std::string what("Caught unexpected and unknown error");
+            controlWrite(Message::Error, hdr.requestID, what.c_str(), what.length());
+        }
+    }
+}
+
+void CatalogueHandler::index(const MessageHeader& hdr) {
+    NOTIMP;
+}
+
+void CatalogueHandler::read(const MessageHeader& hdr) {
+    NOTIMP;
+}
+
+void CatalogueHandler::flush(const MessageHeader& hdr) {
+    Buffer payload(receivePayload(hdr, controlSocket_));
+    MemoryStream s(payload);
+
+    //fdb5::Key dbKey(s);
+
+    size_t numArchived;
+    s >> numArchived;
+
+    ASSERT(numArchived == 0 || archiveFuture_.valid());
+
+    if (archiveFuture_.valid()) {
+        // Ensure that the expected number of fields have been written, and that the
+        // archive worker processes have been cleanly wound up.
+        size_t n = archiveFuture_.get();
+        ASSERT(numArchived == n); // XXX Currently this will fail if there is more than one database in request.
+
+        // Do the actual flush!
+        Log::info() << "Flushing" << std::endl;
+        Log::status() << "Flushing" << std::endl;
+        //if (dbKey.empty()) {
+            for (auto it = catalogues_.begin(); it != catalogues_.end(); it++) {
+                it->second->flush();
+            }
+        // } else {
+        //     store(dbKey).flush();
+        // }
+        Log::info() << "Flush complete" << std::endl;
+        Log::status() << "Flush complete" << std::endl;
+    }
+}
+
+void CatalogueHandler::archive(const MessageHeader& hdr) {
+    // NOTE identical to RemoteCatalogueWriter::archive()
+
+    if(!archiveFuture_.valid()) {
+        Log::debug() << "WIP CatalogueHandler::archive start threadloop" << std::endl;
+        // Start archive worker thread
+        archiveFuture_ = std::async(std::launch::async, [this] { return archiveThreadLoop(); });
+    }
+}
+
+size_t CatalogueHandler::archiveThreadLoop() {
+    size_t totalArchived = 0;
+
+    // Create a worker that will do the actual archiving
+
+    static size_t queueSize(eckit::Resource("fdbServerMaxQueueSize", 32));
+    // using ArchiveQueue = eckit::Queue>>>;
+    // ArchiveQueue queue(queueSize);
+    eckit::Queue> queue(queueSize);
+
+
+    std::future worker = std::async(std::launch::async, [this, &queue] {
+        size_t totalArchived = 0;
+        std::pair elem = std::make_pair(0, Buffer{0});
+
+        try {
+            long queuelen;
+            while ((queuelen = queue.pop(elem)) != -1) {
+                uint32_t id = elem.first;
+                // Key key = elem.second.first;
+                eckit::Buffer payload = std::move(elem.second);
+                MemoryStream s(payload);
+                Key dbKey(s);
+                Key idxKey(s);
+                InspectionKey key; // xxx no stream constructor for inspection key?
+                s >> key;
+                std::unique_ptr location(eckit::Reanimator::reanimate(s));
+                Log::debug() << "CatalogueHandler::archiveThreadLoop message has dbKey: " << dbKey 
+                    << " idxKey: " << idxKey << " key: " << key << " and location.uri" << location->uri() << std::endl;
+        
+                CatalogueWriter& cat = catalogue(dbKey);
+                cat.selectIndex(idxKey);
+                cat.archive(key, std::move(location)); /// xxx currently this is making too many indexes.
+                totalArchived += 1;
+            }
+        }
+        catch (...) {
+            // Ensure exception propagates across the queue back to the parent thread.
+            queue.interrupt(std::current_exception());
+            throw;
+        }
+
+        return totalArchived;
+    });
+
+    try {
+        // The archive loop is the only thing that can listen on the data socket,
+        // so we don't need to to anything clever here.
+
+        // n.b. we also don't need to lock on read. We are the only thing that reads.
+
+        while (true) {
+            MessageHeader hdr;
+            Log::debug() << "CatalogueHandler::archiveThreadLoop awaiting message from client..." << std::endl;
+            socketRead(&hdr, sizeof(hdr), dataSocket_);
+            Log::debug() << "CatalogueHandler::archiveThreadLoop got a message from client!" << std::endl;
+
+            ASSERT(hdr.marker == StartMarker);
+            ASSERT(hdr.version == CurrentVersion);
+            //ASSERT(hdr.requestID == id);
+
+            // Have we been told that we are done yet?
+            if (hdr.message == Message::Flush)
+                break;
+
+            ASSERT(hdr.message == Message::Blob); // no multiblobs for catalogue
+
+            Buffer payload(receivePayload(hdr, dataSocket_));
+
+            eckit::FixedString<4> tail;
+            socketRead(&tail, sizeof(tail), dataSocket_);
+            ASSERT(tail == EndMarker);
+
+            size_t queuelen = queue.emplace(
+                std::make_pair(hdr.requestID, std::move(payload))
+            );
+        }
+
+        // Trigger cleanup of the workers
+        queue.close();
+
+        // Complete reading the Flush instruction
+
+        eckit::FixedString<4> tail;
+        socketRead(&tail, sizeof(tail), dataSocket_);
+        ASSERT(tail == EndMarker);
+
+        // Ensure worker is done
+
+        ASSERT(worker.valid());
+        totalArchived = worker.get();  // n.b. use of async, get() propagates any exceptions.
+    }
+    catch (std::exception& e) {
+        // n.b. more general than eckit::Exception
+        std::string what(e.what());
+        dataWrite(Message::Error, 0, what.c_str(), what.length());
+        queue.interrupt(std::current_exception());
+        throw;
+    }
+    catch (...) {
+        std::string what("Caught unexpected, unknown exception in retrieve worker");
+        dataWrite(Message::Error, 0, what.c_str(), what.length());
+        queue.interrupt(std::current_exception());
+        throw;
+    }
+
+    return totalArchived;
+}
+
+void CatalogueHandler::list(const MessageHeader& hdr) {
+    NOTIMP;
+}
+
+void CatalogueHandler::inspect(const MessageHeader& hdr) {
+    NOTIMP;
+}
+
+void CatalogueHandler::schema(const MessageHeader& hdr) {
+
+    // 1. Read dbkey to select catalogue
+    Buffer payload(receivePayload(hdr, controlSocket_));
+    MemoryStream s(payload);
+    Key dbKey(s);
+
+    // 2. Get catalogue
+    Catalogue& cat = catalogue(dbKey);
+    const Schema& schema = cat.schema();
+    eckit::Buffer schemaBuffer(1024*1024);
+    eckit::MemoryStream stream(schemaBuffer);
+    stream << schema;
+
+    // todo: maybe more appropriate control write instead.
+    dataWrite(Message::Schema, hdr.requestID, schemaBuffer, stream.position());
+}
+
+CatalogueWriter& CatalogueHandler::catalogue(Key dbKey) {
+    auto it = catalogues_.find(dbKey);
+    if (it != catalogues_.end()) {
+        return *(it->second);
+    }
+
+    // xxx specifically catalogue writer right now.
+    return *(catalogues_[dbKey] = CatalogueWriterFactory::instance().build(dbKey, config_));
+}
+
+}  // namespace fdb5::remote
diff --git a/src/fdb5/remote/CatalogueHandler.h b/src/fdb5/remote/server/CatalogueHandler.h
similarity index 63%
rename from src/fdb5/remote/CatalogueHandler.h
rename to src/fdb5/remote/server/CatalogueHandler.h
index a334fc9ff..11e15c333 100644
--- a/src/fdb5/remote/CatalogueHandler.h
+++ b/src/fdb5/remote/server/CatalogueHandler.h
@@ -11,12 +11,12 @@
 #ifndef fdb5_remote_CatalogueHandler_H
 #define fdb5_remote_CatalogueHandler_H
 
-#include "fdb5/remote/DecoupledHandler.h"
+#include "fdb5/remote/server/ServerConnection.h"
 
-namespace fdb5 {
-namespace remote {
+
+namespace fdb5::remote {
 //----------------------------------------------------------------------------------------------------------------------
-class CatalogueHandler : public DecoupledHandler {
+class CatalogueHandler : public ServerConnection {
 public:  // methods
 
     CatalogueHandler(eckit::net::TCPSocket& socket, const Config& config);
@@ -29,18 +29,29 @@ class CatalogueHandler : public DecoupledHandler {
     void initialiseConnections() override;
     void index(const MessageHeader& hdr);
 
+    void read(const MessageHeader& hdr);
+    void flush(const MessageHeader& hdr);
+    void archive(const MessageHeader& hdr);
+    void list(const MessageHeader& hdr);
+    void inspect(const MessageHeader& hdr);
+    void schema(const MessageHeader& hdr);
+    
+
+    CatalogueWriter& catalogue(Key dbKey);
+    size_t archiveThreadLoop();
+
     // API functionality
-    template 
-    void forwardApiCall(const MessageHeader& hdr);
+    // template 
+    // void forwardApiCall(const MessageHeader& hdr);
 
 private:  // member
 
+   std::map> catalogues_;
 //    std::unique_ptr catalogue_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
 
-}  // namespace remote
-}  // namespace fdb5
+}  // namespace fdb5::remote
 
 #endif  // fdb5_remote_CatalogueHandler_H
diff --git a/src/fdb5/toc/TocCatalogueWriter.cc b/src/fdb5/toc/TocCatalogueWriter.cc
index 9974c7464..21e8bed4e 100644
--- a/src/fdb5/toc/TocCatalogueWriter.cc
+++ b/src/fdb5/toc/TocCatalogueWriter.cc
@@ -228,6 +228,11 @@ const Index& TocCatalogueWriter::currentIndex() {
     return current_;
 }
 
+const Key TocCatalogueWriter::currentIndexKey() {
+    currentIndex();
+    return currentIndexKey_;
+}
+
 const TocSerialisationVersion& TocCatalogueWriter::serialisationVersion() const {
     return TocHandler::serialisationVersion();
 }
diff --git a/src/fdb5/toc/TocCatalogueWriter.h b/src/fdb5/toc/TocCatalogueWriter.h
index d97570330..268282517 100644
--- a/src/fdb5/toc/TocCatalogueWriter.h
+++ b/src/fdb5/toc/TocCatalogueWriter.h
@@ -58,6 +58,7 @@ class TocCatalogueWriter : public TocCatalogue, public CatalogueWriter {
     bool enabled(const ControlIdentifier& controlIdentifier) const override;
 
     const Index& currentIndex() override;
+    const Key currentIndexKey() override;
     const TocSerialisationVersion& serialisationVersion() const;
 
 protected: // methods

From d6a000324f08c17cd3d2d583d0da13897588a6d3 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 27 Oct 2023 00:27:59 +0100
Subject: [PATCH 010/186] fdb remote - fix flush of multiple DBs

---
 src/fdb5/remote/RemoteStore.cc                | 277 ++++++++++--------
 src/fdb5/remote/RemoteStore.h                 |  15 +-
 src/fdb5/remote/client/Client.cc              |  22 +-
 src/fdb5/remote/client/Client.h               |  29 +-
 src/fdb5/remote/client/ClientConnection.cc    |   7 +
 src/fdb5/remote/client/ClientConnection.h     |  31 +-
 .../remote/client/ClientConnectionRouter.cc   | 147 +++-------
 .../remote/client/ClientConnectionRouter.h    |  46 +--
 src/fdb5/remote/server/CatalogueHandler.h     |   2 +-
 src/fdb5/remote/server/ServerConnection.h     |   2 +-
 src/fdb5/remote/server/StoreHandler.cc        |  19 +-
 11 files changed, 257 insertions(+), 340 deletions(-)

diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index aa307ddf0..eb53a6642 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -47,23 +47,162 @@ namespace fdb5::remote {
 
 //----------------------------------------------------------------------------------------------------------------------
 
+class ArchivalRequest {
+
+public:
+
+    ArchivalRequest() :
+        id_(0), store_(nullptr), key_(Key()), buffer_(Buffer()) {}
+
+    ArchivalRequest(uint32_t id, RemoteStore* store, const fdb5::Key& key, const void *data, eckit::Length length) :
+        id_(id), store_(store), key_(key), buffer_(Buffer(reinterpret_cast(data), length)) {}
+
+    uint32_t id_;
+    RemoteStore* store_;
+    fdb5::Key key_;
+    eckit::Buffer buffer_;
+};
+
+class Archiver {
+
+    using ArchiveQueue = eckit::Queue;
+
+public:
+
+    static Archiver& get(const eckit::net::Endpoint& endpoint) {
+
+        static std::map > archivers_;
+
+        auto it = archivers_.find(endpoint.hostport());
+        if (it == archivers_.end()) {
+            // auto arch = (archivers_[endpoint.hostport()] = Archiver());
+            it = archivers_.emplace(endpoint.hostport(), new Archiver()).first;
+        }
+        ASSERT(it != archivers_.end());
+        return *(it->second);
+    }
+
+    bool valid() {
+        return archiveFuture_.valid();
+    }
+
+    void start() {
+        // if there is no archiving thread active, then start one.
+        // n.b. reset the archiveQueue_ after a potential flush() cycle.
+        if (!archiveFuture_.valid()) {
+
+            {
+                // Reset the queue after previous done/errors
+                std::lock_guard lock(archiveQueuePtrMutex_);
+                ASSERT(!archiveQueue_);
+                archiveQueue_.reset(new ArchiveQueue(maxArchiveQueueLength_));
+            }
+
+            archiveFuture_ = std::async(std::launch::async, [this] { return archiveThreadLoop(); });
+        }
+    }
+
+    void emplace(uint32_t id, RemoteStore* store, const Key& key, const void *data, eckit::Length length) {
+
+        std::lock_guard lock(archiveQueuePtrMutex_);
+        ASSERT(archiveQueue_);
+        archiveQueue_->emplace(id, store, key, data, length);
+    }
+
+    FDBStats flush(RemoteStore* store) {
+
+        ASSERT(archiveQueue_);
+        std::lock_guard lock(archiveQueuePtrMutex_);
+        archiveQueue_->close();
+
+        FDBStats stats = archiveFuture_.get();
+        ASSERT(!archiveQueue_);
+
+        ASSERT(stats.numFlush() == 0);
+        size_t numArchive = stats.numArchive();
+
+        Buffer sendBuf(4096);
+        MemoryStream s(sendBuf);
+        s << numArchive;
+
+        // The flush call is blocking
+        store->controlWriteCheckResponse(fdb5::remote::Message::Flush, sendBuf, s.position());
+
+        return stats;
+    }
+
+private:
+
+    Archiver() : maxArchiveQueueLength_(eckit::Resource("fdbRemoteArchiveQueueLength;$FDB_REMOTE_ARCHIVE_QUEUE_LENGTH", 200)) {}
+
+    FDBStats archiveThreadLoop() {
+        FDBStats localStats;
+        eckit::Timer timer;
+
+        ArchivalRequest element;
+
+        try {
+
+            ASSERT(archiveQueue_);
+            long popped;
+            while ((popped = archiveQueue_->pop(element)) != -1) {
+
+                timer.start();
+                element.store_->sendArchiveData(element.id_, element.key_, element.buffer_.data(), element.buffer_.size());
+                timer.stop();
+
+                localStats.addArchive(element.buffer_.size(), timer, 1);
+            }
+
+            // And note that we are done. (don't time this, as already being blocked
+            // on by the ::flush() routine)
+
+            // MessageHeader hdr(Message::Flush, 0, 0, 0);
+            element.store_->dataWrite(Message::Flush, nullptr, 0);
+    //       dataWrite(id, &EndMarker, sizeof(EndMarker));
+
+    //        archiveID_ = 0;
+            archiveQueue_.reset();
+
+        } catch (...) {
+            archiveQueue_->interrupt(std::current_exception());
+            throw;
+        }
+
+        return localStats;
+
+        // We are inside an async, so don't need to worry about exceptions escaping.
+        // They will be released when flush() is called.
+    }
+
+private:
+
+    std::mutex archiveQueuePtrMutex_;
+    size_t maxArchiveQueueLength_;
+    std::unique_ptr archiveQueue_;
+    std::future archiveFuture_;
+
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
 RemoteStore::RemoteStore(const Key& dbKey, const Config& config) :
-    Client(std::vector{eckit::net::Endpoint("localhost", 7000)}),
+    Client(eckit::net::Endpoint("localhost", 7000)),
 //    Client(std::vector{ClientConnectionRouter::instance().connectStore(dbKey_, std::vector{eckit::net::Endpoint("localhost", 7000)})}),
 //    ClientConnection(eckit::net::Endpoint(config.getString("storeHost"), config.getInt("storePort")), config),
     dbKey_(dbKey), config_(config), dirty_(false), //archiveID_(0),
-    maxArchiveQueueLength_(eckit::Resource("fdbRemoteArchiveQueueLength;$FDB_REMOTE_ARCHIVE_QUEUE_LENGTH", 200)),
     retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
-    maxArchiveBatchSize_(config.getInt("maxBatchSize", 1)) {
+    maxArchiveBatchSize_(config.getInt("maxBatchSize", 1)),
+    archiver_(Archiver::get(controlEndpoint())) {
 }
 
 RemoteStore::RemoteStore(const eckit::URI& uri, const Config& config) :
-    Client(std::vector{eckit::net::Endpoint(uri.hostport())}),
+    Client(eckit::net::Endpoint(uri.hostport())),
 //    Client(std::vector{ClientConnectionRouter::instance().connectStore(Key(), std::vector{eckit::net::Endpoint("localhost", 7000)})}),
     dbKey_(Key()), config_(config), dirty_(false), //archiveID_(0),
-    maxArchiveQueueLength_(eckit::Resource("fdbRemoteArchiveQueueLength;$FDB_REMOTE_ARCHIVE_QUEUE_LENGTH", 200)),
     retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
-    maxArchiveBatchSize_(config.getInt("maxBatchSize", 1)) {
+    maxArchiveBatchSize_(config.getInt("maxBatchSize", 1)),
+    archiver_(Archiver::get(controlEndpoint())) {
 
     // std::cout << "RemoteStore ctor " << uri.asRawString() << std::endl;
 
@@ -76,7 +215,7 @@ RemoteStore::~RemoteStore() {
     // If we have launched a thread with an async and we manage to get here, this is
     // an error. n.b. if we don't do something, we will block in the destructor
     // of std::future.
-    if (archiveFuture_.valid()) {
+    if (archiver_.valid()) {
         Log::error() << "Attempting to destruct DecoupledFDB with active archive thread" << std::endl;
         eckit::Main::instance().terminate();
     }
@@ -103,31 +242,14 @@ std::future > RemoteStore::archive(const Key& key
     // uint32_t id = generateRequestID();
 
 //    connect();
-
     // if there is no archiving thread active, then start one.
     // n.b. reset the archiveQueue_ after a potential flush() cycle.
-    if (!archiveFuture_.valid()) {
-
-        // Start the archival request on the remote side
+    archiver_.start();
 
-        // Reset the queue after previous done/errors
-        {
-            std::lock_guard lock(archiveQueuePtrMutex_);
-            ASSERT(!archiveQueue_);
-            archiveQueue_.reset(new ArchiveQueue(maxArchiveQueueLength_));
-        }
-
-        archiveFuture_ = std::async(std::launch::async, [this] { return archiveThreadLoop(); });
-    }
+    ASSERT(archiver_.valid());
 
-    ASSERT(archiveFuture_.valid());
-
-    uint32_t id = controlWriteCheckResponse(Message::Archive);
-    std::lock_guard lock(archiveQueuePtrMutex_);
-    ASSERT(archiveQueue_);
-    // std::vector combinedKey = {dbKey_, key};
-    // std::cout << "archiveID_: " << archiveID_ << "  Archiving " << combinedKey << std::endl;
-    archiveQueue_->emplace(std::make_pair(id, std::make_pair(key, Buffer(reinterpret_cast(data), length))));
+    uint32_t id = controlWriteCheckResponse(Message::Archive, nullptr, 0);
+    archiver_.emplace(id, this, key, data, length);
 
     std::promise > loc;
     auto futureLocation = loc.get_future();
@@ -147,25 +269,12 @@ void RemoteStore::flush() {
 
     timer.start();
 
-    // Flush only does anything if there is an ongoing archive();
-    if (archiveFuture_.valid()) {
-
-        ASSERT(archiveQueue_);
-        std::lock_guard lock(archiveQueuePtrMutex_);
-        archiveQueue_->close();
-
-        FDBStats stats = archiveFuture_.get();
-        ASSERT(!archiveQueue_);
+    size_t numArchive = 0;
 
-        ASSERT(stats.numFlush() == 0);
-        size_t numArchive = stats.numArchive();
-
-        Buffer sendBuf(4096);
-        MemoryStream s(sendBuf);
-        s << numArchive;
+    // Flush only does anything if there is an ongoing archive();
+    if (archiver_.valid()) {
 
-        // The flush call is blocking
-        controlWriteCheckResponse(fdb5::remote::Message::Flush, sendBuf, s.position());
+        archiver_.flush(this);
 
 //        internalStats_ += stats;
     }
@@ -260,55 +369,6 @@ void RemoteStore::print(std::ostream &out) const {
 
 
 
-FDBStats RemoteStore::archiveThreadLoop() {
-    FDBStats localStats;
-    eckit::Timer timer;
-
-    // We can pop multiple elements off the archive queue simultaneously, if
-    // configured
-    std::pair > element;
-    // for (size_t i = 0; i < maxArchiveBatchSize_; ++i) {
-    //     elements.emplace_back(std::make_pair(Key{}, Buffer{0}));
-    // }
-
-    try {
-
-        ASSERT(archiveQueue_);
-//        ASSERT(archiveID_ != 0);
-
-//        std::cout << "RemoteStore::archiveThreadLoop \n";
-        long popped;
-        while ((popped = archiveQueue_->pop(element)) != -1) {
-
-            timer.start();
-//        std::cout << "RemoteStore::archiveThreadLoop - sendArchiveData " <<  element.second.second.size() << std::endl;
-            sendArchiveData(element.first, element.second.first, element.second.second.data(), element.second.second.size());
-            timer.stop();
-
-            localStats.addArchive(element.second.second.size(), timer, 1);
-        }
-
-        // And note that we are done. (don't time this, as already being blocked
-        // on by the ::flush() routine)
-
-       //MessageHeader hdr(Message::Flush, 0, 0, 0);
-       dataWrite(Message::Flush, nullptr, 0);
-//       dataWrite(id, &EndMarker, sizeof(EndMarker));
-
-//        archiveID_ = 0;
-        archiveQueue_.reset();
-
-    } catch (...) {
-        archiveQueue_->interrupt(std::current_exception());
-        throw;
-    }
-
-    return localStats;
-
-    // We are inside an async, so don't need to worry about exceptions escaping.
-    // They will be released when flush() is called.
-}
-
 
 bool RemoteStore::handle(Message message, uint32_t requestID) {
 //    std::cout << "RemoteStore::handle received message: " << ((uint) message) << " - requestID: " << requestID << std::endl;
@@ -339,31 +399,10 @@ void RemoteStore::handleException(std::exception_ptr e) {
 
 void RemoteStore::flush(FDBStats& internalStats) {
 
-    // std::cout << "###################################      RemoteStore::flush\n";
     // Flush only does anything if there is an ongoing archive();
-    if (! archiveFuture_.valid()) return;
-
-//    ASSERT(archiveID_ != 0);
-    {
-        ASSERT(archiveQueue_);
-        std::lock_guard lock(archiveQueuePtrMutex_);
-        archiveQueue_->close();
-    }
-    FDBStats stats = archiveFuture_.get();
-    ASSERT(!archiveQueue_);
-//    archiveID_ = 0;
-
-    ASSERT(stats.numFlush() == 0);
-    size_t numArchive = stats.numArchive();
-
-    Buffer sendBuf(4096);
-    MemoryStream s(sendBuf);
-    s << numArchive;
-
-    // The flush call is blocking
-    controlWriteCheckResponse(Message::Flush, sendBuf, s.position());
+    if (! archiver_.valid()) return;
 
-    internalStats += stats;
+    internalStats += archiver_.flush(this);
 }
 
 
@@ -618,8 +657,6 @@ eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation) {
 
 eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation, const Key& remapKey) {
 
-    ASSERT(endpoints_.size()==1);
-    //connect();
 
     Buffer encodeBuffer(4096);
     MemoryStream s(encodeBuffer);
@@ -628,7 +665,7 @@ eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation, c
 
     uint32_t id = controlWriteCheckResponse(fdb5::remote::Message::Read, encodeBuffer, s.position());
 
-    return new FDBRemoteDataHandle(id, retrieveMessageQueue_, endpoints_.at(0));
+    return new FDBRemoteDataHandle(id, retrieveMessageQueue_, endpoint_);
 }
 
 
diff --git a/src/fdb5/remote/RemoteStore.h b/src/fdb5/remote/RemoteStore.h
index b6614403e..4a7f53280 100644
--- a/src/fdb5/remote/RemoteStore.h
+++ b/src/fdb5/remote/RemoteStore.h
@@ -22,6 +22,8 @@
 
 namespace fdb5::remote {
 
+class Archiver;
+
 //----------------------------------------------------------------------------------------------------------------------
 
 /// Store that connects to a remote Store
@@ -30,7 +32,6 @@ class RemoteStore : public Store, public Client {
 
 public: // types
 
-    using ArchiveQueue = eckit::Queue > >;
     using StoredMessage = std::pair;
     using MessageQueue = eckit::Queue;
 
@@ -59,6 +60,8 @@ class RemoteStore : public Store, public Client {
 
    const Config& config() { return config_; }
    
+    void sendArchiveData(uint32_t id, const Key& key, const void* data, size_t length);
+
 protected: // methods
 
     std::string type() const override { return "remote"; }
@@ -81,12 +84,10 @@ class RemoteStore : public Store, public Client {
     bool handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) override;
     void handleException(std::exception_ptr e) override;
 
-    FDBStats archiveThreadLoop();
     // Listen to the dataClient for incoming messages, and push them onto
     // appropriate queues.
 //    void listeningThreadLoop() override;
 //    long sendArchiveData(uint32_t id, const std::vector>& elements, size_t count);
-    void sendArchiveData(uint32_t id, const Key& key, const void* data, size_t length);
     void flush(FDBStats& stats);
 
 //     void connect();
@@ -98,11 +99,7 @@ class RemoteStore : public Store, public Client {
     const Config& config_;
 
     bool dirty_;
-    std::future archiveFuture_;
-//    uint32_t archiveID_;
-    std::mutex archiveQueuePtrMutex_;
-    std::unique_ptr archiveQueue_;
-    size_t maxArchiveQueueLength_;
+
     size_t maxArchiveBatchSize_;
     // @note This is a map of requestID:MessageQueue. At the point that a request is
     // complete, errored or otherwise killed, it needs to be removed from the map.
@@ -114,6 +111,8 @@ class RemoteStore : public Store, public Client {
 
     std::map > > locations_;
 
+    Archiver& archiver_;
+
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index 6f45b4911..b7d008dbf 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -15,33 +15,21 @@ namespace fdb5::remote {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-Client::Client(eckit::net::Endpoint endpoint) :
-    endpoints_(std::vector{endpoint}) {}
-
-Client::Client(std::vector&& endpoints) :
-    endpoints_(std::move(endpoints)) {}
+Client::Client(const eckit::net::Endpoint& endpoint) :
+    endpoint_(endpoint) {}
 
 uint32_t Client::controlWriteCheckResponse(Message msg, const void* payload, uint32_t payloadLength) {
-    return ClientConnectionRouter::instance().controlWriteCheckResponse(endpoints_, *this, msg, payload, payloadLength);
+    return ClientConnectionRouter::instance().controlWriteCheckResponse(*this, msg, payload, payloadLength);
 }
 uint32_t Client::controlWrite(Message msg, const void* payload, uint32_t payloadLength) {
-    return ClientConnectionRouter::instance().controlWrite(endpoints_, *this, msg, payload, payloadLength);
+    return ClientConnectionRouter::instance().controlWrite(*this, msg, payload, payloadLength);
 }
-// void Client::controlWrite(uint32_t requestId, const void* data, size_t length) {
-//     ClientConnectionRouter::instance().controlWrite(*this, requestId, data, length);
-// }
-// void Client::controlRead(uint32_t requestId, void* data, size_t length) {
-//     ClientConnectionRouter::instance().controlRead(*this, requestId, data, length);
-// }
 
 uint32_t Client::dataWrite(Message msg, const void* payload, uint32_t payloadLength) {
-    return ClientConnectionRouter::instance().dataWrite(endpoints_, *this, msg, payload, payloadLength);
+    return ClientConnectionRouter::instance().dataWrite(*this, msg, payload, payloadLength);
 }
 void Client::dataWrite(uint32_t requestId, const void* data, size_t length){
     ClientConnectionRouter::instance().dataWrite(*this, requestId, data, length);
 }
-// void Client::dataRead(uint32_t requestId, void* data, size_t length){
-//     ClientConnectionRouter::instance().dataRead(*this, requestId, data, length);
-// }
 
 }
\ No newline at end of file
diff --git a/src/fdb5/remote/client/Client.h b/src/fdb5/remote/client/Client.h
index 55bb54ffa..28997c2e8 100644
--- a/src/fdb5/remote/client/Client.h
+++ b/src/fdb5/remote/client/Client.h
@@ -25,26 +25,17 @@ namespace fdb5::remote {
 class Connection;
 //----------------------------------------------------------------------------------------------------------------------
 
-// typedef eckit::net::Endpoint Connection;
-// typedef uint32_t ClientID;
-// typedef uint32_t DataLinkID;
-// typedef uint32_t HandlerID;
-// typedef uint32_t RequestID;
-
-// abstract class, gives ClientConnectionRouter access to RemoteCatalogue & RemoteStore handlers
 class Client : eckit::NonCopyable {
 public:
-    Client(eckit::net::Endpoint endpoint);
-    Client(std::vector&& endpoints);
+    Client(const eckit::net::Endpoint& endpoint);
+
+    const eckit::net::Endpoint& controlEndpoint() { return endpoint_; }
 
     uint32_t controlWriteCheckResponse(Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
     uint32_t controlWrite(Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
-    // void controlWrite(uint32_t requestId, const void* data, size_t length);
-    // void controlRead(uint32_t requestId, void* data, size_t length);
 
     uint32_t dataWrite(Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
     void dataWrite(uint32_t requestId, const void* data, size_t length);
-    // void dataRead(uint32_t requestId, void* data, size_t length);
 
     // handlers for incoming messages - to be defined in the client class
     virtual bool handle(Message message, uint32_t requestID) = 0;
@@ -55,20 +46,8 @@ class Client : eckit::NonCopyable {
 
 protected:
 
-    std::vector endpoints_;
-    // Client(DataLinkID dataLinkID, RemoteHandlerID remoteHandlerID);
-
-    // ClientConnectionProxy() {} ///< private constructor only used by singleton
-
-    // eckit::Mutex mutex_;
-
-    // // a Client is either a CatalogueProxy or a StoreProxy (local),
-    // // willing to communicate with a remote CatalogueHandler or StoreHandler at a given endpoint
-    // std::map > clientLinks_;
-
-    // std::map > connections_;
+    eckit::net::Endpoint endpoint_;
 
-    // std::map requests_;
 };
 
 }
\ No newline at end of file
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index 0063dc60b..c13389c60 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -343,6 +343,13 @@ void ClientConnection::listeningThreadLoop() {
             if (hdr.message == Message::Exit) {
                 return;
             }
+            if (hdr.message == Message::Error) {
+                std::stringstream ss;
+                ss << "ERROR: Server-side error. ABORTING";
+                Log::status() << ss.str() << std::endl;
+                Log::error() << "Retrieving... " << ss.str() << std::endl;
+                throw SeriousBug(ss.str(), Here());
+            }            
 
             bool handled = false;
 
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index 25ce1a853..055255ddc 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -12,15 +12,16 @@
 
 #include 
 
+
+#include "eckit/config/LocalConfiguration.h"
 #include "eckit/container/Queue.h"
 #include "eckit/io/Buffer.h"
+#include "eckit/io/Length.h"
 #include "eckit/net/Endpoint.h"
 #include "eckit/net/TCPClient.h"
 #include "eckit/net/TCPStream.h"
 #include "eckit/runtime/SessionID.h"
 
-#include "fdb5/api/FDB.h"
-#include "fdb5/api/FDBFactory.h"
 #include "fdb5/remote/Messages.h"
 #include "eckit/utils/Translator.h"
 
@@ -29,14 +30,14 @@ namespace eckit {
 
 class Buffer;
 
-// xxx can we give this code a better home?
-template<> struct Translator {
-    std::string operator()(const net::Endpoint& e) {
-        std::stringstream ss;
-        ss << e;
-        return ss.str();
-    }
-};
+// // xxx can we give this code a better home?
+// template<> struct Translator {
+//     std::string operator()(const net::Endpoint& e) {
+//         std::stringstream ss;
+//         ss << e;
+//         return ss.str();
+//     }
+// };
 
 }
 
@@ -67,7 +68,6 @@ class ClientConnection : eckit::NonCopyable {
 
 protected: // methods
 
-
     const eckit::net::Endpoint& controlEndpoint() const;
     const eckit::net::Endpoint& dataEndpoint() const;
 
@@ -79,10 +79,8 @@ class ClientConnection : eckit::NonCopyable {
     // construct dictionary for protocol negotiation - to be defined in the client class
     virtual eckit::LocalConfiguration availableFunctionality() const;
 
-    
 private: // methods
 
-
     void writeControlStartupMessage();
     void writeDataStartupMessage(const eckit::SessionID& serverSession);
 
@@ -105,10 +103,7 @@ class ClientConnection : eckit::NonCopyable {
     std::thread listeningThread_;
 
     bool connected_;
-
-    eckit::Length freeSpace_;
-    float freeRatio_;
-
+    
 };
 
 //----------------------------------------------------------------------------------------------------------------------
@@ -128,7 +123,7 @@ class ClientConnection : eckit::NonCopyable {
 class DecoupledFDBException : public eckit::RemoteException {
 public:
     DecoupledFDBException(const std::string& msg, const eckit::net::Endpoint& endpoint):
-        eckit::RemoteException(msg, eckit::Translator()(endpoint)) {}
+        eckit::RemoteException(msg, endpoint.hostport()) {}
 };
 
 }  // namespace fdb5::remote
\ No newline at end of file
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.cc b/src/fdb5/remote/client/ClientConnectionRouter.cc
index 6925e0408..f9a65bd2a 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.cc
+++ b/src/fdb5/remote/client/ClientConnectionRouter.cc
@@ -2,7 +2,6 @@
 #include 
 
 #include "fdb5/remote/client/ClientConnectionRouter.h"
-// #include "fdb5/remote/RemoteStore.h"
 
 using namespace eckit;
 using namespace eckit::net;
@@ -50,38 +49,56 @@ namespace fdb5::remote {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-uint32_t ClientConnectionRouter::controlWriteCheckResponse(std::vector& endpoints, Client& client, Message msg, const void* payload, uint32_t payloadLength) {
+RemoteStore& ClientConnectionRouter::store(const eckit::URI& uri) {
+    const std::string& endpoint = uri.hostport();
+    auto it = readStores_.find(endpoint);
+    if (it != readStores_.end()) {
+        return *(it->second);
+    }
+
+    return *(readStores_[endpoint] = std::unique_ptr(new RemoteStore(uri, Config())));
+}
+
+uint32_t ClientConnectionRouter::createConnection(Client& client, ClientConnection*& conn) {
+
+    // pick the first endpoint - To be improved with DataStoreStrategies
+    const eckit::net::Endpoint& endpoint = client.controlEndpoint();
+    uint32_t id = generateRequestID();
+
+    auto it = connections_.find(endpoint.hostport());
+    if (it != connections_.end()) {
+        conn = it->second.get();
+    } else {
+        conn = (connections_[endpoint.hostport()] = std::unique_ptr(new ClientConnection(endpoint, LocalConfiguration()))).get();
+        conn->connect();
+    }
+
+    ASSERT(conn);
+    requests_[id] = {conn, &client};
+
+    return id;
+}
+
+uint32_t ClientConnectionRouter::controlWriteCheckResponse(Client& client, Message msg, const void* payload, uint32_t payloadLength) {
     //std::cout << "ClientConnectionRouter::controlWriteCheckResponse " << endpoints.size() << std::endl;
     ClientConnection* conn;
-    uint32_t id = createConnection(endpoints, client, conn);
+    uint32_t id = createConnection(client, conn);
 
     conn->controlWriteCheckResponse(msg, id, payload, payloadLength);
 
     return id;
 }
-uint32_t ClientConnectionRouter::controlWrite(std::vector& endpoints, Client& client, Message msg, const void* payload, uint32_t payloadLength) {
+uint32_t ClientConnectionRouter::controlWrite(Client& client, Message msg, const void* payload, uint32_t payloadLength) {
     //std::cout << "ClientConnectionRouter::controlWrite " << endpoints.size() << std::endl;
     ClientConnection* conn;
-    uint32_t id = createConnection(endpoints, client, conn);
+    uint32_t id = createConnection(client, conn);
 
     conn->controlWrite(msg, id, payload, payloadLength);
 
     return id;
 }
-// void ClientConnectionRouter::controlWrite(Client& client, uint32_t requestId, const void* payload, size_t payloadLength) {
-//     auto it = requests_.find(requestId);
-//     ASSERT(it != requests_.end());
-//     ASSERT(it->second.second == &client); // check if the client owns the request
-
-//     it->second.first->controlWrite(payload, payloadLength);
-// }
-    // void controlRead(Client& client, uint32_t requestId, void* data, size_t length);
-
-    // void dataWrite(Client& client, uint32_t requestId, const void* data, size_t length);
-    // void dataRead(Client& client, uint32_t requestId, void* data, size_t length);
-
 void ClientConnectionRouter::controlRead(Client& client, uint32_t requestId, void* payload, size_t payloadLength) {
-    // std::cout << "ClientConnectionRouter::controlRead " << requestId << std::endl;
+
     auto it = requests_.find(requestId);
     ASSERT(it != requests_.end());
     ASSERT(it->second.second == &client); // check if the client owns the request
@@ -90,47 +107,9 @@ void ClientConnectionRouter::controlRead(Client& client, uint32_t requestId, voi
 
 }
 
-// void ClientConnectionRouter::controlRead(Connection& connection, Client& client, void* data, size_t length) {
-
-// }
-
-RemoteStore& ClientConnectionRouter::store(const eckit::URI& uri) {
-    const std::string& endpoint = uri.hostport();
-    auto it = readStores_.find(endpoint);
-    if (it != readStores_.end()) {
-        return *(it->second);
-    }
-
-    return *(readStores_[endpoint] = std::unique_ptr(new RemoteStore(uri, Config())));
-}
-
-
-uint32_t ClientConnectionRouter::createConnection(std::vector& endpoints, Client& client, ClientConnection*& conn) {
-
-    ASSERT(endpoints.size() > 0);
-
-    // pick the first endpoint - To be improved with DataStoreStrategies
-    eckit::net::Endpoint& endpoint = endpoints.at(0);
-    uint32_t id = generateRequestID();
-
-    auto it = connections_.find(endpoint.host());
-    if (it == connections_.end()) {
-        conn = (connections_[endpoint.host()] = std::unique_ptr(new ClientConnection(endpoint, LocalConfiguration()))).get();
-        conn->connect();
-        // client.key();
-        // connections_[endpoint] = std::make_pair, std::map >(std::move(conn))
-    } else {
-        conn = it->second.get();
-    }
-
-    ASSERT(conn);
-    requests_[id] = {conn, &client};
-
-    return id;
-}
-uint32_t ClientConnectionRouter::dataWrite(std::vector& endpoints, Client& client, Message msg, const void* payload, uint32_t payloadLength) {
+uint32_t ClientConnectionRouter::dataWrite(Client& client, Message msg, const void* payload, uint32_t payloadLength) {
     ClientConnection* conn;
-    uint32_t id = createConnection(endpoints, client, conn);
+    uint32_t id = createConnection(client, conn);
 
     conn->dataWrite(msg, id, payload, payloadLength);
 
@@ -143,10 +122,13 @@ void ClientConnectionRouter::dataWrite(Client& client, uint32_t requestId, const
 
     it->second.first->dataWrite(payload, payloadLength);
 }
-// void ClientConnectionRouter::dataRead(Connection& connection, Client& client, void* data, size_t length) {
 
-// }
+// std::future& ClientConnectionRouter::archiveFuture(Client& client) {
+//     ClientConnection* conn;
+//     uint32_t id = createConnection(client, conn);
 
+//     return conn->archiveFuture();
+// }
 
 bool ClientConnectionRouter::handle(Message message, uint32_t requestID) {
     if (requestID != 0) {
@@ -159,8 +141,8 @@ bool ClientConnectionRouter::handle(Message message, uint32_t requestID) {
         return true;
     }
 }
-
 bool ClientConnectionRouter::handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) {
+
     auto it = requests_.find(requestID);
     ASSERT(it != requests_.end());
 
@@ -173,49 +155,10 @@ void ClientConnectionRouter::handleException(std::exception_ptr e) {
     // it->second->handle(message, requestID);
 }
 
-
 ClientConnectionRouter& ClientConnectionRouter::instance()
 {
-    static ClientConnectionRouter proxy;
-    return proxy;
+    static ClientConnectionRouter router;
+    return router;
 }
 
-// void ClientConnectionRouter::add(const std::string& name, const FDBBuilderBase* b)
-// {
-//     eckit::AutoLock lock(mutex_);
-
-//     ASSERT(registry_.find(name) == registry_.end());
-
-//     registry_[name] = b;
-// }
-
-// std::unique_ptr FDBFactory::build(const Config& config) {
-
-//     // Allow expanding of the config to make use of fdb_home supplied in a previous
-//     // configuration file, or to pick up the default configuration from ~fdb/etc/fdb/...
-
-//     Config actualConfig = config.expandConfig();
-
-//     /// We use "local" as a default type if not otherwise configured.
-
-//     std::string key = actualConfig.getString("type", "local");
-
-//     eckit::Log::debug() << "Selecting FDB implementation: " << key << std::endl;
-
-//     eckit::AutoLock lock(mutex_);
-
-//     auto it = registry_.find(key);
-
-//     if (it == registry_.end()) {
-//         std::stringstream ss;
-//         ss << "FDB factory \"" << key << "\" not found";
-//         throw eckit::SeriousBug(ss.str(), Here());
-//     }
-
-//     std::unique_ptr ret = it->second->make(actualConfig);
-//     eckit::Log::debug() << "Constructed FDB implementation: " << *ret << std::endl;
-//     return ret;
-// }
-
-
 }  // namespace fdb5::remote
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.h b/src/fdb5/remote/client/ClientConnectionRouter.h
index 9fb77b9d6..f0c968bf6 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.h
+++ b/src/fdb5/remote/client/ClientConnectionRouter.h
@@ -46,27 +46,18 @@ class ClientConnectionRouter : eckit::NonCopyable {
 public:
 
     static ClientConnectionRouter& instance();
-//    Connection connectCatalogue(Key dbKey, const eckit::Configuration& config);
-//    Connection connectStore(Key dbKey, const std::vector& endpoints);
     RemoteStore& store(const eckit::URI& uri);
 
 // protected:
 
-//     friend class Client;
-
-    uint32_t controlWriteCheckResponse(std::vector& endpoints, Client& client, Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
-    uint32_t controlWrite(std::vector& connections, Client& client, Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
-//    void controlWrite(Client& client, uint32_t requestId, const void* payload, size_t payloadLength);
-
-    // uint32_t controlWriteCheckResponse(Connection& connection, Client& client, Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
-    // uint32_t controlWrite(Connection& connection, Client& client, Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
-    // void controlWrite(Connection& connection, Client& client, const void* data, size_t length);
+    uint32_t controlWriteCheckResponse(Client& client, Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
+    uint32_t controlWrite(Client& client, Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
     void controlRead(Client& client, uint32_t requestId, void* payload, size_t payloadLength);
 
-    uint32_t dataWrite(std::vector& endpoints, Client& client, Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
+    uint32_t dataWrite(Client& client, Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
     void dataWrite(Client& client, uint32_t requestId, const void* data, size_t length);
-    // void dataRead(Client& client, uint32_t requestId, void* data, size_t length);
 
+//    std::future& archiveFuture(Client& client);
 
     // handlers for incoming messages - to be defined in the client class
     bool handle(Message message, uint32_t requestID);
@@ -77,38 +68,19 @@ class ClientConnectionRouter : eckit::NonCopyable {
 
     ClientConnectionRouter() {} ///< private constructor only used by singleton
 
-    uint32_t createConnection(std::vector& endpoints, Client& client, ClientConnection*& conn);
+    uint32_t createConnection(Client& client, ClientConnection*& conn);
 
     eckit::Mutex mutex_;
 
-    // endpoint --> datalink
-    // remoteId --> datalink, remoteHandler
-    // a Client is either a CatalogueProxy or a StoreProxy (local),
-    // willing to communicate with a remote CatalogueHandler or StoreHandler at a given endpoint
-    // std::map > clientLinks_;
-
-    // std::map > connections_;
-
-    // std::map requests_;
-
-
-
-
-
-    // // key --> [endpoint1, endpoint2, ...]
-    // std::map > storeEndpoints_;
-
-    // requestID --> <, Client>
     std::map > requests_;
 
-    // endpoint ->  remoteId>
-//    std::map, std::map > connections_;
+    // endpoint -> connection
     std::map > connections_;
 
+    // endpoint -> Store (one read store for each connection. Do not need to have one for each key)
     std::map > readStores_;
-    
-    // // not owning
-    // ClientConnection* catalogueConnection_;
+
+//    std::map > > archiveFutures_;
 };
 
 }
\ No newline at end of file
diff --git a/src/fdb5/remote/server/CatalogueHandler.h b/src/fdb5/remote/server/CatalogueHandler.h
index 11e15c333..5aa8d41ad 100644
--- a/src/fdb5/remote/server/CatalogueHandler.h
+++ b/src/fdb5/remote/server/CatalogueHandler.h
@@ -47,7 +47,7 @@ class CatalogueHandler : public ServerConnection {
 private:  // member
 
    std::map> catalogues_;
-//    std::unique_ptr catalogue_;
+
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/server/ServerConnection.h b/src/fdb5/remote/server/ServerConnection.h
index 0a25c6182..e6d3f5474 100644
--- a/src/fdb5/remote/server/ServerConnection.h
+++ b/src/fdb5/remote/server/ServerConnection.h
@@ -96,9 +96,9 @@ class ServerConnection : private eckit::NonCopyable {
     eckit::LocalConfiguration agreedConf_;
     std::mutex dataWriteMutex_;
     std::map> workerThreads_;
-    std::future archiveFuture_;
     std::thread readLocationWorker_;
     
+    std::future archiveFuture_;
 
 };
 
diff --git a/src/fdb5/remote/server/StoreHandler.cc b/src/fdb5/remote/server/StoreHandler.cc
index 2aee6d7cf..f02602991 100644
--- a/src/fdb5/remote/server/StoreHandler.cc
+++ b/src/fdb5/remote/server/StoreHandler.cc
@@ -392,8 +392,6 @@ void StoreHandler::flush(const MessageHeader& hdr) {
     Buffer payload(receivePayload(hdr, controlSocket_));
     MemoryStream s(payload);
 
-    //fdb5::Key dbKey(s);
-
     size_t numArchived;
     s >> numArchived;
 
@@ -408,16 +406,15 @@ void StoreHandler::flush(const MessageHeader& hdr) {
         // Do the actual flush!
         Log::info() << "Flushing" << std::endl;
         Log::status() << "Flushing" << std::endl;
-        //if (dbKey.empty()) {
-            for (auto it = stores_.begin(); it != stores_.end(); it++) {
-                it->second->flush();
-            }
-        // } else {
-        //     store(dbKey).flush();
-        // }
-        Log::info() << "Flush complete" << std::endl;
-        Log::status() << "Flush complete" << std::endl;
+
+        for (auto it = stores_.begin(); it != stores_.end(); it++) {
+            it->second->flush();
+        }
+
     }
+
+    Log::info() << "Flush complete" << std::endl;
+    Log::status() << "Flush complete" << std::endl;
 }
 
 }  // namespace remote

From cee3a23ec369696d514e01e0891d6fb191ceb94e Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 27 Oct 2023 09:08:01 +0100
Subject: [PATCH 011/186] fdb remote - read

---
 src/fdb5/remote/Handler.cc                    |   4 -
 src/fdb5/remote/Messages.h                    |   1 -
 src/fdb5/remote/RemoteCatalogue.cc            |   2 +-
 src/fdb5/remote/RemoteCatalogue.h             |   3 +-
 src/fdb5/remote/RemoteStore.cc                | 602 +++++++++---------
 src/fdb5/remote/RemoteStore.h                 |  12 +-
 src/fdb5/remote/client/Client.h               |   2 +-
 src/fdb5/remote/client/ClientConnection.cc    |   9 +-
 .../remote/client/ClientConnectionRouter.cc   |  47 +-
 .../remote/client/ClientConnectionRouter.h    |  20 -
 src/fdb5/remote/server/CatalogueHandler.cc    |  21 +-
 src/fdb5/toc/TocIndex.cc                      |   1 -
 12 files changed, 316 insertions(+), 408 deletions(-)

diff --git a/src/fdb5/remote/Handler.cc b/src/fdb5/remote/Handler.cc
index a9983c7e3..dd5816200 100644
--- a/src/fdb5/remote/Handler.cc
+++ b/src/fdb5/remote/Handler.cc
@@ -419,10 +419,6 @@ void RemoteHandler::handle() {
                     archive(hdr);
                     break;
 
-                // case Message::Store:
-                //     store(hdr);
-                //     break;
-
                 default: {
                     std::stringstream ss;
                     ss << "ERROR: Unexpected message recieved (" << static_cast(hdr.message)
diff --git a/src/fdb5/remote/Messages.h b/src/fdb5/remote/Messages.h
index a4330e66c..1565fe7a4 100644
--- a/src/fdb5/remote/Messages.h
+++ b/src/fdb5/remote/Messages.h
@@ -62,7 +62,6 @@ enum class Message : uint16_t {
     Inspect,
     Read,
     Move,
-    Store,
     Schema,
 
     // Responses
diff --git a/src/fdb5/remote/RemoteCatalogue.cc b/src/fdb5/remote/RemoteCatalogue.cc
index 4bcb0fc94..32f5a59af 100644
--- a/src/fdb5/remote/RemoteCatalogue.cc
+++ b/src/fdb5/remote/RemoteCatalogue.cc
@@ -25,7 +25,7 @@ namespace fdb5::remote {
 RemoteCatalogue::RemoteCatalogue(const Key& key, const Config& config):
     CatalogueImpl(key, ControlIdentifiers(), config), // xxx what are control identifiers? Setting empty here...
     Client(eckit::net::Endpoint("localhost", 7001)), // xxx hardcoded endpoint
-    dbKey_(key), config_(config), schema_(nullptr),
+    config_(config), schema_(nullptr),
     maxArchiveQueueLength_(eckit::Resource("fdbRemoteArchiveQueueLength;$FDB_REMOTE_ARCHIVE_QUEUE_LENGTH", 200))
     {
         loadSchema();
diff --git a/src/fdb5/remote/RemoteCatalogue.h b/src/fdb5/remote/RemoteCatalogue.h
index 82dc10a7f..66a97b15b 100644
--- a/src/fdb5/remote/RemoteCatalogue.h
+++ b/src/fdb5/remote/RemoteCatalogue.h
@@ -71,7 +71,7 @@ class RemoteCatalogue : public CatalogueWriter, public CatalogueImpl, public Cli
     bool handle(Message message, uint32_t requestID) override;
     bool handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) override;
     void handleException(std::exception_ptr e) override;
-    Key key() override { return dbKey_; } // xxx this is seperately declared by Client and CatalogueImpl
+    const Key& key() const override { return CatalogueImpl::key(); }
 
 
     // note: in common with remotestore
@@ -80,7 +80,6 @@ class RemoteCatalogue : public CatalogueWriter, public CatalogueImpl, public Cli
 
 protected:
 
-    Key dbKey_;
     Config config_;
     ControlIdentifiers controlIdentifiers_;
 
diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index eb53a6642..e87534ac7 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -29,24 +29,10 @@
 
 using namespace eckit;
 
-namespace {
-    // // n.b. if we get integer overflow, we reuse the IDs. This is not a
-// //      big deal. The idea that we could be on the 2.1 billionth (successful)
-// //      request, and still have an ongoing request 0 is ... laughable.
-static uint32_t generateRequestID() {
-
-    static std::mutex m;
-    static uint32_t id = 0;
-
-    std::lock_guard lock(m);
-    return ++id;
-}
+//----------------------------------------------------------------------------------------------------------------------
 
-}
 namespace fdb5::remote {
 
-//----------------------------------------------------------------------------------------------------------------------
-
 class ArchivalRequest {
 
 public:
@@ -63,119 +49,36 @@ class ArchivalRequest {
     eckit::Buffer buffer_;
 };
 
-class Archiver {
-
-    using ArchiveQueue = eckit::Queue;
-
-public:
-
-    static Archiver& get(const eckit::net::Endpoint& endpoint) {
-
-        static std::map > archivers_;
-
-        auto it = archivers_.find(endpoint.hostport());
-        if (it == archivers_.end()) {
-            // auto arch = (archivers_[endpoint.hostport()] = Archiver());
-            it = archivers_.emplace(endpoint.hostport(), new Archiver()).first;
-        }
-        ASSERT(it != archivers_.end());
-        return *(it->second);
-    }
-
-    bool valid() {
-        return archiveFuture_.valid();
-    }
-
-    void start() {
-        // if there is no archiving thread active, then start one.
-        // n.b. reset the archiveQueue_ after a potential flush() cycle.
-        if (!archiveFuture_.valid()) {
-
-            {
-                // Reset the queue after previous done/errors
-                std::lock_guard lock(archiveQueuePtrMutex_);
-                ASSERT(!archiveQueue_);
-                archiveQueue_.reset(new ArchiveQueue(maxArchiveQueueLength_));
-            }
-
-            archiveFuture_ = std::async(std::launch::async, [this] { return archiveThreadLoop(); });
-        }
-    }
-
-    void emplace(uint32_t id, RemoteStore* store, const Key& key, const void *data, eckit::Length length) {
-
-        std::lock_guard lock(archiveQueuePtrMutex_);
-        ASSERT(archiveQueue_);
-        archiveQueue_->emplace(id, store, key, data, length);
-    }
-
-    FDBStats flush(RemoteStore* store) {
-
-        ASSERT(archiveQueue_);
-        std::lock_guard lock(archiveQueuePtrMutex_);
-        archiveQueue_->close();
-
-        FDBStats stats = archiveFuture_.get();
-        ASSERT(!archiveQueue_);
-
-        ASSERT(stats.numFlush() == 0);
-        size_t numArchive = stats.numArchive();
-
-        Buffer sendBuf(4096);
-        MemoryStream s(sendBuf);
-        s << numArchive;
-
-        // The flush call is blocking
-        store->controlWriteCheckResponse(fdb5::remote::Message::Flush, sendBuf, s.position());
-
-        return stats;
-    }
-
-private:
-
-    Archiver() : maxArchiveQueueLength_(eckit::Resource("fdbRemoteArchiveQueueLength;$FDB_REMOTE_ARCHIVE_QUEUE_LENGTH", 200)) {}
+//----------------------------------------------------------------------------------------------------------------------
 
-    FDBStats archiveThreadLoop() {
-        FDBStats localStats;
-        eckit::Timer timer;
+class Archiver {
 
-        ArchivalRequest element;
+public: // types
 
-        try {
+    using ArchiveQueue = eckit::Queue;
 
-            ASSERT(archiveQueue_);
-            long popped;
-            while ((popped = archiveQueue_->pop(element)) != -1) {
+public: // methods
 
-                timer.start();
-                element.store_->sendArchiveData(element.id_, element.key_, element.buffer_.data(), element.buffer_.size());
-                timer.stop();
+    static Archiver& get(const eckit::net::Endpoint& endpoint);
 
-                localStats.addArchive(element.buffer_.size(), timer, 1);
-            }
+    bool valid() { return archiveFuture_.valid(); }
+    bool dirty() { return dirty_; }
 
-            // And note that we are done. (don't time this, as already being blocked
-            // on by the ::flush() routine)
+    void start();
+    void error(Buffer&& payload, const eckit::net::Endpoint& endpoint);
+    
+    void emplace(uint32_t id, RemoteStore* store, const Key& key, const void *data, eckit::Length length);
+    FDBStats flush(RemoteStore* store);
 
-            // MessageHeader hdr(Message::Flush, 0, 0, 0);
-            element.store_->dataWrite(Message::Flush, nullptr, 0);
-    //       dataWrite(id, &EndMarker, sizeof(EndMarker));
+private: // methods
 
-    //        archiveID_ = 0;
-            archiveQueue_.reset();
+    Archiver() : dirty_(false), maxArchiveQueueLength_(eckit::Resource("fdbRemoteArchiveQueueLength;$FDB_REMOTE_ARCHIVE_QUEUE_LENGTH", 200)) {}
 
-        } catch (...) {
-            archiveQueue_->interrupt(std::current_exception());
-            throw;
-        }
+    FDBStats archiveThreadLoop();
 
-        return localStats;
-
-        // We are inside an async, so don't need to worry about exceptions escaping.
-        // They will be released when flush() is called.
-    }
+private: // members
 
-private:
+    bool dirty_;
 
     std::mutex archiveQueuePtrMutex_;
     size_t maxArchiveQueueLength_;
@@ -184,225 +87,118 @@ class Archiver {
 
 };
 
-//----------------------------------------------------------------------------------------------------------------------
-
-RemoteStore::RemoteStore(const Key& dbKey, const Config& config) :
-    Client(eckit::net::Endpoint("localhost", 7000)),
-//    Client(std::vector{ClientConnectionRouter::instance().connectStore(dbKey_, std::vector{eckit::net::Endpoint("localhost", 7000)})}),
-//    ClientConnection(eckit::net::Endpoint(config.getString("storeHost"), config.getInt("storePort")), config),
-    dbKey_(dbKey), config_(config), dirty_(false), //archiveID_(0),
-    retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
-    maxArchiveBatchSize_(config.getInt("maxBatchSize", 1)),
-    archiver_(Archiver::get(controlEndpoint())) {
-}
-
-RemoteStore::RemoteStore(const eckit::URI& uri, const Config& config) :
-    Client(eckit::net::Endpoint(uri.hostport())),
-//    Client(std::vector{ClientConnectionRouter::instance().connectStore(Key(), std::vector{eckit::net::Endpoint("localhost", 7000)})}),
-    dbKey_(Key()), config_(config), dirty_(false), //archiveID_(0),
-    retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
-    maxArchiveBatchSize_(config.getInt("maxBatchSize", 1)),
-    archiver_(Archiver::get(controlEndpoint())) {
-
-    // std::cout << "RemoteStore ctor " << uri.asRawString() << std::endl;
+Archiver& Archiver::get(const eckit::net::Endpoint& endpoint) {
 
-    ASSERT(uri.scheme() == "fdb");
-    // std::vector endpoints{eckit::net::Endpoint(uri.hostport())};
-    // connection_ = ClientConnectionRouter::instance().connectStore(dbKey_, endpoints);
-}
+    static std::map > archivers_;
 
-RemoteStore::~RemoteStore() {
-    // If we have launched a thread with an async and we manage to get here, this is
-    // an error. n.b. if we don't do something, we will block in the destructor
-    // of std::future.
-    if (archiver_.valid()) {
-        Log::error() << "Attempting to destruct DecoupledFDB with active archive thread" << std::endl;
-        eckit::Main::instance().terminate();
+    auto it = archivers_.find(endpoint.hostport());
+    if (it == archivers_.end()) {
+        // auto arch = (archivers_[endpoint.hostport()] = Archiver());
+        it = archivers_.emplace(endpoint.hostport(), new Archiver()).first;
     }
-
-    ASSERT(!dirty_);
-//    disconnect();
+    ASSERT(it != archivers_.end());
+    return *(it->second);
 }
 
-eckit::URI RemoteStore::uri() const {
-    return URI("fdb", "");
-}
-
-bool RemoteStore::exists() const {
-    return true; // directory_.exists();
-}
-
-eckit::DataHandle* RemoteStore::retrieve(Field& field) const {
-    //return field.dataHandle();
-    return nullptr;
-}
-
-std::future > RemoteStore::archive(const Key& key, const void *data, eckit::Length length) {
-    // dirty_ = true;
-    // uint32_t id = generateRequestID();
-
-//    connect();
+void Archiver::start() {
     // if there is no archiving thread active, then start one.
     // n.b. reset the archiveQueue_ after a potential flush() cycle.
-    archiver_.start();
-
-    ASSERT(archiver_.valid());
+    if (!archiveFuture_.valid()) {
 
-    uint32_t id = controlWriteCheckResponse(Message::Archive, nullptr, 0);
-    archiver_.emplace(id, this, key, data, length);
-
-    std::promise > loc;
-    auto futureLocation = loc.get_future();
-    locations_[id] = std::move(loc);
-    return futureLocation;
-}
+        {
+            // Reset the queue after previous done/errors
+            std::lock_guard lock(archiveQueuePtrMutex_);
+            ASSERT(!archiveQueue_);
+            archiveQueue_.reset(new ArchiveQueue(maxArchiveQueueLength_));
+        }
 
-bool RemoteStore::open() {
-//    connect();
-    return true;
+        archiveFuture_ = std::async(std::launch::async, [this] { return archiveThreadLoop(); });
+    }
 }
 
+void Archiver::error(Buffer&& payload, const eckit::net::Endpoint& endpoint) {
 
-void RemoteStore::flush() {
+    std::lock_guard lock(archiveQueuePtrMutex_);
 
-    Timer timer;
-
-    timer.start();
-
-    size_t numArchive = 0;
-
-    // Flush only does anything if there is an ongoing archive();
-    if (archiver_.valid()) {
-
-        archiver_.flush(this);
+    if (archiveQueue_) {
+        std::string msg;
+        if (payload.size() > 0) {
+            msg.resize(payload.size(), ' ');
+            payload.copy(&msg[0], payload.size());
+        }
 
-//        internalStats_ += stats;
+        archiveQueue_->interrupt(std::make_exception_ptr(DecoupledFDBException(msg, endpoint)));
     }
-
-    timer.stop();
-//    internalStats_.addFlush(timer);
 }
 
-void RemoteStore::close() {
-//    disconnect();
-}
+void Archiver::emplace(uint32_t id, RemoteStore* store, const Key& key, const void *data, eckit::Length length) {
 
-void RemoteStore::remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const {
-    NOTIMP;    
-}
+    dirty_ = true;
 
-void RemoteStore::remove(const Key& key) const {
-    NOTIMP;
+    std::lock_guard lock(archiveQueuePtrMutex_);
+    ASSERT(archiveQueue_);
+    archiveQueue_->emplace(id, store, key, data, length);
 }
 
+FDBStats Archiver::flush(RemoteStore* store) {
 
+    std::lock_guard lock(archiveQueuePtrMutex_);
+    ASSERT(archiveQueue_);
+    archiveQueue_->close();
 
+    FDBStats stats = archiveFuture_.get();
+    ASSERT(!archiveQueue_);
 
-//     eckit::PathName src_db = directory_ / key.valuesToString();
-        
-//     DIR* dirp = ::opendir(src_db.asString().c_str());
-//     struct dirent* dp;
-//     while ((dp = ::readdir(dirp)) != NULL) {
-//         if (strstr( dp->d_name, ".data")) {
-//             eckit::PathName dataFile = src_db / dp->d_name;
-//             eckit::Log::debug() << "Removing " << dataFile << std::endl;
-//             dataFile.unlink(false);
-//         }
-//     }
-//     closedir(dirp);
-// }
+    ASSERT(stats.numFlush() == 0);
+    size_t numArchive = stats.numArchive();
 
-// void RemoteStore::connect() {
-//     ASSERT(remote_);
-//     remote_->connect();
-// }
+    Buffer sendBuf(4096);
+    MemoryStream s(sendBuf);
+    s << numArchive;
 
-// void RemoteStore::disconnect() {
-//     if (remote_) {
-//         remote_->disconnect();
-//     }
-// }
+    // The flush call is blocking
+    store->controlWriteCheckResponse(fdb5::remote::Message::Flush, sendBuf, s.position());
 
-void RemoteStore::print(std::ostream &out) const {
-    out << "RemoteStore(host=" << /*controlEndpoint() << ", data=" << dataEndpoint() << */ ")";
-}
-
-
-
-
-
-// void RemoteStore::flush() {
-
-//     Timer timer;
-
-//     timer.start();
+    dirty_ = false;
 
-//     // Flush only does anything if there is an ongoing archive();
-//     if (archiveFuture_.valid()) {
-
-//         ASSERT(archiveID_ != 0);
-//         {
-//             ASSERT(archiveQueue_);
-//             std::lock_guard lock(archiveQueuePtrMutex_);
-//             archiveQueue_->close();
-//         }
-//         FDBStats stats = archiveFuture_.get();
-//         ASSERT(!archiveQueue_);
-//         archiveID_ = 0;
+    return stats;
+}
 
-//         ASSERT(stats.numFlush() == 0);
-//         size_t numArchive = stats.numArchive();
+FDBStats Archiver::archiveThreadLoop() {
+    FDBStats localStats;
+    eckit::Timer timer;
 
-//         Buffer sendBuf(4096);
-//         MemoryStream s(sendBuf);
-//         s << numArchive;
+    ArchivalRequest element;
 
-//         // The flush call is blocking
-//         controlWriteCheckResponse(fdb5::remote::Message::Flush, generateRequestID(), sendBuf, s.position());
+    try {
 
-//         internalStats_ += stats;
-//     }
+        ASSERT(archiveQueue_);
+        long popped;
+        while ((popped = archiveQueue_->pop(element)) != -1) {
 
-//     timer.stop();
-//     internalStats_.addFlush(timer);
-// }
+            timer.start();
+            element.store_->sendArchiveData(element.id_, element.key_, element.buffer_.data(), element.buffer_.size());
+            timer.stop();
 
+            localStats.addArchive(element.buffer_.size(), timer, 1);
+        }
 
+        // And note that we are done. (don't time this, as already being blocked
+        // on by the ::flush() routine)
 
+        element.store_->dataWrite(Message::Flush, nullptr, 0);
 
-bool RemoteStore::handle(Message message, uint32_t requestID) {
-//    std::cout << "RemoteStore::handle received message: " << ((uint) message) << " - requestID: " << requestID << std::endl;
-    return false;
+        archiveQueue_.reset();
 
-}
-bool RemoteStore::handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) {
-
-//    std::cout << "RemoteStore::handle received message: " << ((uint) message) << " - requestID: " << requestID << std::endl;
-    if (message == Message::Archive) {
-        auto it = locations_.find(requestID);
-        if (it != locations_.end()) {
-            MemoryStream s(payload);
-            std::unique_ptr location(eckit::Reanimator::reanimate(s));
-            // std::cout <<  "RemoteStore::handle - " << location->uri().asRawString() << " " << location->length() << std::endl;
-            std::unique_ptr remoteLocation = std::unique_ptr(new RemoteFieldLocation(endpoint, *location));
-
-            it->second.set_value(std::move(remoteLocation));
-        }
-        return true;
+    } catch (...) {
+        archiveQueue_->interrupt(std::current_exception());
+        throw;
     }
-    return false;
-}
-void RemoteStore::handleException(std::exception_ptr e) {
-    std::cout << "RemoteStore::handleException" << std::endl;
-}
 
+    return localStats;
 
-void RemoteStore::flush(FDBStats& internalStats) {
-
-    // Flush only does anything if there is an ongoing archive();
-    if (! archiver_.valid()) return;
-
-    internalStats += archiver_.flush(this);
+    // We are inside an async, so don't need to worry about exceptions escaping.
+    // They will be released when flush() is called.
 }
 
 
@@ -419,8 +215,7 @@ void RemoteStore::flush(FDBStats& internalStats) {
 ///       in the stream
 ///
 /// --> Retrieve is a _streaming_ service.
-
-namespace {
+    
 
 class FDBRemoteDataHandle : public DataHandle {
 
@@ -460,34 +255,24 @@ class FDBRemoteDataHandle : public DataHandle {
 
         // If we are in the DataHandle, then there MUST be data to read
 
-        RemoteStore::StoredMessage msg = std::make_pair(remote::MessageHeader{}, eckit::Buffer{0});
+        RemoteStore::StoredMessage msg = std::make_pair(remote::Message{}, eckit::Buffer{0});
         ASSERT(queue_.pop(msg) != -1);
 
-        // TODO; Error handling in the retrieve pathway
-
-        const MessageHeader& hdr(msg.first);
-
-        ASSERT(hdr.marker == StartMarker);
-        ASSERT(hdr.version == CurrentVersion);
-
         // Handle any remote errors communicated from the server
-
-        if (hdr.message == fdb5::remote::Message::Error) {
+        if (msg.first == Message::Error) {
             std::string errmsg(static_cast(msg.second), msg.second.size());
             throw DecoupledFDBException(errmsg, remoteEndpoint_);
         }
 
-        // Are we now complete
-
-        if (hdr.message == fdb5::remote::Message::Complete) {
+        // Are we now complete?
+        if (msg.first == Message::Complete) {
             complete_ = 0;
             return 0;
         }
 
-        ASSERT(hdr.message == fdb5::remote::Message::Blob);
+        ASSERT(msg.first == Message::Blob);
 
         // Otherwise return the data!
-
         std::swap(currentBuffer_, msg.second);
 
         return bufferRead(pos, sz);
@@ -538,8 +323,202 @@ class FDBRemoteDataHandle : public DataHandle {
     bool complete_;
 };
 
+//----------------------------------------------------------------------------------------------------------------------
+
+RemoteStore::RemoteStore(const Key& dbKey, const Config& config) :
+    Client(eckit::net::Endpoint("localhost", 7000)),
+    dbKey_(dbKey), config_(config),
+    retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
+    archiver_(Archiver::get(controlEndpoint())) {
+}
+
+RemoteStore::RemoteStore(const eckit::URI& uri, const Config& config) :
+    Client(eckit::net::Endpoint(uri.hostport())),
+    dbKey_(Key()), config_(config),
+    retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
+    archiver_(Archiver::get(controlEndpoint())) {
+
+    ASSERT(uri.scheme() == "fdb");
+}
+
+RemoteStore::~RemoteStore() {
+    // If we have launched a thread with an async and we manage to get here, this is
+    // an error. n.b. if we don't do something, we will block in the destructor
+    // of std::future.
+    if (archiver_.valid()) {
+        Log::error() << "Attempting to destruct DecoupledFDB with active archive thread" << std::endl;
+        eckit::Main::instance().terminate();
+    }
+
+    ASSERT(!archiver_.dirty());
+//    disconnect();
+}
+
+eckit::URI RemoteStore::uri() const {
+    return URI("fdb", "");
+}
+
+bool RemoteStore::exists() const {
+    return true; // directory_.exists();
+}
+
+eckit::DataHandle* RemoteStore::retrieve(Field& field) const {
+    return field.dataHandle();
+}
+
+std::future > RemoteStore::archive(const Key& key, const void *data, eckit::Length length) {
+
+    // if there is no archiving thread active, then start one.
+    // n.b. reset the archiveQueue_ after a potential flush() cycle.
+    archiver_.start();
+
+    ASSERT(archiver_.valid());
+
+    uint32_t id = controlWriteCheckResponse(Message::Archive, nullptr, 0);
+    archiver_.emplace(id, this, key, data, length);
+
+    std::promise > loc;
+    auto futureLocation = loc.get_future();
+    locations_[id] = std::move(loc);
+    return futureLocation;
+}
+
+bool RemoteStore::open() {
+    return true;
+}
+
+void RemoteStore::flush() {
+
+    Timer timer;
+
+    timer.start();
+
+    size_t numArchive = 0;
+
+    // Flush only does anything if there is an ongoing archive();
+    if (archiver_.valid()) {
+        archiver_.flush(this);
+//        internalStats_ += stats;
+    }
+
+    timer.stop();
+//    internalStats_.addFlush(timer);
+}
+
+void RemoteStore::close() {
+//    disconnect();
+}
+
+void RemoteStore::remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const {
+    NOTIMP;    
+}
+
+void RemoteStore::remove(const Key& key) const {
+    NOTIMP;
+}
+
+void RemoteStore::print(std::ostream &out) const {
+    out << "RemoteStore(host=" << /*controlEndpoint() << ", data=" << dataEndpoint() << */ ")";
 }
 
+bool RemoteStore::handle(Message message, uint32_t requestID) {
+
+    switch (message) {  
+        case Message::Complete: {
+            auto it = messageQueues_.find(requestID);
+            if (it != messageQueues_.end()) {
+                it->second->close();
+
+                // Remove entry (shared_ptr --> message queue will be destroyed when it
+                // goes out of scope in the worker thread).
+                messageQueues_.erase(it);
+
+            } else {
+                retrieveMessageQueue_.emplace(std::make_pair(message, Buffer(0)));
+            }
+            return true;
+        }
+        case Message::Error: {
+
+            auto it = messageQueues_.find(requestID);
+            if (it != messageQueues_.end()) {
+                it->second->interrupt(std::make_exception_ptr(DecoupledFDBException("", endpoint_)));
+
+                // Remove entry (shared_ptr --> message queue will be destroyed when it
+                // goes out of scope in the worker thread).
+                messageQueues_.erase(it);
+                return true;
+
+            }
+            return false;
+        }
+        default:
+            return false;
+    }
+}
+bool RemoteStore::handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) {
+
+    switch (message) {
+    
+        case Message::Archive: {
+            auto it = locations_.find(requestID);
+            if (it != locations_.end()) {
+                MemoryStream s(payload);
+                std::unique_ptr location(eckit::Reanimator::reanimate(s));
+                // std::cout <<  "RemoteStore::handle - " << location->uri().asRawString() << " " << location->length() << std::endl;
+                std::unique_ptr remoteLocation = std::unique_ptr(new RemoteFieldLocation(endpoint, *location));
+                it->second.set_value(std::move(remoteLocation));
+            }
+            return true;
+        }
+        case Message::Blob: {
+            auto it = messageQueues_.find(requestID);
+            if (it != messageQueues_.end()) {
+                it->second->emplace(message, std::move(payload));
+            } else {
+                retrieveMessageQueue_.emplace(std::make_pair(message, std::move(payload)));
+            }
+            return true;
+        }
+        case Message::Error: {
+
+            auto it = messageQueues_.find(requestID);
+            if (it != messageQueues_.end()) {
+                std::string msg;
+                msg.resize(payload.size(), ' ');
+                payload.copy(&msg[0], payload.size());
+                it->second->interrupt(std::make_exception_ptr(DecoupledFDBException(msg, endpoint)));
+
+                // Remove entry (shared_ptr --> message queue will be destroyed when it
+                // goes out of scope in the worker thread).
+                messageQueues_.erase(it);
+
+            } else {
+                auto it = locations_.find(requestID);
+                if (it != locations_.end()) {
+                    archiver_.error(std::move(payload), endpoint);
+                } else {
+                    retrieveMessageQueue_.emplace(std::make_pair(message, std::move(payload)));
+                }
+            }
+            return true;
+        }
+        default:
+            return false;
+    }
+}
+void RemoteStore::handleException(std::exception_ptr e) {
+    std::cout << "RemoteStore::handleException" << std::endl;
+}
+
+void RemoteStore::flush(FDBStats& internalStats) {
+    // Flush only does anything if there is an ongoing archive();
+    if (! archiver_.valid()) return;
+
+    internalStats += archiver_.flush(this);
+}
+
+
 // // Here we do (asynchronous) retrieving related stuff
 
 // //DataHandle* RemoteStore::retrieve(const metkit::mars::MarsRequest& request) {
@@ -657,7 +636,6 @@ eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation) {
 
 eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation, const Key& remapKey) {
 
-
     Buffer encodeBuffer(4096);
     MemoryStream s(encodeBuffer);
     s << fieldLocation;
diff --git a/src/fdb5/remote/RemoteStore.h b/src/fdb5/remote/RemoteStore.h
index 4a7f53280..02f1ce8da 100644
--- a/src/fdb5/remote/RemoteStore.h
+++ b/src/fdb5/remote/RemoteStore.h
@@ -20,6 +20,7 @@
 #include "fdb5/database/Store.h"
 #include "fdb5/remote/client/Client.h"
 
+
 namespace fdb5::remote {
 
 class Archiver;
@@ -32,7 +33,7 @@ class RemoteStore : public Store, public Client {
 
 public: // types
 
-    using StoredMessage = std::pair;
+    using StoredMessage = std::pair;
     using MessageQueue = eckit::Queue;
 
 public: // methods
@@ -77,7 +78,7 @@ class RemoteStore : public Store, public Client {
 
 private: // methods
 
-    Key key() override { return dbKey_; }
+    const Key& key() const override { return dbKey_; }
 
     // handlers for incoming messages - to be defined in the client class
     bool handle(Message message, uint32_t requestID) override;
@@ -96,21 +97,18 @@ class RemoteStore : public Store, public Client {
 private: // members
 
     Key dbKey_;
+
     const Config& config_;
 
-    bool dirty_;
+    std::map > > locations_;
 
-    size_t maxArchiveBatchSize_;
     // @note This is a map of requestID:MessageQueue. At the point that a request is
     // complete, errored or otherwise killed, it needs to be removed from the map.
     // The shared_ptr allows this removal to be asynchronous with the actual task
     // cleaning up and returning to the client.
-
     std::map> messageQueues_;
     MessageQueue retrieveMessageQueue_;
 
-    std::map > > locations_;
-
     Archiver& archiver_;
 
 };
diff --git a/src/fdb5/remote/client/Client.h b/src/fdb5/remote/client/Client.h
index 28997c2e8..1e30dc71e 100644
--- a/src/fdb5/remote/client/Client.h
+++ b/src/fdb5/remote/client/Client.h
@@ -42,7 +42,7 @@ class Client : eckit::NonCopyable {
     virtual bool handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) = 0;
     virtual void handleException(std::exception_ptr e) = 0;
 
-    virtual Key key() = 0;
+    virtual const Key& key() const = 0;
 
 protected:
 
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index c13389c60..ef2f75f47 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -343,13 +343,6 @@ void ClientConnection::listeningThreadLoop() {
             if (hdr.message == Message::Exit) {
                 return;
             }
-            if (hdr.message == Message::Error) {
-                std::stringstream ss;
-                ss << "ERROR: Server-side error. ABORTING";
-                Log::status() << ss.str() << std::endl;
-                Log::error() << "Retrieving... " << ss.str() << std::endl;
-                throw SeriousBug(ss.str(), Here());
-            }            
 
             bool handled = false;
 
@@ -360,7 +353,7 @@ void ClientConnection::listeningThreadLoop() {
                 Buffer payload(hdr.payloadSize);
                 dataRead(payload, hdr.payloadSize);
 
-                handled = ClientConnectionRouter::instance().handle(hdr.message, hdr.requestID, controlEndpoint_, std::move(payload));
+                handled = ClientConnectionRouter::instance().handle(hdr.message, hdr.requestID, dataEndpoint_, std::move(payload));
             }
 
             if (!handled) {
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.cc b/src/fdb5/remote/client/ClientConnectionRouter.cc
index f9a65bd2a..b5360bed2 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.cc
+++ b/src/fdb5/remote/client/ClientConnectionRouter.cc
@@ -9,30 +9,6 @@ using namespace eckit::net;
 
 namespace {
 
-// static ClientID generateClientID() {
-
-//     static std::mutex m;
-//     static ClientID id = 0;
-
-//     std::lock_guard lock(m);
-//     return ++id;
-// }
-// static DataLinkID generateDataLinkID() {
-
-//     static std::mutex m;
-//     static DataLinkID id = 0;
-
-//     std::lock_guard lock(m);
-//     return ++id;
-// }
-// static HandlerID generateHandlerID() {
-
-//     static std::mutex m;
-//     static HandlerID id = 0;
-
-//     std::lock_guard lock(m);
-//     return ++id;
-// }
 static uint32_t generateRequestID() {
 
     static std::mutex m;
@@ -123,30 +99,29 @@ void ClientConnectionRouter::dataWrite(Client& client, uint32_t requestId, const
     it->second.first->dataWrite(payload, payloadLength);
 }
 
-// std::future& ClientConnectionRouter::archiveFuture(Client& client) {
-//     ClientConnection* conn;
-//     uint32_t id = createConnection(client, conn);
-
-//     return conn->archiveFuture();
-// }
-
 bool ClientConnectionRouter::handle(Message message, uint32_t requestID) {
+
     if (requestID != 0) {
         auto it = requests_.find(requestID);
         ASSERT(it != requests_.end());
 
         return it->second.second->handle(message, requestID);
     } else {
-        std::cout << "ClientConnectionRouter::handle [message=" << ((uint) message) << ",requestID=" << requestID << "]" << std::endl;
+        Log::error() << "ClientConnectionRouter::handle [message=" << ((uint) message) << ",requestID=" << requestID << "]" << std::endl;
         return true;
     }
 }
 bool ClientConnectionRouter::handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) {
 
-    auto it = requests_.find(requestID);
-    ASSERT(it != requests_.end());
-
-    return it->second.second->handle(message, requestID, endpoint, std::move(payload));
+    if (requestID != 0) {
+        auto it = requests_.find(requestID);
+        ASSERT(it != requests_.end());
+    
+        return it->second.second->handle(message, requestID, endpoint, std::move(payload));
+    } else {
+        Log::error() << "ClientConnectionRouter::handle [message=" << ((uint) message) << ",requestID=" << requestID << "]" << std::endl;
+        return true;
+    }
 }
 void ClientConnectionRouter::handleException(std::exception_ptr e) {
     // auto it = requests_.find(requestID);
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.h b/src/fdb5/remote/client/ClientConnectionRouter.h
index f0c968bf6..ccc18b9bf 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.h
+++ b/src/fdb5/remote/client/ClientConnectionRouter.h
@@ -27,29 +27,12 @@ namespace fdb5::remote {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-// class Connection {
-// public:
-//     Connection(ClientConnection* clientConnection, uint32_t remoteID) : clientConnection_(clientConnection), remoteID_(remoteID) {}
-//     Connection(Connection& other) : clientConnection_(other.clientConnection_), remoteID_(other.remoteID_) {}
-    
-//     ClientConnection* clientConnection_;
-//     uint32_t remoteID_;
-// };
-
-// typedef eckit::net::Endpoint Connection;
-// typedef uint32_t ClientID;
-// typedef uint32_t DataLinkID;
-// typedef uint32_t HandlerID;
-// typedef uint32_t RequestID;
-
 class ClientConnectionRouter : eckit::NonCopyable {
 public:
 
     static ClientConnectionRouter& instance();
     RemoteStore& store(const eckit::URI& uri);
 
-// protected:
-
     uint32_t controlWriteCheckResponse(Client& client, Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
     uint32_t controlWrite(Client& client, Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
     void controlRead(Client& client, uint32_t requestId, void* payload, size_t payloadLength);
@@ -57,8 +40,6 @@ class ClientConnectionRouter : eckit::NonCopyable {
     uint32_t dataWrite(Client& client, Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
     void dataWrite(Client& client, uint32_t requestId, const void* data, size_t length);
 
-//    std::future& archiveFuture(Client& client);
-
     // handlers for incoming messages - to be defined in the client class
     bool handle(Message message, uint32_t requestID);
     bool handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload);
@@ -80,7 +61,6 @@ class ClientConnectionRouter : eckit::NonCopyable {
     // endpoint -> Store (one read store for each connection. Do not need to have one for each key)
     std::map > readStores_;
 
-//    std::map > > archiveFutures_;
 };
 
 }
\ No newline at end of file
diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc
index e6c16468f..866f7357f 100644
--- a/src/fdb5/remote/server/CatalogueHandler.cc
+++ b/src/fdb5/remote/server/CatalogueHandler.cc
@@ -145,10 +145,6 @@ void CatalogueHandler::handle() {
                     schema(hdr);
                     break;
 
-                // case Message::Store:
-                //     store(hdr);
-                //     break;
-
                 default: {
                     std::stringstream ss;
                     ss << "ERROR: Unexpected message recieved (" << static_cast(hdr.message)
@@ -190,8 +186,6 @@ void CatalogueHandler::flush(const MessageHeader& hdr) {
     Buffer payload(receivePayload(hdr, controlSocket_));
     MemoryStream s(payload);
 
-    //fdb5::Key dbKey(s);
-
     size_t numArchived;
     s >> numArchived;
 
@@ -206,16 +200,13 @@ void CatalogueHandler::flush(const MessageHeader& hdr) {
         // Do the actual flush!
         Log::info() << "Flushing" << std::endl;
         Log::status() << "Flushing" << std::endl;
-        //if (dbKey.empty()) {
-            for (auto it = catalogues_.begin(); it != catalogues_.end(); it++) {
-                it->second->flush();
-            }
-        // } else {
-        //     store(dbKey).flush();
-        // }
-        Log::info() << "Flush complete" << std::endl;
-        Log::status() << "Flush complete" << std::endl;
+
+        for (auto it = catalogues_.begin(); it != catalogues_.end(); it++) {
+            it->second->flush();
+        }
     }
+    Log::info() << "Flush complete" << std::endl;
+    Log::status() << "Flush complete" << std::endl;
 }
 
 void CatalogueHandler::archive(const MessageHeader& hdr) {
diff --git a/src/fdb5/toc/TocIndex.cc b/src/fdb5/toc/TocIndex.cc
index 0c37ab70c..b5aa13bac 100644
--- a/src/fdb5/toc/TocIndex.cc
+++ b/src/fdb5/toc/TocIndex.cc
@@ -85,7 +85,6 @@ bool TocIndex::get(const InspectionKey &key, const Key &remapKey, Field &field)
     bool found = btree_->get(key.valuesToString(), ref);
     if ( found ) {
         const eckit::URI& uri = uris_.get(ref.uriId());
-        std::cout << "TocIndex::get " << uri << std::endl;
         FieldLocation* loc = FieldLocationFactory::instance().build(uri.scheme(), uri, ref.offset(), ref.length(), remapKey);
         field = Field(std::move(*loc), timestamp_, ref.details());
         delete(loc);

From 618e3da967a607b022286b7c938363460bf1b4fc Mon Sep 17 00:00:00 2001
From: Chris Bradley 
Date: Fri, 27 Oct 2023 12:01:28 +0100
Subject: [PATCH 012/186] Remote catalogue list and api forwarding

---
 src/fdb5/CMakeLists.txt                    |   4 +-
 src/fdb5/api/LocalFDB.cc                   |   2 +-
 src/fdb5/api/RemoteFDB.cc                  | 121 +++++++++++++++++++++
 src/fdb5/api/RemoteFDB.h                   |  96 ++++++++++++++++
 src/fdb5/database/Catalogue.cc             |   4 +-
 src/fdb5/remote/RemoteCatalogue.cc         |  23 +++-
 src/fdb5/remote/RemoteCatalogue.h          |   7 +-
 src/fdb5/remote/server/CatalogueHandler.cc |  85 ++++++++++++++-
 src/fdb5/remote/server/CatalogueHandler.h  |   6 +-
 9 files changed, 332 insertions(+), 16 deletions(-)
 create mode 100644 src/fdb5/api/RemoteFDB.cc
 create mode 100644 src/fdb5/api/RemoteFDB.h

diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt
index 474b54fbb..3bb4bc1f3 100644
--- a/src/fdb5/CMakeLists.txt
+++ b/src/fdb5/CMakeLists.txt
@@ -210,8 +210,8 @@ list( APPEND fdb5_srcs
 
 if(HAVE_FDB_REMOTE)
     list( APPEND fdb5_srcs 
-        # api/RemoteFDB.cc
-        # api/RemoteFDB.h
+        api/RemoteFDB.cc
+        api/RemoteFDB.h
         remote/RemoteStore.cc
         remote/RemoteStore.h
         remote/RemoteCatalogue.h
diff --git a/src/fdb5/api/LocalFDB.cc b/src/fdb5/api/LocalFDB.cc
index a9d161d76..b8644ac33 100644
--- a/src/fdb5/api/LocalFDB.cc
+++ b/src/fdb5/api/LocalFDB.cc
@@ -136,7 +136,7 @@ void LocalFDB::print(std::ostream &s) const {
 
 
 static FDBBuilder localFdbBuilder("local");
-
+static FDBBuilder builder("catalogue"); // Enable type=catalogue to build localFDB (serverside).
 //----------------------------------------------------------------------------------------------------------------------
 
 } // namespace fdb5
diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
new file mode 100644
index 000000000..de294da96
--- /dev/null
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -0,0 +1,121 @@
+#include "eckit/io/Buffer.h"
+#include "eckit/log/Log.h"
+#include "eckit/serialisation/MemoryStream.h"
+#include "fdb5/api/helpers/FDBToolRequest.h"
+
+#include "fdb5/api/RemoteFDB.h"
+#include "fdb5/database/Archiver.h"
+#include "fdb5/LibFdb5.h"
+
+#include  // xxx debug / development
+#include  // xxx debug / development
+
+
+using namespace fdb5::remote;
+using namespace eckit;
+namespace fdb5 {
+
+// TODO: Contact Catalogue to get parent schema.
+
+RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name):
+    FDBBase(config, name),
+    Client(std::vector{eckit::net::Endpoint("localhost", 7001)}) /*<--catalogue endpoint*/ 
+    { 
+}
+
+// archive -- same as localFDB. Archiver will build a RemoteCatalogueWriter and RemoteStore.
+// currently RemoteCatalogueWriter is selected by setting `engine` to remote.
+void RemoteFDB::archive(const Key& key, const void* data, size_t length) {
+    if (!archiver_) {
+        eckit::Log::debug() << *this << ": Constructing new archiver" << std::endl;
+        archiver_.reset(new Archiver(config_));
+    }
+
+    archiver_->archive(key, data, length);
+}
+
+ListIterator RemoteFDB::inspect(const metkit::mars::MarsRequest& request) {NOTIMP;}
+
+
+ListIterator RemoteFDB::list(const FDBToolRequest& request) {
+
+    dolist_ = true;
+    
+    // worker that does nothing but exposes the AsyncIterator's queue.
+    auto async_worker = [this] (Queue& queue) {
+        listqueue_ = &queue;
+        while(!queue.closed()) std::this_thread::sleep_for(std::chrono::milliseconds(10));
+    };
+
+    // construct iterator before contacting remote, because we want to point to the asynciterator's queue
+    auto iter = new APIAsyncIterator(async_worker); 
+
+    Buffer encodeBuffer(4096);
+    MemoryStream s(encodeBuffer);
+    s << request;
+
+    controlWriteCheckResponse(fdb5::remote::Message::List, encodeBuffer, s.position());
+    return APIIterator(iter);
+}
+
+DumpIterator RemoteFDB::dump(const FDBToolRequest& request, bool simple) {NOTIMP;}
+
+StatusIterator RemoteFDB::status(const FDBToolRequest& request) {NOTIMP;}
+
+WipeIterator RemoteFDB::wipe(const FDBToolRequest& request, bool doit, bool porcelain, bool unsafeWipeAll) {NOTIMP;}
+
+PurgeIterator RemoteFDB::purge(const FDBToolRequest& request, bool doit, bool porcelain) {NOTIMP;}
+
+StatsIterator RemoteFDB::stats(const FDBToolRequest& request) {NOTIMP;}
+
+ControlIterator RemoteFDB::control(const FDBToolRequest& request,
+                        ControlAction action,
+                        ControlIdentifiers identifiers) {NOTIMP;}
+
+MoveIterator RemoteFDB::move(const FDBToolRequest& request, const eckit::URI& dest) {NOTIMP;}
+
+void RemoteFDB::flush() {
+    if (archiver_) {
+        archiver_->flush();
+    }
+}
+
+void RemoteFDB::print(std::ostream& s) const {
+    s << "RemoteFDB(...)";
+}
+
+FDBStats RemoteFDB::stats() const {NOTIMP;}
+
+
+// Client
+
+bool RemoteFDB::handle(remote::Message message, uint32_t requestID){
+    if (message == Message::Complete) {
+        listqueue_->close();
+        return true;
+    }
+    return false;
+}
+bool RemoteFDB::handle(remote::Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload){
+
+    eckit::Log::debug()<< "RemoteFDB::handle received message: " << ((uint) message) << " - requestID: " << requestID << std::endl;
+    if (message == Message::Blob) {
+        eckit::Log::debug() << "RemoteFDB::handle received payload size: " << payload.size() << std::endl;
+        MemoryStream s(payload);
+        if (dolist_){
+            ListElement elem(s);
+            listqueue_->push(elem);
+        }
+        else {
+            NOTIMP;
+        }
+        return true;
+    }
+    return false;
+}
+void RemoteFDB::handleException(std::exception_ptr e){NOTIMP;}
+Key RemoteFDB::key(){NOTIMP;}
+
+static FDBBuilder builder("remote");
+
+} // namespace fdb5
diff --git a/src/fdb5/api/RemoteFDB.h b/src/fdb5/api/RemoteFDB.h
new file mode 100644
index 000000000..5c0e5c8b0
--- /dev/null
+++ b/src/fdb5/api/RemoteFDB.h
@@ -0,0 +1,96 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/*
+ * This software was developed as part of the EC H2020 funded project NextGenIO
+ * (Project ID: 671951) www.nextgenio.eu
+ */
+
+/// @author Simon Smart
+/// @date   Mar 2018
+
+#ifndef fdb5_remote_RemoteFDB_H
+#define fdb5_remote_RemoteFDB_H
+
+#include 
+#include 
+
+#include "fdb5/api/FDB.h"
+#include "fdb5/api/FDBFactory.h"
+#include "fdb5/remote/client/Client.h"
+
+namespace fdb5 {
+
+//----------------------------------------------------------------------------------------------------------------------
+class Archiver;
+
+class RemoteFDB : public FDBBase, public remote::Client {
+
+public: // types
+
+public: // method
+
+    using FDBBase::stats;
+
+    RemoteFDB(const eckit::Configuration& config, const std::string& name);
+
+    /// Archive writes data into aggregation buffer
+    void archive(const Key& key, const void* data, size_t length) override;
+
+    ListIterator inspect(const metkit::mars::MarsRequest& request) override;
+
+    ListIterator list(const FDBToolRequest& request) override;
+
+    DumpIterator dump(const FDBToolRequest& request, bool simple) override;
+
+    StatusIterator status(const FDBToolRequest& request) override;
+
+    WipeIterator wipe(const FDBToolRequest& request, bool doit, bool porcelain, bool unsafeWipeAll) override;
+
+    PurgeIterator purge(const FDBToolRequest& request, bool doit, bool porcelain) override;
+
+    StatsIterator stats(const FDBToolRequest& request) override;
+
+    ControlIterator control(const FDBToolRequest& request,
+                            ControlAction action,
+                            ControlIdentifiers identifiers) override;
+
+    MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) override;
+
+    void flush() override;
+
+    // Client
+
+    bool handle(remote::Message message, uint32_t requestID) override;
+    bool handle(remote::Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) override;
+    void handleException(std::exception_ptr e) override;
+
+    Key key() override;
+
+private: // methods
+
+    virtual void print(std::ostream& s) const override;
+
+    virtual FDBStats stats() const override;
+
+private: // members
+
+    std::unique_ptr archiver_;
+
+    bool dolist_ = false;
+    eckit::Queue* listqueue_;
+
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
+} // namespace fdb5
+
+#endif // fdb5_remote_RemoteFDB_H
\ No newline at end of file
diff --git a/src/fdb5/database/Catalogue.cc b/src/fdb5/database/Catalogue.cc
index 67dadf10f..6b8e25de5 100644
--- a/src/fdb5/database/Catalogue.cc
+++ b/src/fdb5/database/Catalogue.cc
@@ -91,7 +91,7 @@ std::unique_ptr CatalogueReaderFactory::build(const Key& dbKey,
     eckit::AutoLock lock(mutex_);
     auto j = builders_.find(nameLowercase);
 
-    eckit::Log::debug() << "Looking for CatalogueReaderBuilder [" << nameLowercase << "]" << std::endl;
+    eckit::Log::debug() << "Looking for CatalogueReaderBuilder [" << nameLowercase << "]" << std::endl;
 
     if (j == builders_.end()) {
         eckit::Log::error() << "No CatalogueReaderBuilder for [" << nameLowercase << "]" << std::endl;
@@ -111,7 +111,7 @@ std::unique_ptr CatalogueReaderFactory::build(const eckit::URI&
     eckit::AutoLock lock(mutex_);
     auto j = builders_.find(nameLowercase);
 
-    eckit::Log::debug() << "Looking for CatalogueReaderBuilder [" << nameLowercase << "]" << std::endl;
+    eckit::Log::debug() << "Looking for CatalogueReaderBuilder [" << nameLowercase << "]" << std::endl;
 
     if (j == builders_.end()) {
         eckit::Log::error() << "No CatalogueReaderBuilder for [" << nameLowercase << "]" << std::endl;
diff --git a/src/fdb5/remote/RemoteCatalogue.cc b/src/fdb5/remote/RemoteCatalogue.cc
index 4bcb0fc94..07ea4c2eb 100644
--- a/src/fdb5/remote/RemoteCatalogue.cc
+++ b/src/fdb5/remote/RemoteCatalogue.cc
@@ -201,7 +201,7 @@ void RemoteCatalogue::loadSchema() {
     // NB we're at the db level, so get the db schema. We will want to get the master schema beforehand.
     // (outside of the catalogue) 
 
-    eckit::Log::debug() << "RemoteCatalogueWriter::loadSchema()" << std::endl;
+    eckit::Log::debug() << "RemoteCatalogue::loadSchema()" << std::endl;
 
     // destroy previous schema if it exists
     schema_.reset(nullptr);
@@ -213,7 +213,7 @@ void RemoteCatalogue::loadSchema() {
     keyStream << dbKey_;
     
     uint32_t id = controlWriteCheckResponse(Message::Schema, keyBuffer, keyStream.position());
-    eckit::Log::debug() << "RemoteCatalogueWriter::loadSchema() received id: " << id << std::endl;
+    eckit::Log::debug() << "RemoteCatalogue::loadSchema() received id: " << id << std::endl;
 
     // Should block - Use control connection.
     // XXX This is a temporary work around while control read is being reimplemented.
@@ -243,6 +243,17 @@ void RemoteCatalogue::handleException(std::exception_ptr e) {
     NOTIMP;
 }
 
+// Catalogue Reader
+// DbStats RemoteCatalogue::stats() const {
+//     NOTIMP;
+// }
+// bool RemoteCatalogue::axis(const std::string& keyword, eckit::StringSet& s) const {
+//     NOTIMP;
+// }
+// bool RemoteCatalogue::retrieve(const InspectionKey& key, Field& field) const{
+//     NOTIMP;
+// }
+
 void RemoteCatalogue::overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) {NOTIMP;}
 void RemoteCatalogue::index(const InspectionKey& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) {NOTIMP;}
 void RemoteCatalogue::reconsolidate(){NOTIMP;}
@@ -258,8 +269,12 @@ std::vector RemoteCatalogue::indexes(bool sorted) const {NOTIMP;}
 void RemoteCatalogue::maskIndexEntry(const Index& index) const {NOTIMP;}
 void RemoteCatalogue::allMasked(std::set>& metadata, std::set& data) const {NOTIMP;}
 void RemoteCatalogue::print( std::ostream &out ) const {NOTIMP;}
-std::string RemoteCatalogue::type() const {NOTIMP;}
-bool RemoteCatalogue::open() {NOTIMP;}
+std::string RemoteCatalogue::type() const {
+    NOTIMP;
+}
+bool RemoteCatalogue::open() {
+    NOTIMP;
+}
 
 static CatalogueWriterBuilder builder("remote");
 } // namespace fdb5::remote
\ No newline at end of file
diff --git a/src/fdb5/remote/RemoteCatalogue.h b/src/fdb5/remote/RemoteCatalogue.h
index 82dc10a7f..aeb741a04 100644
--- a/src/fdb5/remote/RemoteCatalogue.h
+++ b/src/fdb5/remote/RemoteCatalogue.h
@@ -31,6 +31,11 @@ class RemoteCatalogue : public CatalogueWriter, public CatalogueImpl, public Cli
     void index(const InspectionKey& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) override;
     void reconsolidate() override;
 
+    //From CatalogueReader
+    // DbStats stats() const override;
+    // bool axis(const std::string& keyword, eckit::StringSet& s) const override;
+    // bool retrieve(const InspectionKey& key, Field& field) const override;
+
     // From Catalogue
     bool selectIndex(const Key& idxKey) override;
     const Key currentIndexKey() override;
@@ -58,10 +63,8 @@ class RemoteCatalogue : public CatalogueWriter, public CatalogueImpl, public Cli
     void checkUID() const override;
     eckit::URI uri() const override;
 
-
 protected:
 
-    Schema getSchema();
     void loadSchema() override;
 
 private:
diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc
index e6c16468f..d7a588ba1 100644
--- a/src/fdb5/remote/server/CatalogueHandler.cc
+++ b/src/fdb5/remote/server/CatalogueHandler.cc
@@ -69,6 +69,86 @@ void CatalogueHandler::initialiseConnections() {
     Log::info() << " but... todo... " << std::endl;
 }
 
+
+// API forwarding logic, adapted from original remoteHandler
+// Used for Inspect and List
+// ***************************************************************************************
+// All of the standard API functions behave in roughly the same manner. The Helper Classes
+// described here capture the manner in which their behaviours differ.
+//
+// See forwardApiCall() for how these helpers are used to forward an API call using a
+// worker thread.
+// ***************************************************************************************
+template 
+struct BaseHelper {
+    virtual size_t encodeBufferSize(const ValueType&) const { return 4096; }
+    void extraDecode(eckit::Stream&) {}
+    ValueType apiCall(FDB&, const FDBToolRequest&) const { NOTIMP; }
+
+    struct Encoded {
+        size_t position;
+        eckit::Buffer buf;
+    };
+
+    Encoded encode(const ValueType& elem, const CatalogueHandler&) const {
+        eckit::Buffer encodeBuffer(encodeBufferSize(elem));
+        MemoryStream s(encodeBuffer);
+        s << elem;
+        return {s.position(), std::move(encodeBuffer)};
+    }
+};
+
+struct ListHelper : public BaseHelper {
+    ListIterator apiCall(FDB& fdb, const FDBToolRequest& request) const {
+        return fdb.list(request);
+    }
+};
+
+struct InspectHelper : public BaseHelper {
+    ListIterator apiCall(FDB& fdb, const FDBToolRequest& request) const {
+        return fdb.inspect(request.request());
+    }
+};
+
+template 
+void CatalogueHandler::forwardApiCall(const MessageHeader& hdr) {
+    HelperClass helper;
+
+    Buffer payload(receivePayload(hdr, controlSocket_));
+    MemoryStream s(payload);
+
+    FDBToolRequest request(s);
+    helper.extraDecode(s);
+
+    // Construct worker thread to feed responses back to client
+
+    ASSERT(workerThreads_.find(hdr.requestID) == workerThreads_.end());
+
+    workerThreads_.emplace(
+        hdr.requestID, std::async(std::launch::async, [request, hdr, helper, this]() {
+
+            try {
+                auto iterator = helper.apiCall(fdb_, request);
+                typename decltype(iterator)::value_type elem;
+                while (iterator.next(elem)) {
+                    auto encoded(helper.encode(elem, *this));
+                    dataWrite(Message::Blob, hdr.requestID, encoded.buf, encoded.position);
+                }
+                dataWrite(Message::Complete, hdr.requestID);
+            }
+            catch (std::exception& e) {
+                // n.b. more general than eckit::Exception
+                std::string what(e.what());
+                dataWrite(Message::Error, hdr.requestID, what.c_str(), what.length());
+            }
+            catch (...) {
+                // We really don't want to std::terminate the thread
+                std::string what("Caught unexpected, unknown exception in worker");
+                dataWrite(Message::Error, hdr.requestID, what.c_str(), what.length());
+            }
+        }));
+}
+
 void CatalogueHandler::handle() {
     initialiseConnections();
 
@@ -96,7 +176,6 @@ void CatalogueHandler::handle() {
                     return;
 
                 case Message::List:
-                    // forwardApiCall(hdr);
                     list(hdr);
                     break;
 
@@ -125,7 +204,6 @@ void CatalogueHandler::handle() {
                     break;
 
                 case Message::Inspect:
-                    // forwardApiCall(hdr);
                     inspect(hdr);
                     break;
 
@@ -338,10 +416,11 @@ size_t CatalogueHandler::archiveThreadLoop() {
 }
 
 void CatalogueHandler::list(const MessageHeader& hdr) {
-    NOTIMP;
+    forwardApiCall(hdr);
 }
 
 void CatalogueHandler::inspect(const MessageHeader& hdr) {
+    // forwardApiCall(hdr);
     NOTIMP;
 }
 
diff --git a/src/fdb5/remote/server/CatalogueHandler.h b/src/fdb5/remote/server/CatalogueHandler.h
index 11e15c333..debead500 100644
--- a/src/fdb5/remote/server/CatalogueHandler.h
+++ b/src/fdb5/remote/server/CatalogueHandler.h
@@ -12,6 +12,7 @@
 #define fdb5_remote_CatalogueHandler_H
 
 #include "fdb5/remote/server/ServerConnection.h"
+#include "fdb5/api/FDB.h"
 
 
 namespace fdb5::remote {
@@ -41,13 +42,14 @@ class CatalogueHandler : public ServerConnection {
     size_t archiveThreadLoop();
 
     // API functionality
-    // template 
-    // void forwardApiCall(const MessageHeader& hdr);
+    template 
+    void forwardApiCall(const MessageHeader& hdr);
 
 private:  // member
 
    std::map> catalogues_;
 //    std::unique_ptr catalogue_;
+    FDB fdb_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------

From 2e8d795a2095a9162ee5e688f285c6f498885180 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 27 Oct 2023 15:52:11 +0100
Subject: [PATCH 013/186] receive store config from catalogue

---
 src/fdb5/api/RemoteFDB.cc                     | 16 +++++-
 src/fdb5/api/RemoteFDB.h                      |  1 +
 src/fdb5/config/Config.cc                     | 15 +++++
 src/fdb5/config/Config.h                      |  2 +
 src/fdb5/remote/FdbServer.cc                  |  4 +-
 src/fdb5/remote/Messages.h                    |  1 +
 src/fdb5/remote/RemoteCatalogue.cc            |  4 +-
 src/fdb5/remote/client/ClientConnection.cc    | 25 ++++++++
 src/fdb5/remote/client/ClientConnection.h     |  2 +
 .../remote/client/ClientConnectionRouter.cc   |  8 +++
 .../remote/client/ClientConnectionRouter.h    |  1 +
 src/fdb5/remote/server/CatalogueHandler.cc    | 57 +++++++++++++------
 12 files changed, 113 insertions(+), 23 deletions(-)

diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index 9114d792b..580b86b93 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -7,6 +7,8 @@
 #include "fdb5/database/Archiver.h"
 #include "fdb5/LibFdb5.h"
 
+#include "fdb5/remote/client/ClientConnectionRouter.h"
+
 #include  // xxx debug / development
 #include  // xxx debug / development
 
@@ -19,8 +21,18 @@ namespace fdb5 {
 
 RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name):
     FDBBase(config, name),
-    Client(eckit::net::Endpoint("localhost", 7001)) /*<--catalogue endpoint*/ 
-    { 
+    Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port"))) {
+
+    uint32_t payloadLength = 102400;
+    eckit::Buffer buf(payloadLength);
+
+    ClientConnectionRouter::instance().controlReadResponse(*this, Message::MasterSchema, buf.data(), payloadLength);
+    MemoryStream s(buf);
+    storeEndpoint_ = eckit::net::Endpoint(s);
+    fdb5::Schema* schema = eckit::Reanimator::reanimate(s);
+    config_.overrideSchema("masterSchema", schema);
+    config_.set("storeHost", storeEndpoint_.host());
+    config_.set("storePort", storeEndpoint_.port());
 }
 
 // archive -- same as localFDB. Archiver will build a RemoteCatalogueWriter and RemoteStore.
diff --git a/src/fdb5/api/RemoteFDB.h b/src/fdb5/api/RemoteFDB.h
index 4af696f66..029c8800e 100644
--- a/src/fdb5/api/RemoteFDB.h
+++ b/src/fdb5/api/RemoteFDB.h
@@ -83,6 +83,7 @@ class RemoteFDB : public FDBBase, public remote::Client {
 private: // members
 
     std::unique_ptr archiver_;
+    eckit::net::Endpoint storeEndpoint_;
 
     bool dolist_ = false;
     eckit::Queue* listqueue_;
diff --git a/src/fdb5/config/Config.cc b/src/fdb5/config/Config.cc
index 17c311072..10421894c 100644
--- a/src/fdb5/config/Config.cc
+++ b/src/fdb5/config/Config.cc
@@ -37,6 +37,12 @@ class SchemaRegistry {
         return me;
     }
 
+    const Schema& add(const PathName& path, Schema* schema) {
+        ASSERT(schema);
+        schemas_[path] = std::unique_ptr(schema);
+        return *schemas_[path];
+    }
+
     const Schema& get(const PathName& path) {
         std::lock_guard lock(m_);
         auto it = schemas_.find(path);
@@ -176,6 +182,15 @@ const PathName& Config::schemaPath() const {
     return schemaPath_;
 }
 
+void Config::overrideSchema(const eckit::PathName& schemaPath, Schema* schema) {
+    ASSERT(schema);
+
+    SchemaRegistry::instance().add("masterSchema", schema);
+    schemaPath_=schemaPath;
+
+    schemaPathInitialised_ = true;
+}
+
 void Config::initializeSchemaPath() const {
 
     if (schemaPathInitialised_) {
diff --git a/src/fdb5/config/Config.h b/src/fdb5/config/Config.h
index f8d8e1d94..be89d4700 100644
--- a/src/fdb5/config/Config.h
+++ b/src/fdb5/config/Config.h
@@ -47,6 +47,8 @@ class Config : public eckit::LocalConfiguration {
     /// then do the expansion in here.
     eckit::PathName expandPath(const std::string& path) const;
 
+
+    void overrideSchema(const eckit::PathName& schemaPath, Schema* schema);
     const eckit::PathName& schemaPath() const;
     eckit::PathName configPath() const;
 
diff --git a/src/fdb5/remote/FdbServer.cc b/src/fdb5/remote/FdbServer.cc
index 5bc2c27be..e8d6d70cc 100644
--- a/src/fdb5/remote/FdbServer.cc
+++ b/src/fdb5/remote/FdbServer.cc
@@ -105,8 +105,8 @@ void FDBServerThread::run() {
 
     if (config_.getString("type", "local") == "catalogue" || (::getenv("FDB_IS_CAT") && ::getenv("FDB_IS_CAT")[0] == '1')) {
         eckit::Log::info() << "FDB using Catalogue Handler" << std::endl;
-        // CatalogueHandler handler(socket_, config_);
-        // handler.handle();
+        CatalogueHandler handler(socket_, config_);
+        handler.handle();
     } 
     else if (config_.getString("type", "local") == "store" || (::getenv("FDB_IS_STORE") && ::getenv("FDB_IS_STORE")[0] == '1')) {
         eckit::Log::info() << "FDB using Store Handler" << std::endl;
diff --git a/src/fdb5/remote/Messages.h b/src/fdb5/remote/Messages.h
index 1565fe7a4..0c6fbf916 100644
--- a/src/fdb5/remote/Messages.h
+++ b/src/fdb5/remote/Messages.h
@@ -47,6 +47,7 @@ enum class Message : uint16_t {
     Exit,
     Startup,
     Error,
+    MasterSchema,
 
     // API calls to forward
     Flush = 100,
diff --git a/src/fdb5/remote/RemoteCatalogue.cc b/src/fdb5/remote/RemoteCatalogue.cc
index c2e9b1681..87e1f2d0e 100644
--- a/src/fdb5/remote/RemoteCatalogue.cc
+++ b/src/fdb5/remote/RemoteCatalogue.cc
@@ -24,7 +24,7 @@ namespace fdb5::remote {
 
 RemoteCatalogue::RemoteCatalogue(const Key& key, const Config& config):
     CatalogueImpl(key, ControlIdentifiers(), config), // xxx what are control identifiers? Setting empty here...
-    Client(eckit::net::Endpoint("localhost", 7001)), // xxx hardcoded endpoint
+    Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port"))), // xxx hardcoded endpoint
     config_(config), schema_(nullptr),
     maxArchiveQueueLength_(eckit::Resource("fdbRemoteArchiveQueueLength;$FDB_REMOTE_ARCHIVE_QUEUE_LENGTH", 200))
     {
@@ -32,7 +32,7 @@ RemoteCatalogue::RemoteCatalogue(const Key& key, const Config& config):
     }
 
 RemoteCatalogue::RemoteCatalogue(const eckit::URI& uri, const Config& config):
-    Client(eckit::net::Endpoint("localhost", 7001)),  schema_(nullptr),
+    Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port"))),  schema_(nullptr),
     maxArchiveQueueLength_(eckit::Resource("fdbRemoteArchiveQueueLength;$FDB_REMOTE_ARCHIVE_QUEUE_LENGTH", 200))
     {
         NOTIMP;
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index ef2f75f47..86e438124 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -174,6 +174,31 @@ void ClientConnection::controlWriteCheckResponse(Message msg, uint32_t requestID
     ASSERT(tail == EndMarker);
 }
 
+void ClientConnection::controlReadResponse(remote::Message msg, uint32_t requestID, void* payload, uint32_t& payloadLength) {
+    
+    controlWrite(msg, requestID, nullptr, 0);
+
+    // Wait for the receipt acknowledgement
+
+    MessageHeader response;
+    controlRead(&response, sizeof(MessageHeader));
+
+    handleError(response);
+
+    ASSERT(response.marker == StartMarker);
+    ASSERT(response.version == CurrentVersion);
+    ASSERT(response.message == Message::Received);
+    ASSERT(response.requestID == requestID);
+    
+    ASSERT(response.payloadSize <= payloadLength);
+    payloadLength = response.payloadSize;
+    controlRead(payload, payloadLength);
+
+    eckit::FixedString<4> tail;
+    controlRead(&tail, sizeof(tail));
+    ASSERT(tail == EndMarker);
+}
+
 void ClientConnection::controlWrite(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
     //std::cout << "ClientConnection::controlWrite " << this << std::endl;
 
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index 055255ddc..158dc2276 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -55,11 +55,13 @@ class ClientConnection : eckit::NonCopyable {
     virtual ~ClientConnection();
 
     void controlWriteCheckResponse(remote::Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
+    void controlReadResponse(remote::Message msg, uint32_t requestID, void* payload, uint32_t& payloadLength);
     void controlWrite(remote::Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
     void dataWrite(remote::Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
 
     void controlRead(void* data, size_t length);
     void controlWrite(const void* data, size_t length);
+
     void dataWrite(const void* data, size_t length);
     void dataRead(void* data, size_t length);
 
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.cc b/src/fdb5/remote/client/ClientConnectionRouter.cc
index b5360bed2..1efa679e7 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.cc
+++ b/src/fdb5/remote/client/ClientConnectionRouter.cc
@@ -64,6 +64,14 @@ uint32_t ClientConnectionRouter::controlWriteCheckResponse(Client& client, Messa
 
     return id;
 }
+
+void ClientConnectionRouter::controlReadResponse(Client& client, remote::Message msg, void* payload, uint32_t& payloadLength) {
+    ClientConnection* conn;
+    uint32_t id = createConnection(client, conn);
+
+    conn->controlReadResponse(msg, id, payload, payloadLength);
+}
+
 uint32_t ClientConnectionRouter::controlWrite(Client& client, Message msg, const void* payload, uint32_t payloadLength) {
     //std::cout << "ClientConnectionRouter::controlWrite " << endpoints.size() << std::endl;
     ClientConnection* conn;
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.h b/src/fdb5/remote/client/ClientConnectionRouter.h
index ccc18b9bf..a99dd3c17 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.h
+++ b/src/fdb5/remote/client/ClientConnectionRouter.h
@@ -34,6 +34,7 @@ class ClientConnectionRouter : eckit::NonCopyable {
     RemoteStore& store(const eckit::URI& uri);
 
     uint32_t controlWriteCheckResponse(Client& client, Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
+    void controlReadResponse(Client& client, remote::Message msg, void* payload, uint32_t& payloadLength);
     uint32_t controlWrite(Client& client, Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
     void controlRead(Client& client, uint32_t requestId, void* payload, size_t payloadLength);
 
diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc
index 21db6aab9..a315e779d 100644
--- a/src/fdb5/remote/server/CatalogueHandler.cc
+++ b/src/fdb5/remote/server/CatalogueHandler.cc
@@ -39,34 +39,57 @@ CatalogueHandler::~CatalogueHandler() {}
 void CatalogueHandler::initialiseConnections() {
     ServerConnection::initialiseConnections();
 
-    std::string storeHost = ::getenv("FDB_STORE_HOST") ? ::getenv("FDB_STORE_HOST") : "localhost";
-    std::string storePort = ::getenv("FDB_STORE_PORT") ? ::getenv("FDB_STORE_PORT") : "7000";
-    net::Endpoint storeEndpoint(storeHost, std::stoi(storePort));
+    MessageHeader hdr;
+    eckit::FixedString<4> tail;
 
+    socketRead(&hdr, sizeof(hdr), controlSocket_);
 
-    // Log::info() << "Sending store endpoint to client: " << storeEndpoint << std::endl;
-    // {
-    //     Buffer startupBuffer(1024);
-    //     MemoryStream s(startupBuffer);
+    ASSERT(hdr.marker == StartMarker);
+    ASSERT(hdr.version == CurrentVersion);
+    ASSERT(hdr.message == Message::MasterSchema);
 
-    //     s << clientSession;
-    //     s << sessionID_;
-    //     s << storeEndpoint;
+    // Ensure we have consumed exactly the correct amount from the socket.
+    socketRead(&tail, sizeof(tail), controlSocket_);
+    ASSERT(tail == EndMarker);
 
-    //     // s << storeEndpoint; // xxx single-store case only: we cant do this with multiple stores // For now, dont send the store endpoint to the client 
+    std::vector stores;
 
-    //     Log::debug() << "Protocol negotiation - configuration: " << agreedConf_ < endpoints = config_.getStringVector("stores");
+        for (const std::string& endpoint: endpoints) {
+            stores.push_back(eckit::net::Endpoint(endpoint));
+        }
+    }
 
+    ASSERT(stores.size() > 0);
+    // TODO randomise
+    eckit::net::Endpoint endpoint = stores.at(0);
 
-    //     controlWrite(Message::Startup, 0, startupBuffer.data(), s.position());
-    // }
+    Log::info() << "Sending store endpoint to client: " << endpoint << std::endl;
+    {
+        Buffer startupBuffer(102400);
+        MemoryStream s(startupBuffer);
+
+        // s << clientSession;
+        // s << sessionID_;
+        s << endpoint;
+        s << config_.schema();
+
+//        Log::debug() << "Protocol negotiation - configuration: " << agreedConf_ <
Date: Fri, 27 Oct 2023 16:01:27 +0100
Subject: [PATCH 014/186] Remote retrieve and archive using control endpoint

---
 src/fdb5/api/RemoteFDB.cc                  | 34 ++++++++++++++++++----
 src/fdb5/api/RemoteFDB.h                   |  2 ++
 src/fdb5/remote/client/ClientConnection.cc |  2 +-
 src/fdb5/remote/server/CatalogueHandler.cc |  3 +-
 4 files changed, 32 insertions(+), 9 deletions(-)

diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index 9114d792b..dc1370e65 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -34,7 +34,26 @@ void RemoteFDB::archive(const Key& key, const void* data, size_t length) {
     archiver_->archive(key, data, length);
 }
 
-ListIterator RemoteFDB::inspect(const metkit::mars::MarsRequest& request) {NOTIMP;}
+ListIterator RemoteFDB::inspect(const metkit::mars::MarsRequest& request) {
+    doinspect_ = true;
+
+    // worker that does nothing but exposes the AsyncIterator's queue.
+    auto async_worker = [this] (Queue& queue) {
+        inspectqueue_ = &queue;
+        while(!queue.closed()) std::this_thread::sleep_for(std::chrono::milliseconds(10));
+    };
+
+    // construct iterator before contacting remote, because we want to point to the asynciterator's queue
+    auto iter = new APIAsyncIterator(async_worker); 
+
+    auto req = FDBToolRequest(request);
+    Buffer encodeBuffer(4096);
+    MemoryStream s(encodeBuffer);
+    s << req;
+    controlWriteCheckResponse(Message::Inspect, encodeBuffer, s.position());
+
+    return APIIterator(iter);
+}
 
 
 ListIterator RemoteFDB::list(const FDBToolRequest& request) {
@@ -54,7 +73,7 @@ ListIterator RemoteFDB::list(const FDBToolRequest& request) {
     MemoryStream s(encodeBuffer);
     s << request;
 
-    controlWriteCheckResponse(fdb5::remote::Message::List, encodeBuffer, s.position());
+    controlWriteCheckResponse(Message::List, encodeBuffer, s.position());
     return APIIterator(iter);
 }
 
@@ -91,7 +110,8 @@ FDBStats RemoteFDB::stats() const {NOTIMP;}
 
 bool RemoteFDB::handle(remote::Message message, uint32_t requestID){
     if (message == Message::Complete) {
-        listqueue_->close();
+        if (dolist_) listqueue_->close();
+        if (doinspect_) inspectqueue_->close();
         return true;
     }
     return false;
@@ -105,11 +125,13 @@ bool RemoteFDB::handle(remote::Message message, uint32_t requestID, eckit::net::
         if (dolist_){
             ListElement elem(s);
             listqueue_->push(elem);
+            return true;
         }
-        else {
-            NOTIMP;
+        else if (doinspect_){
+            ListElement elem(s);
+            inspectqueue_->push(elem);
+            return true;
         }
-        return true;
     }
     return false;
 }
diff --git a/src/fdb5/api/RemoteFDB.h b/src/fdb5/api/RemoteFDB.h
index 4af696f66..99e016c33 100644
--- a/src/fdb5/api/RemoteFDB.h
+++ b/src/fdb5/api/RemoteFDB.h
@@ -87,6 +87,8 @@ class RemoteFDB : public FDBBase, public remote::Client {
     bool dolist_ = false;
     eckit::Queue* listqueue_;
 
+    bool doinspect_ = false;
+    eckit::Queue* inspectqueue_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index ef2f75f47..0063dc60b 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -353,7 +353,7 @@ void ClientConnection::listeningThreadLoop() {
                 Buffer payload(hdr.payloadSize);
                 dataRead(payload, hdr.payloadSize);
 
-                handled = ClientConnectionRouter::instance().handle(hdr.message, hdr.requestID, dataEndpoint_, std::move(payload));
+                handled = ClientConnectionRouter::instance().handle(hdr.message, hdr.requestID, controlEndpoint_, std::move(payload));
             }
 
             if (!handled) {
diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc
index 21db6aab9..ef8fa2dd7 100644
--- a/src/fdb5/remote/server/CatalogueHandler.cc
+++ b/src/fdb5/remote/server/CatalogueHandler.cc
@@ -411,8 +411,7 @@ void CatalogueHandler::list(const MessageHeader& hdr) {
 }
 
 void CatalogueHandler::inspect(const MessageHeader& hdr) {
-    // forwardApiCall(hdr);
-    NOTIMP;
+    forwardApiCall(hdr);
 }
 
 void CatalogueHandler::schema(const MessageHeader& hdr) {

From 6c7c24beb89f3147caf35f8bc3254f9ff3cd2313 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 27 Oct 2023 16:14:55 +0100
Subject: [PATCH 015/186] lazy creation of remote::Archiver

---
 src/fdb5/remote/RemoteStore.cc | 36 ++++++++++++++++++----------------
 src/fdb5/remote/RemoteStore.h  |  4 ++--
 2 files changed, 21 insertions(+), 19 deletions(-)

diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index e87534ac7..712a04849 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -59,7 +59,7 @@ class Archiver {
 
 public: // methods
 
-    static Archiver& get(const eckit::net::Endpoint& endpoint);
+    static Archiver* get(const eckit::net::Endpoint& endpoint);
 
     bool valid() { return archiveFuture_.valid(); }
     bool dirty() { return dirty_; }
@@ -87,7 +87,7 @@ class Archiver {
 
 };
 
-Archiver& Archiver::get(const eckit::net::Endpoint& endpoint) {
+Archiver* Archiver::get(const eckit::net::Endpoint& endpoint) {
 
     static std::map > archivers_;
 
@@ -97,7 +97,7 @@ Archiver& Archiver::get(const eckit::net::Endpoint& endpoint) {
         it = archivers_.emplace(endpoint.hostport(), new Archiver()).first;
     }
     ASSERT(it != archivers_.end());
-    return *(it->second);
+    return it->second.get();
 }
 
 void Archiver::start() {
@@ -328,15 +328,13 @@ class FDBRemoteDataHandle : public DataHandle {
 RemoteStore::RemoteStore(const Key& dbKey, const Config& config) :
     Client(eckit::net::Endpoint("localhost", 7000)),
     dbKey_(dbKey), config_(config),
-    retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
-    archiver_(Archiver::get(controlEndpoint())) {
+    retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)) {
 }
 
 RemoteStore::RemoteStore(const eckit::URI& uri, const Config& config) :
     Client(eckit::net::Endpoint(uri.hostport())),
     dbKey_(Key()), config_(config),
-    retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
-    archiver_(Archiver::get(controlEndpoint())) {
+    retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)) {
 
     ASSERT(uri.scheme() == "fdb");
 }
@@ -345,12 +343,12 @@ RemoteStore::~RemoteStore() {
     // If we have launched a thread with an async and we manage to get here, this is
     // an error. n.b. if we don't do something, we will block in the destructor
     // of std::future.
-    if (archiver_.valid()) {
+    if (archiver_ && archiver_->valid()) {
         Log::error() << "Attempting to destruct DecoupledFDB with active archive thread" << std::endl;
         eckit::Main::instance().terminate();
     }
 
-    ASSERT(!archiver_.dirty());
+    ASSERT(!archiver_ || !archiver_->dirty());
 //    disconnect();
 }
 
@@ -370,12 +368,16 @@ std::future > RemoteStore::archive(const Key& key
 
     // if there is no archiving thread active, then start one.
     // n.b. reset the archiveQueue_ after a potential flush() cycle.
-    archiver_.start();
+    if (!archiver_) {
+        archiver_ = Archiver::get(controlEndpoint());
+    }
+    ASSERT(archiver_);
+    archiver_->start();
 
-    ASSERT(archiver_.valid());
+    ASSERT(archiver_->valid());
 
     uint32_t id = controlWriteCheckResponse(Message::Archive, nullptr, 0);
-    archiver_.emplace(id, this, key, data, length);
+    archiver_->emplace(id, this, key, data, length);
 
     std::promise > loc;
     auto futureLocation = loc.get_future();
@@ -396,8 +398,8 @@ void RemoteStore::flush() {
     size_t numArchive = 0;
 
     // Flush only does anything if there is an ongoing archive();
-    if (archiver_.valid()) {
-        archiver_.flush(this);
+    if (archiver_->valid()) {
+        archiver_->flush(this);
 //        internalStats_ += stats;
     }
 
@@ -496,7 +498,7 @@ bool RemoteStore::handle(Message message, uint32_t requestID, eckit::net::Endpoi
             } else {
                 auto it = locations_.find(requestID);
                 if (it != locations_.end()) {
-                    archiver_.error(std::move(payload), endpoint);
+                    archiver_->error(std::move(payload), endpoint);
                 } else {
                     retrieveMessageQueue_.emplace(std::make_pair(message, std::move(payload)));
                 }
@@ -513,9 +515,9 @@ void RemoteStore::handleException(std::exception_ptr e) {
 
 void RemoteStore::flush(FDBStats& internalStats) {
     // Flush only does anything if there is an ongoing archive();
-    if (! archiver_.valid()) return;
+    if (! archiver_->valid()) return;
 
-    internalStats += archiver_.flush(this);
+    internalStats += archiver_->flush(this);
 }
 
 
diff --git a/src/fdb5/remote/RemoteStore.h b/src/fdb5/remote/RemoteStore.h
index 02f1ce8da..4559b06f7 100644
--- a/src/fdb5/remote/RemoteStore.h
+++ b/src/fdb5/remote/RemoteStore.h
@@ -109,8 +109,8 @@ class RemoteStore : public Store, public Client {
     std::map> messageQueues_;
     MessageQueue retrieveMessageQueue_;
 
-    Archiver& archiver_;
-
+    // not owning
+    Archiver* archiver_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------

From 9894117e8180c5f794025d674bf038f287f8ebf5 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Sun, 29 Oct 2023 08:56:13 +0000
Subject: [PATCH 016/186] list/retrieve/archive with both catalogue and store
 remote

---
 src/fdb5/CMakeLists.txt                       |  11 +-
 src/fdb5/api/LocalFDB.h                       |  10 +-
 src/fdb5/api/RemoteFDB.cc                     | 285 ++++--
 src/fdb5/api/RemoteFDB.h                      |  61 +-
 src/fdb5/config/Config.cc                     |   5 +-
 src/fdb5/database/Engine.h                    |   8 +-
 src/fdb5/database/EntryVisitMechanism.cc      |  24 +-
 src/fdb5/database/Manager.cc                  |  32 +-
 src/fdb5/database/Manager.h                   |   6 +-
 src/fdb5/database/MultiRetrieveVisitor.cc     |   2 +-
 src/fdb5/remote/DecoupledHandler.cc           | 355 -------
 src/fdb5/remote/DecoupledHandler.h            | 113 ---
 src/fdb5/remote/FdbServer.cc                  |   6 +-
 src/fdb5/remote/FdbServer.h                   |  14 +-
 src/fdb5/remote/Handler.cc                    | 867 ------------------
 src/fdb5/remote/Handler.h                     | 134 ---
 src/fdb5/remote/Messages.h                    |   1 -
 src/fdb5/remote/RemoteCatalogue.cc            | 331 ++++---
 src/fdb5/remote/RemoteCatalogue.h             |  29 +-
 src/fdb5/remote/RemoteEngine.cc               |  89 ++
 src/fdb5/remote/RemoteEngine.h                |  50 +
 src/fdb5/remote/RemoteStore.cc                | 158 +---
 src/fdb5/remote/RemoteStore.h                 |   6 +-
 src/fdb5/remote/client/Client.cc              |  11 +-
 src/fdb5/remote/client/Client.h               |  19 +-
 src/fdb5/remote/client/ClientConnection.cc    |  20 +-
 src/fdb5/remote/client/ClientConnection.h     |   9 +-
 .../remote/client/ClientConnectionRouter.cc   |  45 +-
 .../remote/client/ClientConnectionRouter.h    |   7 +-
 src/fdb5/remote/server/CatalogueHandler.cc    |  50 +-
 src/fdb5/remote/server/ServerConnection.cc    |  29 +-
 src/fdb5/remote/server/StoreHandler.cc        |   4 +-
 src/fdb5/rules/Schema.h                       |   3 +
 src/fdb5/toc/TocCatalogue.cc                  |   2 +-
 src/fdb5/toc/TocCatalogue.h                   |   1 -
 src/fdb5/toc/TocEngine.cc                     |  35 +-
 src/fdb5/toc/TocEngine.h                      |   6 -
 37 files changed, 783 insertions(+), 2055 deletions(-)
 delete mode 100644 src/fdb5/remote/DecoupledHandler.cc
 delete mode 100644 src/fdb5/remote/DecoupledHandler.h
 delete mode 100644 src/fdb5/remote/Handler.cc
 delete mode 100644 src/fdb5/remote/Handler.h
 create mode 100644 src/fdb5/remote/RemoteEngine.cc
 create mode 100644 src/fdb5/remote/RemoteEngine.h

diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt
index 3bb4bc1f3..09159aa5b 100644
--- a/src/fdb5/CMakeLists.txt
+++ b/src/fdb5/CMakeLists.txt
@@ -210,12 +210,16 @@ list( APPEND fdb5_srcs
 
 if(HAVE_FDB_REMOTE)
     list( APPEND fdb5_srcs 
+        api/ClientFDB.cc
+        api/ClientFDB.h
         api/RemoteFDB.cc
         api/RemoteFDB.h
         remote/RemoteStore.cc
         remote/RemoteStore.h
         remote/RemoteCatalogue.h
         remote/RemoteCatalogue.cc
+        # remote/RemoteEngine.h
+        # remote/RemoteEngine.cc
 
         remote/client/Client.h
         remote/client/Client.cc
@@ -235,17 +239,10 @@ if(HAVE_FDB_REMOTE)
         remote/RemoteFieldLocation.cc
         remote/Messages.h
         remote/Messages.cc
-        # remote/Handler.h
-        # remote/Handler.cc
         remote/AvailablePortList.cc
         remote/AvailablePortList.h
         remote/FdbServer.h
         remote/FdbServer.cc
-
-        # remote/StoreHandler.h
-        # remote/StoreHandler.cc
-        # remote/DecoupledHandler.h
-        # remote/DecoupledHandler.cc
     )
 endif()
 
diff --git a/src/fdb5/api/LocalFDB.h b/src/fdb5/api/LocalFDB.h
index e7b226c78..75a813cba 100644
--- a/src/fdb5/api/LocalFDB.h
+++ b/src/fdb5/api/LocalFDB.h
@@ -61,14 +61,16 @@ class LocalFDB : public FDBBase {
 
     void flush() override;
 
-private: // methods
-
-    void print(std::ostream& s) const override;
+protected: // methods
 
     template 
     APIIterator queryInternal(const FDBToolRequest& request, Ts ... args);
 
-private: // members
+private: // methods
+
+    void print(std::ostream& s) const override;
+
+protected: // members
 
     std::string home_;
 
diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index de552486a..3dfdcb0b1 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -1,3 +1,6 @@
+#include 
+#include 
+
 #include "eckit/io/Buffer.h"
 #include "eckit/log/Log.h"
 #include "eckit/serialisation/MemoryStream.h"
@@ -5,147 +8,247 @@
 
 #include "fdb5/api/RemoteFDB.h"
 #include "fdb5/database/Archiver.h"
+#include "fdb5/database/Inspector.h"
 #include "fdb5/LibFdb5.h"
 
 #include "fdb5/remote/client/ClientConnectionRouter.h"
 
-#include  // xxx debug / development
-#include  // xxx debug / development
-
-
 using namespace fdb5::remote;
 using namespace eckit;
-namespace fdb5 {
 
-// TODO: Contact Catalogue to get parent schema.
+namespace {
 
-RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name):
-    FDBBase(config, name),
-    Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port"))) {
+template 
+struct BaseAPIHelper {
 
-    uint32_t payloadLength = 102400;
-    eckit::Buffer buf(payloadLength);
+    typedef T ValueType;
 
-    ClientConnectionRouter::instance().controlReadResponse(*this, Message::MasterSchema, buf.data(), payloadLength);
-    MemoryStream s(buf);
-    storeEndpoint_ = eckit::net::Endpoint(s);
-    fdb5::Schema* schema = eckit::Reanimator::reanimate(s);
-    config_.overrideSchema("masterSchema", schema);
-    config_.set("storeHost", storeEndpoint_.host());
-    config_.set("storePort", storeEndpoint_.port());
-}
+    static size_t bufferSize() { return 4096; }
+    static size_t queueSize() { return 100; }
+    static fdb5::remote::Message message() { return msgID; }
 
-// archive -- same as localFDB. Archiver will build a RemoteCatalogueWriter and RemoteStore.
-// currently RemoteCatalogueWriter is selected by setting `engine` to remote.
-void RemoteFDB::archive(const Key& key, const void* data, size_t length) {
-    if (!archiver_) {
-        eckit::Log::debug() << *this << ": Constructing new archiver" << std::endl;
-        archiver_.reset(new Archiver(config_));
-    }
+    void encodeExtra(eckit::Stream& s) const {}
+    static ValueType valueFromStream(eckit::Stream& s, fdb5::RemoteFDB* fdb) { return ValueType(s); }
+};
+
+using ListHelper = BaseAPIHelper;
+
+//using InspectHelper = BaseAPIHelper;
+
+struct InspectHelper : BaseAPIHelper {
+
+//     static fdb5::ListElement valueFromStream(eckit::Stream& s, fdb5::RemoteFDB* fdb) {
+//         fdb5::ListElement elem(s);
+//         return elem;
 
-    archiver_->archive(key, data, length);
+// //        return ListElement(elem.key(), RemoteFieldLocation(fdb, elem.location()).make_shared(), elem.timestamp());
+//     }
+};
 }
 
-ListIterator RemoteFDB::inspect(const metkit::mars::MarsRequest& request) {
-    doinspect_ = true;
+namespace fdb5 {
 
-    // worker that does nothing but exposes the AsyncIterator's queue.
-    auto async_worker = [this] (Queue& queue) {
-        inspectqueue_ = &queue;
-        while(!queue.closed()) std::this_thread::sleep_for(std::chrono::milliseconds(10));
-    };
+RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name):
+    LocalFDB(config, name),
+    Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port"))) {
 
-    // construct iterator before contacting remote, because we want to point to the asynciterator's queue
-    auto iter = new APIAsyncIterator(async_worker); 
+    eckit::Buffer buf = controlWriteReadResponse(remote::Message::Schema);
+    eckit::MemoryStream s(buf);
+    size_t numStores;
+    s >> numStores;
+    for (size_t i=0; i(iter);
+    fdb5::Schema* schema = eckit::Reanimator::reanimate(s);
+
+    // eckit::Log::debug() << *this << " - Received Store endpoint: " << storeEndpoint_ << " and master schema: " << std::endl;
+    // schema->dump(eckit::Log::debug());
+    
+    config_.overrideSchema(endpoint_.hostport()+"/schema", schema);
+    config_.set("storeHost", storeEndpoint.host());
+    config_.set("storePort", storeEndpoint.port());
 }
 
+// -----------------------------------------------------------------------------------------------------
 
-ListIterator RemoteFDB::list(const FDBToolRequest& request) {
+// forwardApiCall captures the asynchronous behaviour:
+//
+// i) Set up a Queue to receive the messages as they come in
+// ii) Encode the request+arguments and send them to the server
+// iii) Return an AsyncIterator that pulls messages off the queue, and returns them to the caller.
 
-    dolist_ = true;
-    
-    // worker that does nothing but exposes the AsyncIterator's queue.
-    auto async_worker = [this] (Queue& queue) {
-        listqueue_ = &queue;
-        while(!queue.closed()) std::this_thread::sleep_for(std::chrono::milliseconds(10));
-    };
 
-    // construct iterator before contacting remote, because we want to point to the asynciterator's queue
-    auto iter = new APIAsyncIterator(async_worker); 
+template 
+auto RemoteFDB::forwardApiCall(const HelperClass& helper, const FDBToolRequest& request) -> APIIterator {
+
+    using ValueType = typename HelperClass::ValueType;
+    using IteratorType = APIIterator;
+    using AsyncIterator = APIAsyncIterator;
+
+    // Ensure we have an entry in the message queue before we trigger anything that
+    // will result in return messages
 
-    Buffer encodeBuffer(4096);
+    uint32_t id = generateRequestID();
+    auto entry = messageQueues_.emplace(id, std::make_shared(HelperClass::queueSize()));
+    ASSERT(entry.second);
+    std::shared_ptr messageQueue(entry.first->second);
+
+    // Encode the request and send it to the server
+
+    Buffer encodeBuffer(HelperClass::bufferSize());
     MemoryStream s(encodeBuffer);
     s << request;
+    helper.encodeExtra(s);
+
+    controlWriteCheckResponse(HelperClass::message(), encodeBuffer, s.position(), id);
+
+    // Return an AsyncIterator to allow the messages to be retrieved in the API
+
+    RemoteFDB* remoteFDB = this;
+    return IteratorType(
+        // n.b. Don't worry about catching exceptions in lambda, as
+        // this is handled in the AsyncIterator.
+        new AsyncIterator([messageQueue, remoteFDB](eckit::Queue& queue) {
+            eckit::Buffer msg{0};
+                        while (true) {
+                            if (messageQueue->pop(msg) == -1) {
+                                break;
+                            } else {
+                                MemoryStream s(msg);
+                                queue.emplace(HelperClass::valueFromStream(s, remoteFDB));
+                            }
+                        }
+                        // messageQueue goes out of scope --> destructed
+                    }
+                )
+           );
+}
 
-    controlWriteCheckResponse(Message::List, encodeBuffer, s.position());
-    return APIIterator(iter);
+ListIterator RemoteFDB::list(const FDBToolRequest& request) {
+    return forwardApiCall(ListHelper(), request);
 }
 
-DumpIterator RemoteFDB::dump(const FDBToolRequest& request, bool simple) {NOTIMP;}
+ListIterator RemoteFDB::inspect(const metkit::mars::MarsRequest& request) {
+    return forwardApiCall(InspectHelper(), request);
+}
 
-StatusIterator RemoteFDB::status(const FDBToolRequest& request) {NOTIMP;}
 
-WipeIterator RemoteFDB::wipe(const FDBToolRequest& request, bool doit, bool porcelain, bool unsafeWipeAll) {NOTIMP;}
+// ListIterator RemoteFDB::inspect(const metkit::mars::MarsRequest& request) {
+//     doinspect_ = true;
 
-PurgeIterator RemoteFDB::purge(const FDBToolRequest& request, bool doit, bool porcelain) {NOTIMP;}
+//     // worker that does nothing but exposes the AsyncIterator's queue.
+//     auto async_worker = [this] (Queue& queue) {
+//         inspectqueue_ = &queue;
+//         while(!queue.closed()) std::this_thread::sleep_for(std::chrono::milliseconds(10));
+//     };
 
-StatsIterator RemoteFDB::stats(const FDBToolRequest& request) {NOTIMP;}
+//     // construct iterator before contacting remote, because we want to point to the asynciterator's queue
+//     auto iter = new APIAsyncIterator(async_worker); 
 
-ControlIterator RemoteFDB::control(const FDBToolRequest& request,
-                        ControlAction action,
-                        ControlIdentifiers identifiers) {NOTIMP;}
+//     auto req = FDBToolRequest(request);
+//     Buffer encodeBuffer(4096);
+//     MemoryStream s(encodeBuffer);
+//     s << req;
+//     uint32_t id = controlWriteCheckResponse(Message::Inspect, encodeBuffer, s.position());
 
-MoveIterator RemoteFDB::move(const FDBToolRequest& request, const eckit::URI& dest) {NOTIMP;}
+//     return APIIterator(iter);
+// }
 
-void RemoteFDB::flush() {
-    if (archiver_) {
-        archiver_->flush();
-    }
-}
+
+// ListIterator RemoteFDB::list(const FDBToolRequest& request) {
+
+//     dolist_ = true;
+    
+//     // worker that does nothing but exposes the AsyncIterator's queue.
+//     auto async_worker = [this] (Queue& queue) {
+//         listqueue_ = &queue;
+//         while(!queue.closed()) std::this_thread::sleep_for(std::chrono::milliseconds(10));
+//     };
+
+//     // construct iterator before contacting remote, because we want to point to the asynciterator's queue
+//     auto iter = new APIAsyncIterator(async_worker); 
+
+//     Buffer encodeBuffer(4096);
+//     MemoryStream s(encodeBuffer);
+//     s << request;
+
+//     controlWriteCheckResponse(Message::List, encodeBuffer, s.position());
+//     return APIIterator(iter);
+// }
 
 void RemoteFDB::print(std::ostream& s) const {
     s << "RemoteFDB(...)";
 }
 
-FDBStats RemoteFDB::stats() const {NOTIMP;}
-
-
 // Client
+bool RemoteFDB::handle(remote::Message message, uint32_t requestID) {
+    
+    switch (message) {
+        case fdb5::remote::Message::Complete: {
+
+            auto it = messageQueues_.find(requestID);
+            if (it == messageQueues_.end()) {
+                return false;
+            }
+
+            it->second->close();
+            // Remove entry (shared_ptr --> message queue will be destroyed when it
+            // goes out of scope in the worker thread).
+            messageQueues_.erase(it);
+            return true;
+        }
+        case fdb5::remote::Message::Error: {
 
-bool RemoteFDB::handle(remote::Message message, uint32_t requestID){
-    if (message == Message::Complete) {
-        if (dolist_) listqueue_->close();
-        if (doinspect_) inspectqueue_->close();
-        return true;
+            auto it = messageQueues_.find(requestID);
+            if (it == messageQueues_.end()) {
+                return false;
+            }
+
+            it->second->interrupt(std::make_exception_ptr(RemoteFDBException("", controlEndpoint())));
+            // Remove entry (shared_ptr --> message queue will be destroyed when it
+            // goes out of scope in the worker thread).
+            messageQueues_.erase(it);
+            return true;
+        }
+        default:
+            return false;
     }
-    return false;
 }
-bool RemoteFDB::handle(remote::Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload){
-
-    eckit::Log::debug()<< "RemoteFDB::handle received message: " << ((uint) message) << " - requestID: " << requestID << std::endl;
-    if (message == Message::Blob) {
-        eckit::Log::debug() << "RemoteFDB::handle received payload size: " << payload.size() << std::endl;
-        MemoryStream s(payload);
-        if (dolist_){
-            ListElement elem(s);
-            listqueue_->push(elem);
+bool RemoteFDB::handle(remote::Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) {
+
+    switch (message) {
+        case fdb5::remote::Message::Blob: {
+            auto it = messageQueues_.find(requestID);
+            if (it == messageQueues_.end()) {
+                return false;
+            }
+
+            it->second->emplace(std::move(payload));
             return true;
         }
-        else if (doinspect_){
-            ListElement elem(s);
-            inspectqueue_->push(elem);
+
+        case fdb5::remote::Message::Error: {
+
+            auto it = messageQueues_.find(requestID);
+            if (it == messageQueues_.end()) {
+                return false;
+            }
+            std::string msg;
+            msg.resize(payload.size(), ' ');
+            payload.copy(&msg[0], payload.size());
+            it->second->interrupt(std::make_exception_ptr(RemoteFDBException(msg, controlEndpoint())));
+            // Remove entry (shared_ptr --> message queue will be destroyed when it
+            // goes out of scope in the worker thread).
+            messageQueues_.erase(it);
             return true;
         }
+        default:
+            return false;
     }
-    return false;
 }
 void RemoteFDB::handleException(std::exception_ptr e){NOTIMP;}
 const Key& RemoteFDB::key() const {NOTIMP;}
diff --git a/src/fdb5/api/RemoteFDB.h b/src/fdb5/api/RemoteFDB.h
index 4bf83d29e..13a57fb20 100644
--- a/src/fdb5/api/RemoteFDB.h
+++ b/src/fdb5/api/RemoteFDB.h
@@ -14,16 +14,16 @@
  */
 
 /// @author Simon Smart
+/// @author Emanuele Danovaro
+/// @author Chris Bradley
 /// @date   Mar 2018
 
-#ifndef fdb5_remote_RemoteFDB_H
-#define fdb5_remote_RemoteFDB_H
+#pragma once
 
 #include 
 #include 
 
-#include "fdb5/api/FDB.h"
-#include "fdb5/api/FDBFactory.h"
+#include "fdb5/api/LocalFDB.h"
 #include "fdb5/remote/client/Client.h"
 
 namespace fdb5 {
@@ -31,40 +31,45 @@ namespace fdb5 {
 //----------------------------------------------------------------------------------------------------------------------
 class Archiver;
 
-class RemoteFDB : public FDBBase, public remote::Client {
+class RemoteFDB : public LocalFDB, public remote::Client {
 
 public: // types
 
-public: // method
+//    using StoredMessage = std::pair;
+    using MessageQueue = eckit::Queue;
 
-    using FDBBase::stats;
+public: // method
 
     RemoteFDB(const eckit::Configuration& config, const std::string& name);
 
-    /// Archive writes data into aggregation buffer
-    void archive(const Key& key, const void* data, size_t length) override;
-
     ListIterator inspect(const metkit::mars::MarsRequest& request) override;
 
     ListIterator list(const FDBToolRequest& request) override;
 
-    DumpIterator dump(const FDBToolRequest& request, bool simple) override;
+    DumpIterator dump(const FDBToolRequest& request, bool simple) override { NOTIMP; }
 
-    StatusIterator status(const FDBToolRequest& request) override;
+    StatusIterator status(const FDBToolRequest& request) override { NOTIMP; }
 
-    WipeIterator wipe(const FDBToolRequest& request, bool doit, bool porcelain, bool unsafeWipeAll) override;
+    WipeIterator wipe(const FDBToolRequest& request, bool doit, bool porcelain, bool unsafeWipeAll) override { NOTIMP; }
 
-    PurgeIterator purge(const FDBToolRequest& request, bool doit, bool porcelain) override;
+    PurgeIterator purge(const FDBToolRequest& request, bool doit, bool porcelain) override { NOTIMP; }
 
-    StatsIterator stats(const FDBToolRequest& request) override;
+    StatsIterator stats(const FDBToolRequest& request) override { NOTIMP; }
 
     ControlIterator control(const FDBToolRequest& request,
                             ControlAction action,
-                            ControlIdentifiers identifiers) override;
+                            ControlIdentifiers identifiers) override { NOTIMP; }
+
+    MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) override { NOTIMP; }
 
-    MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) override;
+private: // methods
+
+    template 
+    auto forwardApiCall(const HelperClass& helper, const FDBToolRequest& request) -> APIIterator;
+
+    virtual void print(std::ostream& s) const override;
 
-    void flush() override;
+    virtual FDBStats stats() const override { NOTIMP; }
 
     // Client
 
@@ -74,26 +79,20 @@ class RemoteFDB : public FDBBase, public remote::Client {
 
     const Key& key() const override;
 
-private: // methods
-
-    virtual void print(std::ostream& s) const override;
-
-    virtual FDBStats stats() const override;
-
 private: // members
 
     std::unique_ptr archiver_;
+    std::vector stores_;
     eckit::net::Endpoint storeEndpoint_;
 
-    bool dolist_ = false;
-    eckit::Queue* listqueue_;
-
-    bool doinspect_ = false;
-    eckit::Queue* inspectqueue_;
+    // Where do we put received messages
+    // @note This is a map of requestID:MessageQueue. At the point that a request is
+    // complete, errored or otherwise killed, it needs to be removed from the map.
+    // The shared_ptr allows this removal to be asynchronous with the actual task
+    // cleaning up and returning to the client.
+    std::map> messageQueues_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
 
 } // namespace fdb5
-
-#endif // fdb5_remote_RemoteFDB_H
\ No newline at end of file
diff --git a/src/fdb5/config/Config.cc b/src/fdb5/config/Config.cc
index 10421894c..67b05dec5 100644
--- a/src/fdb5/config/Config.cc
+++ b/src/fdb5/config/Config.cc
@@ -185,9 +185,10 @@ const PathName& Config::schemaPath() const {
 void Config::overrideSchema(const eckit::PathName& schemaPath, Schema* schema) {
     ASSERT(schema);
 
-    SchemaRegistry::instance().add("masterSchema", schema);
-    schemaPath_=schemaPath;
+    schema->path_ = schemaPath;
+    SchemaRegistry::instance().add(schemaPath, schema);
 
+    schemaPath_=schemaPath;
     schemaPathInitialised_ = true;
 }
 
diff --git a/src/fdb5/database/Engine.h b/src/fdb5/database/Engine.h
index d49c66c1d..867303fed 100644
--- a/src/fdb5/database/Engine.h
+++ b/src/fdb5/database/Engine.h
@@ -50,11 +50,11 @@ class Engine : private eckit::NonCopyable {
     /// @returns if an Engine is capable of opening this path
     virtual bool canHandle(const eckit::URI& uri) const = 0;
 
-    /// Uniquely selects a location where the Key will be put or already exists
-    virtual eckit::URI location(const Key &key, const Config& config) const = 0;
+    // /// Uniquely selects a location where the Key will be put or already exists
+    // virtual eckit::URI location(const Key &key, const Config& config) const = 0;
 
-    /// Lists the roots that can be visited given a DB key
-    virtual std::vector allLocations(const Key& key, const Config& config) const = 0;
+    // /// Lists the roots that can be visited given a DB key
+    // virtual std::vector allLocations(const Key& key, const Config& config) const = 0;
 
     /// Lists the roots that can be visited given a DB key
     virtual std::vector visitableLocations(const Config& config) const = 0;
diff --git a/src/fdb5/database/EntryVisitMechanism.cc b/src/fdb5/database/EntryVisitMechanism.cc
index e243396b3..f8f70fb58 100644
--- a/src/fdb5/database/EntryVisitMechanism.cc
+++ b/src/fdb5/database/EntryVisitMechanism.cc
@@ -108,28 +108,16 @@ void EntryVisitMechanism::visit(const FDBToolRequest& request, EntryVisitor& vis
         // n.b. it is not an error if nothing is found (especially in a sub-fdb).
 
         // And do the visitation
+        for (const URI& uri : uris) {
 
-        for (URI uri : uris) {
+            Log::debug() << "FDB processing URI " << uri << std::endl;
 
-            PathName path(uri.path());
-            if (path.exists()) {
-                if (!path.isDir())
-                    path = path.dirName();
-                path = path.realName();
+            std::unique_ptr catalogue = CatalogueReaderFactory::instance().build(uri, dbConfig_);
+            ASSERT(catalogue->open());
 
-                Log::debug() << "FDB processing Path " << path << std::endl;
+            eckit::AutoCloser closer(*catalogue);
 
-                std::unique_ptr catalogue = CatalogueReaderFactory::instance().build(eckit::URI(uri.scheme(), path), dbConfig_);
-                ASSERT(catalogue->open());
-
-                // std::unique_ptr store = catalogue->buildStore();
-                // ASSERT(store->open());
-
-                eckit::AutoCloser closer(*catalogue);
-                // eckit::AutoCloser storeCloser(*store);
-
-                catalogue->visitEntries(visitor, /* *store, */ false);
-            }
+            catalogue->visitEntries(visitor, /* *store, */ false);
         }
 
     } catch (eckit::UserError&) {
diff --git a/src/fdb5/database/Manager.cc b/src/fdb5/database/Manager.cc
index f431b6e49..0998c856c 100644
--- a/src/fdb5/database/Manager.cc
+++ b/src/fdb5/database/Manager.cc
@@ -269,29 +269,29 @@ std::string Manager::engine(const URI& uri)
     throw eckit::BadParameter(oss.str(), Here());
 }
 
-eckit::URI Manager::location(const Key& key) {
+// eckit::URI Manager::location(const Key& key) {
 
-    const std::string& name = Manager::engine(key);
+//     const std::string& name = Manager::engine(key);
 
-    return Engine::backend(name).location(key, config_);
-}
+//     return Engine::backend(name).location(key, config_);
+// }
 
-std::vector Manager::allLocations(const Key& key)
-{
-    std::set engines = Manager::engines(key);
+// std::vector Manager::allLocations(const Key& key)
+// {
+//     std::set engines = Manager::engines(key);
 
-    Log::debug() << "Matching engines for key " << key << " -> " << engines << std::endl;
+//     Log::debug() << "Matching engines for key " << key << " -> " << engines << std::endl;
 
-    std::vector r; // union of all locations
+//     std::vector r; // union of all locations
 
-    for(std::set::const_iterator i = engines.begin(); i != engines.end(); ++i) {
-        Log::debug() << "Selected FDB engine " << *i << std::endl;
-        std::vector p = Engine::backend(*i).allLocations(key, config_);
-        r.insert(r.end(), p.begin(), p.end());
-    }
+//     for(std::set::const_iterator i = engines.begin(); i != engines.end(); ++i) {
+//         Log::debug() << "Selected FDB engine " << *i << std::endl;
+//         std::vector p = Engine::backend(*i).allLocations(key, config_);
+//         r.insert(r.end(), p.begin(), p.end());
+//     }
 
-    return r;
-}
+//     return r;
+// }
 
 
 std::vector Manager::visitableLocations(const metkit::mars::MarsRequest& rq, bool all) {
diff --git a/src/fdb5/database/Manager.h b/src/fdb5/database/Manager.h
index 41e859e16..73d4e6f6e 100644
--- a/src/fdb5/database/Manager.h
+++ b/src/fdb5/database/Manager.h
@@ -48,10 +48,10 @@ class Manager  {
     std::string engine(const eckit::URI& uri);
 
     /// Uniquely selects a location where the Key will be put or already exists
-    eckit::URI location(const Key &key);
+    // eckit::URI location(const Key &key);
 
-    /// Lists the roots that can be visited given a DB key
-    std::vector allLocations(const Key& key);
+    // /// Lists the roots that can be visited given a DB key
+    // std::vector allLocations(const Key& key);
 
     /// Lists the roots that can be visited given a DB key
     std::vector visitableLocations(const metkit::mars::MarsRequest& request, bool all);
diff --git a/src/fdb5/database/MultiRetrieveVisitor.cc b/src/fdb5/database/MultiRetrieveVisitor.cc
index 07e3b76fa..3dcc88db5 100644
--- a/src/fdb5/database/MultiRetrieveVisitor.cc
+++ b/src/fdb5/database/MultiRetrieveVisitor.cc
@@ -44,7 +44,7 @@ MultiRetrieveVisitor::~MultiRetrieveVisitor() {
 
 bool MultiRetrieveVisitor::selectDatabase(const Key& key, const Key&) {
 
-	eckit::Log::debug() << "FDB5 selectDatabase " << key  << std::endl;
+	eckit::Log::debug() << "FDB5 selectDatabase " << key  << std::endl;
 
     /* is it the current DB ? */
 
diff --git a/src/fdb5/remote/DecoupledHandler.cc b/src/fdb5/remote/DecoupledHandler.cc
deleted file mode 100644
index d41adc56a..000000000
--- a/src/fdb5/remote/DecoupledHandler.cc
+++ /dev/null
@@ -1,355 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-#include 
-
-#include "eckit/config/Resource.h"
-#include "eckit/maths/Functions.h"
-#include "eckit/net/Endpoint.h"
-#include "eckit/runtime/Main.h"
-#include "eckit/runtime/SessionID.h"
-#include "eckit/serialisation/MemoryStream.h"
-
-#include "metkit/mars/MarsRequest.h"
-
-#include "fdb5/LibFdb5.h"
-#include "fdb5/fdb5_version.h"
-#include "fdb5/api/helpers/FDBToolRequest.h"
-#include "fdb5/database/Key.h"
-#include "fdb5/remote/AvailablePortList.h"
-#include "fdb5/remote/Messages.h"
-#include "fdb5/remote/RemoteFieldLocation.h"
-#include "fdb5/api/FDB.h"
-
-#include "fdb5/remote/DecoupledHandler.h"
-
-using namespace eckit;
-using metkit::mars::MarsRequest;
-
-namespace fdb5 {
-namespace remote {
-
-// helpers
-namespace {
-
-class TCPException : public eckit::Exception {
-public:
-    TCPException(const std::string& msg, const eckit::CodeLocation& here) :
-        eckit::Exception(std::string("TCPException: ") + msg, here) {}
-};
-
-std::vector intersection(const LocalConfiguration& c1, const LocalConfiguration& c2, const std::string& field){
-
-    std::vector v1 = c1.getIntVector(field);
-    std::vector v2 = c2.getIntVector(field);
-    std::vector v3;
-
-    std::sort(v1.begin(), v1.end());
-    std::sort(v2.begin(), v2.end());
-
-    std::set_intersection(v1.begin(),v1.end(),
-                          v2.begin(),v2.end(),
-                          back_inserter(v3));
-    return v3;
-}
-} // namespace
-
-DecoupledHandler::DecoupledHandler(eckit::net::TCPSocket& socket, const Config& config) :
-        config_(config),
-        controlSocket_(socket),
-        dataSocket_(selectDataPort()),
-        dataListenHostname_(config.getString("dataListenHostname", "")),
-        // fdb_(config),
-        readLocationQueue_(eckit::Resource("fdbRetrieveQueueSize", 10000)) {
-            Log::debug() << "DecoupledHandler::DecoupledHandler initialized" << std::endl;
-    }
-
-DecoupledHandler::~DecoupledHandler() {
-    // We don't want to die before the worker threads are cleaned up
-    waitForWorkers();
-
-    // And notify the client that we are done.
-    Log::info() << "Sending exit message to client" << std::endl;
-    dataWrite(Message::Exit, 0);
-    Log::info() << "Done" << std::endl;
-}
-
-eckit::LocalConfiguration DecoupledHandler::availableFunctionality() const {
-    eckit::LocalConfiguration conf;
-//    Add to the configuration all the components that require to be versioned, as in the following example, with a vector of supported version numbers
-    std::vector remoteFieldLocationVersions = {1};
-    conf.set("RemoteFieldLocation", remoteFieldLocationVersions);
-    return conf;
-}
-
-void DecoupledHandler::initialiseConnections() {
-    // Read the startup message from the client. Check that it all checks out.
-
-    MessageHeader hdr;
-    socketRead(&hdr, sizeof(hdr), controlSocket_);
-
-    ASSERT(hdr.marker == StartMarker);
-    ASSERT(hdr.version == CurrentVersion);
-    ASSERT(hdr.message == Message::Startup);
-    ASSERT(hdr.requestID == 0);
-
-    Buffer payload1 = receivePayload(hdr, controlSocket_);
-    eckit::FixedString<4> tail;
-    socketRead(&tail, sizeof(tail), controlSocket_);
-    ASSERT(tail == EndMarker);
-
-    MemoryStream s1(payload1);
-    SessionID clientSession(s1);
-    net::Endpoint endpointFromClient(s1);
-    unsigned int remoteProtocolVersion = 0;
-    std::string errorMsg;
-
-    try {
-        s1 >> remoteProtocolVersion;
-    } catch (...) {
-        errorMsg = "Error retrieving client protocol version";
-    }
-
-    if (errorMsg.empty() && !LibFdb5::instance().remoteProtocolVersion().check(remoteProtocolVersion, false)) {
-        std::stringstream ss;
-        ss << "FDB server version " << fdb5_version_str() << " - remote protocol version not supported:" << std::endl;
-        ss << "    versions supported by server: " << LibFdb5::instance().remoteProtocolVersion().supportedStr() << std::endl;
-        ss << "    version requested by client: " << remoteProtocolVersion << std::endl;
-        errorMsg = ss.str();
-    }
-
-    if (errorMsg.empty()) {
-        LocalConfiguration clientAvailableFunctionality(s1);
-        LocalConfiguration serverConf = availableFunctionality();
-        agreedConf_ = LocalConfiguration();
-
-        // agree on a common functionality by intersecting server and client version numbers
-         std::vector rflCommon = intersection(clientAvailableFunctionality, serverConf, "RemoteFieldLocation");
-         if (rflCommon.size() > 0) {
-             Log::debug() << "Protocol negotiation - RemoteFieldLocation version " << rflCommon.back() << std::endl;
-             agreedConf_.set("RemoteFieldLocation", rflCommon.back());
-         }
-         else {
-             std::stringstream ss;
-             ss << "FDB server version " << fdb5_version_str() << " - failed protocol negotiation with FDB client" << std::endl;
-             ss << "    server functionality: " << serverConf << std::endl;
-             ss << "    client functionality: " << clientAvailableFunctionality << std::endl;
-             errorMsg = ss.str();
-         }
-    }
-
-    // We want a data connection too. Send info to RemoteFDB, and wait for connection
-    // n.b. FDB-192: we use the host communicated from the client endpoint. This
-    //               ensures that if a specific interface has been selected and the
-    //               server has multiple, then we use that on, whilst retaining
-    //               the capacity in the protocol for the server to make a choice.
-
-    int dataport = dataSocket_.localPort();
-    net::Endpoint dataEndpoint(endpointFromClient.hostname(), dataport);
-
-    Log::info() << "Sending data endpoint to client: " << dataEndpoint << std::endl;
-    {
-        Buffer startupBuffer(1024);
-        MemoryStream s(startupBuffer);
-
-        s << clientSession;
-        s << sessionID_;
-        s << dataEndpoint;
-        s << agreedConf_.get();
-        // s << storeEndpoint; // xxx single-store case only: we cant do this with multiple stores // For now, dont send the store endpoint to the client 
-
-        Log::debug() << "Protocol negotiation - configuration: " << agreedConf_ <() << "DecoupledHandler::dataWrite.  requestID: " << requestID << " payloadLength: " << payloadLength
-                            << " msg: " << static_cast(msg) << std::endl;
-
-    ASSERT((payload == nullptr) == (payloadLength == 0));
-
-    MessageHeader message(msg, requestID, payloadLength);
-
-    std::lock_guard lock(dataWriteMutex_);
-
-    dataWriteUnsafe(&message, sizeof(message));
-    if (payload) {
-        dataWriteUnsafe(payload, payloadLength);
-    }
-    dataWriteUnsafe(&EndMarker, sizeof(EndMarker));
-}
-
-void DecoupledHandler::dataWriteUnsafe(const void* data, size_t length) {
-    size_t written = dataSocket_.write(data, length);
-    if (length != written) {
-        std::stringstream ss;
-        ss << "Write error. Expected " << length << " bytes, wrote " << written;
-        throw TCPException(ss.str(), Here());
-    }
-}
-
-Buffer DecoupledHandler::receivePayload(const MessageHeader& hdr, net::TCPSocket& socket) {
-    Buffer payload(hdr.payloadSize);
-
-    ASSERT(hdr.payloadSize > 0);
-    socketRead(payload, hdr.payloadSize, socket);
-
-    return payload;
-}
-
-void DecoupledHandler::tidyWorkers() {
-    std::map>::iterator it = workerThreads_.begin();
-
-    for (; it != workerThreads_.end(); /* empty */) {
-        std::future_status stat = it->second.wait_for(std::chrono::milliseconds(0));
-
-        if (stat == std::future_status::ready) {
-            Log::info() << "Tidying up worker for request ID: " << it->first << std::endl;
-            workerThreads_.erase(it++);
-        }
-        else {
-            ++it;
-        }
-    }
-}
-
-void DecoupledHandler::waitForWorkers() {
-    readLocationQueue_.close();
-
-    tidyWorkers();
-
-    for (auto& it : workerThreads_) {
-        Log::error() << "Worker thread still alive for request ID: " << it.first << std::endl;
-        Log::error() << "Joining ..." << std::endl;
-        it.second.get();
-        Log::error() << "Thread complete" << std::endl;
-    }
-
-    if (readLocationWorker_.joinable()) {
-        readLocationWorker_.join();
-    }
-}
-
-
-void DecoupledHandler::read(const MessageHeader& hdr) {
-    // store only
-    NOTIMP;
-}
-
-void DecoupledHandler::archive(const MessageHeader& hdr) {
-    // catalogue only
-    NOTIMP;
-}
-
-// void DecoupledHandler::store(const MessageHeader& hdr) {
-//     // store only
-//     NOTIMP;
-// }
-
-void DecoupledHandler::flush(const MessageHeader& hdr) {
-    // store only
-    NOTIMP;
-}
-
-
-}  // namespace remote
-}  // namespace fdb5
diff --git a/src/fdb5/remote/DecoupledHandler.h b/src/fdb5/remote/DecoupledHandler.h
deleted file mode 100644
index 63c044b65..000000000
--- a/src/fdb5/remote/DecoupledHandler.h
+++ /dev/null
@@ -1,113 +0,0 @@
-
-
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-#ifndef fdb5_remote_DecoupledHandler_H
-#define fdb5_remote_DecoupledHandler_H
-
-#include 
-#include 
-
-#include "eckit/io/Buffer.h"
-#include "eckit/io/DataHandle.h"
-#include "eckit/net/TCPServer.h"
-#include "eckit/net/TCPSocket.h"
-#include "eckit/runtime/SessionID.h"
-
-#include "metkit/mars/MarsRequest.h"
-
-#include "fdb5/api/FDBFactory.h"
-#include "fdb5/config/Config.h"
-#include "fdb5/database/Key.h"
-#include "fdb5/remote/Messages.h"
-
-namespace fdb5 {
-
-class Config;
-
-namespace remote {
-
-struct MessageHeader;
-
-//----------------------------------------------------------------------------------------------------------------------
-// Base class for CatalogueHandler and StoreHandler
-
-class DecoupledHandler : private eckit::NonCopyable {
-public:  // methods
-    DecoupledHandler(eckit::net::TCPSocket& socket, const Config& config);
-    ~DecoupledHandler();
-
-    virtual void handle() = 0;
-
-    std::string host() const { return controlSocket_.localHost(); }
-    int port() const { return controlSocket_.localPort(); }
-    const eckit::LocalConfiguration& agreedConf() const { return agreedConf_; }
-
-protected:
-
-    // socket methods
-    int selectDataPort();
-    virtual void initialiseConnections();
-    eckit::LocalConfiguration availableFunctionality() const;
-
-    void controlWrite(Message msg, uint32_t requestID, const void* payload = nullptr, uint32_t payloadLength = 0);
-    void controlWrite(const void* data, size_t length);
-    void socketRead(void* data, size_t length, eckit::net::TCPSocket& socket);
-    
-    // dataWrite is protected using a mutex, as we may have multiple workers.
-    void dataWrite(Message msg, uint32_t requestID, const void* payload = nullptr, uint32_t payloadLength = 0);
-    eckit::Buffer receivePayload(const MessageHeader& hdr, eckit::net::TCPSocket& socket);
-
-    // socket methods
-    void dataWriteUnsafe(const void* data, size_t length);
-
-    // Worker functionality
-    void tidyWorkers();
-    void waitForWorkers();
-
-
-    virtual void read(const MessageHeader& hdr);
-    virtual void archive(const MessageHeader& hdr);
-    // virtual void store(const MessageHeader& hdr);
-    virtual void flush(const MessageHeader& hdr);
-
-protected:
-
-    Config config_;
-    eckit::net::TCPSocket controlSocket_;
-    eckit::net::EphemeralTCPServer dataSocket_;
-    std::string dataListenHostname_;
-    // FDB fdb_;
-    eckit::Queue>> readLocationQueue_;
-
-    eckit::SessionID sessionID_;
-    eckit::LocalConfiguration agreedConf_;
-    std::mutex dataWriteMutex_;
-    std::map> workerThreads_;
-    std::future archiveFuture_;
-    std::thread readLocationWorker_;
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-}  // namespace remote
-}  // namespace fdb5
-
-
-//----------------------------------------------------------------------------------------------------------------------
-
-
-#endif  // fdb5_remote_DecoupledHandler_H
diff --git a/src/fdb5/remote/FdbServer.cc b/src/fdb5/remote/FdbServer.cc
index e8d6d70cc..739dfcdcf 100644
--- a/src/fdb5/remote/FdbServer.cc
+++ b/src/fdb5/remote/FdbServer.cc
@@ -29,8 +29,7 @@
 using namespace eckit;
 
 
-namespace fdb5 {
-namespace remote {
+namespace fdb5::remote {
 
 //----------------------------------------------------------------------------------------------------------------------
 
@@ -211,5 +210,4 @@ void FdbServer::hookUnique() {}
 
 //----------------------------------------------------------------------------------------------------------------------
 
-} // namespace remote
-} // namespace fdb5
+} // namespace fdb5::remote
diff --git a/src/fdb5/remote/FdbServer.h b/src/fdb5/remote/FdbServer.h
index 80fdf46f6..bbc6535eb 100644
--- a/src/fdb5/remote/FdbServer.h
+++ b/src/fdb5/remote/FdbServer.h
@@ -12,8 +12,7 @@
 /// @author Tiagop Quintino
 /// @date   Nov 2019
 
-#ifndef fdb5_remote_FdbServer_H
-#define fdb5_remote_FdbServer_H
+#pragma once
 
 #include 
 #include 
@@ -30,13 +29,9 @@
 #include "fdb5/config/Config.h"
 #include "fdb5/LibFdb5.h"
 
-
 #include "fdb5/remote/AvailablePortList.h"
-#include "fdb5/remote/Handler.h"
-
 
-namespace fdb5 {
-namespace remote {
+namespace fdb5::remote {
 
 //----------------------------------------------------------------------------------------------------------------------
 
@@ -96,7 +91,4 @@ class FdbServer : public eckit::Application, public FdbServerBase {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-} // namespace remote
-} // namespace fdb5
-
-#endif // fdb5_remote_FdbServer_H
+} // namespace fdb5::remote
diff --git a/src/fdb5/remote/Handler.cc b/src/fdb5/remote/Handler.cc
deleted file mode 100644
index dd5816200..000000000
--- a/src/fdb5/remote/Handler.cc
+++ /dev/null
@@ -1,867 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-#include 
-
-#include "eckit/config/Resource.h"
-#include "eckit/maths/Functions.h"
-#include "eckit/net/Endpoint.h"
-#include "eckit/runtime/Main.h"
-#include "eckit/runtime/SessionID.h"
-#include "eckit/serialisation/MemoryStream.h"
-
-#include "metkit/mars/MarsRequest.h"
-
-#include "fdb5/LibFdb5.h"
-#include "fdb5/fdb5_version.h"
-#include "fdb5/api/helpers/FDBToolRequest.h"
-#include "fdb5/database/Key.h"
-#include "fdb5/remote/AvailablePortList.h"
-#include "fdb5/remote/Handler.h"
-#include "fdb5/remote/Messages.h"
-#include "fdb5/remote/RemoteFieldLocation.h"
-
-using namespace eckit;
-using metkit::mars::MarsRequest;
-
-namespace fdb5 {
-namespace remote {
-
-//----------------------------------------------------------------------------------------------------------------------
-
-namespace {
-class TCPException : public Exception {
-public:
-    TCPException(const std::string& msg, const CodeLocation& here) :
-        Exception(std::string("TCPException: ") + msg, here) {}
-};
-}  // namespace
-
-//----------------------------------------------------------------------------------------------------------------------
-
-// ***************************************************************************************
-// All of the standard API functions behave in roughly the same manner. The Helper Classes
-// described here capture the manner in which their behaviours differ.
-//
-// See forwardApiCall() for how these helpers are used to forward an API call using a
-// worker thread.
-//
-// ***************************************************************************************
-//
-// Note that we use the "control" and "data" connections in a specific way, although these
-// may not be the optimal names for them. "control" is used for blocking requests,
-// and "data" is used for non-blocking activity.
-//
-// ***************************************************************************************
-
-namespace {
-
-template 
-struct BaseHelper {
-    virtual size_t encodeBufferSize(const ValueType&) const { return 4096; }
-    void extraDecode(eckit::Stream&) {}
-    ValueType apiCall(FDB&, const FDBToolRequest&) const { NOTIMP; }
-
-    struct Encoded {
-        size_t position;
-        eckit::Buffer buf;
-    };
-
-    Encoded encode(const ValueType& elem, const RemoteHandler&) const {
-        eckit::Buffer encodeBuffer(encodeBufferSize(elem));
-        MemoryStream s(encodeBuffer);
-        s << elem;
-        return {s.position(), std::move(encodeBuffer)};
-    }
-};
-
-struct ListHelper : public BaseHelper {
-    ListIterator apiCall(FDB& fdb, const FDBToolRequest& request) const {
-        return fdb.list(request);
-    }
-};
-
-struct InspectHelper : public BaseHelper {
-    ListIterator apiCall(FDB& fdb, const FDBToolRequest& request) const {
-        return fdb.inspect(request.request());
-    }
-};
-
-
-struct DumpHelper : public BaseHelper {
-    void extraDecode(eckit::Stream& s) { s >> simple_; }
-
-    DumpIterator apiCall(FDB& fdb, const FDBToolRequest& request) const {
-        return fdb.dump(request, simple_);
-    }
-
-private:
-    bool simple_;
-};
-
-
-struct PurgeHelper : public BaseHelper {
-    void extraDecode(eckit::Stream& s) {
-        s >> doit_;
-        s >> porcelain_;
-    }
-
-    PurgeIterator apiCall(FDB& fdb, const FDBToolRequest& request) const {
-        return fdb.purge(request, doit_, porcelain_);
-    }
-
-private:
-    bool doit_;
-    bool porcelain_;
-};
-
-
-struct StatsHelper : public BaseHelper {
-    StatsIterator apiCall(FDB& fdb, const FDBToolRequest& request) const {
-        return fdb.stats(request);
-    }
-};
-
-struct StatusHelper : public BaseHelper {
-    StatusIterator apiCall(FDB& fdb, const FDBToolRequest& request) const {
-        return fdb.status(request);
-    }
-};
-
-struct WipeHelper : public BaseHelper {
-    void extraDecode(eckit::Stream& s) {
-        s >> doit_;
-        s >> porcelain_;
-        s >> unsafeWipeAll_;
-    }
-
-    WipeIterator apiCall(FDB& fdb, const FDBToolRequest& request) const {
-        return fdb.wipe(request, doit_, porcelain_, unsafeWipeAll_);
-    }
-
-private:
-    bool doit_;
-    bool porcelain_;
-    bool unsafeWipeAll_;
-};
-
-struct ControlHelper : public BaseHelper {
-    void extraDecode(eckit::Stream& s) {
-        s >> action_;
-        identifiers_ = ControlIdentifiers(s);
-    }
-
-    StatusIterator apiCall(FDB& fdb, const FDBToolRequest& request) const {
-        return fdb.control(request, action_, identifiers_);
-    }
-
-private:
-    ControlAction action_;
-    ControlIdentifiers identifiers_;
-};
-
-}  // namespace
-
-//----------------------------------------------------------------------------------------------------------------------
-
-// n.b. by default the retrieve queue is big -- we are only queueing the requests, and it
-// is a common idiom to queue _many_ requests behind each other (and then aggregate the
-// results in a MultiHandle/HandleGatherer).
-
-RemoteHandler::RemoteHandler(eckit::net::TCPSocket& socket, const Config& config) :
-    config_(config),
-    controlSocket_(socket),
-    dataSocket_(selectDataPort()),
-    dataListenHostname_(config.getString("dataListenHostname", "")),
-    fdb_(config),
-    readLocationQueue_(eckit::Resource("fdbRetrieveQueueSize", 10000)) {}
-
-RemoteHandler::~RemoteHandler() {
-    // We don't want to die before the worker threads are cleaned up
-
-    waitForWorkers();
-
-    // And notify the client that we are done.
-
-    Log::info() << "Sending exit message to client" << std::endl;
-    dataWrite(Message::Exit, 0);
-    Log::info() << "Done" << std::endl;
-}
-
-
-eckit::LocalConfiguration RemoteHandler::availableFunctionality() const {
-    eckit::LocalConfiguration conf;
-//    Add to the configuration all the components that require to be versioned, as in the following example, with a vector of supported version numbers
-    std::vector remoteFieldLocationVersions = {1};
-    conf.set("RemoteFieldLocation", remoteFieldLocationVersions);
-    return conf;
-}
-
-std::vector intersection(const LocalConfiguration& c1, const LocalConfiguration& c2, const std::string& field){
-
-    std::vector v1 = c1.getIntVector(field);
-    std::vector v2 = c2.getIntVector(field);
-    std::vector v3;
-
-    std::sort(v1.begin(), v1.end());
-    std::sort(v2.begin(), v2.end());
-
-    std::set_intersection(v1.begin(),v1.end(),
-                          v2.begin(),v2.end(),
-                          back_inserter(v3));
-    return v3;
-}
-
-void RemoteHandler::initialiseConnections() {
-    // Read the startup message from the client. Check that it all checks out.
-
-    MessageHeader hdr;
-    socketRead(&hdr, sizeof(hdr), controlSocket_);
-
-    ASSERT(hdr.marker == StartMarker);
-    ASSERT(hdr.version == CurrentVersion);
-    ASSERT(hdr.message == Message::Startup);
-    ASSERT(hdr.requestID == 0);
-
-    Buffer payload1 = receivePayload(hdr, controlSocket_);
-    eckit::FixedString<4> tail;
-    socketRead(&tail, sizeof(tail), controlSocket_);
-    ASSERT(tail == EndMarker);
-
-    MemoryStream s1(payload1);
-    SessionID clientSession(s1);
-    net::Endpoint endpointFromClient(s1);
-    unsigned int remoteProtocolVersion = 0;
-    std::string errorMsg;
-
-    try {
-        s1 >> remoteProtocolVersion;
-    } catch (...) {
-        errorMsg = "Error retrieving client protocol version";
-    }
-
-    if (errorMsg.empty() && !LibFdb5::instance().remoteProtocolVersion().check(remoteProtocolVersion, false)) {
-        std::stringstream ss;
-        ss << "FDB server version " << fdb5_version_str() << " - remote protocol version not supported:" << std::endl;
-        ss << "    versions supported by server: " << LibFdb5::instance().remoteProtocolVersion().supportedStr() << std::endl;
-        ss << "    version requested by client: " << remoteProtocolVersion << std::endl;
-        errorMsg = ss.str();
-    }
-
-    if (errorMsg.empty()) {
-        LocalConfiguration clientAvailableFunctionality(s1);
-        LocalConfiguration serverConf = availableFunctionality();
-        agreedConf_ = LocalConfiguration();
-
-        // agree on a common functionality by intersecting server and client version numbers
-         std::vector rflCommon = intersection(clientAvailableFunctionality, serverConf, "RemoteFieldLocation");
-         if (rflCommon.size() > 0) {
-             Log::debug() << "Protocol negotiation - RemoteFieldLocation version " << rflCommon.back() << std::endl;
-             agreedConf_.set("RemoteFieldLocation", rflCommon.back());
-         }
-         else {
-             std::stringstream ss;
-             ss << "FDB server version " << fdb5_version_str() << " - failed protocol negotiation with FDB client" << std::endl;
-             ss << "    server functionality: " << serverConf << std::endl;
-             ss << "    client functionality: " << clientAvailableFunctionality << std::endl;
-             errorMsg = ss.str();
-         }
-    }
-
-    // We want a data connection too. Send info to RemoteFDB, and wait for connection
-    // n.b. FDB-192: we use the host communicated from the client endpoint. This
-    //               ensures that if a specific interface has been selected and the
-    //               server has multiple, then we use that on, whilst retaining
-    //               the capacity in the protocol for the server to make a choice.
-
-    int dataport = dataSocket_.localPort();
-    // std::string host = dataListenHostname_.empty() ? dataSocket_.localHost() :
-    // dataListenHostname_;
-    net::Endpoint dataEndpoint(endpointFromClient.hostname(), dataport);
-
-    Log::info() << "Sending data endpoint to client: " << dataEndpoint << std::endl;
-
-    {
-        Buffer startupBuffer(1024);
-        MemoryStream s(startupBuffer);
-
-        s << clientSession;
-        s << sessionID_;
-        s << dataEndpoint;
-
-        s << agreedConf_.get();
-
-        Log::debug() << "Protocol negotiation - configuration: " << agreedConf_ < tail;
-
-    while (true) {
-        tidyWorkers();
-
-        socketRead(&hdr, sizeof(hdr), controlSocket_);
-
-        ASSERT(hdr.marker == StartMarker);
-        ASSERT(hdr.version == CurrentVersion);
-        Log::debug() << "Got message with request ID: " << hdr.requestID << std::endl;
-
-        try {
-            switch (hdr.message) {
-                case Message::Exit:
-                    Log::status() << "Exiting" << std::endl;
-                    Log::info() << "Exiting" << std::endl;
-                    return;
-
-                case Message::List:
-                    forwardApiCall(hdr);
-                    break;
-
-                case Message::Dump:
-                    forwardApiCall(hdr);
-                    break;
-
-                case Message::Purge:
-                    forwardApiCall(hdr);
-                    break;
-
-                case Message::Stats:
-                    forwardApiCall(hdr);
-                    break;
-
-                case Message::Status:
-                    forwardApiCall(hdr);
-                    break;
-
-                case Message::Wipe:
-                    forwardApiCall(hdr);
-                    break;
-
-                case Message::Control:
-                    forwardApiCall(hdr);
-                    break;
-
-                case Message::Inspect:
-                    forwardApiCall(hdr);
-                    break;
-
-                case Message::Read:
-                    read(hdr);
-                    break;
-
-                case Message::Flush:
-                    flush(hdr);
-                    break;
-
-                case Message::Archive:
-                    archive(hdr);
-                    break;
-
-                default: {
-                    std::stringstream ss;
-                    ss << "ERROR: Unexpected message recieved (" << static_cast(hdr.message)
-                       << "). ABORTING";
-                    Log::status() << ss.str() << std::endl;
-                    Log::error() << "Retrieving... " << ss.str() << std::endl;
-                    throw SeriousBug(ss.str(), Here());
-                }
-            }
-
-            // Ensure we have consumed exactly the correct amount from the socket.
-
-            socketRead(&tail, sizeof(tail), controlSocket_);
-            ASSERT(tail == EndMarker);
-
-            // Acknowledge receipt of command
-
-            controlWrite(Message::Received, hdr.requestID);
-        }
-        catch (std::exception& e) {
-            // n.b. more general than eckit::Exception
-            std::string what(e.what());
-            controlWrite(Message::Error, hdr.requestID, what.c_str(), what.length());
-        }
-        catch (...) {
-            std::string what("Caught unexpected and unknown error");
-            controlWrite(Message::Error, hdr.requestID, what.c_str(), what.length());
-        }
-    }
-}
-
-int RemoteHandler::selectDataPort() {
-    eckit::Log::info() << "SelectDataPort: " << std::endl;
-    eckit::Log::info() << config_ << std::endl;
-    if (config_.has("dataPortStart")) {
-        ASSERT(config_.has("dataPortCount"));
-        return AvailablePortList(config_.getInt("dataPortStart"), config_.getLong("dataPortCount"))
-            .acquire();
-    }
-
-    // Use a system assigned port
-    return 0;
-}
-
-void RemoteHandler::controlWrite(Message msg, uint32_t requestID, const void* payload,
-                                 uint32_t payloadLength) {
-    ASSERT((payload == nullptr) == (payloadLength == 0));
-
-    MessageHeader message(msg, requestID, payloadLength);
-    controlWrite(&message, sizeof(message));
-    if (payload) {
-        controlWrite(payload, payloadLength);
-    }
-    controlWrite(&EndMarker, sizeof(EndMarker));
-}
-
-void RemoteHandler::controlWrite(const void* data, size_t length) {
-    size_t written = controlSocket_.write(data, length);
-    if (length != written) {
-        std::stringstream ss;
-        ss << "Write error. Expected " << length << " bytes, wrote " << written;
-        throw TCPException(ss.str(), Here());
-    }
-}
-
-void RemoteHandler::socketRead(void* data, size_t length, eckit::net::TCPSocket& socket) {
-    size_t read = socket.read(data, length);
-    if (length != read) {
-        std::stringstream ss;
-        ss << "Read error. Expected " << length << " bytes, read " << read;
-        throw TCPException(ss.str(), Here());
-    }
-}
-
-void RemoteHandler::dataWrite(Message msg, uint32_t requestID, const void* payload,
-                              uint32_t payloadLength) {
-    ASSERT((payload == nullptr) == (payloadLength == 0));
-
-    MessageHeader message(msg, requestID, payloadLength);
-
-    std::lock_guard lock(dataWriteMutex_);
-
-    dataWriteUnsafe(&message, sizeof(message));
-    if (payload) {
-        dataWriteUnsafe(payload, payloadLength);
-    }
-    dataWriteUnsafe(&EndMarker, sizeof(EndMarker));
-}
-
-void RemoteHandler::dataWriteUnsafe(const void* data, size_t length) {
-    size_t written = dataSocket_.write(data, length);
-    if (length != written) {
-        std::stringstream ss;
-        ss << "Write error. Expected " << length << " bytes, wrote " << written;
-        throw TCPException(ss.str(), Here());
-    }
-}
-
-
-Buffer RemoteHandler::receivePayload(const MessageHeader& hdr, net::TCPSocket& socket) {
-    Buffer payload(hdr.payloadSize);
-
-    ASSERT(hdr.payloadSize > 0);
-    socketRead(payload, hdr.payloadSize, socket);
-
-    return payload;
-}
-
-void RemoteHandler::tidyWorkers() {
-    std::map>::iterator it = workerThreads_.begin();
-
-    for (; it != workerThreads_.end(); /* empty */) {
-        std::future_status stat = it->second.wait_for(std::chrono::milliseconds(0));
-
-        if (stat == std::future_status::ready) {
-            Log::info() << "Tidying up worker for request ID: " << it->first << std::endl;
-            workerThreads_.erase(it++);
-        }
-        else {
-            ++it;
-        }
-    }
-}
-
-void RemoteHandler::waitForWorkers() {
-    readLocationQueue_.close();
-
-    tidyWorkers();
-
-    for (auto& it : workerThreads_) {
-        Log::error() << "Worker thread still alive for request ID: " << it.first << std::endl;
-        Log::error() << "Joining ..." << std::endl;
-        it.second.get();
-        Log::error() << "Thread complete" << std::endl;
-    }
-
-    if (readLocationWorker_.joinable()) {
-        readLocationWorker_.join();
-    }
-}
-
-
-// TODO: This can be an absolutely standard handler based on the ListElement type.
-
-template 
-void RemoteHandler::forwardApiCall(const MessageHeader& hdr) {
-    HelperClass helper;
-
-    Buffer payload(receivePayload(hdr, controlSocket_));
-    MemoryStream s(payload);
-
-    FDBToolRequest request(s);
-    helper.extraDecode(s);
-
-    // Construct worker thread to feed responses back to client
-
-    ASSERT(workerThreads_.find(hdr.requestID) == workerThreads_.end());
-
-    workerThreads_.emplace(
-        hdr.requestID, std::async(std::launch::async, [request, hdr, helper, this]() {
-            try {
-                auto iterator = helper.apiCall(fdb_, request);
-
-                typename decltype(iterator)::value_type elem;
-                while (iterator.next(elem)) {
-                    auto encoded(helper.encode(elem, *this));
-                    dataWrite(Message::Blob, hdr.requestID, encoded.buf, encoded.position);
-                }
-
-                dataWrite(Message::Complete, hdr.requestID);
-            }
-            catch (std::exception& e) {
-                // n.b. more general than eckit::Exception
-                std::string what(e.what());
-                dataWrite(Message::Error, hdr.requestID, what.c_str(), what.length());
-            }
-            catch (...) {
-                // We really don't want to std::terminate the thread
-                std::string what("Caught unexpected, unknown exception in worker");
-                dataWrite(Message::Error, hdr.requestID, what.c_str(), what.length());
-            }
-        }));
-}
-
-
-void RemoteHandler::archive(const MessageHeader& hdr) {
-    ASSERT(hdr.payloadSize == 0);
-
-    // Ensure that we aren't already running an store()
-
-    ASSERT(!archiveFuture_.valid());
-
-    // Start archive worker thread
-
-    uint32_t id    = hdr.requestID;
-    archiveFuture_ = std::async(std::launch::async, [this, id] { return archiveThreadLoop(id); });
-}
-
-
-// A helper function to make archiveThreadLoop a bit cleaner
-
-static void archiveBlobPayload(FDB& fdb, const void* data, size_t length) {
-    MemoryStream s(data, length);
-
-    fdb5::Key key(s);
-
-    std::stringstream ss_key;
-    ss_key << key;
-
-    const char* charData = static_cast(data);  // To allow pointer arithmetic
-    Log::status() << "Archiving data: " << ss_key.str() << std::endl;
-    fdb.archive(key, charData + s.position(), length - s.position());
-    Log::status() << "Archiving done: " << ss_key.str() << std::endl;
-}
-
-
-size_t RemoteHandler::archiveThreadLoop(uint32_t id) {
-    size_t totalArchived = 0;
-
-    // Create a worker that will do the actual archiving
-
-    static size_t queueSize(eckit::Resource("fdbServerMaxQueueSize", 32));
-    eckit::Queue> queue(queueSize);
-
-    std::future worker = std::async(std::launch::async, [this, &queue, id] {
-        size_t totalArchived = 0;
-
-        std::pair elem = std::make_pair(Buffer{0}, false);
-
-        try {
-            long queuelen;
-            while ((queuelen = queue.pop(elem)) != -1) {
-                if (elem.second) {
-                    // Handle MultiBlob
-
-                    const char* firstData =
-                        static_cast(elem.first.data());  // For pointer arithmetic
-                    const char* charData = firstData;
-                    while (size_t(charData - firstData) < elem.first.size()) {
-                        const MessageHeader* hdr =
-                            static_cast(static_cast(charData));
-                        ASSERT(hdr->marker == StartMarker);
-                        ASSERT(hdr->version == CurrentVersion);
-                        ASSERT(hdr->message == Message::Blob);
-                        ASSERT(hdr->requestID == id);
-                        charData += sizeof(MessageHeader);
-
-                        const void* payloadData = charData;
-                        charData += hdr->payloadSize;
-
-                        const decltype(EndMarker)* e = static_cast(
-                            static_cast(charData));
-                        ASSERT(*e == EndMarker);
-                        charData += sizeof(EndMarker);
-
-                        archiveBlobPayload(fdb_, payloadData, hdr->payloadSize);
-                        totalArchived += 1;
-                    }
-                }
-                else {
-                    // Handle single blob
-                    archiveBlobPayload(fdb_, elem.first.data(), elem.first.size());
-                    totalArchived += 1;
-                }
-            }
-        }
-        catch (...) {
-            // Ensure exception propagates across the queue back to the parent thread.
-            queue.interrupt(std::current_exception());
-            throw;
-        }
-
-        return totalArchived;
-    });
-
-    try {
-        // The archive loop is the only thing that can listen on the data socket,
-        // so we don't need to to anything clever here.
-
-        // n.b. we also don't need to lock on read. We are the only thing that reads.
-
-        while (true) {
-            MessageHeader hdr;
-            socketRead(&hdr, sizeof(hdr), dataSocket_);
-
-            ASSERT(hdr.marker == StartMarker);
-            ASSERT(hdr.version == CurrentVersion);
-            ASSERT(hdr.requestID == id);
-
-            // Have we been told that we are done yet?
-            if (hdr.message == Message::Flush)
-                break;
-
-            ASSERT(hdr.message == Message::Blob || hdr.message == Message::MultiBlob);
-
-            Buffer payload(receivePayload(hdr, dataSocket_));
-            MemoryStream s(payload);
-
-            eckit::FixedString<4> tail;
-            socketRead(&tail, sizeof(tail), dataSocket_);
-            ASSERT(tail == EndMarker);
-
-            // Queueing payload
-
-            size_t sz = payload.size();
-            Log::debug() << "Queueing data: " << sz << std::endl;
-            size_t queuelen = queue.emplace(
-                std::make_pair(std::move(payload), hdr.message == Message::MultiBlob));
-            Log::status() << "Queued data (" << queuelen << ", size=" << sz << ")" << std::endl;
-            ;
-            Log::debug() << "Queued data (" << queuelen << ", size=" << sz << ")"
-                                  << std::endl;
-            ;
-        }
-
-        // Trigger cleanup of the workers
-        queue.close();
-
-        // Complete reading the Flush instruction
-
-        eckit::FixedString<4> tail;
-        socketRead(&tail, sizeof(tail), dataSocket_);
-        ASSERT(tail == EndMarker);
-
-        // Ensure worker is done
-
-        ASSERT(worker.valid());
-        totalArchived = worker.get();  // n.b. use of async, get() propagates any exceptions.
-    }
-    catch (std::exception& e) {
-        // n.b. more general than eckit::Exception
-        std::string what(e.what());
-        dataWrite(Message::Error, id, what.c_str(), what.length());
-        queue.interrupt(std::current_exception());
-        throw;
-    }
-    catch (...) {
-        std::string what("Caught unexpected, unknown exception in retrieve worker");
-        dataWrite(Message::Error, id, what.c_str(), what.length());
-        queue.interrupt(std::current_exception());
-        throw;
-    }
-
-    return totalArchived;
-}
-
-void RemoteHandler::flush(const MessageHeader& hdr) {
-    Buffer payload(receivePayload(hdr, controlSocket_));
-    MemoryStream s(payload);
-
-    size_t numArchived;
-    s >> numArchived;
-
-    ASSERT(numArchived == 0 || archiveFuture_.valid());
-
-    if (archiveFuture_.valid()) {
-        // Ensure that the expected number of fields have been written, and that the
-        // archive worker processes have been cleanly wound up.
-        size_t n = archiveFuture_.get();
-        ASSERT(numArchived == n);
-
-        // Do the actual flush!
-        Log::info() << "Flushing" << std::endl;
-        Log::status() << "Flushing" << std::endl;
-        fdb_.flush();
-        Log::info() << "Flush complete" << std::endl;
-        Log::status() << "Flush complete" << std::endl;
-    }
-}
-
-void RemoteHandler::read(const MessageHeader& hdr) {
-
-    if (!readLocationWorker_.joinable()) {
-        readLocationWorker_ = std::thread([this] { readLocationThreadLoop(); });
-    }
-
-    Buffer payload(receivePayload(hdr, controlSocket_));
-    MemoryStream s(payload);
-
-    std::unique_ptr location(eckit::Reanimator::reanimate(s));
-
-    Log::debug() << "Queuing for read: " << hdr.requestID << " " << *location << std::endl;
-
-    std::unique_ptr dh;
-    dh.reset(location->dataHandle());
-
-    readLocationQueue_.emplace(std::make_pair(hdr.requestID, std::move(dh)));
-}
-
-void RemoteHandler::writeToParent(const uint32_t requestID, std::unique_ptr dh) {
-    try {
-        Log::status() << "Reading: " << requestID << std::endl;
-        // Write the data to the parent, in chunks if necessary.
-
-        Buffer writeBuffer(10 * 1024 * 1024);
-        long dataRead;
-
-        dh->openForRead();
-        Log::debug() << "Reading: " << requestID << " dh size: " << dh->size()
-                              << std::endl;
-
-        while ((dataRead = dh->read(writeBuffer, writeBuffer.size())) != 0) {
-            dataWrite(Message::Blob, requestID, writeBuffer, dataRead);
-        }
-
-        // And when we are done, add a complete message.
-
-        Log::debug() << "Writing retrieve complete message: " << requestID
-                              << std::endl;
-
-        dataWrite(Message::Complete, requestID);
-
-        Log::status() << "Done retrieve: " << requestID << std::endl;
-        Log::debug() << "Done retrieve: " << requestID << std::endl;
-    }
-    catch (std::exception& e) {
-        // n.b. more general than eckit::Exception
-        std::string what(e.what());
-        dataWrite(Message::Error, requestID, what.c_str(), what.length());
-    }
-    catch (...) {
-        // We really don't want to std::terminate the thread
-        std::string what("Caught unexpected, unknown exception in retrieve worker");
-        dataWrite(Message::Error, requestID, what.c_str(), what.length());
-    }
-}
-
-void RemoteHandler::readLocationThreadLoop() {
-    std::pair> elem;
-
-    while (readLocationQueue_.pop(elem) != -1) {
-        // Get the next MarsRequest in sequence to work on, do the retrieve, and
-        // send the data back to the client.
-
-        const uint32_t requestID(elem.first);
-
-        // Forward the API call
-        writeToParent(elem.first, std::move(elem.second));
-    }
-}
-
-
-//----------------------------------------------------------------------------------------------------------------------
-
-}  // namespace remote
-}  // namespace fdb5
diff --git a/src/fdb5/remote/Handler.h b/src/fdb5/remote/Handler.h
deleted file mode 100644
index 01b819293..000000000
--- a/src/fdb5/remote/Handler.h
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Simon Smart
-/// @author Tiago Quintino
-/// @date   Apr 2018
-
-#ifndef fdb5_remote_Handler_H
-#define fdb5_remote_Handler_H
-
-#include 
-#include 
-
-#include "eckit/io/Buffer.h"
-#include "eckit/io/DataHandle.h"
-#include "eckit/net/TCPServer.h"
-#include "eckit/net/TCPSocket.h"
-#include "eckit/runtime/SessionID.h"
-
-#include "metkit/mars/MarsRequest.h"
-
-#include "fdb5/api/FDB.h"
-#include "fdb5/config/Config.h"
-#include "fdb5/database/Key.h"
-#include "fdb5/remote/Messages.h"
-
-namespace fdb5 {
-
-class Config;
-
-namespace remote {
-
-struct MessageHeader;
-
-//----------------------------------------------------------------------------------------------------------------------
-
-class RemoteHandler : private eckit::NonCopyable {
-public:  // methods
-    RemoteHandler(eckit::net::TCPSocket& socket, const Config& config);
-    ~RemoteHandler();
-
-    void handle();
-
-    std::string host() const { return controlSocket_.localHost(); }
-    int port() const { return controlSocket_.localPort(); }
-    const eckit::LocalConfiguration& agreedConf() const { return agreedConf_; }
-
-private:  // methods
-    // Socket methods
-
-    int selectDataPort();
-    void initialiseConnections();
-    eckit::LocalConfiguration availableFunctionality() const;
-
-    void controlWrite(Message msg, uint32_t requestID, const void* payload = nullptr,
-                      uint32_t payloadLength = 0);
-    void controlWrite(const void* data, size_t length);
-    void socketRead(void* data, size_t length, eckit::net::TCPSocket& socket);
-
-    // dataWrite is protected using a mutex, as we may have multiple workers.
-    void dataWrite(Message msg, uint32_t requestID, const void* payload = nullptr,
-                   uint32_t payloadLength = 0);
-    void dataWriteUnsafe(const void* data, size_t length);
-
-    eckit::Buffer receivePayload(const MessageHeader& hdr, eckit::net::TCPSocket& socket);
-
-    // Worker functionality
-
-    void tidyWorkers();
-    void waitForWorkers();
-
-    // API functionality
-
-    template 
-    void forwardApiCall(const MessageHeader& hdr);
-
-//    void list(const MessageHeader& hdr);
-//    void dump(const MessageHeader& hdr);
-
-    void flush(const MessageHeader& hdr);
-//    void store(const MessageHeader& hdr);
-    void archive(const MessageHeader& hdr);
-    void retrieve(const MessageHeader& hdr);
-    void read(const MessageHeader& hdr);
-
-    void writeToParent(const uint32_t requestID, std::unique_ptr dh);
-
-    size_t archiveThreadLoop(uint32_t id);
-    void readLocationThreadLoop();
-
-private:  // members
-    Config config_;
-    eckit::SessionID sessionID_;
-
-    eckit::LocalConfiguration agreedConf_;
-
-    eckit::net::TCPSocket controlSocket_;
-    eckit::net::EphemeralTCPServer dataSocket_;
-    std::string dataListenHostname_;
-    std::mutex dataWriteMutex_;
-
-    // API helpers
-
-    FDB fdb_;
-    std::map> workerThreads_;
-
-    // Archive helpers
-
-    std::future archiveFuture_;
-
-    // Retrieve helpers
-
-    std::thread readLocationWorker_;
-    eckit::Queue>> readLocationQueue_;
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-}  // namespace remote
-}  // namespace fdb5
-
-#endif  // fdb5_remote_Handler_H
diff --git a/src/fdb5/remote/Messages.h b/src/fdb5/remote/Messages.h
index 0c6fbf916..1565fe7a4 100644
--- a/src/fdb5/remote/Messages.h
+++ b/src/fdb5/remote/Messages.h
@@ -47,7 +47,6 @@ enum class Message : uint16_t {
     Exit,
     Startup,
     Error,
-    MasterSchema,
 
     // API calls to forward
     Flush = 100,
diff --git a/src/fdb5/remote/RemoteCatalogue.cc b/src/fdb5/remote/RemoteCatalogue.cc
index 87e1f2d0e..c29a61610 100644
--- a/src/fdb5/remote/RemoteCatalogue.cc
+++ b/src/fdb5/remote/RemoteCatalogue.cc
@@ -10,10 +10,11 @@
 
 #include "eckit/config/Resource.h"
 #include "eckit/log/Log.h"
+#include "eckit/serialisation/MemoryStream.h"
 
 #include "fdb5/LibFdb5.h"
 #include "fdb5/remote/RemoteCatalogue.h"
-#include "eckit/serialisation/MemoryStream.h"
+#include "fdb5/remote/RemoteEngine.h"
 
 #include  // xxx debug / development
 #include  // xxx debug / development
@@ -22,69 +23,162 @@
 using namespace eckit;
 namespace fdb5::remote {
 
-RemoteCatalogue::RemoteCatalogue(const Key& key, const Config& config):
-    CatalogueImpl(key, ControlIdentifiers(), config), // xxx what are control identifiers? Setting empty here...
-    Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port"))), // xxx hardcoded endpoint
-    config_(config), schema_(nullptr),
-    maxArchiveQueueLength_(eckit::Resource("fdbRemoteArchiveQueueLength;$FDB_REMOTE_ARCHIVE_QUEUE_LENGTH", 200))
-    {
-        loadSchema();
+class CatalogueArchivalRequest {
+
+public:
+
+    CatalogueArchivalRequest() :
+        id_(0), catalogue_(nullptr), key_(Key()) {}
+
+    CatalogueArchivalRequest(uint32_t id, RemoteCatalogue* catalogue, const fdb5::Key& key, std::unique_ptr location) :
+        id_(id), catalogue_(catalogue), key_(key), location_(std::move(location)) {}
+
+    std::pair > >  element;
+
+    uint32_t id_;
+    RemoteCatalogue* catalogue_;
+    fdb5::Key key_;
+    std::unique_ptr location_;
+};
+
+
+class RemoteCatalogueArchiver {
+
+public: // types
+
+    using CatalogueArchiveQueue = eckit::Queue;
+
+public: // methods
+
+    static RemoteCatalogueArchiver* get(const eckit::net::Endpoint& endpoint);
+
+    bool valid() { return archiveFuture_.valid(); }
+    bool dirty() { return dirty_; }
+
+    void start();
+    void error(eckit::Buffer payload, const eckit::net::Endpoint& endpoint);
+    
+    void emplace(uint32_t id, RemoteCatalogue* catalogue, const Key& key, std::unique_ptr location);
+    FDBStats flush(RemoteCatalogue* catalogue);
+
+private: // methods
+
+    RemoteCatalogueArchiver() : dirty_(false), maxArchiveQueueLength_(eckit::Resource("fdbRemoteArchiveQueueLength;$FDB_REMOTE_ARCHIVE_QUEUE_LENGTH", 200)) {}
+
+    FDBStats archiveThreadLoop();
+
+private: // members
+
+    bool dirty_;
+
+    std::mutex archiveQueuePtrMutex_;
+    size_t maxArchiveQueueLength_;
+    std::unique_ptr archiveQueue_;
+    std::future archiveFuture_;
+
+};
+
+RemoteCatalogueArchiver* RemoteCatalogueArchiver::get(const eckit::net::Endpoint& endpoint) {
+
+    static std::map > archivers_;
+
+    auto it = archivers_.find(endpoint.hostport());
+    if (it == archivers_.end()) {
+        // auto arch = (archivers_[endpoint.hostport()] = RemoteCatalogueArchiver());
+        it = archivers_.emplace(endpoint.hostport(), new RemoteCatalogueArchiver()).first;
     }
+    ASSERT(it != archivers_.end());
+    return it->second.get();
+}
 
-RemoteCatalogue::RemoteCatalogue(const eckit::URI& uri, const Config& config):
-    Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port"))),  schema_(nullptr),
-    maxArchiveQueueLength_(eckit::Resource("fdbRemoteArchiveQueueLength;$FDB_REMOTE_ARCHIVE_QUEUE_LENGTH", 200))
-    {
-        NOTIMP;
+void RemoteCatalogueArchiver::start() {
+    // if there is no archiving thread active, then start one.
+    // n.b. reset the archiveQueue_ after a potential flush() cycle.
+    if (!archiveFuture_.valid()) {
+
+        {
+            // Reset the queue after previous done/errors
+            std::lock_guard lock(archiveQueuePtrMutex_);
+            ASSERT(!archiveQueue_);
+            archiveQueue_.reset(new CatalogueArchiveQueue(maxArchiveQueueLength_));
+        }
+
+        archiveFuture_ = std::async(std::launch::async, [this] { return archiveThreadLoop(); });
     }
+}
 
+void RemoteCatalogueArchiver::error(eckit::Buffer payload, const eckit::net::Endpoint& endpoint) {
 
-void RemoteCatalogue::sendArchiveData(uint32_t id, const Key& key, std::unique_ptr fieldLocation)
-{
-    ASSERT(!dbKey_.empty());
-    ASSERT(!currentIndexKey_.empty());
-    ASSERT(!key.empty());
-    ASSERT(fieldLocation);
+    std::lock_guard lock(archiveQueuePtrMutex_);
 
-    Buffer keyBuffer(4096);
-    MemoryStream keyStream(keyBuffer);
-    keyStream << dbKey_;
-    keyStream << currentIndexKey_;
-    keyStream << key;
+    if (archiveQueue_) {
+        std::string msg;
+        if (payload.size() > 0) {
+            msg.resize(payload.size(), ' ');
+            payload.copy(&msg[0], payload.size());
+        }
 
-    Buffer locBuffer(4096);
-    MemoryStream locStream(locBuffer);
-    locStream << *fieldLocation;
+        archiveQueue_->interrupt(std::make_exception_ptr(RemoteFDBException(msg, endpoint)));
+    }
+}
 
-    MessageHeader message(Message::Blob, 0, id, keyStream.position() + locStream.position());
-    dataWrite(id, &message, sizeof(message));
-    dataWrite(id, keyBuffer, keyStream.position());
-    dataWrite(id, locBuffer, locStream.position());
-    dataWrite(id, &EndMarker, sizeof(EndMarker));
+void RemoteCatalogueArchiver::emplace(uint32_t id, RemoteCatalogue* catalogue, const Key& key, std::unique_ptr location) {
+
+    dirty_ = true;
+
+    std::lock_guard lock(archiveQueuePtrMutex_);
+    ASSERT(archiveQueue_);
+    archiveQueue_->emplace(id, catalogue, key, std::move(location));
+}
+
+FDBStats RemoteCatalogueArchiver::flush(RemoteCatalogue* catalogue) {
+
+    std::lock_guard lock(archiveQueuePtrMutex_);
+    ASSERT(archiveQueue_);
+    archiveQueue_->close();
+
+    FDBStats stats = archiveFuture_.get();
+    ASSERT(!archiveQueue_);
+
+    ASSERT(stats.numFlush() == 0);
+    size_t numArchive = stats.numArchive();
+
+    Buffer sendBuf(4096);
+    MemoryStream s(sendBuf);
+    s << numArchive;
+
+    // The flush call is blocking
+    catalogue->controlWriteCheckResponse(Message::Flush, sendBuf, s.position());
+
+    dirty_ = false;
+
+    return stats;
 }
 
-FDBStats RemoteCatalogue::archiveThreadLoop(){
+FDBStats RemoteCatalogueArchiver::archiveThreadLoop() {
     FDBStats localStats;
     eckit::Timer timer;
 
-    std::pair > >  element;
+    CatalogueArchivalRequest element;
 
     try {
 
         ASSERT(archiveQueue_);
         long popped;
         while ((popped = archiveQueue_->pop(element)) != -1) {
+
             timer.start();
-            eckit::Log::debug() << " RemoteCatalogue::archiveThreadLoop() popped: " << popped << " id: " << element.first << 
-                " key: " << element.second.first << " fieldLocation: " << element.second.second->uri() << std::endl;
-            sendArchiveData(element.first, element.second.first, std::move(element.second.second));
+            element.catalogue_->sendArchiveData(element.id_, element.key_, std::move(element.location_));
             timer.stop();
 
             localStats.addArchive(0, timer, 1);
-
         }
 
-        dataWrite(Message::Flush, nullptr, 0);
+        // And note that we are done. (don't time this, as already being blocked
+        // on by the ::flush() routine)
+
+        element.catalogue_->dataWrite(Message::Flush, nullptr, 0);
+
         archiveQueue_.reset();
 
     } catch (...) {
@@ -93,41 +187,71 @@ FDBStats RemoteCatalogue::archiveThreadLoop(){
     }
 
     return localStats;
+
+    // We are inside an async, so don't need to worry about exceptions escaping.
+    // They will be released when flush() is called.
 }
 
-void RemoteCatalogue::archive(const InspectionKey& key, std::unique_ptr fieldLocation) {
-    
-    // if there is no archiving thread active, then start one.
-    // n.b. reset the archiveQueue_ after a potential flush() cycle.
-    if (!archiveFuture_.valid()) {
 
-        // Start the archival request on the remote side
 
-        // Reset the queue after previous done/errors
-        {
-            std::lock_guard lock(archiveQueuePtrMutex_);
-            ASSERT(!archiveQueue_);
-            archiveQueue_.reset(new ArchiveQueue(maxArchiveQueueLength_));
-        }
 
-        archiveFuture_ = std::async(std::launch::async, [this] { return archiveThreadLoop(); });
+
+
+RemoteCatalogue::RemoteCatalogue(const Key& key, const Config& config):
+    CatalogueImpl(key, ControlIdentifiers(), config), // xxx what are control identifiers? Setting empty here...
+    Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port"))),
+    config_(config), schema_(nullptr) {
+
+    loadSchema();
+}
+
+RemoteCatalogue::RemoteCatalogue(const eckit::URI& uri, const Config& config):
+    Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port"))), config_(config), schema_(nullptr)
+    {
+        NOTIMP;
     }
 
-    ASSERT(archiveFuture_.valid());
 
-    // xxx I don't think we need to to this controlwrite everytime? On the remote side, the first call starts the archive thread loop.
-    // xxx repeated calls dont do anything? Its already listening for archive requests on the data connection after the first call.
-    // xxx so, try only doing this call on the first archive request.
-    uint32_t id = controlWriteCheckResponse(Message::Archive); 
+void RemoteCatalogue::sendArchiveData(uint32_t id, const Key& key, std::unique_ptr fieldLocation)
+{
+    ASSERT(!dbKey_.empty());
+    ASSERT(!currentIndexKey_.empty());
+    ASSERT(!key.empty());
+    ASSERT(fieldLocation);
 
-    std::lock_guard lock(archiveQueuePtrMutex_);
-    ASSERT(archiveQueue_);
+    Buffer keyBuffer(4096);
+    MemoryStream keyStream(keyBuffer);
+    keyStream << dbKey_;
+    keyStream << currentIndexKey_;
+    keyStream << key;
+
+    Buffer locBuffer(4096);
+    MemoryStream locStream(locBuffer);
+    locStream << *fieldLocation;
+
+    MessageHeader message(Message::Blob, 0, id, keyStream.position() + locStream.position());
+    dataWrite(id, &message, sizeof(message));
+    dataWrite(id, keyBuffer, keyStream.position());
+    dataWrite(id, locBuffer, locStream.position());
+    dataWrite(id, &EndMarker, sizeof(EndMarker));
+}
+
+void RemoteCatalogue::archive(const InspectionKey& key, std::unique_ptr fieldLocation) {
+
+    // if there is no archiving thread active, then start one.
+    // n.b. reset the archiveQueue_ after a potential flush() cycle.
+    if (!archiver_) {
+        archiver_ = RemoteCatalogueArchiver::get(controlEndpoint());
+    }
+    ASSERT(archiver_);
+    archiver_->start();
 
-    eckit::Log::debug() << " RemoteCatalogue::archive() added to queue with id: " << id << 
-        " key: " << key << " fieldLocation: " << fieldLocation->uri() << std::endl;
+    ASSERT(archiver_->valid());
 
-    archiveQueue_->emplace(std::make_pair(id, std::make_pair(key, std::move(fieldLocation))));
+    uint32_t id = controlWriteCheckResponse(Message::Archive, nullptr, 0);
+    eckit::Log::debug() << " RemoteCatalogue::archive - adding to queue [id=" << id << ",key=" << key << ",fieldLocation=" << fieldLocation->uri() << "]" << std::endl;
 
+    archiver_->emplace(id, this, key, std::move(fieldLocation));
 }
 
 bool RemoteCatalogue::selectIndex(const Key& idxKey) {
@@ -151,31 +275,14 @@ const Schema& RemoteCatalogue::schema() const {
 }
 
 void RemoteCatalogue::flush() {
-    
+
     Timer timer;
 
     timer.start();
 
     // Flush only does anything if there is an ongoing archive();
-    if (archiveFuture_.valid()) {
-
-        ASSERT(archiveQueue_);
-        std::lock_guard lock(archiveQueuePtrMutex_);
-        archiveQueue_->close();
-
-        FDBStats stats = archiveFuture_.get();
-        ASSERT(!archiveQueue_);
-
-        ASSERT(stats.numFlush() == 0);
-        size_t numArchive = stats.numArchive();
-
-        Buffer sendBuf(4096);
-        MemoryStream s(sendBuf);
-        s << numArchive;
-
-        // The flush call is blocking
-        controlWriteCheckResponse(fdb5::remote::Message::Flush, sendBuf, s.position());
-
+    if (archiver_->valid()) {
+        archiver_->flush(this);
 //        internalStats_ += stats;
     }
 
@@ -189,53 +296,45 @@ void RemoteCatalogue::close() {NOTIMP;}
 
 bool RemoteCatalogue::exists() const {NOTIMP;}
 
-void RemoteCatalogue::checkUID() const {NOTIMP;}
+void RemoteCatalogue::checkUID() const {}
 
-// xxx what is this uri used for?
 eckit::URI RemoteCatalogue::uri() const {
+    return eckit::URI(/*RemoteEngine::typeName()*/ "fdb", endpoint_.host(), endpoint_.port());
     // return eckit::URI(TocEngine::typeName(), basePath());
-    return eckit::URI("remotecat", ""); // todo
 }
 
 void RemoteCatalogue::loadSchema() {
     // NB we're at the db level, so get the db schema. We will want to get the master schema beforehand.
     // (outside of the catalogue) 
 
-    eckit::Log::debug() << "RemoteCatalogue::loadSchema()" << std::endl;
-
-    // destroy previous schema if it exists
-    schema_.reset(nullptr);
-
-    // send dbkey to remote.
-    constexpr uint32_t bufferSize = 4096;
-    eckit::Buffer keyBuffer(bufferSize);
-    eckit::MemoryStream keyStream(keyBuffer);
-    keyStream << dbKey_;
-    
-    uint32_t id = controlWriteCheckResponse(Message::Schema, keyBuffer, keyStream.position());
-    eckit::Log::debug() << "RemoteCatalogue::loadSchema() received id: " << id << std::endl;
-
-    // Should block - Use control connection.
-    // XXX This is a temporary work around while control read is being reimplemented.
-    while (!schema_) {
-        std::this_thread::sleep_for(std::chrono::milliseconds(10));
+    if (!schema_) {
+        eckit::Log::debug() << "RemoteCatalogue::loadSchema()" << std::endl;
+
+        // send dbkey to remote.
+        eckit::Buffer keyBuffer(4096);
+        eckit::MemoryStream keyStream(keyBuffer);
+        keyStream << dbKey_;
+        
+        eckit::Buffer buf = controlWriteReadResponse(Message::Schema, keyBuffer, keyStream.position());
+        eckit::MemoryStream s(buf);
+//        eckit::net::Endpoint storeEndpoint_{s};
+        schema_.reset(eckit::Reanimator::reanimate(s));
     }
-
 }
 
 bool RemoteCatalogue::handle(Message message, uint32_t requestID) {
-    eckit::Log::debug() << "RemoteCatalogue::handle received message: " << ((uint) message) << " - requestID: " << requestID << std::endl;
+    eckit::Log::debug() << *this << " - Received [message=" << ((uint) message) << ",requestID=" << requestID << "]" << std::endl;
     NOTIMP;
     return false;
 }
 bool RemoteCatalogue::handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) {
-    eckit::Log::debug()<< "RemoteCatalogue::handle received message: " << ((uint) message) << " - requestID: " << requestID << std::endl;
-    if (message == Message::Schema) {
-        eckit::Log::debug() << "RemoteCatalogue::handle received payload size: " << payload.size() << std::endl;
-        MemoryStream s(payload);
-        schema_ = std::unique_ptr(eckit::Reanimator::reanimate(s));
-        return true;
-    }
+    eckit::Log::debug() << *this << " - Received [message=" << ((uint) message) << ",requestID=" << requestID << ",payloadSize=" << payload.size() << "]" << std::endl;
+    // if (message == Message::Schema) {
+    //     eckit::Log::debug() << "RemoteCatalogue::handle received payload size: " << payload.size() << std::endl;
+    //     MemoryStream s(payload);
+    //     schema_ = std::unique_ptr(eckit::Reanimator::reanimate(s));
+    //     return true;
+    // }
     return false;
 }
 
@@ -269,12 +368,14 @@ std::vector RemoteCatalogue::indexes(bool sorted) const {NOTIMP;}
 void RemoteCatalogue::maskIndexEntry(const Index& index) const {NOTIMP;}
 void RemoteCatalogue::allMasked(std::set>& metadata, std::set& data) const {NOTIMP;}
 void RemoteCatalogue::print( std::ostream &out ) const {NOTIMP;}
+
 std::string RemoteCatalogue::type() const {
-    NOTIMP;
+    return "remote"; // RemoteEngine::typeName();
 }
 bool RemoteCatalogue::open() {
-    NOTIMP;
+    return true;
 }
 
-static CatalogueWriterBuilder builder("remote");
+static CatalogueReaderBuilder reader("remote");
+static CatalogueWriterBuilder writer("remote");
 } // namespace fdb5::remote
\ No newline at end of file
diff --git a/src/fdb5/remote/RemoteCatalogue.h b/src/fdb5/remote/RemoteCatalogue.h
index 4c3125502..255e6134c 100644
--- a/src/fdb5/remote/RemoteCatalogue.h
+++ b/src/fdb5/remote/RemoteCatalogue.h
@@ -9,16 +9,14 @@
 
 namespace fdb5::remote {
 
+class RemoteCatalogueArchiver;
 //----------------------------------------------------------------------------------------------------------------------
 
-class RemoteCatalogue : public CatalogueWriter, public CatalogueImpl, public Client {
-public: //types
-    using ArchiveQueue = eckit::Queue > > >;
+class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public CatalogueImpl, public Client {
 
 public:
 
     RemoteCatalogue(const Key& key, const Config& config);
-    
     RemoteCatalogue(const eckit::URI& uri, const Config& config);
 
     ~RemoteCatalogue() {}
@@ -32,9 +30,9 @@ class RemoteCatalogue : public CatalogueWriter, public CatalogueImpl, public Cli
     void reconsolidate() override;
 
     //From CatalogueReader
-    // DbStats stats() const override;
-    // bool axis(const std::string& keyword, eckit::StringSet& s) const override;
-    // bool retrieve(const InspectionKey& key, Field& field) const override;
+    DbStats stats() const override { return DbStats(); }
+    bool axis(const std::string& keyword, eckit::StringSet& s) const override { return false; }
+    bool retrieve(const InspectionKey& key, Field& field) const override { return false; }
 
     // From Catalogue
     bool selectIndex(const Key& idxKey) override;
@@ -63,39 +61,32 @@ class RemoteCatalogue : public CatalogueWriter, public CatalogueImpl, public Cli
     void checkUID() const override;
     eckit::URI uri() const override;
 
+    void sendArchiveData(uint32_t id, const Key& key, std::unique_ptr fieldLocation);
+
 protected:
 
     void loadSchema() override;
 
 private:
     // From Client
-
     // handlers for incoming messages - to be defined in the client class
     bool handle(Message message, uint32_t requestID) override;
     bool handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) override;
     void handleException(std::exception_ptr e) override;
     const Key& key() const override { return CatalogueImpl::key(); }
 
-
-    // note: in common with remotestore
-    FDBStats archiveThreadLoop();
-    void sendArchiveData(uint32_t id, const Key& key, std::unique_ptr fieldLocation);
-
 protected:
 
     Config config_;
     ControlIdentifiers controlIdentifiers_;
 
 private:
+
     Key currentIndexKey_;
     std::unique_ptr schema_;
 
-    size_t maxArchiveQueueLength_;
-    std::mutex archiveQueuePtrMutex_;
-    std::future archiveFuture_;
-    std::unique_ptr archiveQueue_;
-
-
+    // not owning
+    RemoteCatalogueArchiver* archiver_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/RemoteEngine.cc b/src/fdb5/remote/RemoteEngine.cc
new file mode 100644
index 000000000..0ee0b54da
--- /dev/null
+++ b/src/fdb5/remote/RemoteEngine.cc
@@ -0,0 +1,89 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+#include "fdb5/remote/RemoteEngine.h"
+
+// #include "eckit/io/Buffer.h"
+// #include "eckit/serialisation/MemoryStream.h"
+
+#include "fdb5/config/Config.h"
+// #include "fdb5/remote/client/Client.h"
+
+
+namespace fdb5::remote {
+
+// class RemoteEngineClient : public Client {
+
+// public:
+
+//     RemoteEngineClient(const Config& config) :
+//         Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port"))) {
+//     }
+
+// private: // methods
+
+//     // Client
+//     bool handle(remote::Message message, uint32_t requestID) override { return false; }
+//     bool handle(remote::Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) override { return false; }
+//     void handleException(std::exception_ptr e) override { NOTIMP; }
+
+//     const Key& key() const override { NOTIMP; }
+
+// private: // members
+
+// };
+
+//----------------------------------------------------------------------------------------------------------------------
+
+std::string RemoteEngine::name() const {
+    return RemoteEngine::typeName();
+}
+
+std::string RemoteEngine::dbType() const {
+    return RemoteEngine::typeName();
+}
+
+bool RemoteEngine::canHandle(const eckit::URI& uri) const
+{
+    return uri.scheme() == "fdb";
+}
+
+std::vector RemoteEngine::visitableLocations(const Config& config) const
+{
+    // if (!client_) {
+    //     client_.reset(new RemoteEngineClient(config));
+    // }
+    // ASSERT(client_);
+
+    return std::vector {}; //databases(Key(), CatalogueRootManager(config).visitableRoots(InspectionKey()), config);
+}
+
+std::vector RemoteEngine::visitableLocations(const metkit::mars::MarsRequest& request, const Config& config) const
+{
+    // if (!client_) {
+    //     client_.reset(new RemoteEngineClient(config));
+    // }
+    // ASSERT(client_);
+
+    return std::vector {}; //databases(request, CatalogueRootManager(config).visitableRoots(request), config);
+}
+
+void RemoteEngine::print(std::ostream& out) const
+{
+    out << "RemoteEngine()";
+}
+
+static EngineBuilder remote_builder;
+
+//static eckit::LocalFileManager manager_toc("toc");
+
+//----------------------------------------------------------------------------------------------------------------------
+
+} // namespace fdb5::remote
diff --git a/src/fdb5/remote/RemoteEngine.h b/src/fdb5/remote/RemoteEngine.h
new file mode 100644
index 000000000..295de620b
--- /dev/null
+++ b/src/fdb5/remote/RemoteEngine.h
@@ -0,0 +1,50 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/// @author Emanuele Danovaro
+/// @date   October 2023
+
+#pragma once
+
+#include "fdb5/database/Engine.h"
+
+namespace fdb5::remote {
+
+class RemoteEngineClient;
+
+//----------------------------------------------------------------------------------------------------------------------
+
+class RemoteEngine : public fdb5::Engine {
+
+public: // methods
+
+    static const char* typeName() { return "remote"; }
+
+protected: // methods
+
+    virtual std::string name() const override;
+
+    virtual std::string dbType() const override;
+
+    virtual bool canHandle(const eckit::URI& path) const override;
+
+    virtual std::vector visitableLocations(const Config& config) const override;
+    virtual std::vector visitableLocations(const metkit::mars::MarsRequest& rq, const Config& config) const override;
+
+    virtual void print( std::ostream &out ) const override;
+
+private:
+    // mutable std::unique_ptr client_;
+
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
+} // namespace fdb5::remote
diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index 712a04849..a0a758e29 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -24,7 +24,6 @@
 #include "fdb5/database/FieldLocation.h"
 #include "fdb5/remote/RemoteStore.h"
 #include "fdb5/remote/RemoteFieldLocation.h"
-#include "fdb5/remote/client/ClientConnectionRouter.h"
 #include "fdb5/io/FDBFileHandle.h"
 
 using namespace eckit;
@@ -33,14 +32,14 @@ using namespace eckit;
 
 namespace fdb5::remote {
 
-class ArchivalRequest {
+class StoreArchivalRequest {
 
 public:
 
-    ArchivalRequest() :
+    StoreArchivalRequest() :
         id_(0), store_(nullptr), key_(Key()), buffer_(Buffer()) {}
 
-    ArchivalRequest(uint32_t id, RemoteStore* store, const fdb5::Key& key, const void *data, eckit::Length length) :
+    StoreArchivalRequest(uint32_t id, RemoteStore* store, const fdb5::Key& key, const void *data, eckit::Length length) :
         id_(id), store_(store), key_(key), buffer_(Buffer(reinterpret_cast(data), length)) {}
 
     uint32_t id_;
@@ -51,28 +50,28 @@ class ArchivalRequest {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-class Archiver {
+class RemoteStoreArchiver {
 
 public: // types
 
-    using ArchiveQueue = eckit::Queue;
+    using StoreArchiveQueue = eckit::Queue;
 
 public: // methods
 
-    static Archiver* get(const eckit::net::Endpoint& endpoint);
+    static RemoteStoreArchiver* get(const eckit::net::Endpoint& endpoint);
 
     bool valid() { return archiveFuture_.valid(); }
     bool dirty() { return dirty_; }
 
     void start();
-    void error(Buffer&& payload, const eckit::net::Endpoint& endpoint);
+    void error(eckit::Buffer&& payload, const eckit::net::Endpoint& endpoint);
     
     void emplace(uint32_t id, RemoteStore* store, const Key& key, const void *data, eckit::Length length);
     FDBStats flush(RemoteStore* store);
 
 private: // methods
 
-    Archiver() : dirty_(false), maxArchiveQueueLength_(eckit::Resource("fdbRemoteArchiveQueueLength;$FDB_REMOTE_ARCHIVE_QUEUE_LENGTH", 200)) {}
+    RemoteStoreArchiver() : dirty_(false), maxArchiveQueueLength_(eckit::Resource("fdbRemoteArchiveQueueLength;$FDB_REMOTE_ARCHIVE_QUEUE_LENGTH", 200)) {}
 
     FDBStats archiveThreadLoop();
 
@@ -82,25 +81,25 @@ class Archiver {
 
     std::mutex archiveQueuePtrMutex_;
     size_t maxArchiveQueueLength_;
-    std::unique_ptr archiveQueue_;
+    std::unique_ptr archiveQueue_;
     std::future archiveFuture_;
 
 };
 
-Archiver* Archiver::get(const eckit::net::Endpoint& endpoint) {
+RemoteStoreArchiver* RemoteStoreArchiver::get(const eckit::net::Endpoint& endpoint) {
 
-    static std::map > archivers_;
+    static std::map > archivers_;
 
     auto it = archivers_.find(endpoint.hostport());
     if (it == archivers_.end()) {
-        // auto arch = (archivers_[endpoint.hostport()] = Archiver());
-        it = archivers_.emplace(endpoint.hostport(), new Archiver()).first;
+        // auto arch = (archivers_[endpoint.hostport()] = RemoteStoreArchiver());
+        it = archivers_.emplace(endpoint.hostport(), new RemoteStoreArchiver()).first;
     }
     ASSERT(it != archivers_.end());
     return it->second.get();
 }
 
-void Archiver::start() {
+void RemoteStoreArchiver::start() {
     // if there is no archiving thread active, then start one.
     // n.b. reset the archiveQueue_ after a potential flush() cycle.
     if (!archiveFuture_.valid()) {
@@ -109,14 +108,14 @@ void Archiver::start() {
             // Reset the queue after previous done/errors
             std::lock_guard lock(archiveQueuePtrMutex_);
             ASSERT(!archiveQueue_);
-            archiveQueue_.reset(new ArchiveQueue(maxArchiveQueueLength_));
+            archiveQueue_.reset(new StoreArchiveQueue(maxArchiveQueueLength_));
         }
 
         archiveFuture_ = std::async(std::launch::async, [this] { return archiveThreadLoop(); });
     }
 }
 
-void Archiver::error(Buffer&& payload, const eckit::net::Endpoint& endpoint) {
+void RemoteStoreArchiver::error(eckit::Buffer&& payload, const eckit::net::Endpoint& endpoint) {
 
     std::lock_guard lock(archiveQueuePtrMutex_);
 
@@ -127,11 +126,11 @@ void Archiver::error(Buffer&& payload, const eckit::net::Endpoint& endpoint) {
             payload.copy(&msg[0], payload.size());
         }
 
-        archiveQueue_->interrupt(std::make_exception_ptr(DecoupledFDBException(msg, endpoint)));
+        archiveQueue_->interrupt(std::make_exception_ptr(RemoteFDBException(msg, endpoint)));
     }
 }
 
-void Archiver::emplace(uint32_t id, RemoteStore* store, const Key& key, const void *data, eckit::Length length) {
+void RemoteStoreArchiver::emplace(uint32_t id, RemoteStore* store, const Key& key, const void *data, eckit::Length length) {
 
     dirty_ = true;
 
@@ -140,7 +139,7 @@ void Archiver::emplace(uint32_t id, RemoteStore* store, const Key& key, const vo
     archiveQueue_->emplace(id, store, key, data, length);
 }
 
-FDBStats Archiver::flush(RemoteStore* store) {
+FDBStats RemoteStoreArchiver::flush(RemoteStore* store) {
 
     std::lock_guard lock(archiveQueuePtrMutex_);
     ASSERT(archiveQueue_);
@@ -157,18 +156,18 @@ FDBStats Archiver::flush(RemoteStore* store) {
     s << numArchive;
 
     // The flush call is blocking
-    store->controlWriteCheckResponse(fdb5::remote::Message::Flush, sendBuf, s.position());
+    store->controlWriteCheckResponse(Message::Flush, sendBuf, s.position());
 
     dirty_ = false;
 
     return stats;
 }
 
-FDBStats Archiver::archiveThreadLoop() {
+FDBStats RemoteStoreArchiver::archiveThreadLoop() {
     FDBStats localStats;
     eckit::Timer timer;
 
-    ArchivalRequest element;
+    StoreArchivalRequest element;
 
     try {
 
@@ -176,6 +175,8 @@ FDBStats Archiver::archiveThreadLoop() {
         long popped;
         while ((popped = archiveQueue_->pop(element)) != -1) {
 
+            eckit::Log::debug() << "RemoteStoreArchiver archiving [id="<sendArchiveData(element.id_, element.key_, element.buffer_.data(), element.buffer_.size());
             timer.stop();
@@ -260,8 +261,8 @@ class FDBRemoteDataHandle : public DataHandle {
 
         // Handle any remote errors communicated from the server
         if (msg.first == Message::Error) {
-            std::string errmsg(static_cast(msg.second), msg.second.size());
-            throw DecoupledFDBException(errmsg, remoteEndpoint_);
+            std::string errmsg(static_cast(msg.second.data()), msg.second.size());
+            throw RemoteFDBException(errmsg, remoteEndpoint_);
         }
 
         // Are we now complete?
@@ -369,11 +370,10 @@ std::future > RemoteStore::archive(const Key& key
     // if there is no archiving thread active, then start one.
     // n.b. reset the archiveQueue_ after a potential flush() cycle.
     if (!archiver_) {
-        archiver_ = Archiver::get(controlEndpoint());
+        archiver_ = RemoteStoreArchiver::get(controlEndpoint());
     }
     ASSERT(archiver_);
     archiver_->start();
-
     ASSERT(archiver_->valid());
 
     uint32_t id = controlWriteCheckResponse(Message::Archive, nullptr, 0);
@@ -395,8 +395,6 @@ void RemoteStore::flush() {
 
     timer.start();
 
-    size_t numArchive = 0;
-
     // Flush only does anything if there is an ongoing archive();
     if (archiver_->valid()) {
         archiver_->flush(this);
@@ -444,7 +442,7 @@ bool RemoteStore::handle(Message message, uint32_t requestID) {
 
             auto it = messageQueues_.find(requestID);
             if (it != messageQueues_.end()) {
-                it->second->interrupt(std::make_exception_ptr(DecoupledFDBException("", endpoint_)));
+                it->second->interrupt(std::make_exception_ptr(RemoteFDBException("", endpoint_)));
 
                 // Remove entry (shared_ptr --> message queue will be destroyed when it
                 // goes out of scope in the worker thread).
@@ -478,7 +476,7 @@ bool RemoteStore::handle(Message message, uint32_t requestID, eckit::net::Endpoi
             if (it != messageQueues_.end()) {
                 it->second->emplace(message, std::move(payload));
             } else {
-                retrieveMessageQueue_.emplace(std::make_pair(message, std::move(payload)));
+                retrieveMessageQueue_.emplace(message, std::move(payload));
             }
             return true;
         }
@@ -489,7 +487,7 @@ bool RemoteStore::handle(Message message, uint32_t requestID, eckit::net::Endpoi
                 std::string msg;
                 msg.resize(payload.size(), ' ');
                 payload.copy(&msg[0], payload.size());
-                it->second->interrupt(std::make_exception_ptr(DecoupledFDBException(msg, endpoint)));
+                it->second->interrupt(std::make_exception_ptr(RemoteFDBException(msg, endpoint)));
 
                 // Remove entry (shared_ptr --> message queue will be destroyed when it
                 // goes out of scope in the worker thread).
@@ -500,7 +498,7 @@ bool RemoteStore::handle(Message message, uint32_t requestID, eckit::net::Endpoi
                 if (it != locations_.end()) {
                     archiver_->error(std::move(payload), endpoint);
                 } else {
-                    retrieveMessageQueue_.emplace(std::make_pair(message, std::move(payload)));
+                    retrieveMessageQueue_.emplace(message, std::move(payload));
                 }
             }
             return true;
@@ -510,7 +508,7 @@ bool RemoteStore::handle(Message message, uint32_t requestID, eckit::net::Endpoi
     }
 }
 void RemoteStore::handleException(std::exception_ptr e) {
-    std::cout << "RemoteStore::handleException" << std::endl;
+    Log::error() << "RemoteStore::handleException " << std::endl;
 }
 
 void RemoteStore::flush(FDBStats& internalStats) {
@@ -520,95 +518,6 @@ void RemoteStore::flush(FDBStats& internalStats) {
     internalStats += archiver_->flush(this);
 }
 
-
-// // Here we do (asynchronous) retrieving related stuff
-
-// //DataHandle* RemoteStore::retrieve(const metkit::mars::MarsRequest& request) {
-
-// //    connect();
-
-// //    Buffer encodeBuffer(4096);
-// //    MemoryStream s(encodeBuffer);
-// //    s << request;
-
-// //    uint32_t id = generateRequestID();
-
-// //    controlWriteCheckResponse(fdb5::remote::Message::Retrieve, id, encodeBuffer, s.position());
-
-// //    return new FDBRemoteDataHandle(id, retrieveMessageQueue_, controlEndpoint_);
-// //}
-
-
-// // Here we do (asynchronous) read related stuff
-
-
-// eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation) {
-//     return dataHandle(fieldLocation, Key());
-// }
-
-// eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation, const Key& remapKey) {
-
-//     connect();
-
-//     Buffer encodeBuffer(4096);
-//     MemoryStream s(encodeBuffer);
-//     s << fieldLocation;
-//     s << remapKey;
-
-//     uint32_t id = generateRequestID();
-
-//     controlWriteCheckResponse(fdb5::remote::Message::Read, id, encodeBuffer, s.position());
-
-//     return new FDBRemoteDataHandle(id, retrieveMessageQueue_, controlEndpoint_);
-// }
-
-
-
-// long RemoteStore::sendArchiveData(uint32_t id, const std::vector>& elements, size_t count) {
-//     if (count == 1) {
-//         sendArchiveData(id, elements[0].first, elements[0].second.data(), elements[0].second.size());
-//         return elements[0].second.size();
-//     }
-
-//     // Serialise the keys
-
-//     std::vector keyBuffers;
-//     std::vector keySizes;
-//     keyBuffers.reserve(count);
-//     keySizes.reserve(count);
-
-//     size_t containedSize = 0;
-
-//     for (size_t i = 0; i < count; ++i) {
-//         keyBuffers.emplace_back(Buffer {4096});
-//         MemoryStream keyStream(keyBuffers.back());
-//         keyStream << elements[i].first;
-//         keySizes.push_back(keyStream.position());
-//         containedSize += (keyStream.position() + elements[i].second.size() +
-//                           sizeof(MessageHeader) + sizeof(EndMarker));
-//     }
-
-//     // Construct the containing message
-
-//     MessageHeader message(Message::MultiBlob, id, containedSize);
-//     dataWrite(&message, sizeof(message));
-
-//     long dataSent = 0;
-
-//     for (size_t i = 0; i < count; ++i) {
-//         MessageHeader containedMessage(Message::Blob, id, elements[i].second.size() + keySizes[i]);
-//         dataWrite(&containedMessage, sizeof(containedMessage));
-//         dataWrite(keyBuffers[i], keySizes[i]);
-//         dataWrite(elements[i].second.data(), elements[i].second.size());
-//         dataWrite(&EndMarker, sizeof(EndMarker));
-//         dataSent += elements[i].second.size();
-//     }
-
-//     dataWrite(&EndMarker, sizeof(EndMarker));
-//     return dataSent;
-// }
-
-
 void RemoteStore::sendArchiveData(uint32_t id, const Key& key, const void* data, size_t length) {
     
     ASSERT(!dbKey_.empty());
@@ -619,7 +528,6 @@ void RemoteStore::sendArchiveData(uint32_t id, const Key& key, const void* data,
 
     Buffer keyBuffer(4096);
     MemoryStream keyStream(keyBuffer);
-
     keyStream << dbKey_;
     keyStream << key;
 
@@ -630,8 +538,6 @@ void RemoteStore::sendArchiveData(uint32_t id, const Key& key, const void* data,
     dataWrite(id, &EndMarker, sizeof(EndMarker));
 }
 
-
-
 eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation) {
     return dataHandle(fieldLocation, Key());
 }
diff --git a/src/fdb5/remote/RemoteStore.h b/src/fdb5/remote/RemoteStore.h
index 4559b06f7..2cbd65ce1 100644
--- a/src/fdb5/remote/RemoteStore.h
+++ b/src/fdb5/remote/RemoteStore.h
@@ -23,7 +23,7 @@
 
 namespace fdb5::remote {
 
-class Archiver;
+class RemoteStoreArchiver;
 
 //----------------------------------------------------------------------------------------------------------------------
 
@@ -33,7 +33,7 @@ class RemoteStore : public Store, public Client {
 
 public: // types
 
-    using StoredMessage = std::pair;
+    using StoredMessage = std::pair;
     using MessageQueue = eckit::Queue;
 
 public: // methods
@@ -110,7 +110,7 @@ class RemoteStore : public Store, public Client {
     MessageQueue retrieveMessageQueue_;
 
     // not owning
-    Archiver* archiver_;
+    RemoteStoreArchiver* archiver_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index b7d008dbf..a44f71d2a 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -18,8 +18,15 @@ namespace fdb5::remote {
 Client::Client(const eckit::net::Endpoint& endpoint) :
     endpoint_(endpoint) {}
 
-uint32_t Client::controlWriteCheckResponse(Message msg, const void* payload, uint32_t payloadLength) {
-    return ClientConnectionRouter::instance().controlWriteCheckResponse(*this, msg, payload, payloadLength);
+uint32_t Client::generateRequestID() {
+    return ClientConnectionRouter::instance().generateRequestID();
+}
+
+uint32_t Client::controlWriteCheckResponse(Message msg, const void* payload, uint32_t payloadLength, uint32_t requestID) {
+    return ClientConnectionRouter::instance().controlWriteCheckResponse(*this, msg, requestID, payload, payloadLength);
+}
+eckit::Buffer Client::controlWriteReadResponse(Message msg, const void* payload, uint32_t payloadLength) {
+    return ClientConnectionRouter::instance().controlWriteReadResponse(*this, msg, payload, payloadLength);
 }
 uint32_t Client::controlWrite(Message msg, const void* payload, uint32_t payloadLength) {
     return ClientConnectionRouter::instance().controlWrite(*this, msg, payload, payloadLength);
diff --git a/src/fdb5/remote/client/Client.h b/src/fdb5/remote/client/Client.h
index 1e30dc71e..0f9821106 100644
--- a/src/fdb5/remote/client/Client.h
+++ b/src/fdb5/remote/client/Client.h
@@ -10,19 +10,23 @@
 
 #pragma once
 
-#include 
-
 #include "eckit/config/Configuration.h"
 #include "eckit/memory/NonCopyable.h"
 #include "eckit/net/Endpoint.h"
-#include "eckit/thread/Mutex.h"
 
-#include "fdb5/database/Store.h"
+#include "fdb5/database/Key.h"
 #include "fdb5/remote/Messages.h"
 
 namespace fdb5::remote {
 
-class Connection;
+//----------------------------------------------------------------------------------------------------------------------
+
+class RemoteFDBException : public eckit::RemoteException {
+public:
+    RemoteFDBException(const std::string& msg, const eckit::net::Endpoint& endpoint):
+        eckit::RemoteException(msg, endpoint.hostport()) {}
+};
+
 //----------------------------------------------------------------------------------------------------------------------
 
 class Client : eckit::NonCopyable {
@@ -31,7 +35,10 @@ class Client : eckit::NonCopyable {
 
     const eckit::net::Endpoint& controlEndpoint() { return endpoint_; }
 
-    uint32_t controlWriteCheckResponse(Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
+    static uint32_t generateRequestID();
+
+    uint32_t controlWriteCheckResponse(Message msg, const void* payload=nullptr, uint32_t payloadLength=0, uint32_t requestID = generateRequestID());
+    eckit::Buffer controlWriteReadResponse(Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
     uint32_t controlWrite(Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
 
     uint32_t dataWrite(Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index 7b9a8720f..a4219e2b0 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -125,7 +125,7 @@ void ClientConnection::disconnect() {
     if (connected_) {
 
         // Send termination message
-        controlWrite(Message::Exit, 0); //xxx why do we generate a request ID here? 
+        controlWrite(Message::Exit, 0);
 
         listeningThread_.join();
 
@@ -174,9 +174,9 @@ void ClientConnection::controlWriteCheckResponse(Message msg, uint32_t requestID
     ASSERT(tail == EndMarker);
 }
 
-void ClientConnection::controlReadResponse(remote::Message msg, uint32_t requestID, void* payload, uint32_t& payloadLength) {
+eckit::Buffer ClientConnection::controlWriteReadResponse(remote::Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
     
-    controlWrite(msg, requestID, nullptr, 0);
+    controlWrite(msg, requestID, payload, payloadLength);
 
     // Wait for the receipt acknowledgement
 
@@ -185,18 +185,20 @@ void ClientConnection::controlReadResponse(remote::Message msg, uint32_t request
 
     handleError(response);
 
+    // std::cout << "ClientConnection::controlWriteReadResponse received " << response.requestID << " | expecting " << requestID << std::endl;
     ASSERT(response.marker == StartMarker);
     ASSERT(response.version == CurrentVersion);
     ASSERT(response.message == Message::Received);
     ASSERT(response.requestID == requestID);
     
-    ASSERT(response.payloadSize <= payloadLength);
-    payloadLength = response.payloadSize;
-    controlRead(payload, payloadLength);
+    eckit::Buffer buf{response.payloadSize};
+    controlRead(buf.data(), buf.size());
 
     eckit::FixedString<4> tail;
     controlRead(&tail, sizeof(tail));
     ASSERT(tail == EndMarker);
+
+    return buf;
 }
 
 void ClientConnection::controlWrite(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
@@ -277,7 +279,7 @@ void ClientConnection::handleError(const MessageHeader& hdr) {
             controlRead(&tail, sizeof(tail));
         } catch (...) {}
 
-        throw DecoupledFDBException(what, controlEndpoint_);
+        throw RemoteFDBException(what, controlEndpoint_);
     }
 }
 
@@ -375,8 +377,8 @@ void ClientConnection::listeningThreadLoop() {
                 handled = ClientConnectionRouter::instance().handle(hdr.message, hdr.requestID);
             }
             else {
-                Buffer payload(hdr.payloadSize);
-                dataRead(payload, hdr.payloadSize);
+                eckit::Buffer payload{hdr.payloadSize};
+                dataRead(payload.data(), hdr.payloadSize);
 
                 handled = ClientConnectionRouter::instance().handle(hdr.message, hdr.requestID, controlEndpoint_, std::move(payload));
             }
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index 158dc2276..0f0df2ff7 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -12,7 +12,6 @@
 
 #include 
 
-
 #include "eckit/config/LocalConfiguration.h"
 #include "eckit/container/Queue.h"
 #include "eckit/io/Buffer.h"
@@ -25,7 +24,6 @@
 #include "fdb5/remote/Messages.h"
 #include "eckit/utils/Translator.h"
 
-
 namespace eckit {
 
 class Buffer;
@@ -55,7 +53,7 @@ class ClientConnection : eckit::NonCopyable {
     virtual ~ClientConnection();
 
     void controlWriteCheckResponse(remote::Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
-    void controlReadResponse(remote::Message msg, uint32_t requestID, void* payload, uint32_t& payloadLength);
+    eckit::Buffer controlWriteReadResponse(remote::Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
     void controlWrite(remote::Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
     void dataWrite(remote::Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
 
@@ -122,10 +120,5 @@ class ClientConnection : eckit::NonCopyable {
 //     return ++id;
 // }
 
-class DecoupledFDBException : public eckit::RemoteException {
-public:
-    DecoupledFDBException(const std::string& msg, const eckit::net::Endpoint& endpoint):
-        eckit::RemoteException(msg, endpoint.hostport()) {}
-};
 
 }  // namespace fdb5::remote
\ No newline at end of file
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.cc b/src/fdb5/remote/client/ClientConnectionRouter.cc
index 1efa679e7..b0b54ed7a 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.cc
+++ b/src/fdb5/remote/client/ClientConnectionRouter.cc
@@ -3,13 +3,14 @@
 
 #include "fdb5/remote/client/ClientConnectionRouter.h"
 
-using namespace eckit;
-using namespace eckit::net;
+// using namespace eckit;
+// using namespace eckit::net;
 
+namespace fdb5::remote {
 
-namespace {
+//----------------------------------------------------------------------------------------------------------------------
 
-static uint32_t generateRequestID() {
+uint32_t ClientConnectionRouter::generateRequestID() {
 
     static std::mutex m;
     static uint32_t id = 0;
@@ -18,13 +19,6 @@ static uint32_t generateRequestID() {
     return ++id;
 }
 
-}
-
-
-namespace fdb5::remote {
-
-//----------------------------------------------------------------------------------------------------------------------
-
 RemoteStore& ClientConnectionRouter::store(const eckit::URI& uri) {
     const std::string& endpoint = uri.hostport();
     auto it = readStores_.find(endpoint);
@@ -35,41 +29,44 @@ RemoteStore& ClientConnectionRouter::store(const eckit::URI& uri) {
     return *(readStores_[endpoint] = std::unique_ptr(new RemoteStore(uri, Config())));
 }
 
-uint32_t ClientConnectionRouter::createConnection(Client& client, ClientConnection*& conn) {
+uint32_t ClientConnectionRouter::createConnection(Client& client, ClientConnection*& conn, bool add, uint32_t id) {
 
     // pick the first endpoint - To be improved with DataStoreStrategies
     const eckit::net::Endpoint& endpoint = client.controlEndpoint();
-    uint32_t id = generateRequestID();
 
     auto it = connections_.find(endpoint.hostport());
     if (it != connections_.end()) {
         conn = it->second.get();
     } else {
-        conn = (connections_[endpoint.hostport()] = std::unique_ptr(new ClientConnection(endpoint, LocalConfiguration()))).get();
+        conn = (connections_[endpoint.hostport()] = std::unique_ptr(new ClientConnection(endpoint, Config()))).get();
         conn->connect();
     }
 
     ASSERT(conn);
-    requests_[id] = {conn, &client};
+    if (add) {
+        requests_[id] = {conn, &client};
+    }
 
     return id;
 }
 
-uint32_t ClientConnectionRouter::controlWriteCheckResponse(Client& client, Message msg, const void* payload, uint32_t payloadLength) {
+uint32_t ClientConnectionRouter::controlWriteCheckResponse(Client& client, Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
     //std::cout << "ClientConnectionRouter::controlWriteCheckResponse " << endpoints.size() << std::endl;
     ClientConnection* conn;
-    uint32_t id = createConnection(client, conn);
+    createConnection(client, conn, true, requestID);
 
-    conn->controlWriteCheckResponse(msg, id, payload, payloadLength);
+    conn->controlWriteCheckResponse(msg, requestID, payload, payloadLength);
 
-    return id;
+    return requestID;
 }
 
-void ClientConnectionRouter::controlReadResponse(Client& client, remote::Message msg, void* payload, uint32_t& payloadLength) {
+
+eckit::Buffer ClientConnectionRouter::controlWriteReadResponse(Client& client, remote::Message msg, const void* payload, uint32_t payloadLength) {
     ClientConnection* conn;
-    uint32_t id = createConnection(client, conn);
+    uint32_t id = createConnection(client, conn, false);
+    //std::cout << "ClientConnectionRouter::controlWriteReadResponse Message: " << ((int) msg) << " ID: " << id << std::endl;
 
-    conn->controlReadResponse(msg, id, payload, payloadLength);
+    return conn->controlWriteReadResponse(msg, id, payload, payloadLength);
 }
 
 uint32_t ClientConnectionRouter::controlWrite(Client& client, Message msg, const void* payload, uint32_t payloadLength) {
@@ -115,7 +112,7 @@ bool ClientConnectionRouter::handle(Message message, uint32_t requestID) {
 
         return it->second.second->handle(message, requestID);
     } else {
-        Log::error() << "ClientConnectionRouter::handle [message=" << ((uint) message) << ",requestID=" << requestID << "]" << std::endl;
+        eckit::Log::error() << "ClientConnectionRouter [message=" << ((uint) message) << ",requestID=" << requestID << "]" << std::endl;
         return true;
     }
 }
@@ -127,7 +124,7 @@ bool ClientConnectionRouter::handle(Message message, uint32_t requestID, eckit::
     
         return it->second.second->handle(message, requestID, endpoint, std::move(payload));
     } else {
-        Log::error() << "ClientConnectionRouter::handle [message=" << ((uint) message) << ",requestID=" << requestID << "]" << std::endl;
+        eckit::Log::error() << "ClientConnectionRouter [message=" << ((uint) message) << ",requestID=" << requestID << "]" << std::endl;
         return true;
     }
 }
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.h b/src/fdb5/remote/client/ClientConnectionRouter.h
index a99dd3c17..08a3dd4a1 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.h
+++ b/src/fdb5/remote/client/ClientConnectionRouter.h
@@ -32,9 +32,10 @@ class ClientConnectionRouter : eckit::NonCopyable {
 
     static ClientConnectionRouter& instance();
     RemoteStore& store(const eckit::URI& uri);
+    static uint32_t generateRequestID();
 
-    uint32_t controlWriteCheckResponse(Client& client, Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
-    void controlReadResponse(Client& client, remote::Message msg, void* payload, uint32_t& payloadLength);
+    uint32_t controlWriteCheckResponse(Client& client, Message msg, uint32_t requestID = generateRequestID(), const void* payload=nullptr, uint32_t payloadLength=0);
+    eckit::Buffer controlWriteReadResponse(Client& client, remote::Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
     uint32_t controlWrite(Client& client, Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
     void controlRead(Client& client, uint32_t requestId, void* payload, size_t payloadLength);
 
@@ -50,7 +51,7 @@ class ClientConnectionRouter : eckit::NonCopyable {
 
     ClientConnectionRouter() {} ///< private constructor only used by singleton
 
-    uint32_t createConnection(Client& client, ClientConnection*& conn);
+    uint32_t createConnection(Client& client, ClientConnection*& conn, bool add=true, uint32_t id = generateRequestID());
 
     eckit::Mutex mutex_;
 
diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc
index f4d6f87a5..391e304dc 100644
--- a/src/fdb5/remote/server/CatalogueHandler.cc
+++ b/src/fdb5/remote/server/CatalogueHandler.cc
@@ -46,7 +46,7 @@ void CatalogueHandler::initialiseConnections() {
 
     ASSERT(hdr.marker == StartMarker);
     ASSERT(hdr.version == CurrentVersion);
-    ASSERT(hdr.message == Message::MasterSchema);
+    ASSERT(hdr.message == Message::Schema);
 
     // Ensure we have consumed exactly the correct amount from the socket.
     socketRead(&tail, sizeof(tail), controlSocket_);
@@ -65,31 +65,18 @@ void CatalogueHandler::initialiseConnections() {
         }
     }
 
-    ASSERT(stores.size() > 0);
-    // TODO randomise
-    eckit::net::Endpoint endpoint = stores.at(0);
-
-    Log::info() << "Sending store endpoint to client: " << endpoint << std::endl;
     {
-        Buffer startupBuffer(102400);
+        Buffer startupBuffer(1024*1024);
         MemoryStream s(startupBuffer);
 
-        // s << clientSession;
-        // s << sessionID_;
-        s << endpoint;
+        s << stores.size();
+        for (const eckit::net::Endpoint& endpoint : stores) {
+            s << endpoint;
+        }
         s << config_.schema();
 
-//        Log::debug() << "Protocol negotiation - configuration: " << agreedConf_ <() << "CatalogueHandler::handle() - pre READ" << std::endl;
         socketRead(&hdr, sizeof(hdr), controlSocket_);
 
         ASSERT(hdr.marker == StartMarker);
         ASSERT(hdr.version == CurrentVersion);
-        Log::debug() << "Got message with request ID: " << hdr.requestID << std::endl;
+        Log::debug() << "CatalogueHandler - got message " << ((int) hdr.message) << " with request ID: " << hdr.requestID << std::endl;
+
+        bool ack = false;
 
         try {
             switch (hdr.message) {
@@ -244,6 +232,7 @@ void CatalogueHandler::handle() {
 
                 case Message::Schema:
                     schema(hdr);
+                    ack = true;
                     break;
 
                 default: {
@@ -260,8 +249,10 @@ void CatalogueHandler::handle() {
             socketRead(&tail, sizeof(tail), controlSocket_);
             ASSERT(tail == EndMarker);
 
-            // Acknowledge receipt of command
-            controlWrite(Message::Received, hdr.requestID);
+            if (!ack) {
+                // Acknowledge receipt of command
+                controlWrite(Message::Received, hdr.requestID);
+            }
         }
         catch (std::exception& e) {
             // n.b. more general than eckit::Exception
@@ -314,7 +305,7 @@ void CatalogueHandler::archive(const MessageHeader& hdr) {
     // NOTE identical to RemoteCatalogueWriter::archive()
 
     if(!archiveFuture_.valid()) {
-        Log::debug() << "WIP CatalogueHandler::archive start threadloop" << std::endl;
+        Log::debug() << "CatalogueHandler::archive start threadloop" << std::endl;
         // Start archive worker thread
         archiveFuture_ = std::async(std::launch::async, [this] { return archiveThreadLoop(); });
     }
@@ -326,8 +317,6 @@ size_t CatalogueHandler::archiveThreadLoop() {
     // Create a worker that will do the actual archiving
 
     static size_t queueSize(eckit::Resource("fdbServerMaxQueueSize", 32));
-    // using ArchiveQueue = eckit::Queue>>>;
-    // ArchiveQueue queue(queueSize);
     eckit::Queue> queue(queueSize);
 
 
@@ -393,9 +382,7 @@ size_t CatalogueHandler::archiveThreadLoop() {
             socketRead(&tail, sizeof(tail), dataSocket_);
             ASSERT(tail == EndMarker);
 
-            size_t queuelen = queue.emplace(
-                std::make_pair(hdr.requestID, std::move(payload))
-            );
+            size_t queuelen = queue.emplace(hdr.requestID, std::move(payload));
         }
 
         // Trigger cleanup of the workers
@@ -443,7 +430,7 @@ void CatalogueHandler::schema(const MessageHeader& hdr) {
     Buffer payload(receivePayload(hdr, controlSocket_));
     MemoryStream s(payload);
     Key dbKey(s);
-
+    
     // 2. Get catalogue
     Catalogue& cat = catalogue(dbKey);
     const Schema& schema = cat.schema();
@@ -451,8 +438,7 @@ void CatalogueHandler::schema(const MessageHeader& hdr) {
     eckit::MemoryStream stream(schemaBuffer);
     stream << schema;
 
-    // todo: maybe more appropriate control write instead.
-    dataWrite(Message::Schema, hdr.requestID, schemaBuffer, stream.position());
+    controlWrite(Message::Received, hdr.requestID, schemaBuffer.data(), stream.position());
 }
 
 CatalogueWriter& CatalogueHandler::catalogue(Key dbKey) {
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index 11d9ce668..dab73ff15 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -73,7 +73,7 @@ ServerConnection::ServerConnection(eckit::net::TCPSocket& socket, const Config&
         dataListenHostname_(config.getString("dataListenHostname", "")),
         // fdb_(config),
         readLocationQueue_(eckit::Resource("fdbRetrieveQueueSize", 10000)) {
-            Log::debug() << "ServerConnection::ServerConnection initialized" << std::endl;
+            eckit::Log::debug() << "ServerConnection::ServerConnection initialized" << std::endl;
     }
 
 ServerConnection::~ServerConnection() {
@@ -138,18 +138,18 @@ void ServerConnection::initialiseConnections() {
         agreedConf_ = LocalConfiguration();
 
         // agree on a common functionality by intersecting server and client version numbers
-         std::vector rflCommon = intersection(clientAvailableFunctionality, serverConf, "RemoteFieldLocation");
-         if (rflCommon.size() > 0) {
-             Log::debug() << "Protocol negotiation - RemoteFieldLocation version " << rflCommon.back() << std::endl;
-             agreedConf_.set("RemoteFieldLocation", rflCommon.back());
-         }
-         else {
-             std::stringstream ss;
-             ss << "FDB server version " << fdb5_version_str() << " - failed protocol negotiation with FDB client" << std::endl;
-             ss << "    server functionality: " << serverConf << std::endl;
-             ss << "    client functionality: " << clientAvailableFunctionality << std::endl;
-             errorMsg = ss.str();
-         }
+        std::vector rflCommon = intersection(clientAvailableFunctionality, serverConf, "RemoteFieldLocation");
+        if (rflCommon.size() > 0) {
+            eckit::Log::debug() << "Protocol negotiation - RemoteFieldLocation version " << rflCommon.back() << std::endl;
+            agreedConf_.set("RemoteFieldLocation", rflCommon.back());
+        }
+        else {
+            std::stringstream ss;
+            ss << "FDB server version " << fdb5_version_str() << " - failed protocol negotiation with FDB client" << std::endl;
+            ss << "    server functionality: " << serverConf << std::endl;
+            ss << "    client functionality: " << clientAvailableFunctionality << std::endl;
+            errorMsg = ss.str();
+        }
     }
 
     // We want a data connection too. Send info to RemoteFDB, and wait for connection
@@ -172,7 +172,7 @@ void ServerConnection::initialiseConnections() {
         s << agreedConf_.get();
         // s << storeEndpoint; // xxx single-store case only: we cant do this with multiple stores // For now, dont send the store endpoint to the client 
 
-        Log::debug() << "Protocol negotiation - configuration: " << agreedConf_ <() << "Protocol negotiation - configuration: " << agreedConf_ <() << "Got message with request ID: " << hdr.requestID << std::endl;
+        Log::debug() << "StoreHandler - got message " << ((int) hdr.message) << " with request ID: " << hdr.requestID << std::endl;
 
         try {
             switch (hdr.message) {
@@ -135,7 +134,6 @@ void StoreHandler::handle() {
 
 void StoreHandler::read(const MessageHeader& hdr) {
 
-    // std::cout << "readLocationThreadLoop\n";
     if (!readLocationWorker_.joinable()) {
         readLocationWorker_ = std::thread([this] { readLocationThreadLoop(); });
     }
diff --git a/src/fdb5/rules/Schema.h b/src/fdb5/rules/Schema.h
index 21e4c956b..2e189c141 100644
--- a/src/fdb5/rules/Schema.h
+++ b/src/fdb5/rules/Schema.h
@@ -25,6 +25,7 @@
 #include "eckit/serialisation/Streamable.h"
 #include "eckit/serialisation/Reanimator.h"
 
+#include "fdb5/config/Config.h"
 #include "fdb5/types/TypesRegistry.h"
 
 namespace metkit { class MarsRequest; }
@@ -97,6 +98,8 @@ class Schema : public eckit::Streamable {
     static eckit::ClassSpec classSpec_;
     static eckit::Reanimator reanimator_;
 
+    friend void Config::overrideSchema(const eckit::PathName& schemaPath, Schema* schema);
+
     TypesRegistry registry_;
     std::vector  rules_;
     std::string path_;
diff --git a/src/fdb5/toc/TocCatalogue.cc b/src/fdb5/toc/TocCatalogue.cc
index 544626c12..c542e9352 100644
--- a/src/fdb5/toc/TocCatalogue.cc
+++ b/src/fdb5/toc/TocCatalogue.cc
@@ -142,7 +142,7 @@ void TocCatalogue::allMasked(std::set>& metadata,
 
 std::string TocCatalogue::type() const
 {
-    return TocCatalogue::catalogueTypeName();
+    return TocEngine::typeName();
 }
 
 void TocCatalogue::checkUID() const {
diff --git a/src/fdb5/toc/TocCatalogue.h b/src/fdb5/toc/TocCatalogue.h
index 39985e34b..738c7a19b 100644
--- a/src/fdb5/toc/TocCatalogue.h
+++ b/src/fdb5/toc/TocCatalogue.h
@@ -38,7 +38,6 @@ class TocCatalogue : public CatalogueImpl, public TocHandler {
 
     ~TocCatalogue() override {}
 
-    static const char* catalogueTypeName() { return TocEngine::typeName(); }
     const eckit::PathName& basePath() const override;
     eckit::URI uri() const override;
     const Key& indexKey() const override { return currentIndexKey_; }
diff --git a/src/fdb5/toc/TocEngine.cc b/src/fdb5/toc/TocEngine.cc
index 2e32a0b4c..9e0de7557 100644
--- a/src/fdb5/toc/TocEngine.cc
+++ b/src/fdb5/toc/TocEngine.cc
@@ -116,11 +116,6 @@ std::string TocEngine::dbType() const {
     return TocEngine::typeName();
 }
 
-eckit::URI TocEngine::location(const Key& key, const Config& config) const
-{
-    return URI("toc", CatalogueRootManager(config).directory(key).directory_);
-}
-
 bool TocEngine::canHandle(const eckit::URI& uri) const
 {
     if (uri.scheme() != "toc")
@@ -209,7 +204,7 @@ std::vector TocEngine::databases(const Key& key,
             TocHandler toc(path, config);
             if (toc.databaseKey().match(key)) {
                 Log::debug() << " found match with " << path << std::endl;
-                result.push_back(eckit::URI("toc", path));
+                result.push_back(eckit::URI(TocEngine::typeName(), path));
             }
         } catch (eckit::Exception& e) {
             eckit::Log::error() <<  "Error loading FDB database from " << path << std::endl;
@@ -234,12 +229,20 @@ std::vector TocEngine::databases(const metkit::mars::MarsRequest& re
     std::set databasesMatchRegex(databases(keys, roots, config));
 
     std::vector result;
-    for (const auto& path : databasesMatchRegex) {
+    for (eckit::PathName path : databasesMatchRegex) {
         try {
-            TocHandler toc(path, config);
-            if (toc.databaseKey().partialMatch(request)) {
-                Log::debug() << " found match with " << path << std::endl;
-                result.push_back(eckit::URI("toc", path));
+            if (path.exists()) {
+                if (!path.isDir())
+                    path = path.dirName();
+                path = path.realName();
+
+                Log::debug() << "FDB processing Path " << path << std::endl;
+
+                TocHandler toc(path, config);
+                if (toc.databaseKey().partialMatch(request)) {
+                    Log::debug() << " found match with " << path << std::endl;
+                    result.push_back(eckit::URI(TocEngine::typeName(), path));
+                }
             }
         } catch (eckit::Exception& e) {
             eckit::Log::error() <<  "Error loading FDB database from " << path << std::endl;
@@ -250,11 +253,6 @@ std::vector TocEngine::databases(const metkit::mars::MarsRequest& re
     return result;
 }
 
-std::vector TocEngine::allLocations(const Key& key, const Config& config) const
-{
-    return databases(key, CatalogueRootManager(config).allRoots(key), config);
-}
-
 std::vector TocEngine::visitableLocations(const Config& config) const
 {
     return databases(Key(), CatalogueRootManager(config).visitableRoots(InspectionKey()), config);
@@ -265,11 +263,6 @@ std::vector TocEngine::visitableLocations(const metkit::mars::MarsRequest&
     return databases(request, CatalogueRootManager(config).visitableRoots(request), config);
 }
 
-// std::vector TocEngine::writableLocations(const Key& key, const Config& config) const
-// {
-//     return databases(key, CatalogueRootManager(config).canArchiveRoots(key), config);
-// }
-
 void TocEngine::print(std::ostream& out) const
 {
     out << "TocEngine()";
diff --git a/src/fdb5/toc/TocEngine.h b/src/fdb5/toc/TocEngine.h
index 5467d1dc2..7550d63c1 100644
--- a/src/fdb5/toc/TocEngine.h
+++ b/src/fdb5/toc/TocEngine.h
@@ -45,17 +45,11 @@ class TocEngine : public fdb5::Engine {
 
     virtual std::string dbType() const override;
 
-    virtual eckit::URI location(const Key &key, const Config& config) const override;
-
     virtual bool canHandle(const eckit::URI& path) const override;
 
-    virtual std::vector allLocations(const Key& key, const Config& config) const override;
-
     virtual std::vector visitableLocations(const Config& config) const override;
     virtual std::vector visitableLocations(const metkit::mars::MarsRequest& rq, const Config& config) const override;
 
-//    virtual std::vector writableLocations(const InspectionKey& key, const Config& config) const override;
-
     virtual void print( std::ostream &out ) const override;
 
 };

From eb715c08e860baa56810d447f2c26d1ba6d2899e Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Sun, 29 Oct 2023 09:06:14 +0000
Subject: [PATCH 017/186] random store

---
 src/fdb5/remote/RemoteStore.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index a0a758e29..c5070f867 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -327,7 +327,7 @@ class FDBRemoteDataHandle : public DataHandle {
 //----------------------------------------------------------------------------------------------------------------------
 
 RemoteStore::RemoteStore(const Key& dbKey, const Config& config) :
-    Client(eckit::net::Endpoint("localhost", 7000)),
+    Client(eckit::net::Endpoint(config.getString("storeHost"), config.getInt("storePort"))),
     dbKey_(dbKey), config_(config),
     retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)) {
 }

From 79e22bfd1fa4549c767caeca4d77527ce5e5304e Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 30 Oct 2023 10:14:39 +0000
Subject: [PATCH 018/186] removed remote::Client::key()

---
 src/fdb5/api/RemoteFDB.cc         |  1 -
 src/fdb5/api/RemoteFDB.h          |  2 --
 src/fdb5/remote/RemoteCatalogue.h |  3 +--
 src/fdb5/remote/RemoteStore.h     | 11 +----------
 src/fdb5/remote/client/Client.h   |  2 --
 5 files changed, 2 insertions(+), 17 deletions(-)

diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index 3dfdcb0b1..fb2acf387 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -251,7 +251,6 @@ bool RemoteFDB::handle(remote::Message message, uint32_t requestID, eckit::net::
     }
 }
 void RemoteFDB::handleException(std::exception_ptr e){NOTIMP;}
-const Key& RemoteFDB::key() const {NOTIMP;}
 
 static FDBBuilder builder("remote");
 
diff --git a/src/fdb5/api/RemoteFDB.h b/src/fdb5/api/RemoteFDB.h
index 13a57fb20..4bf8710b3 100644
--- a/src/fdb5/api/RemoteFDB.h
+++ b/src/fdb5/api/RemoteFDB.h
@@ -77,8 +77,6 @@ class RemoteFDB : public LocalFDB, public remote::Client {
     bool handle(remote::Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) override;
     void handleException(std::exception_ptr e) override;
 
-    const Key& key() const override;
-
 private: // members
 
     std::unique_ptr archiver_;
diff --git a/src/fdb5/remote/RemoteCatalogue.h b/src/fdb5/remote/RemoteCatalogue.h
index 255e6134c..196a9c7dd 100644
--- a/src/fdb5/remote/RemoteCatalogue.h
+++ b/src/fdb5/remote/RemoteCatalogue.h
@@ -73,8 +73,7 @@ class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public C
     bool handle(Message message, uint32_t requestID) override;
     bool handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) override;
     void handleException(std::exception_ptr e) override;
-    const Key& key() const override { return CatalogueImpl::key(); }
-
+    
 protected:
 
     Config config_;
diff --git a/src/fdb5/remote/RemoteStore.h b/src/fdb5/remote/RemoteStore.h
index 2cbd65ce1..a2e8bb374 100644
--- a/src/fdb5/remote/RemoteStore.h
+++ b/src/fdb5/remote/RemoteStore.h
@@ -78,22 +78,13 @@ class RemoteStore : public Store, public Client {
 
 private: // methods
 
-    const Key& key() const override { return dbKey_; }
+    void flush(FDBStats& stats);
 
     // handlers for incoming messages - to be defined in the client class
     bool handle(Message message, uint32_t requestID) override;
     bool handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) override;
     void handleException(std::exception_ptr e) override;
 
-    // Listen to the dataClient for incoming messages, and push them onto
-    // appropriate queues.
-//    void listeningThreadLoop() override;
-//    long sendArchiveData(uint32_t id, const std::vector>& elements, size_t count);
-    void flush(FDBStats& stats);
-
-//     void connect();
-//     void disconnect();
-
 private: // members
 
     Key dbKey_;
diff --git a/src/fdb5/remote/client/Client.h b/src/fdb5/remote/client/Client.h
index 0f9821106..a941eeaf0 100644
--- a/src/fdb5/remote/client/Client.h
+++ b/src/fdb5/remote/client/Client.h
@@ -49,8 +49,6 @@ class Client : eckit::NonCopyable {
     virtual bool handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) = 0;
     virtual void handleException(std::exception_ptr e) = 0;
 
-    virtual const Key& key() const = 0;
-
 protected:
 
     eckit::net::Endpoint endpoint_;

From dfb54b41df20668711a2db19d49ddfaea27ae030 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Sun, 5 Nov 2023 08:23:39 +0000
Subject: [PATCH 019/186] addressed PR comments (blocking requests are using
 data channel - improved concurrency)

---
 src/fdb5/CMakeLists.txt                       |   4 +-
 src/fdb5/api/RemoteFDB.cc                     |   6 +-
 src/fdb5/api/RemoteFDB.h                      |   2 +-
 src/fdb5/database/Archiver.h                  |   2 +-
 src/fdb5/remote/Messages.h                    |  24 +-
 src/fdb5/remote/RemoteCatalogue.cc            |  14 +-
 src/fdb5/remote/RemoteCatalogue.h             |   4 +-
 src/fdb5/remote/RemoteFieldLocation.cc        |   2 +-
 src/fdb5/remote/RemoteStore.cc                |  31 +-
 src/fdb5/remote/RemoteStore.h                 |   5 +-
 src/fdb5/remote/client/Client.cc              |  90 +++++-
 src/fdb5/remote/client/Client.h               |  33 ++-
 src/fdb5/remote/client/ClientConnection.cc    | 280 ++++++++++++------
 src/fdb5/remote/client/ClientConnection.h     |  51 ++--
 .../remote/client/ClientConnectionRouter.cc   | 205 +++++++------
 .../remote/client/ClientConnectionRouter.h    |  64 ++--
 src/fdb5/remote/server/CatalogueHandler.cc    |  16 +-
 src/fdb5/remote/server/CatalogueHandler.h     |   6 +-
 src/fdb5/remote/server/ServerConnection.cc    |  99 +++----
 src/fdb5/remote/server/ServerConnection.h     |  29 +-
 src/fdb5/remote/server/StoreHandler.cc        |   8 +-
 21 files changed, 568 insertions(+), 407 deletions(-)

diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt
index 09159aa5b..a9bd1380a 100644
--- a/src/fdb5/CMakeLists.txt
+++ b/src/fdb5/CMakeLists.txt
@@ -210,8 +210,8 @@ list( APPEND fdb5_srcs
 
 if(HAVE_FDB_REMOTE)
     list( APPEND fdb5_srcs 
-        api/ClientFDB.cc
-        api/ClientFDB.h
+        # api/ClientFDB.cc
+        # api/ClientFDB.h
         api/RemoteFDB.cc
         api/RemoteFDB.h
         remote/RemoteStore.cc
diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index fb2acf387..6394423de 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -92,7 +92,7 @@ auto RemoteFDB::forwardApiCall(const HelperClass& helper, const FDBToolRequest&
     // Ensure we have an entry in the message queue before we trigger anything that
     // will result in return messages
 
-    uint32_t id = generateRequestID();
+    uint32_t id = connection_.generateRequestID();
     auto entry = messageQueues_.emplace(id, std::make_shared(HelperClass::queueSize()));
     ASSERT(entry.second);
     std::shared_ptr messageQueue(entry.first->second);
@@ -104,7 +104,7 @@ auto RemoteFDB::forwardApiCall(const HelperClass& helper, const FDBToolRequest&
     s << request;
     helper.encodeExtra(s);
 
-    controlWriteCheckResponse(HelperClass::message(), encodeBuffer, s.position(), id);
+    controlWriteCheckResponse(HelperClass::message(), id, encodeBuffer, s.position());
 
     // Return an AsyncIterator to allow the messages to be retrieved in the API
 
@@ -218,7 +218,7 @@ bool RemoteFDB::handle(remote::Message message, uint32_t requestID) {
             return false;
     }
 }
-bool RemoteFDB::handle(remote::Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) {
+bool RemoteFDB::handle(remote::Message message, uint32_t requestID, eckit::Buffer&& payload) {
 
     switch (message) {
         case fdb5::remote::Message::Blob: {
diff --git a/src/fdb5/api/RemoteFDB.h b/src/fdb5/api/RemoteFDB.h
index 4bf8710b3..92487553c 100644
--- a/src/fdb5/api/RemoteFDB.h
+++ b/src/fdb5/api/RemoteFDB.h
@@ -74,7 +74,7 @@ class RemoteFDB : public LocalFDB, public remote::Client {
     // Client
 
     bool handle(remote::Message message, uint32_t requestID) override;
-    bool handle(remote::Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) override;
+    bool handle(remote::Message message, uint32_t requestID, eckit::Buffer&& payload) override;
     void handleException(std::exception_ptr e) override;
 
 private: // members
diff --git a/src/fdb5/database/Archiver.h b/src/fdb5/database/Archiver.h
index a1415b57e..3ab9ba442 100644
--- a/src/fdb5/database/Archiver.h
+++ b/src/fdb5/database/Archiver.h
@@ -68,7 +68,7 @@ class Archiver : public eckit::NonCopyable {
 
     friend class BaseArchiveVisitor;
 
-    typedef std::map< Key, std::pair, std::unique_ptr > > > store_t;
+    typedef std::map, std::unique_ptr>>> store_t;
 
     Config dbConfig_;
 
diff --git a/src/fdb5/remote/Messages.h b/src/fdb5/remote/Messages.h
index 1565fe7a4..6a48c6cfc 100644
--- a/src/fdb5/remote/Messages.h
+++ b/src/fdb5/remote/Messages.h
@@ -16,8 +16,7 @@
 /// @author Simon Smart
 /// @date   Apr 2018
 
-#ifndef fdb5_remote_Messages_H
-#define fdb5_remote_Messages_H
+#pragma once
 
 #include "eckit/types/FixedString.h"
 #include "eckit/serialisation/Streamable.h"
@@ -28,9 +27,7 @@ namespace eckit {
     class Stream;
 }
 
-
-namespace fdb5 {
-namespace remote {
+namespace fdb5::remote {
 
 //----------------------------------------------------------------------------------------------------------------------
 
@@ -83,15 +80,13 @@ struct MessageHeader {
     MessageHeader() :
         version(CurrentVersion),
         message(Message::None),
-        remoteID(0),
         requestID(0),
         payloadSize(0) {}
 
-    MessageHeader(Message message, uint32_t remoteID, uint32_t requestID, uint32_t payloadSize=0) :
+    MessageHeader(Message message, uint32_t requestID, uint32_t payloadSize=0) :
         marker(StartMarker),
         version(CurrentVersion),
         message(message),
-        remoteID(remoteID),
         requestID(requestID),
         payloadSize(payloadSize) {}
 
@@ -101,19 +96,14 @@ struct MessageHeader {
 
     Message message;                // 2 bytes  --> 8
 
-    uint32_t remoteID;              // 4 bytes  --> 12
-
-    uint32_t requestID;             // 4 bytes  --> 16
+    uint32_t requestID;             // 4 bytes  --> 12
 
-    uint32_t payloadSize;           // 4 bytes  --> 20
+    uint32_t payloadSize;           // 4 bytes  --> 16
 
-    eckit::FixedString<16> hash;    // 16 bytes --> 36
+    eckit::FixedString<16> hash;    // 16 bytes --> 32
 };
 
 
 //----------------------------------------------------------------------------------------------------------------------
 
-} // namespace remote
-} // namespace fdb5
-
-#endif // fdb5_remote_Messages_H
+} // namespace fdb5::remote
diff --git a/src/fdb5/remote/RemoteCatalogue.cc b/src/fdb5/remote/RemoteCatalogue.cc
index 1fe493c06..98011aaec 100644
--- a/src/fdb5/remote/RemoteCatalogue.cc
+++ b/src/fdb5/remote/RemoteCatalogue.cc
@@ -174,7 +174,7 @@ FDBStats RemoteCatalogueArchiver::archiveThreadLoop() {
         // And note that we are done. (don't time this, as already being blocked
         // on by the ::flush() routine)
 
-        element.catalogue_->dataWrite(Message::Flush, nullptr, 0);
+        element.catalogue_->dataWrite(Message::Flush, 0);
 
         archiveQueue_.reset();
 
@@ -223,11 +223,11 @@ void RemoteCatalogue::sendArchiveData(uint32_t id, const Key& key, std::unique_p
     MemoryStream locStream(locBuffer);
     locStream << *fieldLocation;
 
-    MessageHeader message(Message::Blob, 0, id, keyStream.position() + locStream.position());
-    dataWrite(id, &message, sizeof(message));
-    dataWrite(id, keyBuffer, keyStream.position());
-    dataWrite(id, locBuffer, locStream.position());
-    dataWrite(id, &EndMarker, sizeof(EndMarker));
+    std::vector> payloads;
+    payloads.push_back(std::pair{keyBuffer, keyStream.position()});
+    payloads.push_back(std::pair{locBuffer, locStream.position()});
+
+    dataWrite(Message::Blob, id, payloads);
 }
 
 void RemoteCatalogue::archive(const InspectionKey& key, std::unique_ptr fieldLocation) {
@@ -321,7 +321,7 @@ bool RemoteCatalogue::handle(Message message, uint32_t requestID) {
     NOTIMP;
     return false;
 }
-bool RemoteCatalogue::handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) {
+bool RemoteCatalogue::handle(Message message, uint32_t requestID, eckit::Buffer&& payload) {
     eckit::Log::debug() << *this << " - Received [message=" << ((uint) message) << ",requestID=" << requestID << ",payloadSize=" << payload.size() << "]" << std::endl;
     // if (message == Message::Schema) {
     //     eckit::Log::debug() << "RemoteCatalogue::handle received payload size: " << payload.size() << std::endl;
diff --git a/src/fdb5/remote/RemoteCatalogue.h b/src/fdb5/remote/RemoteCatalogue.h
index 196a9c7dd..044eb6632 100644
--- a/src/fdb5/remote/RemoteCatalogue.h
+++ b/src/fdb5/remote/RemoteCatalogue.h
@@ -71,9 +71,9 @@ class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public C
     // From Client
     // handlers for incoming messages - to be defined in the client class
     bool handle(Message message, uint32_t requestID) override;
-    bool handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) override;
+    bool handle(Message message, uint32_t requestID, eckit::Buffer&& payload) override;
     void handleException(std::exception_ptr e) override;
-    
+
 protected:
 
     Config config_;
diff --git a/src/fdb5/remote/RemoteFieldLocation.cc b/src/fdb5/remote/RemoteFieldLocation.cc
index 35cf2979f..4473f6c1c 100644
--- a/src/fdb5/remote/RemoteFieldLocation.cc
+++ b/src/fdb5/remote/RemoteFieldLocation.cc
@@ -69,7 +69,7 @@ std::shared_ptr RemoteFieldLocation::make_shared() const {
 
 eckit::DataHandle* RemoteFieldLocation::dataHandle() const {
     
-    RemoteStore& store = ClientConnectionRouter::instance().store(uri_);
+    RemoteStore& store = RemoteStore::get(uri_);
     
     const std::string scheme = uri_.query("internalScheme");
     const std::string hostport = uri_.query("internalHost");
diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index c5070f867..a00799c7c 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -187,7 +187,8 @@ FDBStats RemoteStoreArchiver::archiveThreadLoop() {
         // And note that we are done. (don't time this, as already being blocked
         // on by the ::flush() routine)
 
-        element.store_->dataWrite(Message::Flush, nullptr, 0);
+
+        element.store_->dataWrite(Message::Flush, 0);
 
         archiveQueue_.reset();
 
@@ -456,7 +457,7 @@ bool RemoteStore::handle(Message message, uint32_t requestID) {
             return false;
     }
 }
-bool RemoteStore::handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) {
+bool RemoteStore::handle(Message message, uint32_t requestID, eckit::Buffer&& payload) {
 
     switch (message) {
     
@@ -466,7 +467,7 @@ bool RemoteStore::handle(Message message, uint32_t requestID, eckit::net::Endpoi
                 MemoryStream s(payload);
                 std::unique_ptr location(eckit::Reanimator::reanimate(s));
                 // std::cout <<  "RemoteStore::handle - " << location->uri().asRawString() << " " << location->length() << std::endl;
-                std::unique_ptr remoteLocation = std::unique_ptr(new RemoteFieldLocation(endpoint, *location));
+                std::unique_ptr remoteLocation = std::unique_ptr(new RemoteFieldLocation(endpoint_, *location));
                 it->second.set_value(std::move(remoteLocation));
             }
             return true;
@@ -487,7 +488,7 @@ bool RemoteStore::handle(Message message, uint32_t requestID, eckit::net::Endpoi
                 std::string msg;
                 msg.resize(payload.size(), ' ');
                 payload.copy(&msg[0], payload.size());
-                it->second->interrupt(std::make_exception_ptr(RemoteFDBException(msg, endpoint)));
+                it->second->interrupt(std::make_exception_ptr(RemoteFDBException(msg, endpoint_)));
 
                 // Remove entry (shared_ptr --> message queue will be destroyed when it
                 // goes out of scope in the worker thread).
@@ -496,7 +497,7 @@ bool RemoteStore::handle(Message message, uint32_t requestID, eckit::net::Endpoi
             } else {
                 auto it = locations_.find(requestID);
                 if (it != locations_.end()) {
-                    archiver_->error(std::move(payload), endpoint);
+                    archiver_->error(std::move(payload), endpoint_);
                 } else {
                     retrieveMessageQueue_.emplace(message, std::move(payload));
                 }
@@ -531,11 +532,11 @@ void RemoteStore::sendArchiveData(uint32_t id, const Key& key, const void* data,
     keyStream << dbKey_;
     keyStream << key;
 
-    MessageHeader message(Message::Blob, 0, id, length + keyStream.position());
-    dataWrite(id, &message, sizeof(message));
-    dataWrite(id, keyBuffer, keyStream.position());
-    dataWrite(id, data, length);
-    dataWrite(id, &EndMarker, sizeof(EndMarker));
+    std::vector> payloads;
+    payloads.push_back(std::pair{keyBuffer, keyStream.position()});
+    payloads.push_back(std::pair{data, length});
+
+    dataWrite(Message::Blob, id, payloads);
 }
 
 eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation) {
@@ -554,8 +555,18 @@ eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation, c
     return new FDBRemoteDataHandle(id, retrieveMessageQueue_, endpoint_);
 }
 
+RemoteStore& RemoteStore::get(const eckit::URI& uri) {
+    // we memoise one read store for each endpoint. Do not need to have one for each key
+    static std::map > readStores_;
 
+    const std::string& endpoint = uri.hostport();
+    auto it = readStores_.find(endpoint);
+    if (it != readStores_.end()) {
+        return *(it->second);
+    }
 
+    return *(readStores_[endpoint] = std::unique_ptr(new RemoteStore(uri, Config())));
+}
 
 static StoreBuilder builder("remote");
 
diff --git a/src/fdb5/remote/RemoteStore.h b/src/fdb5/remote/RemoteStore.h
index a2e8bb374..3808412cc 100644
--- a/src/fdb5/remote/RemoteStore.h
+++ b/src/fdb5/remote/RemoteStore.h
@@ -20,7 +20,6 @@
 #include "fdb5/database/Store.h"
 #include "fdb5/remote/client/Client.h"
 
-
 namespace fdb5::remote {
 
 class RemoteStoreArchiver;
@@ -44,6 +43,8 @@ class RemoteStore : public Store, public Client {
 
     ~RemoteStore() override;
 
+    static RemoteStore& get(const eckit::URI& uri);
+
     eckit::URI uri() const override;
 
     bool open() override;
@@ -82,7 +83,7 @@ class RemoteStore : public Store, public Client {
 
     // handlers for incoming messages - to be defined in the client class
     bool handle(Message message, uint32_t requestID) override;
-    bool handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) override;
+    bool handle(Message message, uint32_t requestID, eckit::Buffer&& payload) override;
     void handleException(std::exception_ptr e) override;
 
 private: // members
diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index a44f71d2a..82ee2d300 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -8,6 +8,8 @@
  * does it submit to any jurisdiction.
  */
 
+#include 
+
 #include "fdb5/remote/client/Client.h"
 #include "fdb5/remote/client/ClientConnectionRouter.h"
 
@@ -16,27 +18,89 @@ namespace fdb5::remote {
 //----------------------------------------------------------------------------------------------------------------------
 
 Client::Client(const eckit::net::Endpoint& endpoint) :
-    endpoint_(endpoint) {}
+    endpoint_(endpoint),
+    connection_(*(ClientConnectionRouter::instance().connection(*this))),
+    blockingRequestId_(0) {}
+
+// uint32_t Client::generateRequestID() {
+//     return connection_.generateRequestID();
+// }
+
 
-uint32_t Client::generateRequestID() {
-    return ClientConnectionRouter::instance().generateRequestID();
+bool Client::response(uint32_t requestID) {
+    ASSERT(requestID == blockingRequestId_);
+
+    eckit::Buffer buf;
+    payload_.set_value(std::move(buf));
+    return true;
 }
 
-uint32_t Client::controlWriteCheckResponse(Message msg, const void* payload, uint32_t payloadLength, uint32_t requestID) {
-    return ClientConnectionRouter::instance().controlWriteCheckResponse(*this, msg, requestID, payload, payloadLength);
+bool Client::response(uint32_t requestID, eckit::Buffer&& payload) {
+    ASSERT(requestID == blockingRequestId_);
+
+    payload_.set_value(std::move(payload));
+    return true;
 }
-eckit::Buffer Client::controlWriteReadResponse(Message msg, const void* payload, uint32_t payloadLength) {
-    return ClientConnectionRouter::instance().controlWriteReadResponse(*this, msg, payload, payloadLength);
+
+uint32_t Client::controlWriteCheckResponse(Message msg, const void* payload, uint32_t payloadLength) {
+    uint32_t id = connection_.generateRequestID();
+    controlWriteCheckResponse(msg, id, payload, payloadLength);
+    return id;
 }
-uint32_t Client::controlWrite(Message msg, const void* payload, uint32_t payloadLength) {
-    return ClientConnectionRouter::instance().controlWrite(*this, msg, payload, payloadLength);
+
+void Client::controlWriteCheckResponse(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
+    ASSERT(!blockingRequestId_);
+    ASSERT(requestID);
+
+    payload_ = {};
+    blockingRequestId_=requestID;
+    std::future f = payload_.get_future();
+
+    if (payloadLength) {
+        connection_.controlWrite(*this, msg, blockingRequestId_, std::vector>{{payload, payloadLength}});
+    } else {
+        connection_.controlWrite(*this, msg, blockingRequestId_);
+    }
+
+    eckit::Buffer buf = f.get();
+    blockingRequestId_=0;
+    ASSERT(buf.size() == 0);
 }
+eckit::Buffer Client::controlWriteReadResponse(Message msg, const void* payload, uint32_t payloadLength) {
+    ASSERT(!blockingRequestId_);
+
+    payload_ = {};
+    blockingRequestId_=connection_.generateRequestID();
+    std::future f = payload_.get_future();
 
-uint32_t Client::dataWrite(Message msg, const void* payload, uint32_t payloadLength) {
-    return ClientConnectionRouter::instance().dataWrite(*this, msg, payload, payloadLength);
+    if (payloadLength) {
+        connection_.controlWrite(*this, msg, blockingRequestId_, std::vector>{{payload, payloadLength}});
+    } else {
+        connection_.controlWrite(*this, msg, blockingRequestId_);
+    }
+
+    eckit::Buffer buf = f.get();
+    blockingRequestId_=0;
+    return buf;
 }
-void Client::dataWrite(uint32_t requestId, const void* data, size_t length){
-    ClientConnectionRouter::instance().dataWrite(*this, requestId, data, length);
+
+void Client::dataWrite(remote::Message msg, uint32_t requestID, std::vector> data) {
+    connection_.dataWrite(msg, requestID, data);
 }
 
+// uint32_t Client::controlWrite(Message msg, const void* payload, uint32_t payloadLength) {
+//     uint32_t id = connection_.generateRequestID();
+//     connection_.controlWrite(*this, msg, id, payload, payloadLength);
+//     return id;
+// }
+
+// uint32_t Client::dataWrite(Message msg, const void* payload, uint32_t payloadLength) {
+//     uint32_t id = connection_.generateRequestID();
+//     return connection_.dataWrite(msg, id, payload, payloadLength);
+//     return id;
+// }
+// void Client::dataWrite(const void* data, size_t length){
+//     connection_.dataWrite(data, length);
+// }
+
 }
\ No newline at end of file
diff --git a/src/fdb5/remote/client/Client.h b/src/fdb5/remote/client/Client.h
index a941eeaf0..fe2e47b66 100644
--- a/src/fdb5/remote/client/Client.h
+++ b/src/fdb5/remote/client/Client.h
@@ -10,12 +10,15 @@
 
 #pragma once
 
+#include 
+
 #include "eckit/config/Configuration.h"
 #include "eckit/memory/NonCopyable.h"
 #include "eckit/net/Endpoint.h"
 
 #include "fdb5/database/Key.h"
 #include "fdb5/remote/Messages.h"
+#include "fdb5/remote/client/ClientConnection.h"
 
 namespace fdb5::remote {
 
@@ -33,26 +36,38 @@ class Client : eckit::NonCopyable {
 public:
     Client(const eckit::net::Endpoint& endpoint);
 
-    const eckit::net::Endpoint& controlEndpoint() { return endpoint_; }
+    const eckit::net::Endpoint& controlEndpoint() const { return endpoint_; }
 
-    static uint32_t generateRequestID();
+    // uint32_t generateRequestID();
 
-    uint32_t controlWriteCheckResponse(Message msg, const void* payload=nullptr, uint32_t payloadLength=0, uint32_t requestID = generateRequestID());
+    uint32_t controlWriteCheckResponse(Message msg,                     const void* payload=nullptr, uint32_t payloadLength=0);
+    void     controlWriteCheckResponse(Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
     eckit::Buffer controlWriteReadResponse(Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
-    uint32_t controlWrite(Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
-
-    uint32_t dataWrite(Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
-    void dataWrite(uint32_t requestId, const void* data, size_t length);
+//    uint32_t controlWrite(Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
 
+    void dataWrite(remote::Message msg, uint32_t requestID, std::vector> data={});
+    
     // handlers for incoming messages - to be defined in the client class
     virtual bool handle(Message message, uint32_t requestID) = 0;
-    virtual bool handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) = 0;
+    virtual bool handle(Message message, uint32_t requestID, /*eckit::net::Endpoint endpoint,*/ eckit::Buffer&& payload) = 0;
     virtual void handleException(std::exception_ptr e) = 0;
 
-protected:
+    bool response(uint32_t requestID);
+    bool response(uint32_t requestID, eckit::Buffer&& payload);
+    uint32_t blockingRequestId() { return blockingRequestId_; }
 
+protected:
+    
     eckit::net::Endpoint endpoint_;
 
+    ClientConnection& connection_;
+
+
+private:
+
+    uint32_t blockingRequestId_;
+    std::promise payload_;
+
 };
 
 }
\ No newline at end of file
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index a4219e2b0..71f45e287 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -3,32 +3,24 @@
 #include 
 #include 
 
-#include "fdb5/remote/client/ClientConnection.h"
-#include "fdb5/remote/client/ClientConnectionRouter.h"
-
-#include "fdb5/LibFdb5.h"
-#include "fdb5/io/HandleGatherer.h"
-#include "fdb5/remote/Messages.h"
-#include "fdb5/remote/RemoteFieldLocation.h"
-#include "fdb5/api/helpers/FDBToolRequest.h"
-#include "fdb5/database/Key.h"
-
 #include "eckit/config/LocalConfiguration.h"
+#include "eckit/config/Resource.h"
 #include "eckit/io/Buffer.h"
 #include "eckit/log/Bytes.h"
 #include "eckit/log/Log.h"
 #include "eckit/message/Message.h"
-#include "eckit/config/Resource.h"
+#include "eckit/os/BackTrace.h"
+#include "eckit/runtime/Main.h"
 #include "eckit/serialisation/MemoryStream.h"
 #include "eckit/utils/Translator.h"
-#include "eckit/runtime/Main.h"
-#include "eckit/os/BackTrace.h"
 
-#include "metkit/mars/MarsRequest.h"
+#include "fdb5/LibFdb5.h"
+#include "fdb5/remote/Messages.h"
+#include "fdb5/remote/RemoteFieldLocation.h"
+#include "fdb5/remote/client/ClientConnection.h"
+#include "fdb5/remote/client/ClientConnectionRouter.h"
 
-using namespace eckit;
-using namespace eckit::net;
-// using namespace fdb5::remote;
+// using namespace eckit;
 
 namespace fdb5::remote {
 
@@ -48,20 +40,20 @@ ConnectionError::ConnectionError(const int retries) {
     std::ostringstream s;
     s << "Unable to create a connection with the FDB server after " << retries << " retries";
     reason(s.str());
-    Log::status() << what() << std::endl;
+    eckit::Log::status() << what() << std::endl;
 }
 
 ConnectionError::ConnectionError(const int retries, const eckit::net::Endpoint& endpoint) {
     std::ostringstream s;
     s << "Unable to create a connection with the FDB endpoint " << endpoint << " after " << retries << " retries";
     reason(s.str());
-    Log::status() << what() << std::endl;
+    eckit::Log::status() << what() << std::endl;
 }
 
-class TCPException : public Exception {
+class TCPException : public eckit::Exception {
 public:
-    TCPException(const std::string& msg, const CodeLocation& here) :
-        Exception(std::string("TCPException: ") + msg, here) {
+    TCPException(const std::string& msg, const eckit::CodeLocation& here) :
+        eckit::Exception(std::string("TCPException: ") + msg, here) {
 
         eckit::Log::error() << "TCP Exception; backtrace(): " << std::endl;
         eckit::Log::error() << eckit::BackTrace::dump() << std::endl;
@@ -73,10 +65,10 @@ class TCPException : public Exception {
 //----------------------------------------------------------------------------------------------------------------------
 
 
-ClientConnection::ClientConnection(const eckit::net::Endpoint& controlEndpoint, const eckit::Configuration& config):
-    controlEndpoint_(controlEndpoint),
+ClientConnection::ClientConnection(const eckit::net::Endpoint& controlEndpoint):
+    controlEndpoint_(controlEndpoint), id_(0),
     connected_(false) {
-        Log::debug() << "ClientConnection::ClientConnection() controlEndpoint: " << controlEndpoint_ << std::endl;
+        eckit::Log::debug() << "ClientConnection::ClientConnection() controlEndpoint: " << controlEndpoint_ << std::endl;
     }
 
 
@@ -84,10 +76,16 @@ ClientConnection::~ClientConnection() {
     disconnect();
 }
 
+uint32_t ClientConnection::generateRequestID() {
+
+    std::lock_guard lock(idMutex_);
+    return ++id_;
+}
+
 void ClientConnection::connect() {
 
     if (connected_) {
-        Log::warning() << "ClientConnection::connect() called when already connected" << std::endl;
+        eckit::Log::warning() << "ClientConnection::connect() called when already connected" << std::endl;
         return;
     }
 
@@ -96,20 +94,20 @@ void ClientConnection::connect() {
     try {
         // Connect to server, and check that the server is happy on the response
 
-        Log::debug() << "Connecting to host: " << controlEndpoint_ << std::endl;
+        eckit::Log::debug() << "Connecting to host: " << controlEndpoint_ << std::endl;
         controlClient_.connect(controlEndpoint_, fdbMaxConnectRetries);
         writeControlStartupMessage();
-        SessionID serverSession = verifyServerStartupResponse();
+        eckit::SessionID serverSession = verifyServerStartupResponse();
 
         // Connect to the specified data port
-        Log::debug() << "Received data endpoint from host: " << dataEndpoint_ << std::endl;
+        eckit::Log::debug() << "Received data endpoint from host: " << dataEndpoint_ << std::endl;
         dataClient_.connect(dataEndpoint_, fdbMaxConnectRetries);
         writeDataStartupMessage(serverSession);
 
         // And the connections are set up. Let everything start up!
         listeningThread_ = std::thread([this] { listeningThreadLoop(); });
         connected_ = true;
-    } catch(TooManyRetries& e) {
+    } catch(eckit::TooManyRetries& e) {
         if (controlClient_.isConnected()) {
             controlClient_.close();
             throw ConnectionError(fdbMaxConnectRetries, dataEndpoint_);
@@ -119,13 +117,20 @@ void ClientConnection::connect() {
     }
 }
 
+// void ClientConnection::controlWrite(remote::Message msg) {
+//     MessageHeader message(msg, 0, 0);
+
+//     std::lock_guard lock(controlMutex_);
+//     controlWrite(&message, sizeof(message));
+//     controlWrite(&EndMarker, sizeof(EndMarker));
+// }
 
 
 void ClientConnection::disconnect() {
     if (connected_) {
 
         // Send termination message
-        controlWrite(Message::Exit, 0);
+        controlWrite(Message::Exit);
 
         listeningThread_.join();
 
@@ -153,65 +158,109 @@ eckit::LocalConfiguration ClientConnection::availableFunctionality() const {
 
 // -----------------------------------------------------------------------------------------------------
 
-void ClientConnection::controlWriteCheckResponse(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
-//    std::cout << "ClientConnection::controlWriteCheckResponse " << ((uint) msg) << "  " << requestID << "  " << payloadLength << std::endl;
+void ClientConnection::addRequest(Client& client, uint32_t requestId) {
+    ASSERT(requestId);
+    requests_[requestId] = &client;
 
-    controlWrite(msg, requestID, payload, payloadLength);
+//     // pick the first endpoint - To be improved with DataStoreStrategies
+//     const eckit::net::Endpoint& endpoint = client.controlEndpoint();
 
-    // Wait for the receipt acknowledgement
+//     auto it = connections_.find(endpoint.hostport());
+//     if (it != connections_.end()) {
+//         conn = it->second.get();
+//     } else {
+//         conn = (connections_[endpoint.hostport()] = std::unique_ptr(new ClientConnection(endpoint, Config()))).get();
+//         conn->connect();
+//     }
 
-    MessageHeader response;
-    controlRead(&response, sizeof(MessageHeader));
+//     ASSERT(conn);
+//     if (add) {
+//         requests_[id] = {conn, &client};
+//     }
 
-    handleError(response);
+//     return id;
+}
 
-    ASSERT(response.marker == StartMarker);
-    ASSERT(response.version == CurrentVersion);
-    ASSERT(response.message == Message::Received);
+// void ClientConnection::controlWriteCheckResponse(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
+// //    std::cout << "ClientConnection::controlWriteCheckResponse " << ((uint) msg) << "  " << requestID << "  " << payloadLength << std::endl;
 
-    eckit::FixedString<4> tail;
-    controlRead(&tail, sizeof(tail));
-    ASSERT(tail == EndMarker);
-}
+//     controlWrite(msg, requestID, payload, payloadLength);
+
+//     // Wait for the receipt acknowledgement
+
+//     MessageHeader response;
+//     controlRead(&response, sizeof(MessageHeader));
+
+//     handleError(response);
 
-eckit::Buffer ClientConnection::controlWriteReadResponse(remote::Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
+//     ASSERT(response.marker == StartMarker);
+//     ASSERT(response.version == CurrentVersion);
+//     ASSERT(response.message == Message::Received);
+
+//     eckit::FixedString<4> tail;
+//     controlRead(&tail, sizeof(tail));
+//     ASSERT(tail == EndMarker);
+// }
+
+// eckit::Buffer ClientConnection::controlWriteReadResponse(remote::Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
     
-    controlWrite(msg, requestID, payload, payloadLength);
+//     controlWrite(msg, requestID, payload, payloadLength);
 
-    // Wait for the receipt acknowledgement
+//     // Wait for the receipt acknowledgement
 
-    MessageHeader response;
-    controlRead(&response, sizeof(MessageHeader));
+//     MessageHeader response;
+//     controlRead(&response, sizeof(MessageHeader));
 
-    handleError(response);
+//     handleError(response);
 
-    // std::cout << "ClientConnection::controlWriteReadResponse received " << response.requestID << " | expecting " << requestID << std::endl;
-    ASSERT(response.marker == StartMarker);
-    ASSERT(response.version == CurrentVersion);
-    ASSERT(response.message == Message::Received);
-    ASSERT(response.requestID == requestID);
+//     // std::cout << "ClientConnection::controlWriteReadResponse received " << response.requestID << " | expecting " << requestID << std::endl;
+//     ASSERT(response.marker == StartMarker);
+//     ASSERT(response.version == CurrentVersion);
+//     ASSERT(response.message == Message::Received);
+//     ASSERT(response.requestID == requestID);
     
-    eckit::Buffer buf{response.payloadSize};
-    controlRead(buf.data(), buf.size());
+//     eckit::Buffer buf{response.payloadSize};
+//     controlRead(buf.data(), buf.size());
 
-    eckit::FixedString<4> tail;
-    controlRead(&tail, sizeof(tail));
-    ASSERT(tail == EndMarker);
+//     eckit::FixedString<4> tail;
+//     controlRead(&tail, sizeof(tail));
+//     ASSERT(tail == EndMarker);
+
+//     return buf;
+// }
+
+void ClientConnection::controlWrite(Client& client, Message msg, uint32_t requestID, std::vector> data) {
+
+    if (requestID) {
+        addRequest(client, requestID);
+    }
 
-    return buf;
+    controlWrite(msg, requestID, data);
 }
 
-void ClientConnection::controlWrite(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
-    //std::cout << "ClientConnection::controlWrite " << this << std::endl;
+void ClientConnection::controlWrite(Message msg, uint32_t requestID, std::vector> data) {
 
-    ASSERT((payload == nullptr) == (payloadLength == 0));
+    uint32_t payloadLength = 0;
+    for (auto d: data) {
+        payloadLength += d.second;
+    }
+    eckit::Log::debug() << "ClientConnection::controlWrite [message=" << ((int) msg) << ",requestID=" << requestID << ",data=" << data.size() << ",payload=" << payloadLength << "]" << std::endl;
+
+    MessageHeader message(msg, requestID, payloadLength);
 
-    MessageHeader message(msg, 0, requestID, payloadLength);
+    std::lock_guard lock(controlMutex_);
+    eckit::Log::debug() << "ClientConnection::controlWrite 1 - " << sizeof(message) << std::endl;
     controlWrite(&message, sizeof(message));
-    if (payload) {
-        controlWrite(payload, payloadLength);
+    for (auto d: data) {
+        eckit::Log::debug() << "ClientConnection::controlWrite 2 - " << d.second << std::endl;
+        ASSERT(d.first);
+        controlWrite(d.first, d.second);
     }
+    eckit::Log::debug() << "ClientConnection::controlWrite 3 - " << sizeof(EndMarker) << std::endl;
     controlWrite(&EndMarker, sizeof(EndMarker));
+
+    eckit::Log::debug() << "ClientConnection::controlWrite - completed WRITE" << std::endl;
+
 }
 
 void ClientConnection::controlWrite(const void* data, size_t length) {
@@ -233,17 +282,35 @@ void ClientConnection::controlRead(void* data, size_t length) {
     }
 }
 
-void ClientConnection::dataWrite(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
+void ClientConnection::dataWrite(remote::Message msg, uint32_t requestID, std::vector> data) {
+    uint32_t payloadLength = 0;
+    for (auto d: data) {
+        payloadLength += d.second;
+    }
+    MessageHeader message(msg, requestID, payloadLength);
+
+    eckit::Log::debug() << "DATA DATA ClientConnection::dataWrite [message=" << ((int) msg) << ",requestID=" << requestID << ",data=" << data.size() << ",payload=" << payloadLength << "]" << std::endl;
 
-    ASSERT((payload == nullptr) == (payloadLength == 0));
-    MessageHeader message(msg, 0, requestID, payloadLength);
+    std::lock_guard lock(dataMutex_);
     dataWrite(&message, sizeof(message));
-    if (payload) {
-        dataWrite(payload, payloadLength);
+    for (auto d: data) {
+        dataWrite(d.first, d.second);
+        payloadLength += d.second;
     }
     dataWrite(&EndMarker, sizeof(EndMarker));
 }
 
+// void ClientConnection::dataWrite(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
+
+//     ASSERT((payload == nullptr) == (payloadLength == 0));
+//     MessageHeader message(msg, requestID, payloadLength);
+//     dataWrite(&message, sizeof(message));
+//     if (payload) {
+//         dataWrite(payload, payloadLength);
+//     }
+//     dataWrite(&EndMarker, sizeof(EndMarker));
+// }
+
 void ClientConnection::dataWrite(const void* data, size_t length) {
     size_t written = dataClient_.write(data, length);
     if (length != written) {
@@ -287,8 +354,8 @@ void ClientConnection::handleError(const MessageHeader& hdr) {
 
 void ClientConnection::writeControlStartupMessage() {
 
-    Buffer payload(4096);
-    MemoryStream s(payload);
+    eckit::Buffer payload(4096);
+    eckit::MemoryStream s(payload);
     s << sessionID_;
     s << controlEndpoint_;
     s << LibFdb5::instance().remoteProtocolVersion().used();
@@ -299,21 +366,21 @@ void ClientConnection::writeControlStartupMessage() {
     s << availableFunctionality().get();
 
     //std::cout << "writeControlStartupMessage" << std::endl;
-    controlWrite(Message::Startup, 0, payload.data(), s.position());
+    controlWrite(Message::Startup, 0, std::vector>{{payload, s.position()}});
 }
 
 void ClientConnection::writeDataStartupMessage(const eckit::SessionID& serverSession) {
 
-    Buffer payload(1024);
-    MemoryStream s(payload);
+    eckit::Buffer payload(1024);
+    eckit::MemoryStream s(payload);
 
     s << sessionID_;
     s << serverSession;
 
-    dataWrite(Message::Startup, 0, payload.data(), s.position());
+    dataWrite(Message::Startup, 0, std::vector>{{payload, s.position()}});
 }
 
-SessionID ClientConnection::verifyServerStartupResponse() {
+eckit::SessionID ClientConnection::verifyServerStartupResponse() {
 
     MessageHeader hdr;
     controlRead(&hdr, sizeof(hdr));
@@ -323,22 +390,22 @@ SessionID ClientConnection::verifyServerStartupResponse() {
     ASSERT(hdr.message == Message::Startup);
     ASSERT(hdr.requestID == 0);
 
-    Buffer payload(hdr.payloadSize);
+    eckit::Buffer payload(hdr.payloadSize);
     eckit::FixedString<4> tail;
     controlRead(payload, hdr.payloadSize);
     controlRead(&tail, sizeof(tail));
     ASSERT(tail == EndMarker);
 
-    MemoryStream s(payload);
-    SessionID clientSession(s);
-    SessionID serverSession(s);
-    Endpoint dataEndpoint(s);
-    LocalConfiguration serverFunctionality(s);
+    eckit::MemoryStream s(payload);
+    eckit::SessionID clientSession(s);
+    eckit::SessionID serverSession(s);
+    eckit::net::Endpoint dataEndpoint(s);
+    eckit::LocalConfiguration serverFunctionality(s);
 
     dataEndpoint_ = dataEndpoint;
 
     if (dataEndpoint_.hostname() != controlEndpoint_.hostname()) {
-        Log::warning() << "Data and control interface hostnames do not match. "
+        eckit::Log::warning() << "Data and control interface hostnames do not match. "
                        << dataEndpoint_.hostname() << " /= "
                        << controlEndpoint_.hostname() << std::endl;
     }
@@ -347,7 +414,7 @@ SessionID ClientConnection::verifyServerStartupResponse() {
         std::stringstream ss;
         ss << "Session ID does not match session received from server: "
            << sessionID_ << " != " << clientSession;
-        throw BadValue(ss.str(), Here());
+        throw eckit::BadValue(ss.str(), Here());
     }
 
     return serverSession;
@@ -364,6 +431,8 @@ void ClientConnection::listeningThreadLoop() {
 
             dataRead(&hdr, sizeof(hdr));
 
+            eckit::Log::debug() << "ClientConnection::listeningThreadLoop - got message " << ((int) hdr.message) << " with request ID: " << hdr.requestID << std::endl;
+
             ASSERT(hdr.marker == StartMarker);
             ASSERT(hdr.version == CurrentVersion);
 
@@ -372,23 +441,42 @@ void ClientConnection::listeningThreadLoop() {
             }
 
             bool handled = false;
+            auto it = requests_.find(hdr.requestID);
+            if (it == requests_.end()) {
+                std::stringstream ss;
+                ss << "ERROR: Unexpected answer to requestID recieved (" << hdr.requestID << "). ABORTING";
+                eckit::Log::status() << ss.str() << std::endl;
+                eckit::Log::error() << "Retrieving... " << ss.str() << std::endl;
+                throw eckit::SeriousBug(ss.str(), Here());
+
+                ASSERT(false); // todo report the error
+            }
 
             if (hdr.payloadSize == 0) {
-                handled = ClientConnectionRouter::instance().handle(hdr.message, hdr.requestID);
+                if (it->second->blockingRequestId() == hdr.requestID) {
+                    ASSERT(hdr.message == Message::Received);
+                    handled = it->second->response(hdr.requestID);
+                } else {
+                    handled = it->second->handle(hdr.message, hdr.requestID);
+                }
             }
             else {
                 eckit::Buffer payload{hdr.payloadSize};
-                dataRead(payload.data(), hdr.payloadSize);
+                dataRead(payload, hdr.payloadSize);
 
-                handled = ClientConnectionRouter::instance().handle(hdr.message, hdr.requestID, controlEndpoint_, std::move(payload));
+                if (it->second->blockingRequestId() == hdr.requestID) {
+                    handled = it->second->response(hdr.requestID, std::move(payload));
+                } else {
+                    handled = it->second->handle(hdr.message, hdr.requestID, std::move(payload));
+                }
             }
 
             if (!handled) {
                 std::stringstream ss;
                 ss << "ERROR: Unexpected message recieved (" << static_cast(hdr.message) << "). ABORTING";
-                Log::status() << ss.str() << std::endl;
-                Log::error() << "Retrieving... " << ss.str() << std::endl;
-                throw SeriousBug(ss.str(), Here());
+                eckit::Log::status() << ss.str() << std::endl;
+                eckit::Log::error() << "Client Retrieving... " << ss.str() << std::endl;
+                throw eckit::SeriousBug(ss.str(), Here());
             }
 
             // Ensure we have consumed exactly the correct amount from the socket.
@@ -398,9 +486,9 @@ void ClientConnection::listeningThreadLoop() {
 
     // We don't want to let exceptions escape inside a worker thread.
     } catch (const std::exception& e) {
-        ClientConnectionRouter::instance().handleException(std::make_exception_ptr(e));
+//        ClientConnectionRouter::instance().handleException(std::make_exception_ptr(e));
     } catch (...) {
-        ClientConnectionRouter::instance().handleException(std::current_exception());
+//        ClientConnectionRouter::instance().handleException(std::current_exception());
     }
 }
 
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index 0f0df2ff7..22fe327c5 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -22,25 +22,17 @@
 #include "eckit/runtime/SessionID.h"
 
 #include "fdb5/remote/Messages.h"
-#include "eckit/utils/Translator.h"
 
 namespace eckit {
 
 class Buffer;
 
-// // xxx can we give this code a better home?
-// template<> struct Translator {
-//     std::string operator()(const net::Endpoint& e) {
-//         std::stringstream ss;
-//         ss << e;
-//         return ss.str();
-//     }
-// };
-
 }
 
 namespace fdb5::remote {
 
+class Client;
+
 //----------------------------------------------------------------------------------------------------------------------
 
 class ClientConnection : eckit::NonCopyable {
@@ -49,28 +41,24 @@ class ClientConnection : eckit::NonCopyable {
 
 public: // methods
 
-    ClientConnection(const eckit::net::Endpoint& controlEndpoint, const eckit::Configuration& config);
+    ClientConnection(const eckit::net::Endpoint& controlEndpoint);
     virtual ~ClientConnection();
 
-    void controlWriteCheckResponse(remote::Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
-    eckit::Buffer controlWriteReadResponse(remote::Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
-    void controlWrite(remote::Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
-    void dataWrite(remote::Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
-
-    void controlRead(void* data, size_t length);
-    void controlWrite(const void* data, size_t length);
-
-    void dataWrite(const void* data, size_t length);
-    void dataRead(void* data, size_t length);
+    // void controlWriteCheckResponse(remote::Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
+    // eckit::Buffer controlWriteReadResponse(remote::Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
+    void controlWrite(Client& client, remote::Message msg, uint32_t requestID, std::vector> data={});
+    void dataWrite(remote::Message msg, uint32_t requestID, std::vector> data={});
 
     void connect();
     void disconnect();
 
+    uint32_t generateRequestID();
+
 protected: // methods
 
     const eckit::net::Endpoint& controlEndpoint() const;
     const eckit::net::Endpoint& dataEndpoint() const;
-
+    
     // // handlers for incoming messages - to be defined in the client class
     // virtual void handle(Message message, uint32_t requestID) = 0;
     // virtual void handle(Message message, uint32_t requestID, eckit::Buffer&& payload) = 0;
@@ -81,12 +69,20 @@ class ClientConnection : eckit::NonCopyable {
 
 private: // methods
 
+    void controlWrite(Message msg, uint32_t requestID = 0, std::vector> data = {});
+    void controlWrite(const void* data, size_t length);
+    void controlRead (      void* data, size_t length);
+    void dataWrite   (const void* data, size_t length);
+    void dataRead    (      void* data, size_t length);
+ 
+    void addRequest(Client& client, uint32_t requestId);
+
     void writeControlStartupMessage();
     void writeDataStartupMessage(const eckit::SessionID& serverSession);
 
     eckit::SessionID verifyServerStartupResponse();
 
-    void handleError(const remote::MessageHeader& hdr);
+    void handleError(const MessageHeader& hdr);
 
     void listeningThreadLoop();
 
@@ -100,7 +96,16 @@ class ClientConnection : eckit::NonCopyable {
     eckit::net::TCPClient controlClient_;
     eckit::net::TCPClient dataClient_;
 
+    std::map requests_;
+
     std::thread listeningThread_;
+    
+    std::mutex controlMutex_;
+    std::mutex dataMutex_;
+
+    // requestId
+    std::mutex idMutex_;
+    uint32_t id_;
 
     bool connected_;
     
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.cc b/src/fdb5/remote/client/ClientConnectionRouter.cc
index b0b54ed7a..11af270ac 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.cc
+++ b/src/fdb5/remote/client/ClientConnectionRouter.cc
@@ -1,5 +1,5 @@
-#include 
-#include 
+// #include 
+// #include 
 
 #include "fdb5/remote/client/ClientConnectionRouter.h"
 
@@ -10,129 +10,154 @@ namespace fdb5::remote {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-uint32_t ClientConnectionRouter::generateRequestID() {
+// uint32_t ClientConnectionRouter::generateRequestID() {
 
-    static std::mutex m;
-    static uint32_t id = 0;
+//     static std::mutex m;
+//     static uint32_t id = 0;
 
-    std::lock_guard lock(m);
-    return ++id;
-}
-
-RemoteStore& ClientConnectionRouter::store(const eckit::URI& uri) {
-    const std::string& endpoint = uri.hostport();
-    auto it = readStores_.find(endpoint);
-    if (it != readStores_.end()) {
-        return *(it->second);
-    }
-
-    return *(readStores_[endpoint] = std::unique_ptr(new RemoteStore(uri, Config())));
-}
+//     std::lock_guard lock(m);
+//     return ++id;
+// }
 
-uint32_t ClientConnectionRouter::createConnection(Client& client, ClientConnection*& conn, bool add, uint32_t id) {
+ClientConnection* ClientConnectionRouter::connection(Client& client) {
 
     // pick the first endpoint - To be improved with DataStoreStrategies
     const eckit::net::Endpoint& endpoint = client.controlEndpoint();
 
+    std::lock_guard lock(connectionMutex_);
+
     auto it = connections_.find(endpoint.hostport());
     if (it != connections_.end()) {
-        conn = it->second.get();
+        it->second.clients_.insert(&client);
+        return it->second.connection_.get();
     } else {
-        conn = (connections_[endpoint.hostport()] = std::unique_ptr(new ClientConnection(endpoint, Config()))).get();
-        conn->connect();
+        auto it = (connections_.emplace(endpoint.hostport(), Connection(std::unique_ptr(new ClientConnection{endpoint}), client))).first;
+        it->second.connection_->connect();
+        return it->second.connection_.get();
     }
+}
 
-    ASSERT(conn);
-    if (add) {
-        requests_[id] = {conn, &client};
-    }
+// uint32_t ClientConnectionRouter::createConnection(Client& client, ClientConnection*& conn, bool add, uint32_t id) {
 
-    return id;
-}
+//     // pick the first endpoint - To be improved with DataStoreStrategies
+//     const eckit::net::Endpoint& endpoint = client.controlEndpoint();
 
-uint32_t ClientConnectionRouter::controlWriteCheckResponse(Client& client, Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
-    //std::cout << "ClientConnectionRouter::controlWriteCheckResponse " << endpoints.size() << std::endl;
-    ClientConnection* conn;
-    createConnection(client, conn, true, requestID);
+//     auto it = connections_.find(endpoint.hostport());
+//     if (it != connections_.end()) {
+//         conn = it->second.get();
+//     } else {
+//         conn = (connections_[endpoint.hostport()] = std::unique_ptr(new ClientConnection(endpoint, Config()))).get();
+//         conn->connect();
+//     }
 
-    conn->controlWriteCheckResponse(msg, requestID, payload, payloadLength);
+//     ASSERT(conn);
+//     if (add) {
+//         requests_[id] = {conn, &client};
+//     }
 
-    return requestID;
-}
+//     return id;
+// }
 
+// uint32_t ClientConnectionRouter::controlWriteCheckResponse(Client& client, Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
+//     //std::cout << "ClientConnectionRouter::controlWriteCheckResponse " << endpoints.size() << std::endl;
+//     ClientConnection* conn;
+//     createConnection(client, conn, true, requestID);
 
-eckit::Buffer ClientConnectionRouter::controlWriteReadResponse(Client& client, remote::Message msg, const void* payload, uint32_t payloadLength) {
-    ClientConnection* conn;
-    uint32_t id = createConnection(client, conn, false);
-    //std::cout << "ClientConnectionRouter::controlWriteReadResponse Message: " << ((int) msg) << " ID: " << id << std::endl;
+//     conn->controlWriteCheckResponse(msg, requestID, payload, payloadLength);
 
-    return conn->controlWriteReadResponse(msg, id, payload, payloadLength);
-}
+//     return requestID;
+// }
 
-uint32_t ClientConnectionRouter::controlWrite(Client& client, Message msg, const void* payload, uint32_t payloadLength) {
-    //std::cout << "ClientConnectionRouter::controlWrite " << endpoints.size() << std::endl;
-    ClientConnection* conn;
-    uint32_t id = createConnection(client, conn);
 
-    conn->controlWrite(msg, id, payload, payloadLength);
+// eckit::Buffer ClientConnectionRouter::controlWriteReadResponse(Client& client, remote::Message msg, const void* payload, uint32_t payloadLength) {
+//     ClientConnection* conn;
+//     uint32_t id = createConnection(client, conn, false);
+//     //std::cout << "ClientConnectionRouter::controlWriteReadResponse Message: " << ((int) msg) << " ID: " << id << std::endl;
 
-    return id;
-}
-void ClientConnectionRouter::controlRead(Client& client, uint32_t requestId, void* payload, size_t payloadLength) {
+//     return conn->controlWriteReadResponse(msg, id, payload, payloadLength);
+// }
 
-    auto it = requests_.find(requestId);
-    ASSERT(it != requests_.end());
-    ASSERT(it->second.second == &client); // check if the client owns the request
+// uint32_t ClientConnectionRouter::controlWrite(Client& client, Message msg, const void* payload, uint32_t payloadLength) {
+//     //std::cout << "ClientConnectionRouter::controlWrite " << endpoints.size() << std::endl;
+//     ClientConnection* conn;
+//     uint32_t id = createConnection(client, conn);
 
-    it->second.first->controlRead(payload, payloadLength);
+//     conn->controlWrite(msg, id, payload, payloadLength);
 
-}
+//     return id;
+// }
+// void ClientConnectionRouter::controlRead(Client& client, uint32_t requestId, void* payload, size_t payloadLength) {
 
-uint32_t ClientConnectionRouter::dataWrite(Client& client, Message msg, const void* payload, uint32_t payloadLength) {
-    ClientConnection* conn;
-    uint32_t id = createConnection(client, conn);
+//     auto it = requests_.find(requestId);
+//     ASSERT(it != requests_.end());
+//     ASSERT(it->second.second == &client); // check if the client owns the request
 
-    conn->dataWrite(msg, id, payload, payloadLength);
+//     it->second.first->controlRead(payload, payloadLength);
 
-    return id;
-}
-void ClientConnectionRouter::dataWrite(Client& client, uint32_t requestId, const void* payload, size_t payloadLength) {
-    auto it = requests_.find(requestId);
-    ASSERT(it != requests_.end());
-    ASSERT(it->second.second == &client); // check if the client owns the request
+// }
 
-    it->second.first->dataWrite(payload, payloadLength);
-}
+// uint32_t ClientConnectionRouter::dataWrite(Client& client, Message msg, const void* payload, uint32_t payloadLength) {
+//     ClientConnection* conn;
+//     uint32_t id = createConnection(client, conn);
 
-bool ClientConnectionRouter::handle(Message message, uint32_t requestID) {
+//     conn->dataWrite(msg, id, payload, payloadLength);
 
-    if (requestID != 0) {
-        auto it = requests_.find(requestID);
-        ASSERT(it != requests_.end());
+//     return id;
+// }
+// void ClientConnectionRouter::dataWrite(Client& client, uint32_t requestId, const void* payload, size_t payloadLength) {
+//     auto it = requests_.find(requestId);
+//     ASSERT(it != requests_.end());
+//     ASSERT(it->second.second == &client); // check if the client owns the request
 
-        return it->second.second->handle(message, requestID);
-    } else {
-        eckit::Log::error() << "ClientConnectionRouter [message=" << ((uint) message) << ",requestID=" << requestID << "]" << std::endl;
-        return true;
-    }
-}
-bool ClientConnectionRouter::handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) {
+//     it->second.first->dataWrite(payload, payloadLength);
+// }
+
+// bool ClientConnectionRouter::handle(Message message, uint32_t requestID) {
 
-    if (requestID != 0) {
-        auto it = requests_.find(requestID);
-        ASSERT(it != requests_.end());
+//     if (requestID != 0) {
+//         auto it = requests_.find(requestID);
+//         ASSERT(it != requests_.end());
+
+//         return it->second.second->handle(message, requestID);
+//     } else {
+//         eckit::Log::error() << "ClientConnectionRouter [message=" << ((uint) message) << ",requestID=" << requestID << "]" << std::endl;
+//         return true;
+//     }
+// }
+// bool ClientConnectionRouter::handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) {
+
+//     if (requestID != 0) {
+//         auto it = requests_.find(requestID);
+//         ASSERT(it != requests_.end());
     
-        return it->second.second->handle(message, requestID, endpoint, std::move(payload));
-    } else {
-        eckit::Log::error() << "ClientConnectionRouter [message=" << ((uint) message) << ",requestID=" << requestID << "]" << std::endl;
-        return true;
-    }
-}
-void ClientConnectionRouter::handleException(std::exception_ptr e) {
-    // auto it = requests_.find(requestID);
-    // ASSERT(it != requests_.end());
+//         return it->second.second->handle(message, requestID, endpoint, std::move(payload));
+//     } else {
+//         eckit::Log::error() << "ClientConnectionRouter [message=" << ((uint) message) << ",requestID=" << requestID << "]" << std::endl;
+//         return true;
+//     }
+// }
+// void ClientConnectionRouter::handleException(std::exception_ptr e) {
+//     // auto it = requests_.find(requestID);
+//     // ASSERT(it != requests_.end());
+
+//     // it->second->handle(message, requestID);
+// }
+
+void ClientConnectionRouter::deregister(Client& client) {
+    const eckit::net::Endpoint& endpoint = client.controlEndpoint();
 
-    // it->second->handle(message, requestID);
+    std::lock_guard lock(connectionMutex_);
+
+    auto it = connections_.find(endpoint.hostport());
+    ASSERT(it != connections_.end());
+
+    auto clientIt = it->second.clients_.find(&client);
+    ASSERT(clientIt != it->second.clients_.end());
+
+    it->second.clients_.erase(clientIt);
+    if (it->second.clients_.empty()) {
+        connections_.erase(it);
+    }
 }
 
 ClientConnectionRouter& ClientConnectionRouter::instance()
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.h b/src/fdb5/remote/client/ClientConnectionRouter.h
index 08a3dd4a1..0836bf5dd 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.h
+++ b/src/fdb5/remote/client/ClientConnectionRouter.h
@@ -11,58 +11,68 @@
 #pragma once
 
 #include 
+#include 
 
-#include "eckit/config/Configuration.h"
-#include "eckit/memory/NonCopyable.h"
-#include "eckit/net/Endpoint.h"
-#include "eckit/thread/Mutex.h"
+// #include "eckit/config/Configuration.h"
+// #include "eckit/memory/NonCopyable.h"
+// #include "eckit/net/Endpoint.h"
+// #include "eckit/thread/Mutex.h"
 
-#include "fdb5/database/Store.h"
-#include "fdb5/remote/Messages.h"
+// #include "fdb5/remote/Messages.h"
 #include "fdb5/remote/client/Client.h"
 #include "fdb5/remote/client/ClientConnection.h"
-#include "fdb5/remote/RemoteStore.h"
 
 namespace fdb5::remote {
 
 //----------------------------------------------------------------------------------------------------------------------
 
+class Connection {
+public:
+
+    Connection(std::unique_ptr connection, Client& clients)
+        : connection_(std::move(connection)), clients_(std::set{&clients}) {}
+
+    std::unique_ptr connection_;
+    std::set clients_;
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
 class ClientConnectionRouter : eckit::NonCopyable {
 public:
 
     static ClientConnectionRouter& instance();
-    RemoteStore& store(const eckit::URI& uri);
-    static uint32_t generateRequestID();
+    // static uint32_t generateRequestID();
+
+    // uint32_t controlWriteCheckResponse(Client& client, Message msg, uint32_t requestID = generateRequestID(), const void* payload=nullptr, uint32_t payloadLength=0);
+    // eckit::Buffer controlWriteReadResponse(Client& client, remote::Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
+    // uint32_t controlWrite(Client& client, Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
+    // void controlRead(Client& client, uint32_t requestId, void* payload, size_t payloadLength);
+
+    // uint32_t dataWrite(Client& client, Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
+    // void dataWrite(Client& client, uint32_t requestId, const void* data, size_t length);
 
-    uint32_t controlWriteCheckResponse(Client& client, Message msg, uint32_t requestID = generateRequestID(), const void* payload=nullptr, uint32_t payloadLength=0);
-    eckit::Buffer controlWriteReadResponse(Client& client, remote::Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
-    uint32_t controlWrite(Client& client, Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
-    void controlRead(Client& client, uint32_t requestId, void* payload, size_t payloadLength);
+    // // handlers for incoming messages - to be defined in the client class
+    // bool handle(Message message, uint32_t requestID);
+    // bool handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload);
+    // void handleException(std::exception_ptr e);
 
-    uint32_t dataWrite(Client& client, Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
-    void dataWrite(Client& client, uint32_t requestId, const void* data, size_t length);
+    ClientConnection* connection(Client& client);
 
-    // handlers for incoming messages - to be defined in the client class
-    bool handle(Message message, uint32_t requestID);
-    bool handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload);
-    void handleException(std::exception_ptr e);
+    void deregister(Client& client);
 
 private:
 
     ClientConnectionRouter() {} ///< private constructor only used by singleton
 
-    uint32_t createConnection(Client& client, ClientConnection*& conn, bool add=true, uint32_t id = generateRequestID());
 
-    eckit::Mutex mutex_;
+    std::mutex connectionMutex_;
 
-    std::map > requests_;
+    // std::map> currentRequests_;
+    // std::map
 
     // endpoint -> connection
-    std::map > connections_;
-
-    // endpoint -> Store (one read store for each connection. Do not need to have one for each key)
-    std::map > readStores_;
-
+    std::map connections_;
 };
 
 }
\ No newline at end of file
diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc
index 391e304dc..657dbdace 100644
--- a/src/fdb5/remote/server/CatalogueHandler.cc
+++ b/src/fdb5/remote/server/CatalogueHandler.cc
@@ -21,7 +21,6 @@ using metkit::mars::MarsRequest;
 
 namespace fdb5::remote {
 
-
 // ***************************************************************************************
 //
 // Note that we use the "control" and "data" connections in a specific way, although these
@@ -31,9 +30,8 @@ namespace fdb5::remote {
 // ***************************************************************************************
 
 CatalogueHandler::CatalogueHandler(eckit::net::TCPSocket& socket, const Config& config):
-    ServerConnection(socket, config) {
+    ServerConnection(socket, config) {}
 
-}
 CatalogueHandler::~CatalogueHandler() {}
 
 void CatalogueHandler::initialiseConnections() {
@@ -75,7 +73,7 @@ void CatalogueHandler::initialiseConnections() {
         }
         s << config_.schema();
 
-        controlWrite(Message::Received, hdr.requestID, startupBuffer.data(), s.position());
+        dataWrite(Message::Received, hdr.requestID, startupBuffer.data(), s.position());
     }
 }
 
@@ -240,7 +238,7 @@ void CatalogueHandler::handle() {
                     ss << "ERROR: Unexpected message recieved (" << static_cast(hdr.message)
                        << "). ABORTING";
                     Log::status() << ss.str() << std::endl;
-                    Log::error() << "Retrieving... " << ss.str() << std::endl;
+                    Log::error() << "Catalogue Retrieving... " << ss.str() << std::endl;
                     throw SeriousBug(ss.str(), Here());
                 }
             }
@@ -251,17 +249,17 @@ void CatalogueHandler::handle() {
 
             if (!ack) {
                 // Acknowledge receipt of command
-                controlWrite(Message::Received, hdr.requestID);
+                dataWrite(Message::Received, hdr.requestID);
             }
         }
         catch (std::exception& e) {
             // n.b. more general than eckit::Exception
             std::string what(e.what());
-            controlWrite(Message::Error, hdr.requestID, what.c_str(), what.length());
+            dataWrite(Message::Error, hdr.requestID, what.c_str(), what.length());
         }
         catch (...) {
             std::string what("Caught unexpected and unknown error");
-            controlWrite(Message::Error, hdr.requestID, what.c_str(), what.length());
+            dataWrite(Message::Error, hdr.requestID, what.c_str(), what.length());
         }
     }
 }
@@ -438,7 +436,7 @@ void CatalogueHandler::schema(const MessageHeader& hdr) {
     eckit::MemoryStream stream(schemaBuffer);
     stream << schema;
 
-    controlWrite(Message::Received, hdr.requestID, schemaBuffer.data(), stream.position());
+    dataWrite(Message::Received, hdr.requestID, schemaBuffer.data(), stream.position());
 }
 
 CatalogueWriter& CatalogueHandler::catalogue(Key dbKey) {
diff --git a/src/fdb5/remote/server/CatalogueHandler.h b/src/fdb5/remote/server/CatalogueHandler.h
index cb99bc99d..9b31d2db6 100644
--- a/src/fdb5/remote/server/CatalogueHandler.h
+++ b/src/fdb5/remote/server/CatalogueHandler.h
@@ -8,13 +8,11 @@
  * does it submit to any jurisdiction.
  */
 
-#ifndef fdb5_remote_CatalogueHandler_H
-#define fdb5_remote_CatalogueHandler_H
+#pragma once
 
 #include "fdb5/remote/server/ServerConnection.h"
 #include "fdb5/api/FDB.h"
 
-
 namespace fdb5::remote {
 //----------------------------------------------------------------------------------------------------------------------
 class CatalogueHandler : public ServerConnection {
@@ -55,5 +53,3 @@ class CatalogueHandler : public ServerConnection {
 //----------------------------------------------------------------------------------------------------------------------
 
 }  // namespace fdb5::remote
-
-#endif  // fdb5_remote_CatalogueHandler_H
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index dab73ff15..417a52915 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -22,8 +22,6 @@
 #include "eckit/runtime/SessionID.h"
 #include "eckit/serialisation/MemoryStream.h"
 
-#include "metkit/mars/MarsRequest.h"
-
 #include "fdb5/LibFdb5.h"
 #include "fdb5/fdb5_version.h"
 #include "fdb5/api/helpers/FDBToolRequest.h"
@@ -35,11 +33,7 @@
 
 #include "fdb5/remote/server/ServerConnection.h"
 
-using namespace eckit;
-using metkit::mars::MarsRequest;
-
-namespace fdb5 {
-namespace remote {
+namespace fdb5::remote {
 
 // helpers
 namespace {
@@ -50,7 +44,7 @@ class TCPException : public eckit::Exception {
         eckit::Exception(std::string("TCPException: ") + msg, here) {}
 };
 
-std::vector intersection(const LocalConfiguration& c1, const LocalConfiguration& c2, const std::string& field){
+std::vector intersection(const eckit::LocalConfiguration& c1, const eckit::LocalConfiguration& c2, const std::string& field){
 
     std::vector v1 = c1.getIntVector(field);
     std::vector v2 = c2.getIntVector(field);
@@ -64,6 +58,7 @@ std::vector intersection(const LocalConfiguration& c1, const LocalConfigura
                           back_inserter(v3));
     return v3;
 }
+
 } // namespace
 
 ServerConnection::ServerConnection(eckit::net::TCPSocket& socket, const Config& config) :
@@ -71,7 +66,6 @@ ServerConnection::ServerConnection(eckit::net::TCPSocket& socket, const Config&
         controlSocket_(socket),
         dataSocket_(selectDataPort()),
         dataListenHostname_(config.getString("dataListenHostname", "")),
-        // fdb_(config),
         readLocationQueue_(eckit::Resource("fdbRetrieveQueueSize", 10000)) {
             eckit::Log::debug() << "ServerConnection::ServerConnection initialized" << std::endl;
     }
@@ -81,9 +75,9 @@ ServerConnection::~ServerConnection() {
     waitForWorkers();
 
     // And notify the client that we are done.
-    Log::info() << "Sending exit message to client" << std::endl;
+    eckit::Log::info() << "Sending exit message to client" << std::endl;
     dataWrite(Message::Exit, 0);
-    Log::info() << "Done" << std::endl;
+    eckit::Log::info() << "Done" << std::endl;
 }
 
 eckit::LocalConfiguration ServerConnection::availableFunctionality() const {
@@ -100,21 +94,21 @@ void ServerConnection::initialiseConnections() {
     MessageHeader hdr;
     socketRead(&hdr, sizeof(hdr), controlSocket_);
 
-
     ASSERT(hdr.marker == StartMarker);
     ASSERT(hdr.version == CurrentVersion);
     ASSERT(hdr.message == Message::Startup);
-    ASSERT(hdr.remoteID == 0);
     ASSERT(hdr.requestID == 0);
 
-    Buffer payload1 = receivePayload(hdr, controlSocket_);
+    std::cout << "ServerConnection::initialiseConnections() - payload "<< hdr.payloadSize << std::endl;
+
+    eckit::Buffer payload1 = receivePayload(hdr, controlSocket_);
     eckit::FixedString<4> tail;
     socketRead(&tail, sizeof(tail), controlSocket_);
     ASSERT(tail == EndMarker);
 
-    MemoryStream s1(payload1);
-    SessionID clientSession(s1);
-    net::Endpoint endpointFromClient(s1);
+    eckit::MemoryStream s1(payload1);
+    eckit::SessionID clientSession(s1);
+    eckit::net::Endpoint endpointFromClient(s1);
     unsigned int remoteProtocolVersion = 0;
     std::string errorMsg;
 
@@ -133,9 +127,9 @@ void ServerConnection::initialiseConnections() {
     }
 
     if (errorMsg.empty()) {
-        LocalConfiguration clientAvailableFunctionality(s1);
-        LocalConfiguration serverConf = availableFunctionality();
-        agreedConf_ = LocalConfiguration();
+        eckit::LocalConfiguration clientAvailableFunctionality(s1);
+        eckit::LocalConfiguration serverConf = availableFunctionality();
+        agreedConf_ = eckit::LocalConfiguration();
 
         // agree on a common functionality by intersecting server and client version numbers
         std::vector rflCommon = intersection(clientAvailableFunctionality, serverConf, "RemoteFieldLocation");
@@ -159,12 +153,12 @@ void ServerConnection::initialiseConnections() {
     //               the capacity in the protocol for the server to make a choice.
 
     int dataport = dataSocket_.localPort();
-    net::Endpoint dataEndpoint(endpointFromClient.hostname(), dataport);
+    eckit::net::Endpoint dataEndpoint(endpointFromClient.hostname(), dataport);
 
-    Log::info() << "Sending data endpoint to client: " << dataEndpoint << std::endl;
+    eckit::Log::info() << "Sending data endpoint to client: " << dataEndpoint << std::endl;
     {
-        Buffer startupBuffer(1024);
-        MemoryStream s(startupBuffer);
+        eckit::Buffer startupBuffer(1024);
+        eckit::MemoryStream s(startupBuffer);
 
         s << clientSession;
         s << sessionID_;
@@ -174,7 +168,6 @@ void ServerConnection::initialiseConnections() {
 
         eckit::Log::debug() << "Protocol negotiation - configuration: " << agreedConf_ <() << "ServerConnection::dataWrite [message="<< static_cast(msg) << ",requestID=" << requestID << ",payloadLength=" << payloadLength << "]" << std::endl;
 
     ASSERT((payload == nullptr) == (payloadLength == 0));
 
-    MessageHeader message(msg, 0, requestID, payloadLength);
+    MessageHeader message(msg, requestID, payloadLength);
 
     std::lock_guard lock(dataWriteMutex_);
-
     dataWriteUnsafe(&message, sizeof(message));
     if (payload) {
         dataWriteUnsafe(payload, payloadLength);
@@ -289,8 +281,8 @@ void ServerConnection::dataWriteUnsafe(const void* data, size_t length) {
     }
 }
 
-Buffer ServerConnection::receivePayload(const MessageHeader& hdr, net::TCPSocket& socket) {
-    Buffer payload(hdr.payloadSize);
+eckit::Buffer ServerConnection::receivePayload(const MessageHeader& hdr, eckit::net::TCPSocket& socket) {
+    eckit::Buffer payload(hdr.payloadSize);
 
     ASSERT(hdr.payloadSize > 0);
     socketRead(payload, hdr.payloadSize, socket);
@@ -305,7 +297,7 @@ void ServerConnection::tidyWorkers() {
         std::future_status stat = it->second.wait_for(std::chrono::milliseconds(0));
 
         if (stat == std::future_status::ready) {
-            Log::info() << "Tidying up worker for request ID: " << it->first << std::endl;
+            eckit::Log::info() << "Tidying up worker for request ID: " << it->first << std::endl;
             workerThreads_.erase(it++);
         }
         else {
@@ -320,10 +312,10 @@ void ServerConnection::waitForWorkers() {
     tidyWorkers();
 
     for (auto& it : workerThreads_) {
-        Log::error() << "Worker thread still alive for request ID: " << it.first << std::endl;
-        Log::error() << "Joining ..." << std::endl;
+        eckit::Log::error() << "Worker thread still alive for request ID: " << it.first << std::endl;
+        eckit::Log::error() << "Joining ..." << std::endl;
         it.second.get();
-        Log::error() << "Thread complete" << std::endl;
+        eckit::Log::error() << "Thread complete" << std::endl;
     }
 
     if (readLocationWorker_.joinable()) {
@@ -331,27 +323,4 @@ void ServerConnection::waitForWorkers() {
     }
 }
 
-
-// void ServerConnection::read(const MessageHeader& hdr) {
-//     // store only
-//     NOTIMP;
-// }
-
-// void ServerConnection::archive(const MessageHeader& hdr) {
-//     // catalogue only
-//     NOTIMP;
-// }
-
-// // void ServerConnection::store(const MessageHeader& hdr) {
-// //     // store only
-// //     NOTIMP;
-// // }
-
-// void ServerConnection::flush(const MessageHeader& hdr) {
-//     // store only
-//     NOTIMP;
-// }
-
-
-}  // namespace remote
-}  // namespace fdb5
+}  // namespace fdb5::remote
diff --git a/src/fdb5/remote/server/ServerConnection.h b/src/fdb5/remote/server/ServerConnection.h
index e6d3f5474..7823729ab 100644
--- a/src/fdb5/remote/server/ServerConnection.h
+++ b/src/fdb5/remote/server/ServerConnection.h
@@ -26,18 +26,10 @@
 #include "eckit/net/TCPSocket.h"
 #include "eckit/runtime/SessionID.h"
 
-#include "metkit/mars/MarsRequest.h"
-
-#include "fdb5/api/FDBFactory.h"
 #include "fdb5/config/Config.h"
-#include "fdb5/database/Key.h"
 #include "fdb5/remote/Messages.h"
 
-namespace fdb5 {
-
-class Config;
-
-namespace remote {
+namespace fdb5::remote {
 
 struct MessageHeader;
 
@@ -61,11 +53,9 @@ class ServerConnection : private eckit::NonCopyable {
     int selectDataPort();
     virtual void initialiseConnections();
     eckit::LocalConfiguration availableFunctionality() const;
-
-    void controlWrite(Message msg, uint32_t requestID, const void* payload = nullptr, uint32_t payloadLength = 0);
-    void controlWrite(const void* data, size_t length);
-    void socketRead(void* data, size_t length, eckit::net::TCPSocket& socket);
     
+    void socketRead(void* data, size_t length, eckit::net::TCPSocket& socket);
+
     // dataWrite is protected using a mutex, as we may have multiple workers.
     void dataWrite(Message msg, uint32_t requestID, const void* payload = nullptr, uint32_t payloadLength = 0);
     eckit::Buffer receivePayload(const MessageHeader& hdr, eckit::net::TCPSocket& socket);
@@ -77,6 +67,10 @@ class ServerConnection : private eckit::NonCopyable {
     void tidyWorkers();
     void waitForWorkers();
 
+private:
+
+    void controlWrite(Message msg, uint32_t requestID, const void* payload = nullptr, uint32_t payloadLength = 0);
+    void controlWrite(const void* data, size_t length);
 
     // virtual void read(const MessageHeader& hdr);
     // virtual void archive(const MessageHeader& hdr);
@@ -89,7 +83,7 @@ class ServerConnection : private eckit::NonCopyable {
     eckit::net::TCPSocket controlSocket_;
     eckit::net::EphemeralTCPServer dataSocket_;
     std::string dataListenHostname_;
-    // FDB fdb_;
+
     eckit::Queue>> readLocationQueue_;
 
     eckit::SessionID sessionID_;
@@ -104,9 +98,4 @@ class ServerConnection : private eckit::NonCopyable {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-}  // namespace remote
-}  // namespace fdb5
-
-
-//----------------------------------------------------------------------------------------------------------------------
-
+}  // namespace fdb5::remote
diff --git a/src/fdb5/remote/server/StoreHandler.cc b/src/fdb5/remote/server/StoreHandler.cc
index f3bd8612a..61a65b152 100644
--- a/src/fdb5/remote/server/StoreHandler.cc
+++ b/src/fdb5/remote/server/StoreHandler.cc
@@ -108,7 +108,7 @@ void StoreHandler::handle() {
                     ss << "ERROR: Unexpected message recieved (" << static_cast(hdr.message)
                        << "). ABORTING";
                     Log::status() << ss.str() << std::endl;
-                    Log::error() << "Retrieving... " << ss.str() << std::endl;
+                    Log::error() << "Store Retrieving... " << ss.str() << std::endl;
                     throw SeriousBug(ss.str(), Here());
                 }
             }
@@ -118,16 +118,16 @@ void StoreHandler::handle() {
             ASSERT(tail == EndMarker);
 
             // Acknowledge receipt of command
-            controlWrite(Message::Received, hdr.requestID);
+            dataWrite(Message::Received, hdr.requestID);
         }
         catch (std::exception& e) {
             // n.b. more general than eckit::Exception
             std::string what(e.what());
-            controlWrite(Message::Error, hdr.requestID, what.c_str(), what.length());
+            dataWrite(Message::Error, hdr.requestID, what.c_str(), what.length());
         }
         catch (...) {
             std::string what("Caught unexpected and unknown error");
-            controlWrite(Message::Error, hdr.requestID, what.c_str(), what.length());
+            dataWrite(Message::Error, hdr.requestID, what.c_str(), what.length());
         }
     }
 }

From 8ab2cc2ebeca07b85fb39c9f11ebee153c60fcd1 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 6 Nov 2023 07:03:46 +0000
Subject: [PATCH 020/186] cleanup

---
 src/fdb5/remote/client/Client.cc              |  24 +---
 src/fdb5/remote/client/Client.h               |   2 +-
 src/fdb5/remote/client/ClientConnection.cc    |  97 +-------------
 src/fdb5/remote/client/ClientConnection.h     |  20 ---
 .../remote/client/ClientConnectionRouter.cc   | 121 ------------------
 .../remote/client/ClientConnectionRouter.h    |  24 ----
 6 files changed, 6 insertions(+), 282 deletions(-)

diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index 82ee2d300..f32ee6cc1 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -22,10 +22,9 @@ Client::Client(const eckit::net::Endpoint& endpoint) :
     connection_(*(ClientConnectionRouter::instance().connection(*this))),
     blockingRequestId_(0) {}
 
-// uint32_t Client::generateRequestID() {
-//     return connection_.generateRequestID();
-// }
-
+Client::~Client() {
+    ClientConnectionRouter::instance().deregister(*this);
+}
 
 bool Client::response(uint32_t requestID) {
     ASSERT(requestID == blockingRequestId_);
@@ -88,19 +87,4 @@ void Client::dataWrite(remote::Message msg, uint32_t requestID, std::vector> data={});
     
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index 71f45e287..d8d827a2d 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -117,15 +117,6 @@ void ClientConnection::connect() {
     }
 }
 
-// void ClientConnection::controlWrite(remote::Message msg) {
-//     MessageHeader message(msg, 0, 0);
-
-//     std::lock_guard lock(controlMutex_);
-//     controlWrite(&message, sizeof(message));
-//     controlWrite(&EndMarker, sizeof(EndMarker));
-// }
-
-
 void ClientConnection::disconnect() {
     if (connected_) {
 
@@ -141,7 +132,6 @@ void ClientConnection::disconnect() {
     }
 }
 
-
 const eckit::net::Endpoint& ClientConnection::controlEndpoint() const { 
     return controlEndpoint_;
 } 
@@ -161,74 +151,8 @@ eckit::LocalConfiguration ClientConnection::availableFunctionality() const {
 void ClientConnection::addRequest(Client& client, uint32_t requestId) {
     ASSERT(requestId);
     requests_[requestId] = &client;
-
-//     // pick the first endpoint - To be improved with DataStoreStrategies
-//     const eckit::net::Endpoint& endpoint = client.controlEndpoint();
-
-//     auto it = connections_.find(endpoint.hostport());
-//     if (it != connections_.end()) {
-//         conn = it->second.get();
-//     } else {
-//         conn = (connections_[endpoint.hostport()] = std::unique_ptr(new ClientConnection(endpoint, Config()))).get();
-//         conn->connect();
-//     }
-
-//     ASSERT(conn);
-//     if (add) {
-//         requests_[id] = {conn, &client};
-//     }
-
-//     return id;
 }
 
-// void ClientConnection::controlWriteCheckResponse(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
-// //    std::cout << "ClientConnection::controlWriteCheckResponse " << ((uint) msg) << "  " << requestID << "  " << payloadLength << std::endl;
-
-//     controlWrite(msg, requestID, payload, payloadLength);
-
-//     // Wait for the receipt acknowledgement
-
-//     MessageHeader response;
-//     controlRead(&response, sizeof(MessageHeader));
-
-//     handleError(response);
-
-//     ASSERT(response.marker == StartMarker);
-//     ASSERT(response.version == CurrentVersion);
-//     ASSERT(response.message == Message::Received);
-
-//     eckit::FixedString<4> tail;
-//     controlRead(&tail, sizeof(tail));
-//     ASSERT(tail == EndMarker);
-// }
-
-// eckit::Buffer ClientConnection::controlWriteReadResponse(remote::Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
-    
-//     controlWrite(msg, requestID, payload, payloadLength);
-
-//     // Wait for the receipt acknowledgement
-
-//     MessageHeader response;
-//     controlRead(&response, sizeof(MessageHeader));
-
-//     handleError(response);
-
-//     // std::cout << "ClientConnection::controlWriteReadResponse received " << response.requestID << " | expecting " << requestID << std::endl;
-//     ASSERT(response.marker == StartMarker);
-//     ASSERT(response.version == CurrentVersion);
-//     ASSERT(response.message == Message::Received);
-//     ASSERT(response.requestID == requestID);
-    
-//     eckit::Buffer buf{response.payloadSize};
-//     controlRead(buf.data(), buf.size());
-
-//     eckit::FixedString<4> tail;
-//     controlRead(&tail, sizeof(tail));
-//     ASSERT(tail == EndMarker);
-
-//     return buf;
-// }
-
 void ClientConnection::controlWrite(Client& client, Message msg, uint32_t requestID, std::vector> data) {
 
     if (requestID) {
@@ -249,18 +173,12 @@ void ClientConnection::controlWrite(Message msg, uint32_t requestID, std::vector
     MessageHeader message(msg, requestID, payloadLength);
 
     std::lock_guard lock(controlMutex_);
-    eckit::Log::debug() << "ClientConnection::controlWrite 1 - " << sizeof(message) << std::endl;
     controlWrite(&message, sizeof(message));
     for (auto d: data) {
-        eckit::Log::debug() << "ClientConnection::controlWrite 2 - " << d.second << std::endl;
         ASSERT(d.first);
         controlWrite(d.first, d.second);
     }
-    eckit::Log::debug() << "ClientConnection::controlWrite 3 - " << sizeof(EndMarker) << std::endl;
     controlWrite(&EndMarker, sizeof(EndMarker));
-
-    eckit::Log::debug() << "ClientConnection::controlWrite - completed WRITE" << std::endl;
-
 }
 
 void ClientConnection::controlWrite(const void* data, size_t length) {
@@ -300,17 +218,6 @@ void ClientConnection::dataWrite(remote::Message msg, uint32_t requestID, std::v
     dataWrite(&EndMarker, sizeof(EndMarker));
 }
 
-// void ClientConnection::dataWrite(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
-
-//     ASSERT((payload == nullptr) == (payloadLength == 0));
-//     MessageHeader message(msg, requestID, payloadLength);
-//     dataWrite(&message, sizeof(message));
-//     if (payload) {
-//         dataWrite(payload, payloadLength);
-//     }
-//     dataWrite(&EndMarker, sizeof(EndMarker));
-// }
-
 void ClientConnection::dataWrite(const void* data, size_t length) {
     size_t written = dataClient_.write(data, length);
     if (length != written) {
@@ -350,8 +257,6 @@ void ClientConnection::handleError(const MessageHeader& hdr) {
     }
 }
 
-
-
 void ClientConnection::writeControlStartupMessage() {
 
     eckit::Buffer payload(4096);
@@ -431,7 +336,7 @@ void ClientConnection::listeningThreadLoop() {
 
             dataRead(&hdr, sizeof(hdr));
 
-            eckit::Log::debug() << "ClientConnection::listeningThreadLoop - got message " << ((int) hdr.message) << " with request ID: " << hdr.requestID << std::endl;
+            eckit::Log::debug() << "ClientConnection::listeningThreadLoop - got [message=" << ((int) hdr.message) << ",requestID=" << hdr.requestID << "]" << std::endl;
 
             ASSERT(hdr.marker == StartMarker);
             ASSERT(hdr.version == CurrentVersion);
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index 22fe327c5..26c11b75f 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -44,8 +44,6 @@ class ClientConnection : eckit::NonCopyable {
     ClientConnection(const eckit::net::Endpoint& controlEndpoint);
     virtual ~ClientConnection();
 
-    // void controlWriteCheckResponse(remote::Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
-    // eckit::Buffer controlWriteReadResponse(remote::Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
     void controlWrite(Client& client, remote::Message msg, uint32_t requestID, std::vector> data={});
     void dataWrite(remote::Message msg, uint32_t requestID, std::vector> data={});
 
@@ -59,11 +57,6 @@ class ClientConnection : eckit::NonCopyable {
     const eckit::net::Endpoint& controlEndpoint() const;
     const eckit::net::Endpoint& dataEndpoint() const;
     
-    // // handlers for incoming messages - to be defined in the client class
-    // virtual void handle(Message message, uint32_t requestID) = 0;
-    // virtual void handle(Message message, uint32_t requestID, eckit::Buffer&& payload) = 0;
-    // virtual void handleException(std::exception_ptr e) = 0;
-
     // construct dictionary for protocol negotiation - to be defined in the client class
     virtual eckit::LocalConfiguration availableFunctionality() const;
 
@@ -113,17 +106,4 @@ class ClientConnection : eckit::NonCopyable {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-// // // n.b. if we get integer overflow, we reuse the IDs. This is not a
-// // //      big deal. The idea that we could be on the 4.2 billionth (successful)
-// // //      request, and still have an ongoing request 0 is ... laughable.
-// static uint32_t generateRequestID() {
-
-//     static std::mutex m;
-//     static uint32_t id = 0;
-
-//     std::lock_guard lock(m);
-//     return ++id;
-// }
-
-
 }  // namespace fdb5::remote
\ No newline at end of file
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.cc b/src/fdb5/remote/client/ClientConnectionRouter.cc
index 11af270ac..846e0187b 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.cc
+++ b/src/fdb5/remote/client/ClientConnectionRouter.cc
@@ -1,24 +1,9 @@
-// #include 
-// #include 
-
 #include "fdb5/remote/client/ClientConnectionRouter.h"
 
-// using namespace eckit;
-// using namespace eckit::net;
-
 namespace fdb5::remote {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-// uint32_t ClientConnectionRouter::generateRequestID() {
-
-//     static std::mutex m;
-//     static uint32_t id = 0;
-
-//     std::lock_guard lock(m);
-//     return ++id;
-// }
-
 ClientConnection* ClientConnectionRouter::connection(Client& client) {
 
     // pick the first endpoint - To be improved with DataStoreStrategies
@@ -37,112 +22,6 @@ ClientConnection* ClientConnectionRouter::connection(Client& client) {
     }
 }
 
-// uint32_t ClientConnectionRouter::createConnection(Client& client, ClientConnection*& conn, bool add, uint32_t id) {
-
-//     // pick the first endpoint - To be improved with DataStoreStrategies
-//     const eckit::net::Endpoint& endpoint = client.controlEndpoint();
-
-//     auto it = connections_.find(endpoint.hostport());
-//     if (it != connections_.end()) {
-//         conn = it->second.get();
-//     } else {
-//         conn = (connections_[endpoint.hostport()] = std::unique_ptr(new ClientConnection(endpoint, Config()))).get();
-//         conn->connect();
-//     }
-
-//     ASSERT(conn);
-//     if (add) {
-//         requests_[id] = {conn, &client};
-//     }
-
-//     return id;
-// }
-
-// uint32_t ClientConnectionRouter::controlWriteCheckResponse(Client& client, Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
-//     //std::cout << "ClientConnectionRouter::controlWriteCheckResponse " << endpoints.size() << std::endl;
-//     ClientConnection* conn;
-//     createConnection(client, conn, true, requestID);
-
-//     conn->controlWriteCheckResponse(msg, requestID, payload, payloadLength);
-
-//     return requestID;
-// }
-
-
-// eckit::Buffer ClientConnectionRouter::controlWriteReadResponse(Client& client, remote::Message msg, const void* payload, uint32_t payloadLength) {
-//     ClientConnection* conn;
-//     uint32_t id = createConnection(client, conn, false);
-//     //std::cout << "ClientConnectionRouter::controlWriteReadResponse Message: " << ((int) msg) << " ID: " << id << std::endl;
-
-//     return conn->controlWriteReadResponse(msg, id, payload, payloadLength);
-// }
-
-// uint32_t ClientConnectionRouter::controlWrite(Client& client, Message msg, const void* payload, uint32_t payloadLength) {
-//     //std::cout << "ClientConnectionRouter::controlWrite " << endpoints.size() << std::endl;
-//     ClientConnection* conn;
-//     uint32_t id = createConnection(client, conn);
-
-//     conn->controlWrite(msg, id, payload, payloadLength);
-
-//     return id;
-// }
-// void ClientConnectionRouter::controlRead(Client& client, uint32_t requestId, void* payload, size_t payloadLength) {
-
-//     auto it = requests_.find(requestId);
-//     ASSERT(it != requests_.end());
-//     ASSERT(it->second.second == &client); // check if the client owns the request
-
-//     it->second.first->controlRead(payload, payloadLength);
-
-// }
-
-// uint32_t ClientConnectionRouter::dataWrite(Client& client, Message msg, const void* payload, uint32_t payloadLength) {
-//     ClientConnection* conn;
-//     uint32_t id = createConnection(client, conn);
-
-//     conn->dataWrite(msg, id, payload, payloadLength);
-
-//     return id;
-// }
-// void ClientConnectionRouter::dataWrite(Client& client, uint32_t requestId, const void* payload, size_t payloadLength) {
-//     auto it = requests_.find(requestId);
-//     ASSERT(it != requests_.end());
-//     ASSERT(it->second.second == &client); // check if the client owns the request
-
-//     it->second.first->dataWrite(payload, payloadLength);
-// }
-
-// bool ClientConnectionRouter::handle(Message message, uint32_t requestID) {
-
-//     if (requestID != 0) {
-//         auto it = requests_.find(requestID);
-//         ASSERT(it != requests_.end());
-
-//         return it->second.second->handle(message, requestID);
-//     } else {
-//         eckit::Log::error() << "ClientConnectionRouter [message=" << ((uint) message) << ",requestID=" << requestID << "]" << std::endl;
-//         return true;
-//     }
-// }
-// bool ClientConnectionRouter::handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) {
-
-//     if (requestID != 0) {
-//         auto it = requests_.find(requestID);
-//         ASSERT(it != requests_.end());
-    
-//         return it->second.second->handle(message, requestID, endpoint, std::move(payload));
-//     } else {
-//         eckit::Log::error() << "ClientConnectionRouter [message=" << ((uint) message) << ",requestID=" << requestID << "]" << std::endl;
-//         return true;
-//     }
-// }
-// void ClientConnectionRouter::handleException(std::exception_ptr e) {
-//     // auto it = requests_.find(requestID);
-//     // ASSERT(it != requests_.end());
-
-//     // it->second->handle(message, requestID);
-// }
-
 void ClientConnectionRouter::deregister(Client& client) {
     const eckit::net::Endpoint& endpoint = client.controlEndpoint();
 
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.h b/src/fdb5/remote/client/ClientConnectionRouter.h
index 0836bf5dd..a663dd359 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.h
+++ b/src/fdb5/remote/client/ClientConnectionRouter.h
@@ -13,12 +13,6 @@
 #include 
 #include 
 
-// #include "eckit/config/Configuration.h"
-// #include "eckit/memory/NonCopyable.h"
-// #include "eckit/net/Endpoint.h"
-// #include "eckit/thread/Mutex.h"
-
-// #include "fdb5/remote/Messages.h"
 #include "fdb5/remote/client/Client.h"
 #include "fdb5/remote/client/ClientConnection.h"
 
@@ -42,20 +36,6 @@ class ClientConnectionRouter : eckit::NonCopyable {
 public:
 
     static ClientConnectionRouter& instance();
-    // static uint32_t generateRequestID();
-
-    // uint32_t controlWriteCheckResponse(Client& client, Message msg, uint32_t requestID = generateRequestID(), const void* payload=nullptr, uint32_t payloadLength=0);
-    // eckit::Buffer controlWriteReadResponse(Client& client, remote::Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
-    // uint32_t controlWrite(Client& client, Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
-    // void controlRead(Client& client, uint32_t requestId, void* payload, size_t payloadLength);
-
-    // uint32_t dataWrite(Client& client, Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
-    // void dataWrite(Client& client, uint32_t requestId, const void* data, size_t length);
-
-    // // handlers for incoming messages - to be defined in the client class
-    // bool handle(Message message, uint32_t requestID);
-    // bool handle(Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload);
-    // void handleException(std::exception_ptr e);
 
     ClientConnection* connection(Client& client);
 
@@ -65,12 +45,8 @@ class ClientConnectionRouter : eckit::NonCopyable {
 
     ClientConnectionRouter() {} ///< private constructor only used by singleton
 
-
     std::mutex connectionMutex_;
 
-    // std::map> currentRequests_;
-    // std::map
-
     // endpoint -> connection
     std::map connections_;
 };

From e5c17d73097e31aa29a4c797f252edb791184f97 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 7 Nov 2023 14:22:53 +0000
Subject: [PATCH 021/186] wip

---
 src/fdb5/remote/client/ClientConnection.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index d8d827a2d..b2009f7aa 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -168,7 +168,8 @@ void ClientConnection::controlWrite(Message msg, uint32_t requestID, std::vector
     for (auto d: data) {
         payloadLength += d.second;
     }
-    eckit::Log::debug() << "ClientConnection::controlWrite [message=" << ((int) msg) << ",requestID=" << requestID << ",data=" << data.size() << ",payload=" << payloadLength << "]" << std::endl;
+    eckit::Log::debug() << "ClientConnection::controlWrite [endpoint=" << controlEndpoint_.hostport() <<
+        ",message=" << ((int) msg) << ",requestID=" << requestID << ",data=" << data.size() << ",payload=" << payloadLength << "]" << std::endl;
 
     MessageHeader message(msg, requestID, payloadLength);
 
@@ -207,7 +208,8 @@ void ClientConnection::dataWrite(remote::Message msg, uint32_t requestID, std::v
     }
     MessageHeader message(msg, requestID, payloadLength);
 
-    eckit::Log::debug() << "DATA DATA ClientConnection::dataWrite [message=" << ((int) msg) << ",requestID=" << requestID << ",data=" << data.size() << ",payload=" << payloadLength << "]" << std::endl;
+    eckit::Log::debug() << "ClientConnection::dataWrite [endpoint=" << dataEndpoint_.hostport() <<
+        ",message=" << ((int) msg) << ",requestID=" << requestID << ",data=" << data.size() << ",payload=" << payloadLength << "]" << std::endl;
 
     std::lock_guard lock(dataMutex_);
     dataWrite(&message, sizeof(message));

From 8eee95e54af5c8d35dbe60856cb9437319e6a074 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 7 Nov 2023 15:36:28 +0000
Subject: [PATCH 022/186] wip

---
 src/fdb5/remote/server/ServerConnection.cc | 3 ++-
 src/fdb5/remote/server/ServerConnection.h  | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index 417a52915..817058ff2 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -99,7 +99,7 @@ void ServerConnection::initialiseConnections() {
     ASSERT(hdr.message == Message::Startup);
     ASSERT(hdr.requestID == 0);
 
-    std::cout << "ServerConnection::initialiseConnections() - payload "<< hdr.payloadSize << std::endl;
+    // std::cout << "ServerConnection::initialiseConnections() - payload "<< hdr.payloadSize << std::endl;
 
     eckit::Buffer payload1 = receivePayload(hdr, controlSocket_);
     eckit::FixedString<4> tail;
@@ -230,6 +230,7 @@ void ServerConnection::controlWrite(Message msg, uint32_t requestID, const void*
     ASSERT((payload == nullptr) == (payloadLength == 0));
 
     MessageHeader message(msg, requestID, payloadLength);
+    std::lock_guard lock(controlWriteMutex_);
     controlWrite(&message, sizeof(message));
     if (payload) {
         controlWrite(payload, payloadLength);
diff --git a/src/fdb5/remote/server/ServerConnection.h b/src/fdb5/remote/server/ServerConnection.h
index 7823729ab..07c7d4779 100644
--- a/src/fdb5/remote/server/ServerConnection.h
+++ b/src/fdb5/remote/server/ServerConnection.h
@@ -88,6 +88,7 @@ class ServerConnection : private eckit::NonCopyable {
 
     eckit::SessionID sessionID_;
     eckit::LocalConfiguration agreedConf_;
+    std::mutex controlWriteMutex_;
     std::mutex dataWriteMutex_;
     std::map> workerThreads_;
     std::thread readLocationWorker_;

From da0a0f79d9401241df2789720694e9cc2b2407a2 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 7 Nov 2023 16:07:58 +0000
Subject: [PATCH 023/186] wip

---
 src/fdb5/remote/client/ClientConnection.cc | 2 ++
 src/fdb5/remote/client/ClientConnection.h  | 1 +
 2 files changed, 3 insertions(+)

diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index b2009f7aa..cf31ed019 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -150,6 +150,8 @@ eckit::LocalConfiguration ClientConnection::availableFunctionality() const {
 
 void ClientConnection::addRequest(Client& client, uint32_t requestId) {
     ASSERT(requestId);
+
+    std::lock_guard lock(requestMutex_);
     requests_[requestId] = &client;
 }
 
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index 26c11b75f..45fe858ed 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -93,6 +93,7 @@ class ClientConnection : eckit::NonCopyable {
 
     std::thread listeningThread_;
     
+    std::mutex requestMutex_;
     std::mutex controlMutex_;
     std::mutex dataMutex_;
 

From 40fcf662cb55fa29ccc17f0ed2fe1b3490ead2a5 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 7 Nov 2023 16:38:20 +0000
Subject: [PATCH 024/186] wip

---
 src/fdb5/remote/client/Client.cc | 17 ++++++++---------
 src/fdb5/remote/client/Client.h  |  3 ++-
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index f32ee6cc1..692eb6826 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -29,15 +29,15 @@ Client::~Client() {
 bool Client::response(uint32_t requestID) {
     ASSERT(requestID == blockingRequestId_);
 
-    eckit::Buffer buf;
-    payload_.set_value(std::move(buf));
+
+    promise_.set_value(true);
     return true;
 }
 
 bool Client::response(uint32_t requestID, eckit::Buffer&& payload) {
     ASSERT(requestID == blockingRequestId_);
 
-    payload_.set_value(std::move(payload));
+    payloadPromise_.set_value(std::move(payload));
     return true;
 }
 
@@ -51,9 +51,9 @@ void Client::controlWriteCheckResponse(Message msg, uint32_t requestID, const vo
     ASSERT(!blockingRequestId_);
     ASSERT(requestID);
 
-    payload_ = {};
+    promise_ = {};
+    std::future f = promise_.get_future();
     blockingRequestId_=requestID;
-    std::future f = payload_.get_future();
 
     if (payloadLength) {
         connection_.controlWrite(*this, msg, blockingRequestId_, std::vector>{{payload, payloadLength}});
@@ -61,16 +61,15 @@ void Client::controlWriteCheckResponse(Message msg, uint32_t requestID, const vo
         connection_.controlWrite(*this, msg, blockingRequestId_);
     }
 
-    eckit::Buffer buf = f.get();
+    f.get();
     blockingRequestId_=0;
-    ASSERT(buf.size() == 0);
 }
 eckit::Buffer Client::controlWriteReadResponse(Message msg, const void* payload, uint32_t payloadLength) {
     ASSERT(!blockingRequestId_);
 
-    payload_ = {};
+    payloadPromise_ = {};
     blockingRequestId_=connection_.generateRequestID();
-    std::future f = payload_.get_future();
+    std::future f = payloadPromise_.get_future();
 
     if (payloadLength) {
         connection_.controlWrite(*this, msg, blockingRequestId_, std::vector>{{payload, payloadLength}});
diff --git a/src/fdb5/remote/client/Client.h b/src/fdb5/remote/client/Client.h
index 8b013f30b..5e01e2900 100644
--- a/src/fdb5/remote/client/Client.h
+++ b/src/fdb5/remote/client/Client.h
@@ -66,7 +66,8 @@ class Client : eckit::NonCopyable {
 private:
 
     uint32_t blockingRequestId_;
-    std::promise payload_;
+    std::promise promise_;
+    std::promise payloadPromise_;
 
 };
 

From d46037ce5ec81943f324f18bffa23c4bec80c697 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 7 Nov 2023 17:07:06 +0000
Subject: [PATCH 025/186] wip

---
 src/fdb5/remote/RemoteStore.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index a00799c7c..47f3cd699 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -71,7 +71,8 @@ class RemoteStoreArchiver {
 
 private: // methods
 
-    RemoteStoreArchiver() : dirty_(false), maxArchiveQueueLength_(eckit::Resource("fdbRemoteArchiveQueueLength;$FDB_REMOTE_ARCHIVE_QUEUE_LENGTH", 200)) {}
+    RemoteStoreArchiver() : dirty_(false), maxArchiveQueueLength_(eckit::Resource("fdbRemoteArchiveQueueLength;$FDB_REMOTE_ARCHIVE_QUEUE_LENGTH", 200)),
+    archiveQueue_(nullptr) {}
 
     FDBStats archiveThreadLoop();
 
@@ -151,7 +152,7 @@ FDBStats RemoteStoreArchiver::flush(RemoteStore* store) {
     ASSERT(stats.numFlush() == 0);
     size_t numArchive = stats.numArchive();
 
-    Buffer sendBuf(4096);
+    Buffer sendBuf(1024);
     MemoryStream s(sendBuf);
     s << numArchive;
 

From 75e8aa9e6c70a9334c0a9151cf1411783449e9a1 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 8 Nov 2023 14:38:20 +0000
Subject: [PATCH 026/186] added localStores (to strip endpoint before
 archival/inject endpoint on retrieval)

---
 src/fdb5/api/RemoteFDB.cc                  | 44 ++++++++++++++++------
 src/fdb5/api/RemoteFDB.h                   |  6 ++-
 src/fdb5/remote/RemoteFieldLocation.cc     |  4 ++
 src/fdb5/remote/RemoteStore.cc             | 44 +++++++++++++++++++---
 src/fdb5/remote/RemoteStore.h              |  2 +
 src/fdb5/remote/server/CatalogueHandler.cc | 11 ++++++
 6 files changed, 91 insertions(+), 20 deletions(-)

diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index 6394423de..a4c5b7eaa 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -12,6 +12,7 @@
 #include "fdb5/LibFdb5.h"
 
 #include "fdb5/remote/client/ClientConnectionRouter.h"
+#include "fdb5/remote/RemoteFieldLocation.h"
 
 using namespace fdb5::remote;
 using namespace eckit;
@@ -33,21 +34,30 @@ struct BaseAPIHelper {
 
 using ListHelper = BaseAPIHelper;
 
-//using InspectHelper = BaseAPIHelper;
-
 struct InspectHelper : BaseAPIHelper {
 
-//     static fdb5::ListElement valueFromStream(eckit::Stream& s, fdb5::RemoteFDB* fdb) {
-//         fdb5::ListElement elem(s);
-//         return elem;
+    static fdb5::ListElement valueFromStream(eckit::Stream& s, fdb5::RemoteFDB* fdb) {
+        fdb5::ListElement elem(s);
+        // std::cout << "InspectHelper::valueFromStream - ";
+        // elem.location().dump(std::cout);
+        // std::cout << std::endl;
+        if (elem.location().uri().scheme() == "fdb") {
+            return elem;
+        }
 
-// //        return ListElement(elem.key(), RemoteFieldLocation(fdb, elem.location()).make_shared(), elem.timestamp());
-//     }
+        return fdb5::ListElement(elem.key(), fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(), elem.location()).make_shared(), elem.timestamp());
+    }
 };
 }
 
 namespace fdb5 {
 
+eckit::net::Endpoint RemoteFDB::storeEndpoint() {
+    ASSERT(localStores_.size() > 0);
+
+    return localStores_.at(std::rand() % localStores_.size());
+}
+
 RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name):
     LocalFDB(config, name),
     Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port"))) {
@@ -56,12 +66,20 @@ RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name
     eckit::MemoryStream s(buf);
     size_t numStores;
     s >> numStores;
+    std::vector stores;
+    for (size_t i=0; i> numStores;
+    std::vector localStores;
     for (size_t i=0; i::reanimate(s);
 
@@ -69,8 +87,10 @@ RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name
     // schema->dump(eckit::Log::debug());
     
     config_.overrideSchema(endpoint_.hostport()+"/schema", schema);
-    config_.set("storeHost", storeEndpoint.host());
-    config_.set("storePort", storeEndpoint.port());
+    config_.set("stores", stores);
+    config_.set("localStores", localStores);
+    // config_.set("storeHost", storeEndpoint.host());
+    // config_.set("storePort", storeEndpoint.port());
 }
 
 // -----------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/api/RemoteFDB.h b/src/fdb5/api/RemoteFDB.h
index 92487553c..18f412505 100644
--- a/src/fdb5/api/RemoteFDB.h
+++ b/src/fdb5/api/RemoteFDB.h
@@ -62,6 +62,8 @@ class RemoteFDB : public LocalFDB, public remote::Client {
 
     MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) override { NOTIMP; }
 
+    eckit::net::Endpoint storeEndpoint();
+
 private: // methods
 
     template 
@@ -80,8 +82,8 @@ class RemoteFDB : public LocalFDB, public remote::Client {
 private: // members
 
     std::unique_ptr archiver_;
-    std::vector stores_;
-    eckit::net::Endpoint storeEndpoint_;
+    std::vector localStores_;
+    // eckit::net::Endpoint storeEndpoint_;
 
     // Where do we put received messages
     // @note This is a map of requestID:MessageQueue. At the point that a request is
diff --git a/src/fdb5/remote/RemoteFieldLocation.cc b/src/fdb5/remote/RemoteFieldLocation.cc
index 4473f6c1c..1c3fbb2e7 100644
--- a/src/fdb5/remote/RemoteFieldLocation.cc
+++ b/src/fdb5/remote/RemoteFieldLocation.cc
@@ -69,6 +69,10 @@ std::shared_ptr RemoteFieldLocation::make_shared() const {
 
 eckit::DataHandle* RemoteFieldLocation::dataHandle() const {
     
+    // std::cout << "############   retrieve from location ";
+    // dump(std::cout);
+    // std::cout << std::endl;
+
     RemoteStore& store = RemoteStore::get(uri_);
     
     const std::string scheme = uri_.query("internalScheme");
diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index 47f3cd699..85cbcaa13 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -187,8 +187,6 @@ FDBStats RemoteStoreArchiver::archiveThreadLoop() {
 
         // And note that we are done. (don't time this, as already being blocked
         // on by the ::flush() routine)
-
-
         element.store_->dataWrite(Message::Flush, 0);
 
         archiveQueue_.reset();
@@ -326,12 +324,31 @@ class FDBRemoteDataHandle : public DataHandle {
     bool complete_;
 };
 
+eckit::net::Endpoint storeEndpoint(const Config& config) {
+    if (config.has("storeHost") && config.has("storePort")) {
+        return eckit::net::Endpoint(config.getString("storeHost"), config.getInt("storePort"));
+    }
+    ASSERT(config.has("stores"));
+    std::vector stores = config.getStringVector("stores");
+    std::srand(std::time(nullptr));
+    return eckit::net::Endpoint(stores.at(std::rand() % stores.size()));
+}
+
 //----------------------------------------------------------------------------------------------------------------------
 
 RemoteStore::RemoteStore(const Key& dbKey, const Config& config) :
-    Client(eckit::net::Endpoint(config.getString("storeHost"), config.getInt("storePort"))),
+    Client(storeEndpoint(config)),
     dbKey_(dbKey), config_(config),
     retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)) {
+
+    if (config.has("localStores")) {
+        for (const std::string& localStore : config.getStringVector("localStores")) {
+            if (localStore == endpoint_.hostport()) {
+                local_ = true;
+                break;
+            }
+        }
+    }
 }
 
 RemoteStore::RemoteStore(const eckit::URI& uri, const Config& config) :
@@ -339,6 +356,17 @@ RemoteStore::RemoteStore(const eckit::URI& uri, const Config& config) :
     dbKey_(Key()), config_(config),
     retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)) {
 
+    // no need to set the local_ flag on the read path
+
+    // if (config.has("localStores")) {
+    //     for (const std::string& localStore : config.getStringVector("localStores")) {
+    //         if (localStore == endpoint_.hostport()) {
+    //             local_ = true;
+    //             break;
+    //         }
+    //     }
+    // }
+
     ASSERT(uri.scheme() == "fdb");
 }
 
@@ -467,9 +495,13 @@ bool RemoteStore::handle(Message message, uint32_t requestID, eckit::Buffer&& pa
             if (it != locations_.end()) {
                 MemoryStream s(payload);
                 std::unique_ptr location(eckit::Reanimator::reanimate(s));
-                // std::cout <<  "RemoteStore::handle - " << location->uri().asRawString() << " " << location->length() << std::endl;
-                std::unique_ptr remoteLocation = std::unique_ptr(new RemoteFieldLocation(endpoint_, *location));
-                it->second.set_value(std::move(remoteLocation));
+                if (local_) {
+                    it->second.set_value(std::move(location));
+                } else {
+                    // std::cout <<  "RemoteStore::handle - " << location->uri().asRawString() << " " << location->length() << std::endl;
+                    std::unique_ptr remoteLocation = std::unique_ptr(new RemoteFieldLocation(endpoint_, *location));
+                    it->second.set_value(std::move(remoteLocation));
+                }
             }
             return true;
         }
diff --git a/src/fdb5/remote/RemoteStore.h b/src/fdb5/remote/RemoteStore.h
index 3808412cc..9bf102de4 100644
--- a/src/fdb5/remote/RemoteStore.h
+++ b/src/fdb5/remote/RemoteStore.h
@@ -103,6 +103,8 @@ class RemoteStore : public Store, public Client {
 
     // not owning
     RemoteStoreArchiver* archiver_;
+
+    bool local_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc
index 657dbdace..d6c19a464 100644
--- a/src/fdb5/remote/server/CatalogueHandler.cc
+++ b/src/fdb5/remote/server/CatalogueHandler.cc
@@ -51,6 +51,7 @@ void CatalogueHandler::initialiseConnections() {
     ASSERT(tail == EndMarker);
 
     std::vector stores;
+    std::vector localStores;
 
     if (::getenv("FDB_STORE_HOST") && ::getenv("FDB_STORE_PORT")) {
         // override the configuration
@@ -61,6 +62,12 @@ void CatalogueHandler::initialiseConnections() {
         for (const std::string& endpoint: endpoints) {
             stores.push_back(eckit::net::Endpoint(endpoint));
         }
+        if (config_.has("localStores")) {
+            std::vector endpoints = config_.getStringVector("localStores");
+            for (const std::string& endpoint: endpoints) {
+                localStores.push_back(eckit::net::Endpoint(endpoint));
+            }
+        }
     }
 
     {
@@ -71,6 +78,10 @@ void CatalogueHandler::initialiseConnections() {
         for (const eckit::net::Endpoint& endpoint : stores) {
             s << endpoint;
         }
+        s << localStores.size();
+        for (const eckit::net::Endpoint& endpoint : localStores) {
+            s << endpoint;
+        }
         s << config_.schema();
 
         dataWrite(Message::Received, hdr.requestID, startupBuffer.data(), s.position());

From 77af8450f7f85b98fd05ca8f27dfcf0897142846 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 8 Nov 2023 16:59:34 +0000
Subject: [PATCH 027/186] fdb-read not respecting order

---
 src/fdb5/tools/fdb-read.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/fdb5/tools/fdb-read.cc b/src/fdb5/tools/fdb-read.cc
index 161ee3a29..dd897d1a5 100644
--- a/src/fdb5/tools/fdb-read.cc
+++ b/src/fdb5/tools/fdb-read.cc
@@ -82,7 +82,7 @@ void FDBRead::execute(const eckit::option::CmdArgs &args) {
 
     // Evaluate the requests to obtain data handles
 
-    fdb5::HandleGatherer handles(false);
+    fdb5::HandleGatherer handles(true);
 
     fdb5::FDB fdb(config(args));
 

From 5b4c9dcc5557aafa9835d8b68e7dbe477ef826f3 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 9 Nov 2023 14:00:36 +0000
Subject: [PATCH 028/186] Archiver cleanup

---
 src/fdb5/api/FDB.h                     |  2 +-
 src/fdb5/api/RemoteFDB.cc              | 11 +++++----
 src/fdb5/database/Archiver.cc          | 31 ++++++++++++++------------
 src/fdb5/database/Archiver.h           |  9 +++++---
 src/fdb5/remote/RemoteFieldLocation.cc |  8 ++++---
 src/fdb5/remote/server/StoreHandler.cc |  3 ---
 6 files changed, 36 insertions(+), 28 deletions(-)

diff --git a/src/fdb5/api/FDB.h b/src/fdb5/api/FDB.h
index 97f629321..cb9879363 100644
--- a/src/fdb5/api/FDB.h
+++ b/src/fdb5/api/FDB.h
@@ -48,7 +48,7 @@ namespace fdb5 {
 
 class FDBBase;
 class FDBToolRequest;
-class InspectionKey;
+class Key;
 
 //----------------------------------------------------------------------------------------------------------------------
 
diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index a4c5b7eaa..55fef8811 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -38,14 +38,17 @@ struct InspectHelper : BaseAPIHelper() << "InspectHelper::valueFromStream - original location: ";
+        elem.location().dump(eckit::Log::debug());
+        eckit::Log::debug() << std::endl;
+
         if (elem.location().uri().scheme() == "fdb") {
             return elem;
         }
 
-        return fdb5::ListElement(elem.key(), fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(), elem.location()).make_shared(), elem.timestamp());
+        std::shared_ptr remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(), elem.location()).make_shared();
+        return fdb5::ListElement(elem.key(), remoteLocation, elem.timestamp());
     }
 };
 }
diff --git a/src/fdb5/database/Archiver.cc b/src/fdb5/database/Archiver.cc
index 5af12d856..573e0da67 100644
--- a/src/fdb5/database/Archiver.cc
+++ b/src/fdb5/database/Archiver.cc
@@ -27,7 +27,8 @@ namespace fdb5 {
 
 Archiver::Archiver(const Config& dbConfig) :
     dbConfig_(dbConfig),
-    catalogue_(nullptr) {
+    catalogue_(nullptr),
+    store_(nullptr) {
 }
 
 Archiver::~Archiver() {
@@ -60,20 +61,20 @@ void Archiver::archive(const Key &key, BaseArchiveVisitor& visitor) {
 }
 
 void Archiver::flush() {
-    for (store_t::iterator i = databases_.begin(); i != databases_.end(); ++i) {
-        i->second.second.second->flush(); // flush the store
-        i->second.second.first->flush();  // flush the catalogue
+    for (auto i = databases_.begin(); i != databases_.end(); ++i) {
+        i->second.store_->flush();      // flush the store
+        i->second.catalogue_->flush();  // flush the catalogue
     }
 }
 
 void Archiver::selectDatabase(const Key &dbKey) {
 
-    store_t::iterator i = databases_.find(dbKey);
+    auto i = databases_.find(dbKey);
 
     if (i != databases_.end() ) {
-        catalogue_ = i->second.second.first.get();
-        store_ = i->second.second.second.get();
-        i->second.first = ::time(0);
+        catalogue_ = i->second.catalogue_.get();
+        store_ = i->second.store_.get();
+        i->second.time_ = ::time(0);
         return;
     }
 
@@ -83,16 +84,18 @@ void Archiver::selectDatabase(const Key &dbKey) {
         bool found = false;
         time_t oldest = ::time(0) + 24 * 60 * 60;
         Key oldK;
-        for (store_t::iterator i = databases_.begin(); i != databases_.end(); ++i) {
-            if (i->second.first <= oldest) {
+        for (auto i = databases_.begin(); i != databases_.end(); ++i) {
+            if (i->second.time_ <= oldest) {
                 found = true;
                 oldK = i->first;
-                oldest = i->second.first;
+                oldest = i->second.time_;
             }
         }
         if (found) {
-            // what if the catalogue/store are not flashed ???
-            eckit::Log::warning() << "Closing database " << *databases_[oldK].second.first << std::endl;
+            databases_[oldK].store_->flush();
+            databases_[oldK].catalogue_->flush();
+            
+            eckit::Log::info() << "Closing database " << *databases_[oldK].catalogue_ << std::endl;
             databases_.erase(oldK);
         }
     }
@@ -111,7 +114,7 @@ void Archiver::selectDatabase(const Key &dbKey) {
     catalogue_ = cat.get();
     store_ = str.get();
 
-    databases_[dbKey] = std::make_pair(::time(0), std::make_pair(std::move(cat), std::move(str)));
+    databases_[dbKey] = Database{::time(0), std::move(cat), std::move(str)};
 }
 
 void Archiver::print(std::ostream &out) const {
diff --git a/src/fdb5/database/Archiver.h b/src/fdb5/database/Archiver.h
index 3ab9ba442..fa12bc192 100644
--- a/src/fdb5/database/Archiver.h
+++ b/src/fdb5/database/Archiver.h
@@ -38,6 +38,11 @@ class Schema;
 
 //----------------------------------------------------------------------------------------------------------------------
 
+struct Database {
+    time_t                              time_;
+    std::unique_ptr    catalogue_;
+    std::unique_ptr              store_;
+};
 class Archiver : public eckit::NonCopyable {
 
 public: // methods
@@ -68,11 +73,9 @@ class Archiver : public eckit::NonCopyable {
 
     friend class BaseArchiveVisitor;
 
-    typedef std::map, std::unique_ptr>>> store_t;
-
     Config dbConfig_;
 
-    store_t databases_;
+    std::map databases_;
 
     std::vector prev_;
 
diff --git a/src/fdb5/remote/RemoteFieldLocation.cc b/src/fdb5/remote/RemoteFieldLocation.cc
index 1c3fbb2e7..19ac0c9f2 100644
--- a/src/fdb5/remote/RemoteFieldLocation.cc
+++ b/src/fdb5/remote/RemoteFieldLocation.cc
@@ -16,9 +16,11 @@
 #include "fdb5/remote/RemoteFieldLocation.h"
 #include "fdb5/remote/RemoteStore.h"
 #include "fdb5/remote/client/ClientConnectionRouter.h"
+#include "fdb5/LibFdb5.h"
 
 #include "eckit/exception/Exceptions.h"
 #include "eckit/filesystem/URIManager.h"
+#include "eckit/log/Log.h"
 
 namespace fdb5 {
 namespace remote {
@@ -69,9 +71,9 @@ std::shared_ptr RemoteFieldLocation::make_shared() const {
 
 eckit::DataHandle* RemoteFieldLocation::dataHandle() const {
     
-    // std::cout << "############   retrieve from location ";
-    // dump(std::cout);
-    // std::cout << std::endl;
+    eckit::Log::debug() << "RemoteFieldLocation::dataHandle for location: ";
+    dump(eckit::Log::debug());
+    eckit::Log::debug() << std::endl;
 
     RemoteStore& store = RemoteStore::get(uri_);
     
diff --git a/src/fdb5/remote/server/StoreHandler.cc b/src/fdb5/remote/server/StoreHandler.cc
index 61a65b152..f803a1a45 100644
--- a/src/fdb5/remote/server/StoreHandler.cc
+++ b/src/fdb5/remote/server/StoreHandler.cc
@@ -208,12 +208,9 @@ void StoreHandler::archive(const MessageHeader& hdr) {
     ASSERT(hdr.payloadSize == 0);
 
     // Ensure that we aren't already running a store()
-
     if(!archiveFuture_.valid()) {
 
         // Start archive worker thread
-
-    //    uint32_t id    = hdr.requestID;
         archiveFuture_ = std::async(std::launch::async, [this] { return archiveThreadLoop(); });
     }
 }

From 18276940135969e1f3de536aeb3afb1e80798255 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 9 Nov 2023 14:18:32 +0000
Subject: [PATCH 029/186] fix merge

---
 src/fdb5/tools/fdb-list.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/fdb5/tools/fdb-list.cc b/src/fdb5/tools/fdb-list.cc
index 0d575d169..420cb8c0b 100644
--- a/src/fdb5/tools/fdb-list.cc
+++ b/src/fdb5/tools/fdb-list.cc
@@ -148,7 +148,6 @@ void FDBList::execute(const CmdArgs& args) {
                 } else {
                     elem.print(Log::info(), location_, !porcelain_);
                     Log::info() << std::endl;
-                    count++;
                 }
             }
         }

From 06339d38f4518bec9ef7420b571c101f3f9d58c2 Mon Sep 17 00:00:00 2001
From: FDB Dev 
Date: Tue, 14 Nov 2023 16:36:00 +0000
Subject: [PATCH 030/186] empty line

---
 CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 03d8592f5..9b0b76fd9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -127,6 +127,7 @@ add_subdirectory( tests )
 
 ### finalize
 
+
 ecbuild_install_project( NAME fdb )
 
 ecbuild_print_summary()

From 42c4f08ae7a6394121f0cb7cd6b17b137aa5436b Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 16 Nov 2023 10:48:28 +0000
Subject: [PATCH 031/186] archive callback

---
 src/fdb5/api/RemoteFDB.h            |  2 --
 src/fdb5/database/ArchiveVisitor.cc | 14 +++++++++-----
 src/fdb5/database/IndexAxis.cc      |  6 +++---
 src/fdb5/database/IndexAxis.h       |  2 +-
 src/fdb5/database/Report.cc         |  2 +-
 src/fdb5/database/Store.cc          |  8 ++++++++
 src/fdb5/database/Store.h           |  7 ++++---
 src/fdb5/message/MessageDecoder.cc  |  2 +-
 src/fdb5/rados/RadosStore.cc        |  6 ++----
 src/fdb5/rados/RadosStore.h         |  2 +-
 src/fdb5/remote/RemoteCatalogue.cc  |  4 ++--
 src/fdb5/remote/RemoteStore.cc      | 15 ++++++---------
 src/fdb5/remote/RemoteStore.h       |  4 ++--
 src/fdb5/toc/TocCommon.cc           |  2 +-
 src/fdb5/toc/TocHandler.cc          |  2 +-
 src/fdb5/toc/TocStats.cc            |  4 ++--
 src/fdb5/toc/TocStore.cc            |  7 +++----
 src/fdb5/toc/TocStore.h             |  2 +-
 src/fdb5/tools/FDBInspect.cc        |  2 +-
 src/fdb5/tools/FDBVisitTool.cc      |  2 +-
 20 files changed, 50 insertions(+), 45 deletions(-)

diff --git a/src/fdb5/api/RemoteFDB.h b/src/fdb5/api/RemoteFDB.h
index 18f412505..32fbf710b 100644
--- a/src/fdb5/api/RemoteFDB.h
+++ b/src/fdb5/api/RemoteFDB.h
@@ -20,7 +20,6 @@
 
 #pragma once
 
-#include 
 #include 
 
 #include "fdb5/api/LocalFDB.h"
@@ -83,7 +82,6 @@ class RemoteFDB : public LocalFDB, public remote::Client {
 
     std::unique_ptr archiver_;
     std::vector localStores_;
-    // eckit::net::Endpoint storeEndpoint_;
 
     // Where do we put received messages
     // @note This is a map of requestID:MessageQueue. At the point that a request is
diff --git a/src/fdb5/database/ArchiveVisitor.cc b/src/fdb5/database/ArchiveVisitor.cc
index e9802a77d..19f682dde 100644
--- a/src/fdb5/database/ArchiveVisitor.cc
+++ b/src/fdb5/database/ArchiveVisitor.cc
@@ -7,13 +7,18 @@
  * granted to it by virtue of its status as an intergovernmental organisation nor
  * does it submit to any jurisdiction.
  */
-
+#include 
 #include "eckit/exception/Exceptions.h"
 
 #include "fdb5/database/ArchiveVisitor.h"
 #include "fdb5/database/Catalogue.h"
 #include "fdb5/database/Store.h"
 
+namespace {
+void CatalogueCallback(fdb5::CatalogueWriter* catalogue, const fdb5::InspectionKey &key, std::unique_ptr fieldLocation) {
+    catalogue->archive(key, std::move(fieldLocation));
+}
+}
 namespace fdb5 {
 
 ArchiveVisitor::ArchiveVisitor(Archiver &owner, const Key &dataKey, const void *data, size_t size) :
@@ -22,16 +27,15 @@ ArchiveVisitor::ArchiveVisitor(Archiver &owner, const Key &dataKey, const void *
     size_(size) {
 }
 
+
 bool ArchiveVisitor::selectDatum(const InspectionKey &key, const Key &full) {
 
     // eckit::Log::info() << "selectDatum " << key << ", " << full << " " << size_ << std::endl;
     checkMissingKeys(full);
     const Key idxKey = current()->currentIndexKey();
 
-    // here we could create a queue... and keep accepting archival request until the queue is full
-    auto futureLocation = store()->archive(idxKey, data_, size_);
-    current()->archive(key, futureLocation.get());
-
+    store()->archive(idxKey, data_, size_, std::bind(&CatalogueCallback, current(), key, std::placeholders::_1));
+    
     return true;
 }
 
diff --git a/src/fdb5/database/IndexAxis.cc b/src/fdb5/database/IndexAxis.cc
index f535bd9eb..91a4f68be 100755
--- a/src/fdb5/database/IndexAxis.cc
+++ b/src/fdb5/database/IndexAxis.cc
@@ -130,7 +130,7 @@ void IndexAxis::decodeCurrent(eckit::Stream &s, const int version) {
                 ASSERT(n);
                 for (size_t i = 0; i < n; i++) {
                     s >> k;
-                    std::shared_ptr >& values = axis_[k];
+                    std::shared_ptr>& values = axis_[k];
                     values.reset(new eckit::DenseSet);
                     size_t m;
                     s >> m;
@@ -161,7 +161,7 @@ void IndexAxis::decodeLegacy(eckit::Stream& s, const int version) {
 
     for (size_t i = 0; i < n; i++) {
         s >> k;
-        std::shared_ptr >& values = axis_[k];
+        std::shared_ptr>& values = axis_[k];
         values.reset(new eckit::DenseSet);
         size_t m;
         s >> m;
@@ -237,7 +237,7 @@ void IndexAxis::insert(const InspectionKey &key) {
     for (Key::const_iterator i = key.begin(); i  != key.end(); ++i) {
         const std::string &keyword = i->first;
 
-        std::shared_ptr >& axis_set = axis_[keyword];
+        std::shared_ptr>& axis_set = axis_[keyword];
         if (!axis_set)
             axis_set.reset(new eckit::DenseSet);
 
diff --git a/src/fdb5/database/IndexAxis.h b/src/fdb5/database/IndexAxis.h
index 72fe85a88..7ecaf2763 100644
--- a/src/fdb5/database/IndexAxis.h
+++ b/src/fdb5/database/IndexAxis.h
@@ -91,7 +91,7 @@ class IndexAxis : private eckit::NonCopyable {
 
 private: // members
 
-    typedef std::map > > AxisMap;
+    typedef std::map>> AxisMap;
     AxisMap axis_;
 
     bool readOnly_;
diff --git a/src/fdb5/database/Report.cc b/src/fdb5/database/Report.cc
index 1b59643e2..cf3333159 100644
--- a/src/fdb5/database/Report.cc
+++ b/src/fdb5/database/Report.cc
@@ -51,7 +51,7 @@ Report& Report::operator+=(const Report& rhs) {
                    dbtypes_.end(),
                    rhs.dbtypes_.begin(),
                    rhs.dbtypes_.end(),
-                   std::insert_iterator< std::set >(join, join.begin()));
+                   std::insert_iterator< std::set>(join, join.begin()));
 
     std::swap(dbtypes_, join);
 
diff --git a/src/fdb5/database/Store.cc b/src/fdb5/database/Store.cc
index 3c83951e3..243e57f7b 100644
--- a/src/fdb5/database/Store.cc
+++ b/src/fdb5/database/Store.cc
@@ -24,6 +24,14 @@ namespace fdb5 {
 
 //----------------------------------------------------------------------------------------------------------------------
 
+void Store::archive(const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive) {
+    catalogue_archive(archive(key, data, length));
+}
+
+std::unique_ptr Store::archive(const Key& key, const void *data, eckit::Length length) {
+    NOTIMP;
+}
+
 bool Store::canMoveTo(const Key&, const Config&, const eckit::URI& dest) const {
     std::stringstream ss;
     ss << "Store type " << type() << " does not support move" << std::endl;
diff --git a/src/fdb5/database/Store.h b/src/fdb5/database/Store.h
index 97143af83..ecf3e5695 100644
--- a/src/fdb5/database/Store.h
+++ b/src/fdb5/database/Store.h
@@ -14,8 +14,6 @@
 
 #pragma once
 
-#include 
-
 #include "eckit/distributed/Transport.h"
 #include "eckit/filesystem/URI.h"
 #include "eckit/io/DataHandle.h"
@@ -37,7 +35,9 @@ class Store {
     virtual ~Store() {}
 
     virtual eckit::DataHandle* retrieve(Field& field) const = 0;
-    virtual std::future > archive(const Key& key, const void *data, eckit::Length length) = 0;
+//    virtual void archive(const Key& key, const void *data, eckit::Length length, void(*catalogue_archive)(std::unique_ptr fieldLocation)) = 0;
+    virtual void archive(const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive);
+    virtual std::unique_ptr archive(const Key& key, const void *data, eckit::Length length);
 
     virtual void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit = true) const = 0;
 
@@ -58,6 +58,7 @@ class Store {
     virtual void remove(const Key& key) const { NOTIMP; }
 
     virtual eckit::URI uri() const = 0;
+    
 };
 
 
diff --git a/src/fdb5/message/MessageDecoder.cc b/src/fdb5/message/MessageDecoder.cc
index fcbd24bac..cd71d57d0 100755
--- a/src/fdb5/message/MessageDecoder.cc
+++ b/src/fdb5/message/MessageDecoder.cc
@@ -109,7 +109,7 @@ std::vector MessageDecoder::messageToRequests(const e
     eckit::message::Reader reader(path);
     eckit::message::Message msg;
 
-    std::map > s;
+    std::map> s;
 
     while ( (msg = reader.next()) ) {
 
diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc
index 6842dc989..df2338a48 100644
--- a/src/fdb5/rados/RadosStore.cc
+++ b/src/fdb5/rados/RadosStore.cc
@@ -53,7 +53,7 @@ eckit::DataHandle* RadosStore::retrieve(Field& field, Key& remapKey) const {
         field.dataHandle(remapKey);
 }
 
-std::future > RadosStore::archive(const Key& key, const void *data, eckit::Length length) {
+std::unique_ptr RadosStore::archive(const Key& key, const void *data, eckit::Length length) {
     dirty_ = true;
 
     eckit::PathName dataPath = getDataPath(key);
@@ -67,9 +67,7 @@ std::future > RadosStore::archive(const Key& key,
 
     ASSERT(len == length);
 
-    std::promise > loc;
-    loc.set_value(std::unique_ptr(new RadosFieldLocation(dataUri, position, length)));
-    return loc.get_future();
+    return std::unique_ptr(new RadosFieldLocation(dataUri, position, length));
 }
 
 void RadosStore::flush() {
diff --git a/src/fdb5/rados/RadosStore.h b/src/fdb5/rados/RadosStore.h
index f05181ae8..0059c2a89 100644
--- a/src/fdb5/rados/RadosStore.h
+++ b/src/fdb5/rados/RadosStore.h
@@ -51,7 +51,7 @@ class RadosStore : public Store {
     bool exists() const override;
 
     eckit::DataHandle* retrieve(Field& field, Key& remapKey) const override;
-    std::future > archive(const Key& key, const void *data, eckit::Length length) override;
+    std::unique_ptr archive(const Key& key, const void *data, eckit::Length length) override;
 
     void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override;
 
diff --git a/src/fdb5/remote/RemoteCatalogue.cc b/src/fdb5/remote/RemoteCatalogue.cc
index 98011aaec..387bbe3c9 100644
--- a/src/fdb5/remote/RemoteCatalogue.cc
+++ b/src/fdb5/remote/RemoteCatalogue.cc
@@ -30,7 +30,7 @@ class CatalogueArchivalRequest {
     CatalogueArchivalRequest(uint32_t id, RemoteCatalogue* catalogue, const fdb5::Key& key, std::unique_ptr location) :
         id_(id), catalogue_(catalogue), key_(key), location_(std::move(location)) {}
 
-    std::pair > >  element;
+    std::pair>>  element;
 
     uint32_t id_;
     RemoteCatalogue* catalogue_;
@@ -77,7 +77,7 @@ class RemoteCatalogueArchiver {
 
 RemoteCatalogueArchiver* RemoteCatalogueArchiver::get(const eckit::net::Endpoint& endpoint) {
 
-    static std::map > archivers_;
+    static std::map> archivers_;
 
     auto it = archivers_.find(endpoint.hostport());
     if (it == archivers_.end()) {
diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index 85cbcaa13..4d2d07dc7 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -89,7 +89,7 @@ class RemoteStoreArchiver {
 
 RemoteStoreArchiver* RemoteStoreArchiver::get(const eckit::net::Endpoint& endpoint) {
 
-    static std::map > archivers_;
+    static std::map> archivers_;
 
     auto it = archivers_.find(endpoint.hostport());
     if (it == archivers_.end()) {
@@ -395,7 +395,7 @@ eckit::DataHandle* RemoteStore::retrieve(Field& field) const {
     return field.dataHandle();
 }
 
-std::future > RemoteStore::archive(const Key& key, const void *data, eckit::Length length) {
+void RemoteStore::archive(const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive) {
 
     // if there is no archiving thread active, then start one.
     // n.b. reset the archiveQueue_ after a potential flush() cycle.
@@ -409,10 +409,7 @@ std::future > RemoteStore::archive(const Key& key
     uint32_t id = controlWriteCheckResponse(Message::Archive, nullptr, 0);
     archiver_->emplace(id, this, key, data, length);
 
-    std::promise > loc;
-    auto futureLocation = loc.get_future();
-    locations_[id] = std::move(loc);
-    return futureLocation;
+    locations_[id] = catalogue_archive;
 }
 
 bool RemoteStore::open() {
@@ -496,11 +493,11 @@ bool RemoteStore::handle(Message message, uint32_t requestID, eckit::Buffer&& pa
                 MemoryStream s(payload);
                 std::unique_ptr location(eckit::Reanimator::reanimate(s));
                 if (local_) {
-                    it->second.set_value(std::move(location));
+                    it->second(std::move(location));
                 } else {
                     // std::cout <<  "RemoteStore::handle - " << location->uri().asRawString() << " " << location->length() << std::endl;
                     std::unique_ptr remoteLocation = std::unique_ptr(new RemoteFieldLocation(endpoint_, *location));
-                    it->second.set_value(std::move(remoteLocation));
+                    it->second(std::move(remoteLocation));
                 }
             }
             return true;
@@ -590,7 +587,7 @@ eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation, c
 
 RemoteStore& RemoteStore::get(const eckit::URI& uri) {
     // we memoise one read store for each endpoint. Do not need to have one for each key
-    static std::map > readStores_;
+    static std::map> readStores_;
 
     const std::string& endpoint = uri.hostport();
     auto it = readStores_.find(endpoint);
diff --git a/src/fdb5/remote/RemoteStore.h b/src/fdb5/remote/RemoteStore.h
index 9bf102de4..a0aff551a 100644
--- a/src/fdb5/remote/RemoteStore.h
+++ b/src/fdb5/remote/RemoteStore.h
@@ -71,7 +71,7 @@ class RemoteStore : public Store, public Client {
     bool exists() const override;
 
     eckit::DataHandle* retrieve(Field& field) const override;
-    std::future > archive(const Key& key, const void *data, eckit::Length length) override;
+    void archive(const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive) override;
 
     void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override;
 
@@ -92,7 +92,7 @@ class RemoteStore : public Store, public Client {
 
     const Config& config_;
 
-    std::map > > locations_;
+    std::map fieldLocation)>> locations_;
 
     // @note This is a map of requestID:MessageQueue. At the point that a request is
     // complete, errored or otherwise killed, it needs to be removed from the map.
diff --git a/src/fdb5/toc/TocCommon.cc b/src/fdb5/toc/TocCommon.cc
index e913bade4..e1531165d 100644
--- a/src/fdb5/toc/TocCommon.cc
+++ b/src/fdb5/toc/TocCommon.cc
@@ -45,7 +45,7 @@ void TocCommon::checkUID() const {
     }
 
     static std::vector fdbSuperUsers =
-        eckit::Resource >("fdbSuperUsers", "", true);
+        eckit::Resource>("fdbSuperUsers", "", true);
 
     if (dbUID() != userUID_) {
         if (std::find(fdbSuperUsers.begin(), fdbSuperUsers.end(), userName(userUID_)) ==
diff --git a/src/fdb5/toc/TocHandler.cc b/src/fdb5/toc/TocHandler.cc
index 61b646113..3b84c7cdf 100644
--- a/src/fdb5/toc/TocHandler.cc
+++ b/src/fdb5/toc/TocHandler.cc
@@ -201,7 +201,7 @@ void TocHandler::checkUID() const {
         return;
     }
 
-    static std::vector fdbSuperUsers = eckit::Resource >("fdbSuperUsers", "", true);
+    static std::vector fdbSuperUsers = eckit::Resource>("fdbSuperUsers", "", true);
 
     if (dbUID() != userUID_) {
 
diff --git a/src/fdb5/toc/TocStats.cc b/src/fdb5/toc/TocStats.cc
index 19cda5208..25a6e6061 100644
--- a/src/fdb5/toc/TocStats.cc
+++ b/src/fdb5/toc/TocStats.cc
@@ -176,7 +176,7 @@ TocDataStats& TocDataStats::operator+=(const TocDataStats& rhs) {
                    allDataFiles_.end(),
                    rhs.allDataFiles_.begin(),
                    rhs.allDataFiles_.end(),
-                   std::insert_iterator< std::set >(intersect, intersect.begin()));
+                   std::insert_iterator< std::set>(intersect, intersect.begin()));
 
     std::swap(allDataFiles_, intersect);
 
@@ -185,7 +185,7 @@ TocDataStats& TocDataStats::operator+=(const TocDataStats& rhs) {
                    activeDataFiles_.end(),
                    rhs.activeDataFiles_.begin(),
                    rhs.activeDataFiles_.end(),
-                   std::insert_iterator< std::set >(intersect, intersect.begin()));
+                   std::insert_iterator< std::set>(intersect, intersect.begin()));
 
     std::swap(activeDataFiles_, intersect);
 
diff --git a/src/fdb5/toc/TocStore.cc b/src/fdb5/toc/TocStore.cc
index 8c0764947..edd1426c3 100644
--- a/src/fdb5/toc/TocStore.cc
+++ b/src/fdb5/toc/TocStore.cc
@@ -52,7 +52,8 @@ eckit::DataHandle* TocStore::retrieve(Field& field) const {
     return field.dataHandle();
 }
 
-std::future > TocStore::archive(const Key& key, const void *data, eckit::Length length) {
+std::unique_ptr TocStore::archive(const Key& key, const void *data, eckit::Length length) {
+    
     dirty_ = true;
 
     eckit::PathName dataPath = getDataPath(key);
@@ -65,9 +66,7 @@ std::future > TocStore::archive(const Key& key, c
 
     ASSERT(len == length);
 
-    std::promise > loc;
-    loc.set_value(std::unique_ptr(new TocFieldLocation(dataPath, position, length, Key())));
-    return loc.get_future();
+    return std::unique_ptr(new TocFieldLocation(dataPath, position, length, Key()));
 }
 
 void TocStore::flush() {
diff --git a/src/fdb5/toc/TocStore.h b/src/fdb5/toc/TocStore.h
index 3391b6f95..30d7be60f 100644
--- a/src/fdb5/toc/TocStore.h
+++ b/src/fdb5/toc/TocStore.h
@@ -58,7 +58,7 @@ class TocStore : public Store, public TocCommon {
     bool exists() const override;
 
     eckit::DataHandle* retrieve(Field& field) const override;
-    std::future > archive(const Key& key, const void *data, eckit::Length length) override;
+    std::unique_ptr archive(const Key& key, const void *data, eckit::Length length) override;
 
     void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override;
 
diff --git a/src/fdb5/tools/FDBInspect.cc b/src/fdb5/tools/FDBInspect.cc
index e42e02b55..db5d88c58 100644
--- a/src/fdb5/tools/FDBInspect.cc
+++ b/src/fdb5/tools/FDBInspect.cc
@@ -35,7 +35,7 @@ FDBInspect::FDBInspect(int argc, char **argv, std::string minimumKeys) :
     FDBTool(argc, argv),
     fail_(true) {
 
-    minimumKeys_ = Resource >("FDBInspectMinimumKeys", minimumKeys, true);
+    minimumKeys_ = Resource>("FDBInspectMinimumKeys", minimumKeys, true);
 
     if(minimumKeys_.size() == 0) {
         options_.push_back(new eckit::option::SimpleOption("all", "Visit all FDB databases"));
diff --git a/src/fdb5/tools/FDBVisitTool.cc b/src/fdb5/tools/FDBVisitTool.cc
index 15b51d746..04c6cce9d 100644
--- a/src/fdb5/tools/FDBVisitTool.cc
+++ b/src/fdb5/tools/FDBVisitTool.cc
@@ -35,7 +35,7 @@ FDBVisitTool::FDBVisitTool(int argc, char **argv, std::string minimumKeys) :
     all_(false),
     raw_(false) {
 
-    minimumKeys_ = Resource >("FDBInspectMinimumKeys", minimumKeys, true);
+    minimumKeys_ = Resource>("FDBInspectMinimumKeys", minimumKeys, true);
 
     if(minimumKeys_.size() != 0) {
         options_.push_back(new VectorOption("minimum-keys",

From 2943da3b3ee4da84ace682406e7773a585da3818 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 23 Nov 2023 14:40:23 +0000
Subject: [PATCH 032/186] cleanup

---
 src/fdb5/api/FDB.cc | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/src/fdb5/api/FDB.cc b/src/fdb5/api/FDB.cc
index 34c98c2a3..f04547e36 100644
--- a/src/fdb5/api/FDB.cc
+++ b/src/fdb5/api/FDB.cc
@@ -100,7 +100,24 @@ void FDB::archive(const Key& key, const void* data, size_t length) {
     eckit::Timer timer;
     timer.start();
 
-    internal_->archive(key, data, length);
+    auto stepunit = key.find("stepunits");
+    if (stepunit != key.end()) {
+        Key k;
+        for (auto it : key) {
+            if (it.first == "step" && stepunit->second.size()>0 && stepunit->second[0]!='h') {
+                // TODO - enable canonical representation of step (as soon as Metkit supports it)
+                std::string canonicalStep = it.second+stepunit->second; // k.registry().lookupType("step").toKey("step", it.second+stepunit->second);
+                k.set(it.first, canonicalStep);
+            } else {
+                if (it.first != "stepunits") {
+                    k.set(it.first, it.second);
+                }
+            }
+        }
+        internal_->archive(k, data, length);
+    } else {
+        internal_->archive(key, data, length);
+    }
     dirty_ = true;
 
     timer.stop();

From c132accdf4a145e283c3f6cf1ab5372ddf81b370 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 23 Nov 2023 21:27:22 +0000
Subject: [PATCH 033/186] cleanup

---
 src/fdb5/api/RemoteFDB.cc                  | 37 +++++++++++++---------
 src/fdb5/remote/AvailablePortList.cc       |  2 +-
 src/fdb5/remote/RemoteStore.cc             | 25 +++++++++++++++
 src/fdb5/remote/client/Client.cc           |  2 +-
 src/fdb5/remote/client/ClientConnection.cc | 36 +++++++++++++++------
 src/fdb5/remote/client/ClientConnection.h  |  7 ++--
 6 files changed, 80 insertions(+), 29 deletions(-)

diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index 55fef8811..865da2a5b 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -69,31 +69,38 @@ RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name
     eckit::MemoryStream s(buf);
     size_t numStores;
     s >> numStores;
-    std::vector stores;
-    for (size_t i=0; i 0) {
+        std::vector stores;
+        for (size_t i=0; i> numStores;
-    std::vector localStores;
-    for (size_t i=0; i 0) {
+        std::vector localStores;
+        for (size_t i=0; i::reanimate(s);
 
     // eckit::Log::debug() << *this << " - Received Store endpoint: " << storeEndpoint_ << " and master schema: " << std::endl;
     // schema->dump(eckit::Log::debug());
     
     config_.overrideSchema(endpoint_.hostport()+"/schema", schema);
-    config_.set("stores", stores);
-    config_.set("localStores", localStores);
-    // config_.set("storeHost", storeEndpoint.host());
-    // config_.set("storePort", storeEndpoint.port());
 }
 
 // -----------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/AvailablePortList.cc b/src/fdb5/remote/AvailablePortList.cc
index 0e19eef15..03bd0c67f 100644
--- a/src/fdb5/remote/AvailablePortList.cc
+++ b/src/fdb5/remote/AvailablePortList.cc
@@ -53,7 +53,7 @@ std::set readServices() {
         return portsToSkip;
     }
 
-    eckit::Tokenizer parse(" /");
+    eckit::Tokenizer parse("\t /");
 
     eckit::Translator toInt;
 
diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index 4d2d07dc7..f31921bf3 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -324,6 +324,31 @@ class FDBRemoteDataHandle : public DataHandle {
     bool complete_;
 };
 
+// std::vector writeEndpoints(const Config& config) {
+//     if (config.has("storeHost") && config.has("storePort")) {
+//         return std::vector{eckit::net::Endpoint(config.getString("storeHost"), config.getInt("storePort"))};
+//     }
+//     ASSERT(config.has("stores"));
+//     std::vector stores = config.getSubConfigurations("stores");
+//     for (const auto& root : stores) {
+//         spaceRoots.emplace_back(
+//             Root(
+//                 root.getString("path"),
+//                 root.getString("name", ""),
+//                 root.getBool("list", visit),
+//                 root.getBool("retrieve", visit),
+//                 root.getBool("archive", writable),
+//                 root.getBool("wipe", writable)
+//             )
+//         );
+//     }
+
+//     .getStringVector("stores");
+
+//     std::srand(std::time(nullptr));
+//     return eckit::net::Endpoint(stores.at(std::rand() % stores.size()));
+// }
+
 eckit::net::Endpoint storeEndpoint(const Config& config) {
     if (config.has("storeHost") && config.has("storePort")) {
         return eckit::net::Endpoint(config.getString("storeHost"), config.getInt("storePort"));
diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index 692eb6826..589b2264b 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -83,7 +83,7 @@ eckit::Buffer Client::controlWriteReadResponse(Message msg, const void* payload,
 }
 
 void Client::dataWrite(remote::Message msg, uint32_t requestID, std::vector> data) {
-    connection_.dataWrite(msg, requestID, data);
+    connection_.dataWrite(*this, msg, requestID, data);
 }
 
 } // namespace fdb5::remote
\ No newline at end of file
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index cf31ed019..87cde35dc 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -84,24 +84,26 @@ uint32_t ClientConnection::generateRequestID() {
 
 void ClientConnection::connect() {
 
+    std::lock_guard lock(requestMutex_);
+
     if (connected_) {
         eckit::Log::warning() << "ClientConnection::connect() called when already connected" << std::endl;
         return;
     }
 
     static int fdbMaxConnectRetries = eckit::Resource("fdbMaxConnectRetries", 5);
+    static int fdbConnectTimeout = eckit::Resource("fdbConnectTimeout", 2); // No timeout
 
     try {
-        // Connect to server, and check that the server is happy on the response
-
+        // Connect to server, and check that the server is happy on the response 
         eckit::Log::debug() << "Connecting to host: " << controlEndpoint_ << std::endl;
-        controlClient_.connect(controlEndpoint_, fdbMaxConnectRetries);
+        controlClient_.connect(controlEndpoint_, fdbMaxConnectRetries, fdbConnectTimeout);
         writeControlStartupMessage();
         eckit::SessionID serverSession = verifyServerStartupResponse();
 
         // Connect to the specified data port
         eckit::Log::debug() << "Received data endpoint from host: " << dataEndpoint_ << std::endl;
-        dataClient_.connect(dataEndpoint_, fdbMaxConnectRetries);
+        dataClient_.connect(dataEndpoint_, fdbMaxConnectRetries, fdbConnectTimeout);
         writeDataStartupMessage(serverSession);
 
         // And the connections are set up. Let everything start up!
@@ -118,6 +120,9 @@ void ClientConnection::connect() {
 }
 
 void ClientConnection::disconnect() {
+
+    std::lock_guard lock(requestMutex_);
+
     if (connected_) {
 
         // Send termination message
@@ -148,11 +153,11 @@ eckit::LocalConfiguration ClientConnection::availableFunctionality() const {
 
 // -----------------------------------------------------------------------------------------------------
 
-void ClientConnection::addRequest(Client& client, uint32_t requestId) {
-    ASSERT(requestId);
+void ClientConnection::addRequest(Client& client, uint32_t requestID) {
+    ASSERT(requestID);
 
     std::lock_guard lock(requestMutex_);
-    requests_[requestId] = &client;
+    requests_[requestID] = &client;
 }
 
 void ClientConnection::controlWrite(Client& client, Message msg, uint32_t requestID, std::vector> data) {
@@ -168,6 +173,7 @@ void ClientConnection::controlWrite(Message msg, uint32_t requestID, std::vector
 
     uint32_t payloadLength = 0;
     for (auto d: data) {
+        ASSERT(d.first);
         payloadLength += d.second;
     }
     eckit::Log::debug() << "ClientConnection::controlWrite [endpoint=" << controlEndpoint_.hostport() <<
@@ -178,7 +184,6 @@ void ClientConnection::controlWrite(Message msg, uint32_t requestID, std::vector
     std::lock_guard lock(controlMutex_);
     controlWrite(&message, sizeof(message));
     for (auto d: data) {
-        ASSERT(d.first);
         controlWrite(d.first, d.second);
     }
     controlWrite(&EndMarker, sizeof(EndMarker));
@@ -203,9 +208,23 @@ void ClientConnection::controlRead(void* data, size_t length) {
     }
 }
 
+
+void ClientConnection::dataWrite(Client& client, remote::Message msg, uint32_t requestID, std::vector> data) {
+
+    if (requestID) {
+        auto it = requests_.find(requestID);
+        ASSERT(it != requests_.end());
+        ASSERT(it->second == &client);
+    }
+
+    dataWrite(msg, requestID, data);
+}
+
 void ClientConnection::dataWrite(remote::Message msg, uint32_t requestID, std::vector> data) {
+
     uint32_t payloadLength = 0;
     for (auto d: data) {
+        ASSERT(d.first);
         payloadLength += d.second;
     }
     MessageHeader message(msg, requestID, payloadLength);
@@ -217,7 +236,6 @@ void ClientConnection::dataWrite(remote::Message msg, uint32_t requestID, std::v
     dataWrite(&message, sizeof(message));
     for (auto d: data) {
         dataWrite(d.first, d.second);
-        payloadLength += d.second;
     }
     dataWrite(&EndMarker, sizeof(EndMarker));
 }
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index 45fe858ed..d98e1ee83 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -45,7 +45,7 @@ class ClientConnection : eckit::NonCopyable {
     virtual ~ClientConnection();
 
     void controlWrite(Client& client, remote::Message msg, uint32_t requestID, std::vector> data={});
-    void dataWrite(remote::Message msg, uint32_t requestID, std::vector> data={});
+    void dataWrite   (Client& client, remote::Message msg, uint32_t requestID, std::vector> data={});
 
     void connect();
     void disconnect();
@@ -65,10 +65,11 @@ class ClientConnection : eckit::NonCopyable {
     void controlWrite(Message msg, uint32_t requestID = 0, std::vector> data = {});
     void controlWrite(const void* data, size_t length);
     void controlRead (      void* data, size_t length);
+    void dataWrite   (Message msg, uint32_t requestID = 0, std::vector> data = {});
     void dataWrite   (const void* data, size_t length);
     void dataRead    (      void* data, size_t length);
  
-    void addRequest(Client& client, uint32_t requestId);
+    void addRequest(Client& client, uint32_t requestID);
 
     void writeControlStartupMessage();
     void writeDataStartupMessage(const eckit::SessionID& serverSession);
@@ -97,7 +98,7 @@ class ClientConnection : eckit::NonCopyable {
     std::mutex controlMutex_;
     std::mutex dataMutex_;
 
-    // requestId
+    // requestID
     std::mutex idMutex_;
     uint32_t id_;
 

From 4adcf4121298a8b016e2215d53f51dace8eb2fed Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 24 Nov 2023 09:41:31 +0000
Subject: [PATCH 034/186] wip

---
 src/fdb5/remote/RemoteStore.cc             |  3 +--
 src/fdb5/remote/client/Client.cc           | 14 +++++++++-----
 src/fdb5/remote/client/Client.h            |  2 ++
 src/fdb5/remote/client/ClientConnection.cc |  4 ++--
 4 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index f31921bf3..c0e469d65 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -432,9 +432,8 @@ void RemoteStore::archive(const Key& key, const void *data, eckit::Length length
     ASSERT(archiver_->valid());
 
     uint32_t id = controlWriteCheckResponse(Message::Archive, nullptr, 0);
-    archiver_->emplace(id, this, key, data, length);
-
     locations_[id] = catalogue_archive;
+    archiver_->emplace(id, this, key, data, length);
 }
 
 bool RemoteStore::open() {
diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index 589b2264b..10b1d21f8 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -29,7 +29,6 @@ Client::~Client() {
 bool Client::response(uint32_t requestID) {
     ASSERT(requestID == blockingRequestId_);
 
-
     promise_.set_value(true);
     return true;
 }
@@ -48,11 +47,13 @@ uint32_t Client::controlWriteCheckResponse(Message msg, const void* payload, uin
 }
 
 void Client::controlWriteCheckResponse(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
-    ASSERT(!blockingRequestId_);
-    ASSERT(requestID);
 
+    ASSERT(requestID);
+    std::lock_guard lock(blockingRequestMutex_);
+    
     promise_ = {};
     std::future f = promise_.get_future();
+
     blockingRequestId_=requestID;
 
     if (payloadLength) {
@@ -65,12 +66,14 @@ void Client::controlWriteCheckResponse(Message msg, uint32_t requestID, const vo
     blockingRequestId_=0;
 }
 eckit::Buffer Client::controlWriteReadResponse(Message msg, const void* payload, uint32_t payloadLength) {
-    ASSERT(!blockingRequestId_);
+
+    std::lock_guard lock(blockingRequestMutex_);
 
     payloadPromise_ = {};
-    blockingRequestId_=connection_.generateRequestID();
     std::future f = payloadPromise_.get_future();
 
+    blockingRequestId_=connection_.generateRequestID();
+
     if (payloadLength) {
         connection_.controlWrite(*this, msg, blockingRequestId_, std::vector>{{payload, payloadLength}});
     } else {
@@ -79,6 +82,7 @@ eckit::Buffer Client::controlWriteReadResponse(Message msg, const void* payload,
 
     eckit::Buffer buf = f.get();
     blockingRequestId_=0;
+
     return buf;
 }
 
diff --git a/src/fdb5/remote/client/Client.h b/src/fdb5/remote/client/Client.h
index 5e01e2900..506961a50 100644
--- a/src/fdb5/remote/client/Client.h
+++ b/src/fdb5/remote/client/Client.h
@@ -65,6 +65,8 @@ class Client : eckit::NonCopyable {
 
 private:
 
+    std::mutex blockingRequestMutex_;
+
     uint32_t blockingRequestId_;
     std::promise promise_;
     std::promise payloadPromise_;
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index 87cde35dc..38147f142 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -267,7 +267,7 @@ void ClientConnection::handleError(const MessageHeader& hdr) {
         ASSERT(hdr.payloadSize > 9);
 
         std::string what(hdr.payloadSize, ' ');
-        controlRead(&what[0], hdr.payloadSize);
+        dataRead(&what[0], hdr.payloadSize);
         what[hdr.payloadSize] = 0; // Just in case
 
         try {
@@ -358,7 +358,7 @@ void ClientConnection::listeningThreadLoop() {
 
             dataRead(&hdr, sizeof(hdr));
 
-            eckit::Log::debug() << "ClientConnection::listeningThreadLoop - got [message=" << ((int) hdr.message) << ",requestID=" << hdr.requestID << "]" << std::endl;
+            eckit::Log::debug() << "ClientConnection::listeningThreadLoop - got [message=" << ((int) hdr.message) << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
 
             ASSERT(hdr.marker == StartMarker);
             ASSERT(hdr.version == CurrentVersion);

From 20e580a490cbd2c5dde8e9382ac0d1f18175c8ab Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 24 Nov 2023 19:54:02 +0000
Subject: [PATCH 035/186] fix remote archiver

---
 src/fdb5/CMakeLists.txt            |  3 +++
 src/fdb5/remote/RemoteCatalogue.cc | 11 +++++------
 src/fdb5/remote/RemoteStore.cc     | 13 +++++++------
 src/fdb5/remote/client/Client.cc   |  6 ++++++
 4 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt
index a9bd1380a..13b1f346a 100644
--- a/src/fdb5/CMakeLists.txt
+++ b/src/fdb5/CMakeLists.txt
@@ -1,5 +1,8 @@
 ecbuild_generate_config_headers( DESTINATION ${INSTALL_INCLUDE_DIR}/fdb5 )
 
+SET(UNINITIALIZED_COMPILE_FLAGS " -Wuninitialized -Wmaybe-uninitialized ")
+SET(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} ${UNINITIALIZED_COMPILE_FLAGS}")
+
 configure_file( fdb5_config.h.in    fdb5_config.h   )
 configure_file( fdb5_version.h.in   fdb5_version.h  )
 configure_file( fdb5_version.c.in   ${CMAKE_CURRENT_BINARY_DIR}/fdb5_version.c   )
diff --git a/src/fdb5/remote/RemoteCatalogue.cc b/src/fdb5/remote/RemoteCatalogue.cc
index 387bbe3c9..5aa6d87b5 100644
--- a/src/fdb5/remote/RemoteCatalogue.cc
+++ b/src/fdb5/remote/RemoteCatalogue.cc
@@ -78,13 +78,14 @@ class RemoteCatalogueArchiver {
 RemoteCatalogueArchiver* RemoteCatalogueArchiver::get(const eckit::net::Endpoint& endpoint) {
 
     static std::map> archivers_;
+    static std::mutex getArchiverMutex_;
+
+    std::lock_guard lock(getArchiverMutex_);
 
     auto it = archivers_.find(endpoint.hostport());
     if (it == archivers_.end()) {
-        // auto arch = (archivers_[endpoint.hostport()] = RemoteCatalogueArchiver());
         it = archivers_.emplace(endpoint.hostport(), new RemoteCatalogueArchiver()).first;
     }
-    ASSERT(it != archivers_.end());
     return it->second.get();
 }
 
@@ -194,13 +195,13 @@ FDBStats RemoteCatalogueArchiver::archiveThreadLoop() {
 RemoteCatalogue::RemoteCatalogue(const Key& key, const Config& config):
     CatalogueImpl(key, ControlIdentifiers(), config), // xxx what are control identifiers? Setting empty here...
     Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port"))),
-    config_(config), schema_(nullptr) {
+    config_(config), schema_(nullptr), archiver_(nullptr) {
 
     loadSchema();
 }
 
 RemoteCatalogue::RemoteCatalogue(const eckit::URI& uri, const Config& config):
-    Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port"))), config_(config), schema_(nullptr)
+    Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port"))), config_(config), schema_(nullptr), archiver_(nullptr)
     {
         NOTIMP;
     }
@@ -237,9 +238,7 @@ void RemoteCatalogue::archive(const InspectionKey& key, std::unique_ptrstart();
-
     ASSERT(archiver_->valid());
 
     uint32_t id = controlWriteCheckResponse(Message::Archive, nullptr, 0);
diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index c0e469d65..671d1ddcc 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -90,13 +90,14 @@ class RemoteStoreArchiver {
 RemoteStoreArchiver* RemoteStoreArchiver::get(const eckit::net::Endpoint& endpoint) {
 
     static std::map> archivers_;
+    static std::mutex getArchiverMutex_;
+
+    std::lock_guard lock(getArchiverMutex_);
 
     auto it = archivers_.find(endpoint.hostport());
     if (it == archivers_.end()) {
-        // auto arch = (archivers_[endpoint.hostport()] = RemoteStoreArchiver());
         it = archivers_.emplace(endpoint.hostport(), new RemoteStoreArchiver()).first;
     }
-    ASSERT(it != archivers_.end());
     return it->second.get();
 }
 
@@ -355,7 +356,6 @@ eckit::net::Endpoint storeEndpoint(const Config& config) {
     }
     ASSERT(config.has("stores"));
     std::vector stores = config.getStringVector("stores");
-    std::srand(std::time(nullptr));
     return eckit::net::Endpoint(stores.at(std::rand() % stores.size()));
 }
 
@@ -364,7 +364,8 @@ eckit::net::Endpoint storeEndpoint(const Config& config) {
 RemoteStore::RemoteStore(const Key& dbKey, const Config& config) :
     Client(storeEndpoint(config)),
     dbKey_(dbKey), config_(config),
-    retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)) {
+    retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
+    archiver_(nullptr) {
 
     if (config.has("localStores")) {
         for (const std::string& localStore : config.getStringVector("localStores")) {
@@ -379,7 +380,8 @@ RemoteStore::RemoteStore(const Key& dbKey, const Config& config) :
 RemoteStore::RemoteStore(const eckit::URI& uri, const Config& config) :
     Client(eckit::net::Endpoint(uri.hostport())),
     dbKey_(Key()), config_(config),
-    retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)) {
+    retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
+    archiver_(nullptr) {
 
     // no need to set the local_ flag on the read path
 
@@ -427,7 +429,6 @@ void RemoteStore::archive(const Key& key, const void *data, eckit::Length length
     if (!archiver_) {
         archiver_ = RemoteStoreArchiver::get(controlEndpoint());
     }
-    ASSERT(archiver_);
     archiver_->start();
     ASSERT(archiver_->valid());
 
diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index 10b1d21f8..42fb383ba 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -10,6 +10,8 @@
 
 #include 
 
+#include "fdb5/LibFdb5.h"
+
 #include "fdb5/remote/client/Client.h"
 #include "fdb5/remote/client/ClientConnectionRouter.h"
 
@@ -29,6 +31,8 @@ Client::~Client() {
 bool Client::response(uint32_t requestID) {
     ASSERT(requestID == blockingRequestId_);
 
+    eckit::Log::debug() << " response to blocking request " << requestID << std::endl;
+
     promise_.set_value(true);
     return true;
 }
@@ -36,6 +40,8 @@ bool Client::response(uint32_t requestID) {
 bool Client::response(uint32_t requestID, eckit::Buffer&& payload) {
     ASSERT(requestID == blockingRequestId_);
 
+    eckit::Log::debug() << " response to blocking request " << requestID << " - payload size: " << payload.size() << std::endl;
+
     payloadPromise_.set_value(std::move(payload));
     return true;
 }

From 08a4b86495d5a5465731263b9763fff0f41a00d1 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 27 Nov 2023 11:37:45 +0000
Subject: [PATCH 036/186] cleanup

---
 src/fdb5/CMakeLists.txt            |  3 ---
 src/fdb5/remote/RemoteCatalogue.cc |  2 +-
 src/fdb5/remote/RemoteStore.cc     |  2 +-
 src/fdb5/remote/client/Client.h    | 25 ++++++++++++++-----------
 4 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt
index 13b1f346a..a9bd1380a 100644
--- a/src/fdb5/CMakeLists.txt
+++ b/src/fdb5/CMakeLists.txt
@@ -1,8 +1,5 @@
 ecbuild_generate_config_headers( DESTINATION ${INSTALL_INCLUDE_DIR}/fdb5 )
 
-SET(UNINITIALIZED_COMPILE_FLAGS " -Wuninitialized -Wmaybe-uninitialized ")
-SET(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} ${UNINITIALIZED_COMPILE_FLAGS}")
-
 configure_file( fdb5_config.h.in    fdb5_config.h   )
 configure_file( fdb5_version.h.in   fdb5_version.h  )
 configure_file( fdb5_version.c.in   ${CMAKE_CURRENT_BINARY_DIR}/fdb5_version.c   )
diff --git a/src/fdb5/remote/RemoteCatalogue.cc b/src/fdb5/remote/RemoteCatalogue.cc
index 5aa6d87b5..912112a9a 100644
--- a/src/fdb5/remote/RemoteCatalogue.cc
+++ b/src/fdb5/remote/RemoteCatalogue.cc
@@ -241,7 +241,7 @@ void RemoteCatalogue::archive(const InspectionKey& key, std::unique_ptrstart();
     ASSERT(archiver_->valid());
 
-    uint32_t id = controlWriteCheckResponse(Message::Archive, nullptr, 0);
+    uint32_t id = controlWriteCheckResponse(Message::Archive);
     eckit::Log::debug() << " RemoteCatalogue::archive - adding to queue [id=" << id << ",key=" << key << ",fieldLocation=" << fieldLocation->uri() << "]" << std::endl;
 
     archiver_->emplace(id, this, key, std::move(fieldLocation));
diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index 671d1ddcc..4af809042 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -432,7 +432,7 @@ void RemoteStore::archive(const Key& key, const void *data, eckit::Length length
     archiver_->start();
     ASSERT(archiver_->valid());
 
-    uint32_t id = controlWriteCheckResponse(Message::Archive, nullptr, 0);
+    uint32_t id = controlWriteCheckResponse(Message::Archive);
     locations_[id] = catalogue_archive;
     archiver_->emplace(id, this, key, data, length);
 }
diff --git a/src/fdb5/remote/client/Client.h b/src/fdb5/remote/client/Client.h
index 506961a50..3677f9c02 100644
--- a/src/fdb5/remote/client/Client.h
+++ b/src/fdb5/remote/client/Client.h
@@ -39,34 +39,37 @@ class Client : eckit::NonCopyable {
 
     const eckit::net::Endpoint& controlEndpoint() const { return endpoint_; }
 
-    // uint32_t generateRequestID();
-
     uint32_t controlWriteCheckResponse(Message msg,                     const void* payload=nullptr, uint32_t payloadLength=0);
     void     controlWriteCheckResponse(Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
     eckit::Buffer controlWriteReadResponse(Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
 
     void dataWrite(remote::Message msg, uint32_t requestID, std::vector> data={});
     
-    // handlers for incoming messages - to be defined in the client class
+    friend class ClientConnection;
+
+protected:
+
+    // handlers for incoming messages - to be implemented in the client class
     virtual bool handle(Message message, uint32_t requestID) = 0;
-    virtual bool handle(Message message, uint32_t requestID, /*eckit::net::Endpoint endpoint,*/ eckit::Buffer&& payload) = 0;
+    virtual bool handle(Message message, uint32_t requestID, eckit::Buffer&& payload) = 0;
     virtual void handleException(std::exception_ptr e) = 0;
 
-    bool response(uint32_t requestID);
-    bool response(uint32_t requestID, eckit::Buffer&& payload);
-    uint32_t blockingRequestId() { return blockingRequestId_; }
-
 protected:
-    
-    eckit::net::Endpoint endpoint_;
 
+    eckit::net::Endpoint endpoint_;
     ClientConnection& connection_;
 
+private:
+
+    // for blocking requests
+    bool response(uint32_t requestID);
+    bool response(uint32_t requestID, eckit::Buffer&& payload);
+    uint32_t blockingRequestId() { return blockingRequestId_; }
 
 private:
 
+    // for blocking requests
     std::mutex blockingRequestMutex_;
-
     uint32_t blockingRequestId_;
     std::promise promise_;
     std::promise payloadPromise_;

From 399a3447b9eecdde874e48c5cba381c3c8f298b1 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 27 Nov 2023 11:38:54 +0000
Subject: [PATCH 037/186] version bump

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 8fbbecb59..288942e4e 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.25
+5.11.91

From 25724842fa222735a944e597b0611df3f61b1912 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 28 Nov 2023 10:50:10 +0000
Subject: [PATCH 038/186] reduced verbosity on archival

---
 src/fdb5/remote/RemoteCatalogue.cc         | 14 ++--
 src/fdb5/remote/RemoteStore.cc             | 15 ++--
 src/fdb5/remote/client/ClientConnection.cc | 82 ++++++++++++----------
 3 files changed, 66 insertions(+), 45 deletions(-)

diff --git a/src/fdb5/remote/RemoteCatalogue.cc b/src/fdb5/remote/RemoteCatalogue.cc
index 912112a9a..117e51b15 100644
--- a/src/fdb5/remote/RemoteCatalogue.cc
+++ b/src/fdb5/remote/RemoteCatalogue.cc
@@ -145,6 +145,7 @@ FDBStats RemoteCatalogueArchiver::flush(RemoteCatalogue* catalogue) {
     MemoryStream s(sendBuf);
     s << numArchive;
 
+    eckit::Log::debug() << " RemoteCatalogue::flush - flushing " << numArchive << " fields" << std::endl;
     // The flush call is blocking
     catalogue->controlWriteCheckResponse(Message::Flush, sendBuf, s.position());
 
@@ -238,12 +239,15 @@ void RemoteCatalogue::archive(const InspectionKey& key, std::unique_ptrstart();
-    ASSERT(archiver_->valid());
-
-    uint32_t id = controlWriteCheckResponse(Message::Archive);
-    eckit::Log::debug() << " RemoteCatalogue::archive - adding to queue [id=" << id << ",key=" << key << ",fieldLocation=" << fieldLocation->uri() << "]" << std::endl;
+    uint32_t id = connection_.generateRequestID();
+    if (!archiver_->valid()) {
+        archiver_->start();
+        ASSERT(archiver_->valid());
 
+        // std::cout << "Catalogue - controlWriteCheckResponse(Message::Archive)" << std::endl;
+        controlWriteCheckResponse(Message::Archive, id);
+    }
+    // eckit::Log::debug() << " RemoteCatalogue::archive - adding to queue [id=" << id << ",key=" << key << ",fieldLocation=" << fieldLocation->uri() << "]" << std::endl;
     archiver_->emplace(id, this, key, std::move(fieldLocation));
 }
 
diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index 4af809042..90d26ab1a 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -157,6 +157,7 @@ FDBStats RemoteStoreArchiver::flush(RemoteStore* store) {
     MemoryStream s(sendBuf);
     s << numArchive;
 
+    eckit::Log::debug() << " RemoteStoreArchiver::flush - flushing " << numArchive << " fields" << std::endl;
     // The flush call is blocking
     store->controlWriteCheckResponse(Message::Flush, sendBuf, s.position());
 
@@ -367,6 +368,7 @@ RemoteStore::RemoteStore(const Key& dbKey, const Config& config) :
     retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
     archiver_(nullptr) {
 
+    local_ = false;
     if (config.has("localStores")) {
         for (const std::string& localStore : config.getStringVector("localStores")) {
             if (localStore == endpoint_.hostport()) {
@@ -429,10 +431,14 @@ void RemoteStore::archive(const Key& key, const void *data, eckit::Length length
     if (!archiver_) {
         archiver_ = RemoteStoreArchiver::get(controlEndpoint());
     }
-    archiver_->start();
-    ASSERT(archiver_->valid());
+    uint32_t id = connection_.generateRequestID();
+    if (!archiver_->valid()) {
+        archiver_->start();
+        ASSERT(archiver_->valid());
 
-    uint32_t id = controlWriteCheckResponse(Message::Archive);
+        // std::cout << "controlWriteCheckResponse(Message::Archive)" << std::endl;
+        controlWriteCheckResponse(Message::Archive, id);
+    }
     locations_[id] = catalogue_archive;
     archiver_->emplace(id, this, key, data, length);
 }
@@ -518,10 +524,11 @@ bool RemoteStore::handle(Message message, uint32_t requestID, eckit::Buffer&& pa
                 MemoryStream s(payload);
                 std::unique_ptr location(eckit::Reanimator::reanimate(s));
                 if (local_) {
+                    // std::cout << "LOCAL: " << location->uri().asRawString() << std::endl;
                     it->second(std::move(location));
                 } else {
-                    // std::cout <<  "RemoteStore::handle - " << location->uri().asRawString() << " " << location->length() << std::endl;
                     std::unique_ptr remoteLocation = std::unique_ptr(new RemoteFieldLocation(endpoint_, *location));
+                    // std::cout << "REMOTE: " << remoteLocation->uri().asRawString() << std::endl;
                     it->second(std::move(remoteLocation));
                 }
             }
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index 38147f142..1c86f7a29 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -92,7 +92,7 @@ void ClientConnection::connect() {
     }
 
     static int fdbMaxConnectRetries = eckit::Resource("fdbMaxConnectRetries", 5);
-    static int fdbConnectTimeout = eckit::Resource("fdbConnectTimeout", 2); // No timeout
+    static int fdbConnectTimeout = eckit::Resource("fdbConnectTimeout", 0); // No timeout
 
     try {
         // Connect to server, and check that the server is happy on the response 
@@ -213,8 +213,11 @@ void ClientConnection::dataWrite(Client& client, remote::Message msg, uint32_t r
 
     if (requestID) {
         auto it = requests_.find(requestID);
-        ASSERT(it != requests_.end());
-        ASSERT(it->second == &client);
+        if (it != requests_.end()) {
+            ASSERT(it->second == &client);
+        } else {
+            addRequest(client, requestID);
+        }
     }
 
     dataWrite(msg, requestID, data);
@@ -367,43 +370,50 @@ void ClientConnection::listeningThreadLoop() {
                 return;
             }
 
-            bool handled = false;
-            auto it = requests_.find(hdr.requestID);
-            if (it == requests_.end()) {
-                std::stringstream ss;
-                ss << "ERROR: Unexpected answer to requestID recieved (" << hdr.requestID << "). ABORTING";
-                eckit::Log::status() << ss.str() << std::endl;
-                eckit::Log::error() << "Retrieving... " << ss.str() << std::endl;
-                throw eckit::SeriousBug(ss.str(), Here());
-
-                ASSERT(false); // todo report the error
-            }
+            if (hdr.requestID) {
+                bool handled = false;
+                auto it = requests_.find(hdr.requestID);
+                if (it == requests_.end()) {
+                    std::stringstream ss;
+                    ss << "ERROR: Unexpected answer to requestID recieved (" << hdr.requestID << "). ABORTING";
+                    eckit::Log::status() << ss.str() << std::endl;
+                    eckit::Log::error() << "Retrieving... " << ss.str() << std::endl;
+                    throw eckit::SeriousBug(ss.str(), Here());
+
+                    ASSERT(false); // todo report the error
+                }
 
-            if (hdr.payloadSize == 0) {
-                if (it->second->blockingRequestId() == hdr.requestID) {
-                    ASSERT(hdr.message == Message::Received);
-                    handled = it->second->response(hdr.requestID);
-                } else {
-                    handled = it->second->handle(hdr.message, hdr.requestID);
+                if (hdr.payloadSize == 0) {
+                    if (it->second->blockingRequestId() == hdr.requestID) {
+                        ASSERT(hdr.message == Message::Received);
+                        handled = it->second->response(hdr.requestID);
+                    } else {
+                        handled = it->second->handle(hdr.message, hdr.requestID);
+                    }
                 }
-            }
-            else {
-                eckit::Buffer payload{hdr.payloadSize};
-                dataRead(payload, hdr.payloadSize);
-
-                if (it->second->blockingRequestId() == hdr.requestID) {
-                    handled = it->second->response(hdr.requestID, std::move(payload));
-                } else {
-                    handled = it->second->handle(hdr.message, hdr.requestID, std::move(payload));
+                else {
+                    eckit::Buffer payload{hdr.payloadSize};
+                    dataRead(payload, hdr.payloadSize);
+
+                    if (it->second->blockingRequestId() == hdr.requestID) {
+                        handled = it->second->response(hdr.requestID, std::move(payload));
+                    } else {
+                        handled = it->second->handle(hdr.message, hdr.requestID, std::move(payload));
+                    }
                 }
-            }
 
-            if (!handled) {
-                std::stringstream ss;
-                ss << "ERROR: Unexpected message recieved (" << static_cast(hdr.message) << "). ABORTING";
-                eckit::Log::status() << ss.str() << std::endl;
-                eckit::Log::error() << "Client Retrieving... " << ss.str() << std::endl;
-                throw eckit::SeriousBug(ss.str(), Here());
+                if (!handled) {
+                    std::stringstream ss;
+                    ss << "ERROR: Unexpected message recieved (" << static_cast(hdr.message) << "). ABORTING";
+                    eckit::Log::status() << ss.str() << std::endl;
+                    eckit::Log::error() << "Client Retrieving... " << ss.str() << std::endl;
+                    throw eckit::SeriousBug(ss.str(), Here());
+                }
+            } else {
+                if (hdr.payloadSize) {
+                    eckit::Buffer payload{hdr.payloadSize};
+                    dataRead(payload, hdr.payloadSize);
+                }
             }
 
             // Ensure we have consumed exactly the correct amount from the socket.

From 00c3527db27c247755a523a6c87670a4e77b2ebe Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 29 Nov 2023 07:10:10 +0000
Subject: [PATCH 039/186] wip

---
 src/fdb5/api/RemoteFDB.cc          |  3 ++-
 src/fdb5/remote/RemoteCatalogue.cc |  7 ++++---
 src/fdb5/remote/RemoteStore.cc     | 17 ++++-------------
 src/fdb5/remote/RemoteStore.h      |  1 -
 src/fdb5/remote/client/Client.cc   | 15 ++++++++-------
 src/fdb5/remote/client/Client.h    | 30 ++++++++++++------------------
 6 files changed, 30 insertions(+), 43 deletions(-)

diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index 865da2a5b..fdb116c12 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -65,7 +65,8 @@ RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name
     LocalFDB(config, name),
     Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port"))) {
 
-    eckit::Buffer buf = controlWriteReadResponse(remote::Message::Schema);
+    uint32_t id = generateRequestID();
+    eckit::Buffer buf = controlWriteReadResponse(remote::Message::Schema, id);
     eckit::MemoryStream s(buf);
     size_t numStores;
     s >> numStores;
diff --git a/src/fdb5/remote/RemoteCatalogue.cc b/src/fdb5/remote/RemoteCatalogue.cc
index 117e51b15..919998fab 100644
--- a/src/fdb5/remote/RemoteCatalogue.cc
+++ b/src/fdb5/remote/RemoteCatalogue.cc
@@ -124,7 +124,6 @@ void RemoteCatalogueArchiver::emplace(uint32_t id, RemoteCatalogue* catalogue, c
 
     dirty_ = true;
 
-    std::lock_guard lock(archiveQueuePtrMutex_);
     ASSERT(archiveQueue_);
     archiveQueue_->emplace(id, catalogue, key, std::move(location));
 }
@@ -146,8 +145,9 @@ FDBStats RemoteCatalogueArchiver::flush(RemoteCatalogue* catalogue) {
     s << numArchive;
 
     eckit::Log::debug() << " RemoteCatalogue::flush - flushing " << numArchive << " fields" << std::endl;
+    uint32_t id = catalogue->generateRequestID();
     // The flush call is blocking
-    catalogue->controlWriteCheckResponse(Message::Flush, sendBuf, s.position());
+    catalogue->controlWriteCheckResponse(Message::Flush, id, sendBuf, s.position());
 
     dirty_ = false;
 
@@ -312,7 +312,8 @@ void RemoteCatalogue::loadSchema() {
         eckit::MemoryStream keyStream(keyBuffer);
         keyStream << dbKey_;
         
-        eckit::Buffer buf = controlWriteReadResponse(Message::Schema, keyBuffer, keyStream.position());
+        uint32_t id = generateRequestID();
+        eckit::Buffer buf = controlWriteReadResponse(Message::Schema, id, keyBuffer, keyStream.position());
         eckit::MemoryStream s(buf);
 //        eckit::net::Endpoint storeEndpoint_{s};
         schema_.reset(eckit::Reanimator::reanimate(s));
diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index 90d26ab1a..875b1252c 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -136,7 +136,6 @@ void RemoteStoreArchiver::emplace(uint32_t id, RemoteStore* store, const Key& ke
 
     dirty_ = true;
 
-    std::lock_guard lock(archiveQueuePtrMutex_);
     ASSERT(archiveQueue_);
     archiveQueue_->emplace(id, store, key, data, length);
 }
@@ -159,7 +158,8 @@ FDBStats RemoteStoreArchiver::flush(RemoteStore* store) {
 
     eckit::Log::debug() << " RemoteStoreArchiver::flush - flushing " << numArchive << " fields" << std::endl;
     // The flush call is blocking
-    store->controlWriteCheckResponse(Message::Flush, sendBuf, s.position());
+    uint32_t id = store->generateRequestID();
+    store->controlWriteCheckResponse(Message::Flush, id, sendBuf, s.position());
 
     dirty_ = false;
 
@@ -386,16 +386,6 @@ RemoteStore::RemoteStore(const eckit::URI& uri, const Config& config) :
     archiver_(nullptr) {
 
     // no need to set the local_ flag on the read path
-
-    // if (config.has("localStores")) {
-    //     for (const std::string& localStore : config.getStringVector("localStores")) {
-    //         if (localStore == endpoint_.hostport()) {
-    //             local_ = true;
-    //             break;
-    //         }
-    //     }
-    // }
-
     ASSERT(uri.scheme() == "fdb");
 }
 
@@ -612,7 +602,8 @@ eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation, c
     s << fieldLocation;
     s << remapKey;
 
-    uint32_t id = controlWriteCheckResponse(fdb5::remote::Message::Read, encodeBuffer, s.position());
+    uint32_t id = connection_.generateRequestID();
+    controlWriteCheckResponse(fdb5::remote::Message::Read, id, encodeBuffer, s.position());
 
     return new FDBRemoteDataHandle(id, retrieveMessageQueue_, endpoint_);
 }
diff --git a/src/fdb5/remote/RemoteStore.h b/src/fdb5/remote/RemoteStore.h
index a0aff551a..79884dbe8 100644
--- a/src/fdb5/remote/RemoteStore.h
+++ b/src/fdb5/remote/RemoteStore.h
@@ -38,7 +38,6 @@ class RemoteStore : public Store, public Client {
 public: // methods
 
     RemoteStore(const Key& key, const Config& config);
-//    RemoteStore(const Key& key, const Config& config, const eckit::net::Endpoint& controlEndpoint);
     RemoteStore(const eckit::URI& uri, const Config& config);
 
     ~RemoteStore() override;
diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index 42fb383ba..df933b14f 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -46,11 +46,11 @@ bool Client::response(uint32_t requestID, eckit::Buffer&& payload) {
     return true;
 }
 
-uint32_t Client::controlWriteCheckResponse(Message msg, const void* payload, uint32_t payloadLength) {
-    uint32_t id = connection_.generateRequestID();
-    controlWriteCheckResponse(msg, id, payload, payloadLength);
-    return id;
-}
+// uint32_t Client::controlWriteCheckResponse(Message msg, const void* payload, uint32_t payloadLength) {
+//     uint32_t id = connection_.generateRequestID();
+//     controlWriteCheckResponse(msg, id, payload, payloadLength);
+//     return id;
+// }
 
 void Client::controlWriteCheckResponse(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
 
@@ -71,14 +71,15 @@ void Client::controlWriteCheckResponse(Message msg, uint32_t requestID, const vo
     f.get();
     blockingRequestId_=0;
 }
-eckit::Buffer Client::controlWriteReadResponse(Message msg, const void* payload, uint32_t payloadLength) {
+
+eckit::Buffer Client::controlWriteReadResponse(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
 
     std::lock_guard lock(blockingRequestMutex_);
 
     payloadPromise_ = {};
     std::future f = payloadPromise_.get_future();
 
-    blockingRequestId_=connection_.generateRequestID();
+    blockingRequestId_=requestID;
 
     if (payloadLength) {
         connection_.controlWrite(*this, msg, blockingRequestId_, std::vector>{{payload, payloadLength}});
diff --git a/src/fdb5/remote/client/Client.h b/src/fdb5/remote/client/Client.h
index 3677f9c02..8a40eb1f9 100644
--- a/src/fdb5/remote/client/Client.h
+++ b/src/fdb5/remote/client/Client.h
@@ -39,36 +39,30 @@ class Client : eckit::NonCopyable {
 
     const eckit::net::Endpoint& controlEndpoint() const { return endpoint_; }
 
-    uint32_t controlWriteCheckResponse(Message msg,                     const void* payload=nullptr, uint32_t payloadLength=0);
-    void     controlWriteCheckResponse(Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
-    eckit::Buffer controlWriteReadResponse(Message msg, const void* payload=nullptr, uint32_t payloadLength=0);
+    uint32_t generateRequestID() { return connection_.generateRequestID(); }
+    
+    // uint32_t controlWriteCheckResponse(Message msg,                     const void* payload=nullptr, uint32_t payloadLength=0);
+    void          controlWriteCheckResponse(Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
+    eckit::Buffer controlWriteReadResponse (Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
 
     void dataWrite(remote::Message msg, uint32_t requestID, std::vector> data={});
     
-    friend class ClientConnection;
-
-protected:
-
-    // handlers for incoming messages - to be implemented in the client class
+    // handlers for incoming messages - to be defined in the client class
     virtual bool handle(Message message, uint32_t requestID) = 0;
-    virtual bool handle(Message message, uint32_t requestID, eckit::Buffer&& payload) = 0;
+    virtual bool handle(Message message, uint32_t requestID, /*eckit::net::Endpoint endpoint,*/ eckit::Buffer&& payload) = 0;
     virtual void handleException(std::exception_ptr e) = 0;
 
-protected:
-
-    eckit::net::Endpoint endpoint_;
-    ClientConnection& connection_;
-
-private:
-
-    // for blocking requests
     bool response(uint32_t requestID);
     bool response(uint32_t requestID, eckit::Buffer&& payload);
     uint32_t blockingRequestId() { return blockingRequestId_; }
 
+protected:
+    
+    eckit::net::Endpoint endpoint_;
+    ClientConnection& connection_;
+
 private:
 
-    // for blocking requests
     std::mutex blockingRequestMutex_;
     uint32_t blockingRequestId_;
     std::promise promise_;

From fc61ce197606d3f35a475e43633dd2f46a25dc74 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Sun, 3 Dec 2023 08:05:24 +0100
Subject: [PATCH 040/186] fdb remote - connection demultiplexing

---
 src/fdb5/remote/Messages.h                 | 14 ++--
 src/fdb5/remote/RemoteCatalogue.cc         |  2 +-
 src/fdb5/remote/client/Client.cc           |  8 ++-
 src/fdb5/remote/client/Client.h            |  9 ++-
 src/fdb5/remote/client/ClientConnection.cc | 22 +++----
 src/fdb5/remote/client/ClientConnection.h  |  6 +-
 src/fdb5/remote/server/CatalogueHandler.cc | 54 ++++++++--------
 src/fdb5/remote/server/CatalogueHandler.h  |  6 +-
 src/fdb5/remote/server/ServerConnection.cc | 33 ++++++++--
 src/fdb5/remote/server/ServerConnection.h  | 22 ++++++-
 src/fdb5/remote/server/StoreHandler.cc     | 74 ++++++++++++----------
 src/fdb5/remote/server/StoreHandler.h      | 11 ++--
 12 files changed, 162 insertions(+), 99 deletions(-)

diff --git a/src/fdb5/remote/Messages.h b/src/fdb5/remote/Messages.h
index 6a48c6cfc..675900f58 100644
--- a/src/fdb5/remote/Messages.h
+++ b/src/fdb5/remote/Messages.h
@@ -34,7 +34,7 @@ namespace fdb5::remote {
 const static eckit::FixedString<4> StartMarker {"SFDB"};
 const static eckit::FixedString<4> EndMarker {"EFDB"};
 
-constexpr uint16_t CurrentVersion = 10;
+constexpr uint16_t CurrentVersion = 11;
 
 
 enum class Message : uint16_t {
@@ -80,13 +80,15 @@ struct MessageHeader {
     MessageHeader() :
         version(CurrentVersion),
         message(Message::None),
+        clientID(0),
         requestID(0),
         payloadSize(0) {}
 
-    MessageHeader(Message message, uint32_t requestID, uint32_t payloadSize=0) :
+    MessageHeader(Message message, uint32_t clientID, uint32_t requestID, uint32_t payloadSize=0) :
         marker(StartMarker),
         version(CurrentVersion),
         message(message),
+        clientID(clientID),
         requestID(requestID),
         payloadSize(payloadSize) {}
 
@@ -96,11 +98,13 @@ struct MessageHeader {
 
     Message message;                // 2 bytes  --> 8
 
-    uint32_t requestID;             // 4 bytes  --> 12
+    uint32_t clientID;              // 4 bytes  --> 12
 
-    uint32_t payloadSize;           // 4 bytes  --> 16
+    uint32_t requestID;             // 4 bytes  --> 16
 
-    eckit::FixedString<16> hash;    // 16 bytes --> 32
+    uint32_t payloadSize;           // 4 bytes  --> 20
+
+    eckit::FixedString<16> hash;    // 16 bytes --> 36
 };
 
 
diff --git a/src/fdb5/remote/RemoteCatalogue.cc b/src/fdb5/remote/RemoteCatalogue.cc
index 919998fab..4b0899661 100644
--- a/src/fdb5/remote/RemoteCatalogue.cc
+++ b/src/fdb5/remote/RemoteCatalogue.cc
@@ -217,7 +217,7 @@ void RemoteCatalogue::sendArchiveData(uint32_t id, const Key& key, std::unique_p
 
     Buffer keyBuffer(4096);
     MemoryStream keyStream(keyBuffer);
-    keyStream << dbKey_;
+//    keyStream << dbKey_;
     keyStream << currentIndexKey_;
     keyStream << key;
 
diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index df933b14f..075d2dfbe 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -19,10 +19,16 @@ namespace fdb5::remote {
 
 //----------------------------------------------------------------------------------------------------------------------
 
+uint32_t Client::clientId_=0;
+
 Client::Client(const eckit::net::Endpoint& endpoint) :
     endpoint_(endpoint),
     connection_(*(ClientConnectionRouter::instance().connection(*this))),
-    blockingRequestId_(0) {}
+    blockingRequestId_(0) {
+
+    std::lock_guard lock(idMutex_);
+    id_ = ++clientId_;
+}
 
 Client::~Client() {
     ClientConnectionRouter::instance().deregister(*this);
diff --git a/src/fdb5/remote/client/Client.h b/src/fdb5/remote/client/Client.h
index 8a40eb1f9..e913ac3df 100644
--- a/src/fdb5/remote/client/Client.h
+++ b/src/fdb5/remote/client/Client.h
@@ -37,11 +37,12 @@ class Client : eckit::NonCopyable {
     Client(const eckit::net::Endpoint& endpoint);
     ~Client();
 
+    uint32_t id() { return id_; }
     const eckit::net::Endpoint& controlEndpoint() const { return endpoint_; }
 
     uint32_t generateRequestID() { return connection_.generateRequestID(); }
-    
-    // uint32_t controlWriteCheckResponse(Message msg,                     const void* payload=nullptr, uint32_t payloadLength=0);
+
+    // blocking requests
     void          controlWriteCheckResponse(Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
     eckit::Buffer controlWriteReadResponse (Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
 
@@ -62,6 +63,10 @@ class Client : eckit::NonCopyable {
     ClientConnection& connection_;
 
 private:
+    static uint32_t clientId_;
+
+    std::mutex idMutex_;
+    uint32_t id_;
 
     std::mutex blockingRequestMutex_;
     uint32_t blockingRequestId_;
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index 1c86f7a29..b736fedc4 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -66,7 +66,7 @@ class TCPException : public eckit::Exception {
 
 
 ClientConnection::ClientConnection(const eckit::net::Endpoint& controlEndpoint):
-    controlEndpoint_(controlEndpoint), id_(0),
+    controlEndpoint_(controlEndpoint), id_(1),
     connected_(false) {
         eckit::Log::debug() << "ClientConnection::ClientConnection() controlEndpoint: " << controlEndpoint_ << std::endl;
     }
@@ -126,7 +126,7 @@ void ClientConnection::disconnect() {
     if (connected_) {
 
         // Send termination message
-        controlWrite(Message::Exit);
+        controlWrite(Message::Exit, 0);
 
         listeningThread_.join();
 
@@ -166,10 +166,10 @@ void ClientConnection::controlWrite(Client& client, Message msg, uint32_t reques
         addRequest(client, requestID);
     }
 
-    controlWrite(msg, requestID, data);
+    controlWrite(msg, client.id(), requestID, data);
 }
 
-void ClientConnection::controlWrite(Message msg, uint32_t requestID, std::vector> data) {
+void ClientConnection::controlWrite(Message msg, uint32_t clientID, uint32_t requestID, std::vector> data) {
 
     uint32_t payloadLength = 0;
     for (auto d: data) {
@@ -179,7 +179,7 @@ void ClientConnection::controlWrite(Message msg, uint32_t requestID, std::vector
     eckit::Log::debug() << "ClientConnection::controlWrite [endpoint=" << controlEndpoint_.hostport() <<
         ",message=" << ((int) msg) << ",requestID=" << requestID << ",data=" << data.size() << ",payload=" << payloadLength << "]" << std::endl;
 
-    MessageHeader message(msg, requestID, payloadLength);
+    MessageHeader message(msg, clientID, requestID, payloadLength);
 
     std::lock_guard lock(controlMutex_);
     controlWrite(&message, sizeof(message));
@@ -220,17 +220,17 @@ void ClientConnection::dataWrite(Client& client, remote::Message msg, uint32_t r
         }
     }
 
-    dataWrite(msg, requestID, data);
+    dataWrite(msg, client.id(), requestID, data);
 }
 
-void ClientConnection::dataWrite(remote::Message msg, uint32_t requestID, std::vector> data) {
+void ClientConnection::dataWrite(remote::Message msg, uint32_t clientID, uint32_t requestID, std::vector> data) {
 
     uint32_t payloadLength = 0;
     for (auto d: data) {
         ASSERT(d.first);
         payloadLength += d.second;
     }
-    MessageHeader message(msg, requestID, payloadLength);
+    MessageHeader message(msg, clientID, requestID, payloadLength);
 
     eckit::Log::debug() << "ClientConnection::dataWrite [endpoint=" << dataEndpoint_.hostport() <<
         ",message=" << ((int) msg) << ",requestID=" << requestID << ",data=" << data.size() << ",payload=" << payloadLength << "]" << std::endl;
@@ -275,7 +275,7 @@ void ClientConnection::handleError(const MessageHeader& hdr) {
 
         try {
             eckit::FixedString<4> tail;
-            controlRead(&tail, sizeof(tail));
+            dataRead(&tail, sizeof(tail));
         } catch (...) {}
 
         throw RemoteFDBException(what, controlEndpoint_);
@@ -296,7 +296,7 @@ void ClientConnection::writeControlStartupMessage() {
     s << availableFunctionality().get();
 
     //std::cout << "writeControlStartupMessage" << std::endl;
-    controlWrite(Message::Startup, 0, std::vector>{{payload, s.position()}});
+    controlWrite(Message::Startup, 0, 0, std::vector>{{payload, s.position()}});
 }
 
 void ClientConnection::writeDataStartupMessage(const eckit::SessionID& serverSession) {
@@ -307,7 +307,7 @@ void ClientConnection::writeDataStartupMessage(const eckit::SessionID& serverSes
     s << sessionID_;
     s << serverSession;
 
-    dataWrite(Message::Startup, 0, std::vector>{{payload, s.position()}});
+    dataWrite(Message::Startup, 0, 0, std::vector>{{payload, s.position()}});
 }
 
 eckit::SessionID ClientConnection::verifyServerStartupResponse() {
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index d98e1ee83..3fe232f87 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -62,10 +62,10 @@ class ClientConnection : eckit::NonCopyable {
 
 private: // methods
 
-    void controlWrite(Message msg, uint32_t requestID = 0, std::vector> data = {});
+    void controlWrite(Message msg, uint32_t clientID, uint32_t requestID = 0, std::vector> data = {});
     void controlWrite(const void* data, size_t length);
     void controlRead (      void* data, size_t length);
-    void dataWrite   (Message msg, uint32_t requestID = 0, std::vector> data = {});
+    void dataWrite   (Message msg, uint32_t clientID, uint32_t requestID = 0, std::vector> data = {});
     void dataWrite   (const void* data, size_t length);
     void dataRead    (      void* data, size_t length);
  
@@ -90,7 +90,7 @@ class ClientConnection : eckit::NonCopyable {
     eckit::net::TCPClient controlClient_;
     eckit::net::TCPClient dataClient_;
 
-    std::map requests_;
+    std::map requests_;
 
     std::thread listeningThread_;
     
diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc
index d6c19a464..12a5a2cdf 100644
--- a/src/fdb5/remote/server/CatalogueHandler.cc
+++ b/src/fdb5/remote/server/CatalogueHandler.cc
@@ -84,7 +84,7 @@ void CatalogueHandler::initialiseConnections() {
         }
         s << config_.schema();
 
-        dataWrite(Message::Received, hdr.requestID, startupBuffer.data(), s.position());
+        dataWrite(Message::Received, hdr.clientID, hdr.requestID, startupBuffer.data(), s.position());
     }
 }
 
@@ -151,19 +151,19 @@ void CatalogueHandler::forwardApiCall(const MessageHeader& hdr) {
                 typename decltype(iterator)::value_type elem;
                 while (iterator.next(elem)) {
                     auto encoded(helper.encode(elem, *this));
-                    dataWrite(Message::Blob, hdr.requestID, encoded.buf, encoded.position);
+                    dataWrite(Message::Blob, hdr.clientID, hdr.requestID, encoded.buf, encoded.position);
                 }
-                dataWrite(Message::Complete, hdr.requestID);
+                dataWrite(Message::Complete, hdr.clientID, hdr.requestID);
             }
             catch (std::exception& e) {
                 // n.b. more general than eckit::Exception
                 std::string what(e.what());
-                dataWrite(Message::Error, hdr.requestID, what.c_str(), what.length());
+                dataWrite(Message::Error, hdr.clientID, hdr.requestID, what.c_str(), what.length());
             }
             catch (...) {
                 // We really don't want to std::terminate the thread
                 std::string what("Caught unexpected, unknown exception in worker");
-                dataWrite(Message::Error, hdr.requestID, what.c_str(), what.length());
+                dataWrite(Message::Error, hdr.clientID, hdr.requestID, what.c_str(), what.length());
             }
         }));
 }
@@ -260,17 +260,17 @@ void CatalogueHandler::handle() {
 
             if (!ack) {
                 // Acknowledge receipt of command
-                dataWrite(Message::Received, hdr.requestID);
+                dataWrite(Message::Received, hdr.clientID, hdr.requestID);
             }
         }
         catch (std::exception& e) {
             // n.b. more general than eckit::Exception
             std::string what(e.what());
-            dataWrite(Message::Error, hdr.requestID, what.c_str(), what.length());
+            dataWrite(Message::Error, hdr.clientID, hdr.requestID, what.c_str(), what.length());
         }
         catch (...) {
             std::string what("Caught unexpected and unknown error");
-            dataWrite(Message::Error, hdr.requestID, what.c_str(), what.length());
+            dataWrite(Message::Error, hdr.clientID, hdr.requestID, what.c_str(), what.length());
         }
     }
 }
@@ -336,21 +336,20 @@ size_t CatalogueHandler::archiveThreadLoop() {
         try {
             long queuelen;
             while ((queuelen = queue.pop(elem)) != -1) {
-                uint32_t id = elem.first;
+                uint32_t clientID = elem.first;
                 // Key key = elem.second.first;
                 eckit::Buffer payload = std::move(elem.second);
                 MemoryStream s(payload);
-                Key dbKey(s);
                 Key idxKey(s);
                 InspectionKey key; // xxx no stream constructor for inspection key?
                 s >> key;
                 std::unique_ptr location(eckit::Reanimator::reanimate(s));
-                Log::debug() << "CatalogueHandler::archiveThreadLoop message has dbKey: " << dbKey 
-                    << " idxKey: " << idxKey << " key: " << key << " and location.uri" << location->uri() << std::endl;
+                CatalogueWriter& cat = catalogue(clientID);
+                Log::debug() << "CatalogueHandler::archiveThreadLoop message has clientID: " << clientID << " key: " << cat.key()
+                    << idxKey << key << " and location.uri" << location->uri() << std::endl;
         
-                CatalogueWriter& cat = catalogue(dbKey);
                 cat.selectIndex(idxKey);
-                cat.archive(key, std::move(location)); /// xxx currently this is making too many indexes.
+                cat.archive(key, std::move(location));
                 totalArchived += 1;
             }
         }
@@ -391,7 +390,7 @@ size_t CatalogueHandler::archiveThreadLoop() {
             socketRead(&tail, sizeof(tail), dataSocket_);
             ASSERT(tail == EndMarker);
 
-            size_t queuelen = queue.emplace(hdr.requestID, std::move(payload));
+            size_t queuelen = queue.emplace(hdr.clientID, std::move(payload));
         }
 
         // Trigger cleanup of the workers
@@ -411,13 +410,13 @@ size_t CatalogueHandler::archiveThreadLoop() {
     catch (std::exception& e) {
         // n.b. more general than eckit::Exception
         std::string what(e.what());
-        dataWrite(Message::Error, 0, what.c_str(), what.length());
+        dataWrite(Message::Error, 0, 0, what.c_str(), what.length());
         queue.interrupt(std::current_exception());
         throw;
     }
     catch (...) {
         std::string what("Caught unexpected, unknown exception in retrieve worker");
-        dataWrite(Message::Error, 0, what.c_str(), what.length());
+        dataWrite(Message::Error, 0, 0, what.c_str(), what.length());
         queue.interrupt(std::current_exception());
         throw;
     }
@@ -441,23 +440,28 @@ void CatalogueHandler::schema(const MessageHeader& hdr) {
     Key dbKey(s);
     
     // 2. Get catalogue
-    Catalogue& cat = catalogue(dbKey);
+    Catalogue& cat = catalogue(hdr.clientID, dbKey);
     const Schema& schema = cat.schema();
     eckit::Buffer schemaBuffer(1024*1024);
     eckit::MemoryStream stream(schemaBuffer);
     stream << schema;
 
-    dataWrite(Message::Received, hdr.requestID, schemaBuffer.data(), stream.position());
+    dataWrite(Message::Received, hdr.clientID, hdr.requestID, schemaBuffer.data(), stream.position());
 }
 
-CatalogueWriter& CatalogueHandler::catalogue(Key dbKey) {
-    auto it = catalogues_.find(dbKey);
-    if (it != catalogues_.end()) {
-        return *(it->second);
+CatalogueWriter& CatalogueHandler::catalogue(uint32_t id) {
+    auto it = catalogues_.find(id);
+    if (it == catalogues_.end()) {
+        std::string what("Requested unknown catalogue id: " + std::to_string(id));
+        dataWrite(Message::Error, 0, 0, what.c_str(), what.length());
+        throw;
     }
 
-    // xxx specifically catalogue writer right now.
-    return *(catalogues_[dbKey] = CatalogueWriterFactory::instance().build(dbKey, config_));
+    return *(it->second);
+}
+
+CatalogueWriter& CatalogueHandler::catalogue(uint32_t id, const Key& dbKey) {
+    return *(catalogues_[id] = CatalogueWriterFactory::instance().build(dbKey, config_));
 }
 
 }  // namespace fdb5::remote
diff --git a/src/fdb5/remote/server/CatalogueHandler.h b/src/fdb5/remote/server/CatalogueHandler.h
index 9b31d2db6..a8750291c 100644
--- a/src/fdb5/remote/server/CatalogueHandler.h
+++ b/src/fdb5/remote/server/CatalogueHandler.h
@@ -36,7 +36,9 @@ class CatalogueHandler : public ServerConnection {
     void schema(const MessageHeader& hdr);
     
 
-    CatalogueWriter& catalogue(Key dbKey);
+    CatalogueWriter& catalogue(uint32_t id);
+    CatalogueWriter& catalogue(uint32_t id, const Key& dbKey);
+
     size_t archiveThreadLoop();
 
     // API functionality
@@ -45,7 +47,7 @@ class CatalogueHandler : public ServerConnection {
 
 private:  // member
 
-   std::map> catalogues_;
+   std::map> catalogues_;
 
     FDB fdb_;
 };
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index 817058ff2..1b6e3f5f0 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -76,7 +76,7 @@ ServerConnection::~ServerConnection() {
 
     // And notify the client that we are done.
     eckit::Log::info() << "Sending exit message to client" << std::endl;
-    dataWrite(Message::Exit, 0);
+    dataWrite(Message::Exit, 0, 0);
     eckit::Log::info() << "Done" << std::endl;
 }
 
@@ -168,12 +168,12 @@ void ServerConnection::initialiseConnections() {
 
         eckit::Log::debug() << "Protocol negotiation - configuration: " << agreedConf_ <second;
+// }
+
+// Handler& ServerConnection::handler(uint32_t id, eckit::Buffer payload) {
+
+//     ASSERT(handlers_.find(id) == handlers_.end());
+
+
+
+//     auto it = handlers_.find(id);
+//     ASSERT(it != handlers_.end());
+
+//     return *it->second;
+// }
+
 int ServerConnection::selectDataPort() {
     eckit::Log::info() << "SelectDataPort: " << std::endl;
     eckit::Log::info() << config_ << std::endl;
@@ -226,10 +245,10 @@ int ServerConnection::selectDataPort() {
     return 0;
 }
 
-void ServerConnection::controlWrite(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
+void ServerConnection::controlWrite(Message msg, uint32_t clientID, uint32_t requestID, const void* payload, uint32_t payloadLength) {
     ASSERT((payload == nullptr) == (payloadLength == 0));
 
-    MessageHeader message(msg, requestID, payloadLength);
+    MessageHeader message(msg, clientID, requestID, payloadLength);
     std::lock_guard lock(controlWriteMutex_);
     controlWrite(&message, sizeof(message));
     if (payload) {
@@ -257,13 +276,13 @@ void ServerConnection::socketRead(void* data, size_t length, eckit::net::TCPSock
     }
 }
 
-void ServerConnection::dataWrite(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
+void ServerConnection::dataWrite(Message msg, uint32_t clientID, uint32_t requestID, const void* payload, uint32_t payloadLength) {
 
     eckit::Log::debug() << "ServerConnection::dataWrite [message="<< static_cast(msg) << ",requestID=" << requestID << ",payloadLength=" << payloadLength << "]" << std::endl;
 
     ASSERT((payload == nullptr) == (payloadLength == 0));
 
-    MessageHeader message(msg, requestID, payloadLength);
+    MessageHeader message(msg, clientID, requestID, payloadLength);
 
     std::lock_guard lock(dataWriteMutex_);
     dataWriteUnsafe(&message, sizeof(message));
diff --git a/src/fdb5/remote/server/ServerConnection.h b/src/fdb5/remote/server/ServerConnection.h
index 07c7d4779..a31efa8be 100644
--- a/src/fdb5/remote/server/ServerConnection.h
+++ b/src/fdb5/remote/server/ServerConnection.h
@@ -28,6 +28,7 @@
 
 #include "fdb5/config/Config.h"
 #include "fdb5/remote/Messages.h"
+// #include "fdb5/remote/server/Handler.h"
 
 namespace fdb5::remote {
 
@@ -36,6 +37,17 @@ struct MessageHeader;
 //----------------------------------------------------------------------------------------------------------------------
 // Base class for CatalogueHandler and StoreHandler
 
+struct readLocationElem {
+    uint32_t clientID;
+    uint32_t requestID;
+    std::unique_ptr readLocation;
+
+    readLocationElem() : clientID(0), requestID(0), readLocation(nullptr) {}
+    
+    readLocationElem(uint32_t clientID, uint32_t requestID, std::unique_ptr readLocation) :
+        clientID(clientID), requestID(requestID), readLocation(std::move(readLocation)) {}
+};
+
 class ServerConnection : private eckit::NonCopyable {
 public:  // methods
     ServerConnection(eckit::net::TCPSocket& socket, const Config& config);
@@ -49,6 +61,9 @@ class ServerConnection : private eckit::NonCopyable {
 
 protected:
 
+    // Handler& handler(uint32_t id);
+    // virtual Handler& handler(uint32_t id, Buffer& payload) = 0;
+
     // socket methods
     int selectDataPort();
     virtual void initialiseConnections();
@@ -57,7 +72,7 @@ class ServerConnection : private eckit::NonCopyable {
     void socketRead(void* data, size_t length, eckit::net::TCPSocket& socket);
 
     // dataWrite is protected using a mutex, as we may have multiple workers.
-    void dataWrite(Message msg, uint32_t requestID, const void* payload = nullptr, uint32_t payloadLength = 0);
+    void dataWrite(Message msg, uint32_t clientID, uint32_t requestID, const void* payload = nullptr, uint32_t payloadLength = 0);
     eckit::Buffer receivePayload(const MessageHeader& hdr, eckit::net::TCPSocket& socket);
 
     // socket methods
@@ -69,7 +84,7 @@ class ServerConnection : private eckit::NonCopyable {
 
 private:
 
-    void controlWrite(Message msg, uint32_t requestID, const void* payload = nullptr, uint32_t payloadLength = 0);
+    void controlWrite(Message msg, uint32_t clientID, uint32_t requestID, const void* payload = nullptr, uint32_t payloadLength = 0);
     void controlWrite(const void* data, size_t length);
 
     // virtual void read(const MessageHeader& hdr);
@@ -79,12 +94,13 @@ class ServerConnection : private eckit::NonCopyable {
 
 protected:
 
+    // std::map handlers_;
     Config config_;
     eckit::net::TCPSocket controlSocket_;
     eckit::net::EphemeralTCPServer dataSocket_;
     std::string dataListenHostname_;
 
-    eckit::Queue>> readLocationQueue_;
+    eckit::Queue readLocationQueue_;
 
     eckit::SessionID sessionID_;
     eckit::LocalConfiguration agreedConf_;
diff --git a/src/fdb5/remote/server/StoreHandler.cc b/src/fdb5/remote/server/StoreHandler.cc
index f803a1a45..0f8fc3785 100644
--- a/src/fdb5/remote/server/StoreHandler.cc
+++ b/src/fdb5/remote/server/StoreHandler.cc
@@ -118,16 +118,16 @@ void StoreHandler::handle() {
             ASSERT(tail == EndMarker);
 
             // Acknowledge receipt of command
-            dataWrite(Message::Received, hdr.requestID);
+            dataWrite(Message::Received, hdr.clientID, hdr.requestID);
         }
         catch (std::exception& e) {
             // n.b. more general than eckit::Exception
             std::string what(e.what());
-            dataWrite(Message::Error, hdr.requestID, what.c_str(), what.length());
+            dataWrite(Message::Error, hdr.clientID, hdr.requestID, what.c_str(), what.length());
         }
         catch (...) {
             std::string what("Caught unexpected and unknown error");
-            dataWrite(Message::Error, hdr.requestID, what.c_str(), what.length());
+            dataWrite(Message::Error, hdr.clientID, hdr.requestID, what.c_str(), what.length());
         }
     }
 }
@@ -148,24 +148,22 @@ void StoreHandler::read(const MessageHeader& hdr) {
     std::unique_ptr dh;
     dh.reset(location->dataHandle());
 
-    readLocationQueue_.emplace(std::make_pair(hdr.requestID, std::move(dh)));
+    readLocationQueue_.emplace(readLocationElem(hdr.clientID, hdr.requestID, std::move(dh)));
 }
 
 void StoreHandler::readLocationThreadLoop() {
-    std::pair> elem;
+    readLocationElem elem;
 
     while (readLocationQueue_.pop(elem) != -1) {
         // Get the next MarsRequest in sequence to work on, do the retrieve, and
         // send the data back to the client.
 
-        const uint32_t requestID(elem.first);
-
         // Forward the API call
-        writeToParent(elem.first, std::move(elem.second));
+        writeToParent(elem.clientID, elem.requestID, std::move(elem.readLocation));
     }
 }
 
-void StoreHandler::writeToParent(const uint32_t requestID, std::unique_ptr dh) {
+void StoreHandler::writeToParent(const uint32_t clientID, const uint32_t requestID, std::unique_ptr dh) {
    try {
         Log::status() << "Reading: " << requestID << std::endl;
         // Write the data to the parent, in chunks if necessary.
@@ -178,7 +176,7 @@ void StoreHandler::writeToParent(const uint32_t requestID, std::unique_ptrread(writeBuffer, writeBuffer.size())) != 0) {
-            dataWrite(Message::Blob, requestID, writeBuffer, dataRead);
+            dataWrite(Message::Blob, clientID, requestID, writeBuffer, dataRead);
         }
 
         // And when we are done, add a complete message.
@@ -186,7 +184,7 @@ void StoreHandler::writeToParent(const uint32_t requestID, std::unique_ptr() << "Writing retrieve complete message: " << requestID
                               << std::endl;
 
-        dataWrite(Message::Complete, requestID);
+        dataWrite(Message::Complete, clientID, requestID);
 
         Log::status() << "Done retrieve: " << requestID << std::endl;
         Log::debug() << "Done retrieve: " << requestID << std::endl;
@@ -194,12 +192,12 @@ void StoreHandler::writeToParent(const uint32_t requestID, std::unique_ptrsecond);
     }
 
-    return *(stores_[dbKey] = StoreFactory::instance().build(dbKey, config_));
+    return *(stores_[id] = StoreFactory::instance().build(dbKey, config_));
 }
 
 // A helper function to make archiveThreadLoop a bit cleaner
-void StoreHandler::archiveBlobPayload(uint32_t id, const void* data, size_t length) {
+void StoreHandler::archiveBlobPayload(const uint32_t clientID, const uint32_t requestID, const void* data, size_t length) {
     MemoryStream s(data, length);
 
     fdb5::Key dbKey(s);
@@ -237,7 +235,7 @@ void StoreHandler::archiveBlobPayload(uint32_t id, const void* data, size_t leng
     const char* charData = static_cast(data);  // To allow pointer arithmetic
     Log::status() << "Archiving data: " << ss_key.str() << std::endl;
     
-    Store& ss = store(dbKey);
+    Store& ss = store(clientID, dbKey);
 
     auto futureLocation = ss.archive(idxKey, charData + s.position(), length - s.position());
     Log::status() << "Archiving done: " << ss_key.str() << std::endl;
@@ -249,36 +247,49 @@ void StoreHandler::archiveBlobPayload(uint32_t id, const void* data, size_t leng
     eckit::Buffer locBuffer(16 * 1024);
     MemoryStream locStream(locBuffer);
     locStream << (*loc);
-    dataWrite(Message::Archive, id, locBuffer.data(), locStream.position());
+    dataWrite(Message::Archive, clientID, requestID, locBuffer.data(), locStream.position());
 }
 
+struct archiveElem {
+    uint32_t clientID;
+    uint32_t requestID;
+    eckit::Buffer payload;
+    bool multiblob;
+
+    archiveElem() : clientID(0), requestID(0), payload(0), multiblob(false) {}
+
+    archiveElem(uint32_t clientID, uint32_t requestID, eckit::Buffer payload, bool multiblob) :
+        clientID(clientID), requestID(requestID), payload(std::move(payload)), multiblob(multiblob) {}
+};
+
 size_t StoreHandler::archiveThreadLoop() {
     size_t totalArchived = 0;
 
     // Create a worker that will do the actual archiving
 
     static size_t queueSize(eckit::Resource("fdbServerMaxQueueSize", 32));
-    eckit::Queue, bool>> queue(queueSize);
+    eckit::Queue queue(queueSize);
 
     std::future worker = std::async(std::launch::async, [this, &queue] {
         size_t totalArchived = 0;
 
-        std::pair, bool> elem = std::make_pair(std::make_pair(0, Buffer{0}), false);
+        archiveElem elem;
 
         try {
             long queuelen;
             while ((queuelen = queue.pop(elem)) != -1) {
-                if (elem.second) {
+                if (elem.multiblob) {
                     // Handle MultiBlob
 
-                    const char* firstData = static_cast(elem.first.second.data());  // For pointer arithmetic
+                    const char* firstData = static_cast(elem.payload.data());  // For pointer arithmetic
                     const char* charData = firstData;
-                    while (size_t(charData - firstData) < elem.first.second.size()) {
+                    while (size_t(charData - firstData) < elem.payload.size()) {
                         const MessageHeader* hdr = static_cast(static_cast(charData));
                         ASSERT(hdr->marker == StartMarker);
                         ASSERT(hdr->version == CurrentVersion);
                         ASSERT(hdr->message == Message::Blob);
-                        ASSERT(hdr->requestID == elem.first.first);
+                        ASSERT(hdr->clientID == elem.clientID);
+                        ASSERT(hdr->requestID == elem.requestID);
                         charData += sizeof(MessageHeader);
 
                         const void* payloadData = charData;
@@ -288,13 +299,13 @@ size_t StoreHandler::archiveThreadLoop() {
                         ASSERT(*e == EndMarker);
                         charData += sizeof(EndMarker);
 
-                        archiveBlobPayload(elem.first.first, payloadData, hdr->payloadSize);
+                        archiveBlobPayload(elem.clientID, elem.requestID, payloadData, hdr->payloadSize);
                         totalArchived += 1;
                     }
                 }
                 else {
                     // Handle single blob
-                    archiveBlobPayload(elem.first.first, elem.first.second.data(), elem.first.second.size());
+                    archiveBlobPayload(elem.clientID, elem.requestID, elem.payload.data(), elem.payload.size());
                     totalArchived += 1;
                 }
             }
@@ -341,10 +352,7 @@ size_t StoreHandler::archiveThreadLoop() {
 
             size_t sz = payload.size();
             Log::debug() << "Queueing data: " << sz << std::endl;
-            size_t queuelen = queue.emplace(
-                std::make_pair(
-                    std::make_pair(hdr.requestID, std::move(payload)),
-                    hdr.message == Message::MultiBlob));
+            size_t queuelen = queue.emplace(archiveElem(hdr.clientID, hdr.requestID, std::move(payload), hdr.message == Message::MultiBlob));
             Log::status() << "Queued data (" << queuelen << ", size=" << sz << ")" << std::endl;
             ;
             Log::debug() << "Queued data (" << queuelen << ", size=" << sz << ")"
@@ -369,13 +377,13 @@ size_t StoreHandler::archiveThreadLoop() {
     catch (std::exception& e) {
         // n.b. more general than eckit::Exception
         std::string what(e.what());
-        dataWrite(Message::Error, 0, what.c_str(), what.length());
+        dataWrite(Message::Error, 0, 0, what.c_str(), what.length());
         queue.interrupt(std::current_exception());
         throw;
     }
     catch (...) {
         std::string what("Caught unexpected, unknown exception in retrieve worker");
-        dataWrite(Message::Error, 0, what.c_str(), what.length());
+        dataWrite(Message::Error, 0, 0, what.c_str(), what.length());
         queue.interrupt(std::current_exception());
         throw;
     }
diff --git a/src/fdb5/remote/server/StoreHandler.h b/src/fdb5/remote/server/StoreHandler.h
index 02b5ac805..10c77885c 100644
--- a/src/fdb5/remote/server/StoreHandler.h
+++ b/src/fdb5/remote/server/StoreHandler.h
@@ -28,21 +28,20 @@ class StoreHandler : public ServerConnection {
 
     void initialiseConnections() override;
     void readLocationThreadLoop();
-    void writeToParent(const uint32_t requestID, std::unique_ptr dh);
+    void writeToParent(const uint32_t clientID, const uint32_t requestID, std::unique_ptr dh);
 
     void archive(const MessageHeader& hdr);
     size_t archiveThreadLoop();
-    void archiveBlobPayload(uint32_t id, const void* data, size_t length);
+    void archiveBlobPayload(const uint32_t clientID, const uint32_t requestID, const void* data, size_t length);
 
     void flush(const MessageHeader& hdr);
 
-  //  Catalogue& catalogue(Key dbKey);
-    Store& store(Key dbKey);
+    // Store& store(uint32_t id);
+    Store& store(uint32_t id, Key dbKey);
   //  Store& store(eckit::URI uri);
 
 private:  // members
-//    std::map> catalogues_;
-    std::map> stores_;
+    std::map> stores_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------

From 41a34ad4f2ef938d215ecfc63e9147e530181418 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Sun, 3 Dec 2023 08:14:41 +0100
Subject: [PATCH 041/186] version bump

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 288942e4e..484ca723f 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.91
+5.11.92

From 5e9012792d92538cdb172a3600a29bdbde08f993 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 7 Dec 2023 06:52:09 +0000
Subject: [PATCH 042/186] client-defined domain of fdb remote endpoints

---
 src/fdb5/CMakeLists.txt                       |  1 +
 src/fdb5/api/RemoteFDB.cc                     | 31 +++++--
 src/fdb5/api/RemoteFDB.h                      |  8 +-
 src/fdb5/remote/FdbEndpoint.h                 | 41 +++++++++
 src/fdb5/remote/RemoteCatalogue.cc            | 15 +++-
 src/fdb5/remote/RemoteFieldLocation.cc        |  9 +-
 src/fdb5/remote/RemoteFieldLocation.h         |  1 +
 src/fdb5/remote/RemoteStore.cc                | 25 +++---
 src/fdb5/remote/client/Client.cc              | 24 ++++-
 src/fdb5/remote/client/Client.h               | 14 ++-
 src/fdb5/remote/client/ClientConnection.cc    | 60 ++++---------
 src/fdb5/remote/client/ClientConnection.h     | 13 ++-
 .../remote/client/ClientConnectionRouter.cc   | 89 ++++++++++++++++---
 .../remote/client/ClientConnectionRouter.h    |  3 +-
 14 files changed, 245 insertions(+), 89 deletions(-)
 create mode 100644 src/fdb5/remote/FdbEndpoint.h

diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt
index a9bd1380a..647f926e6 100644
--- a/src/fdb5/CMakeLists.txt
+++ b/src/fdb5/CMakeLists.txt
@@ -241,6 +241,7 @@ if(HAVE_FDB_REMOTE)
         remote/Messages.cc
         remote/AvailablePortList.cc
         remote/AvailablePortList.h
+        remote/FdbEndpoint.h
         remote/FdbServer.h
         remote/FdbServer.cc
     )
diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index fdb116c12..6a766825e 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -13,6 +13,7 @@
 
 #include "fdb5/remote/client/ClientConnectionRouter.h"
 #include "fdb5/remote/RemoteFieldLocation.h"
+#include "fdb5/remote/FdbEndpoint.h"
 
 using namespace fdb5::remote;
 using namespace eckit;
@@ -44,10 +45,20 @@ struct InspectHelper : BaseAPIHelper() << std::endl;
 
         if (elem.location().uri().scheme() == "fdb") {
-            return elem;
+            if (fdb->remoteDomain().empty()) {
+                return elem;
+            }
+            fdb5::remote::FdbEndpoint endpoint{elem.location().uri().host(), elem.location().uri().port(), fdb->remoteDomain()};
+            std::shared_ptr remoteLocation = fdb5::remote::RemoteFieldLocation(endpoint, static_cast(elem.location())).make_shared();
+
+            // eckit::Log::debug() << "InspectHelper::valueFromStream - reconstructed location: ";
+            // remoteLocation->dump(eckit::Log::debug());
+            // eckit::Log::debug() << std::endl;
+
+            return fdb5::ListElement(elem.key(), remoteLocation, elem.timestamp());
         }
 
-        std::shared_ptr remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(), elem.location()).make_shared();
+        std::shared_ptr remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->localStoreEndpoint(), elem.location()).make_shared();
         return fdb5::ListElement(elem.key(), remoteLocation, elem.timestamp());
     }
 };
@@ -55,7 +66,7 @@ struct InspectHelper : BaseAPIHelper 0);
 
     return localStores_.at(std::rand() % localStores_.size());
@@ -63,7 +74,9 @@ eckit::net::Endpoint RemoteFDB::storeEndpoint() {
 
 RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name):
     LocalFDB(config, name),
-    Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port"))) {
+    Client(remote::FdbEndpoint(config.getString("host"), config.getInt("port"), config.getString("remoteDomain", ""))) {
+
+    remoteDomain_ = config.getString("remoteDomain", "");
 
     uint32_t id = generateRequestID();
     eckit::Buffer buf = controlWriteReadResponse(remote::Message::Schema, id);
@@ -73,7 +86,9 @@ RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name
     if (numStores > 0) {
         std::vector stores;
         for (size_t i=0; i 0) {
         std::vector localStores;
         for (size_t i=0; i() << *this << " - Received Store endpoint: " << storeEndpoint_ << " and master schema: " << std::endl;
     // schema->dump(eckit::Log::debug());
     
-    config_.overrideSchema(endpoint_.hostport()+"/schema", schema);
+    config_.overrideSchema(controlEndpoint().hostport()+"/schema", schema);
 }
 
 // -----------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/api/RemoteFDB.h b/src/fdb5/api/RemoteFDB.h
index 32fbf710b..b97edde65 100644
--- a/src/fdb5/api/RemoteFDB.h
+++ b/src/fdb5/api/RemoteFDB.h
@@ -61,7 +61,9 @@ class RemoteFDB : public LocalFDB, public remote::Client {
 
     MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) override { NOTIMP; }
 
-    eckit::net::Endpoint storeEndpoint();
+    const eckit::net::Endpoint& localStoreEndpoint() const;
+
+    std::string remoteDomain() { return remoteDomain_; }
 
 private: // methods
 
@@ -80,8 +82,10 @@ class RemoteFDB : public LocalFDB, public remote::Client {
 
 private: // members
 
+    std::string remoteDomain_;
+
     std::unique_ptr archiver_;
-    std::vector localStores_;
+    std::vector localStores_;
 
     // Where do we put received messages
     // @note This is a map of requestID:MessageQueue. At the point that a request is
diff --git a/src/fdb5/remote/FdbEndpoint.h b/src/fdb5/remote/FdbEndpoint.h
new file mode 100644
index 000000000..cd2991978
--- /dev/null
+++ b/src/fdb5/remote/FdbEndpoint.h
@@ -0,0 +1,41 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/// @author Emanuele Danovaro
+/// @date   December 2023
+
+#pragma once
+
+#include "eckit/net/Endpoint.h"
+
+namespace fdb5::remote {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+class FdbEndpoint : public eckit::net::Endpoint {
+
+public: 
+
+    FdbEndpoint(const std::string& endpoint, const std::string& domain = "") :
+        eckit::net::Endpoint(endpoint), domain_(domain) {}
+    FdbEndpoint(const std::string& host, int port, const std::string& domain = "") :
+        eckit::net::Endpoint(host, port), domain_(domain) {}
+    FdbEndpoint(const eckit::net::Endpoint& endpoint, const std::string& domain = "") :
+        eckit::net::Endpoint(endpoint), domain_(domain) {}
+
+    std::string hostname() const override { return host_+domain_; }
+
+private:
+    std::string domain_;
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
+} // namespace fdb5::remote
diff --git a/src/fdb5/remote/RemoteCatalogue.cc b/src/fdb5/remote/RemoteCatalogue.cc
index 4b0899661..76dd6e318 100644
--- a/src/fdb5/remote/RemoteCatalogue.cc
+++ b/src/fdb5/remote/RemoteCatalogue.cc
@@ -192,17 +192,26 @@ FDBStats RemoteCatalogueArchiver::archiveThreadLoop() {
 }
 
 
+std::string host(const eckit::Configuration& config) {
+    std::string host = config.getString("host");
+    std::string remoteDomain = config.getString("remoteDomain", "");
+    if (remoteDomain.empty()) {
+        return host;
+    }
+    return host+remoteDomain;
+}
+
 
 RemoteCatalogue::RemoteCatalogue(const Key& key, const Config& config):
     CatalogueImpl(key, ControlIdentifiers(), config), // xxx what are control identifiers? Setting empty here...
-    Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port"))),
+    Client(eckit::net::Endpoint(host(config), config.getInt("port"))),
     config_(config), schema_(nullptr), archiver_(nullptr) {
 
     loadSchema();
 }
 
 RemoteCatalogue::RemoteCatalogue(const eckit::URI& uri, const Config& config):
-    Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port"))), config_(config), schema_(nullptr), archiver_(nullptr)
+    Client(eckit::net::Endpoint(host(config), config.getInt("port"))), config_(config), schema_(nullptr), archiver_(nullptr)
     {
         NOTIMP;
     }
@@ -296,7 +305,7 @@ bool RemoteCatalogue::exists() const {NOTIMP;}
 void RemoteCatalogue::checkUID() const {}
 
 eckit::URI RemoteCatalogue::uri() const {
-    return eckit::URI(/*RemoteEngine::typeName()*/ "fdb", endpoint_.host(), endpoint_.port());
+    return eckit::URI(/*RemoteEngine::typeName()*/ "fdb", controlEndpoint().host(), controlEndpoint().port());
     // return eckit::URI(TocEngine::typeName(), basePath());
 }
 
diff --git a/src/fdb5/remote/RemoteFieldLocation.cc b/src/fdb5/remote/RemoteFieldLocation.cc
index 19ac0c9f2..749b2d65b 100644
--- a/src/fdb5/remote/RemoteFieldLocation.cc
+++ b/src/fdb5/remote/RemoteFieldLocation.cc
@@ -32,7 +32,7 @@ ::eckit::Reanimator RemoteFieldLocation::reanimator_;
 
 RemoteFieldLocation::RemoteFieldLocation(const eckit::net::Endpoint& endpoint, const FieldLocation& remoteLocation) :
     FieldLocation(
-        eckit::URI("fdb", remoteLocation.uri(), endpoint.host(),  endpoint.port()),
+        eckit::URI("fdb", remoteLocation.uri(), endpoint.hostname(),  endpoint.port()),
         remoteLocation.offset(),
         remoteLocation.length(),
         remoteLocation.remapKey()) {
@@ -46,6 +46,13 @@ RemoteFieldLocation::RemoteFieldLocation(const eckit::net::Endpoint& endpoint, c
     }
 }
 
+RemoteFieldLocation::RemoteFieldLocation(const eckit::net::Endpoint& endpoint, const RemoteFieldLocation& remoteLocation) :
+    FieldLocation(
+        eckit::URI("fdb", remoteLocation.uri(), endpoint.hostname(),  endpoint.port()),
+        remoteLocation.offset(),
+        remoteLocation.length(),
+        remoteLocation.remapKey()) {}
+
 RemoteFieldLocation::RemoteFieldLocation(const eckit::URI& uri) :
     FieldLocation(eckit::URI("fdb", uri)) {
     NOTIMP;
diff --git a/src/fdb5/remote/RemoteFieldLocation.h b/src/fdb5/remote/RemoteFieldLocation.h
index 91cf5b8b9..e16d7e950 100644
--- a/src/fdb5/remote/RemoteFieldLocation.h
+++ b/src/fdb5/remote/RemoteFieldLocation.h
@@ -31,6 +31,7 @@ class RemoteFieldLocation : public FieldLocation {
 public:
 
     RemoteFieldLocation(const eckit::net::Endpoint& endpoint, const FieldLocation& remoteLocation);
+    RemoteFieldLocation(const eckit::net::Endpoint& endpoint, const RemoteFieldLocation& remoteLocation);
     RemoteFieldLocation(const eckit::URI &uri);
     RemoteFieldLocation(const eckit::URI &uri, const eckit::Offset &offset, const eckit::Length &length, const Key& remapKey);
     RemoteFieldLocation(eckit::Stream&);
diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index 875b1252c..6cc876127 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -351,13 +351,17 @@ class FDBRemoteDataHandle : public DataHandle {
 //     return eckit::net::Endpoint(stores.at(std::rand() % stores.size()));
 // }
 
-eckit::net::Endpoint storeEndpoint(const Config& config) {
+std::vector storeEndpoint(const Config& config) {
     if (config.has("storeHost") && config.has("storePort")) {
-        return eckit::net::Endpoint(config.getString("storeHost"), config.getInt("storePort"));
+        return std::vector{FdbEndpoint{config.getString("storeHost"), config.getInt("storePort"), config.getString("remoteDomain", "")}};
     }
     ASSERT(config.has("stores"));
-    std::vector stores = config.getStringVector("stores");
-    return eckit::net::Endpoint(stores.at(std::rand() % stores.size()));
+    std::vector stores;
+    for (auto s : config.getStringVector("stores")) {
+        stores.push_back(FdbEndpoint{s, config.getString("remoteDomain", "")});
+    }
+    return stores;
+//    return eckit::net::Endpoint(stores.at(std::rand() % stores.size()));
 }
 
 //----------------------------------------------------------------------------------------------------------------------
@@ -371,7 +375,7 @@ RemoteStore::RemoteStore(const Key& dbKey, const Config& config) :
     local_ = false;
     if (config.has("localStores")) {
         for (const std::string& localStore : config.getStringVector("localStores")) {
-            if (localStore == endpoint_.hostport()) {
+            if (localStore == controlEndpoint().hostport()) {
                 local_ = true;
                 break;
             }
@@ -490,7 +494,7 @@ bool RemoteStore::handle(Message message, uint32_t requestID) {
 
             auto it = messageQueues_.find(requestID);
             if (it != messageQueues_.end()) {
-                it->second->interrupt(std::make_exception_ptr(RemoteFDBException("", endpoint_)));
+                it->second->interrupt(std::make_exception_ptr(RemoteFDBException("", controlEndpoint())));
 
                 // Remove entry (shared_ptr --> message queue will be destroyed when it
                 // goes out of scope in the worker thread).
@@ -517,7 +521,8 @@ bool RemoteStore::handle(Message message, uint32_t requestID, eckit::Buffer&& pa
                     // std::cout << "LOCAL: " << location->uri().asRawString() << std::endl;
                     it->second(std::move(location));
                 } else {
-                    std::unique_ptr remoteLocation = std::unique_ptr(new RemoteFieldLocation(endpoint_, *location));
+                    eckit::net::Endpoint remoteEndpoint{controlEndpoint().host(), controlEndpoint().port()};
+                    std::unique_ptr remoteLocation = std::unique_ptr(new RemoteFieldLocation(remoteEndpoint, *location));
                     // std::cout << "REMOTE: " << remoteLocation->uri().asRawString() << std::endl;
                     it->second(std::move(remoteLocation));
                 }
@@ -540,7 +545,7 @@ bool RemoteStore::handle(Message message, uint32_t requestID, eckit::Buffer&& pa
                 std::string msg;
                 msg.resize(payload.size(), ' ');
                 payload.copy(&msg[0], payload.size());
-                it->second->interrupt(std::make_exception_ptr(RemoteFDBException(msg, endpoint_)));
+                it->second->interrupt(std::make_exception_ptr(RemoteFDBException(msg, controlEndpoint())));
 
                 // Remove entry (shared_ptr --> message queue will be destroyed when it
                 // goes out of scope in the worker thread).
@@ -549,7 +554,7 @@ bool RemoteStore::handle(Message message, uint32_t requestID, eckit::Buffer&& pa
             } else {
                 auto it = locations_.find(requestID);
                 if (it != locations_.end()) {
-                    archiver_->error(std::move(payload), endpoint_);
+                    archiver_->error(std::move(payload), controlEndpoint());
                 } else {
                     retrieveMessageQueue_.emplace(message, std::move(payload));
                 }
@@ -605,7 +610,7 @@ eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation, c
     uint32_t id = connection_.generateRequestID();
     controlWriteCheckResponse(fdb5::remote::Message::Read, id, encodeBuffer, s.position());
 
-    return new FDBRemoteDataHandle(id, retrieveMessageQueue_, endpoint_);
+    return new FDBRemoteDataHandle(id, retrieveMessageQueue_, controlEndpoint());
 }
 
 RemoteStore& RemoteStore::get(const eckit::URI& uri) {
diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index 075d2dfbe..cea1ddf45 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -21,9 +21,27 @@ namespace fdb5::remote {
 
 uint32_t Client::clientId_=0;
 
-Client::Client(const eckit::net::Endpoint& endpoint) :
-    endpoint_(endpoint),
-    connection_(*(ClientConnectionRouter::instance().connection(*this))),
+// Client::Client(const eckit::net::Endpoint& endpoint, std::string domain) :
+//     endpoint_(endpoint),
+//     fullyQualifiedEndpoint_(endpoint.host()+domain, endpoint.port()),
+//     connection_(*(ClientConnectionRouter::instance().connection(*this))),
+//     blockingRequestId_(0) {
+
+//     std::lock_guard lock(idMutex_);
+//     id_ = ++clientId_;
+// }
+
+Client::Client(const FdbEndpoint& endpoint) :
+    connection_(*(ClientConnectionRouter::instance().connection(*this, endpoint))),
+    blockingRequestId_(0) {
+
+    std::lock_guard lock(idMutex_);
+    id_ = ++clientId_;
+}
+
+
+Client::Client(const std::vector& endpoints) :
+    connection_(*(ClientConnectionRouter::instance().connection(*this, endpoints))),
     blockingRequestId_(0) {
 
     std::lock_guard lock(idMutex_);
diff --git a/src/fdb5/remote/client/Client.h b/src/fdb5/remote/client/Client.h
index e913ac3df..37abc64f3 100644
--- a/src/fdb5/remote/client/Client.h
+++ b/src/fdb5/remote/client/Client.h
@@ -27,18 +27,20 @@ namespace fdb5::remote {
 class RemoteFDBException : public eckit::RemoteException {
 public:
     RemoteFDBException(const std::string& msg, const eckit::net::Endpoint& endpoint):
-        eckit::RemoteException(msg, endpoint.hostport()) {}
+        eckit::RemoteException(msg, endpoint.host()+":"+std::to_string(endpoint.port())) {}
 };
 
 //----------------------------------------------------------------------------------------------------------------------
 
 class Client : eckit::NonCopyable {
 public:
-    Client(const eckit::net::Endpoint& endpoint);
+    Client(const FdbEndpoint& endpoint);
+    Client(const std::vector& endpoints);
     ~Client();
 
     uint32_t id() { return id_; }
-    const eckit::net::Endpoint& controlEndpoint() const { return endpoint_; }
+    const FdbEndpoint& controlEndpoint() const { return connection_.controlEndpoint(); }
+    // const eckit::net::Endpoint& fullyQualifiedControlEndpoint() const { return connection_.controlEndpoint(); }
 
     uint32_t generateRequestID() { return connection_.generateRequestID(); }
 
@@ -59,7 +61,11 @@ class Client : eckit::NonCopyable {
 
 protected:
     
-    eckit::net::Endpoint endpoint_;
+    // std::vector endpoints_;
+    // std::string domain_;
+
+    // eckit::net::Endpoint endpoint_;
+    // eckit::net::Endpoint fullyQualifiedEndpoint_;
     ClientConnection& connection_;
 
 private:
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index b736fedc4..58da92e8f 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -20,35 +20,11 @@
 #include "fdb5/remote/client/ClientConnection.h"
 #include "fdb5/remote/client/ClientConnectionRouter.h"
 
-// using namespace eckit;
-
 namespace fdb5::remote {
 
 //----------------------------------------------------------------------------------------------------------------------
 
 namespace{
-    
-class ConnectionError : public eckit::Exception {
-public:
-    ConnectionError(const int);
-    ConnectionError(const int, const eckit::net::Endpoint&);
-
-    bool retryOnClient() const override { return true; } 
-};
-
-ConnectionError::ConnectionError(const int retries) {
-    std::ostringstream s;
-    s << "Unable to create a connection with the FDB server after " << retries << " retries";
-    reason(s.str());
-    eckit::Log::status() << what() << std::endl;
-}
-
-ConnectionError::ConnectionError(const int retries, const eckit::net::Endpoint& endpoint) {
-    std::ostringstream s;
-    s << "Unable to create a connection with the FDB endpoint " << endpoint << " after " << retries << " retries";
-    reason(s.str());
-    eckit::Log::status() << what() << std::endl;
-}
 
 class TCPException : public eckit::Exception {
 public:
@@ -65,10 +41,10 @@ class TCPException : public eckit::Exception {
 //----------------------------------------------------------------------------------------------------------------------
 
 
-ClientConnection::ClientConnection(const eckit::net::Endpoint& controlEndpoint):
-    controlEndpoint_(controlEndpoint), id_(1),
-    connected_(false) {
-        eckit::Log::debug() << "ClientConnection::ClientConnection() controlEndpoint: " << controlEndpoint_ << std::endl;
+ClientConnection::ClientConnection(const FdbEndpoint& controlEndpoint):
+    controlEndpoint_(controlEndpoint),
+    id_(1), connected_(false) {
+        eckit::Log::debug() << "ClientConnection::ClientConnection() controlEndpoint: " << controlEndpoint.hostport() << std::endl;
     }
 
 
@@ -82,21 +58,20 @@ uint32_t ClientConnection::generateRequestID() {
     return ++id_;
 }
 
-void ClientConnection::connect() {
+bool ClientConnection::connect(bool singleAttempt) {
 
     std::lock_guard lock(requestMutex_);
-
     if (connected_) {
         eckit::Log::warning() << "ClientConnection::connect() called when already connected" << std::endl;
-        return;
+        return connected_;
     }
 
-    static int fdbMaxConnectRetries = eckit::Resource("fdbMaxConnectRetries", 5);
-    static int fdbConnectTimeout = eckit::Resource("fdbConnectTimeout", 0); // No timeout
+    static int fdbMaxConnectRetries = singleAttempt ? 1 : eckit::Resource("fdbMaxConnectRetries", 3);
+    static int fdbConnectTimeout = eckit::Resource("fdbConnectTimeout", singleAttempt?2:5); // 0 = No timeout 
 
     try {
         // Connect to server, and check that the server is happy on the response 
-        eckit::Log::debug() << "Connecting to host: " << controlEndpoint_ << std::endl;
+        eckit::Log::debug() << "Connecting to host: " << controlEndpoint_.hostport() << std::endl;
         controlClient_.connect(controlEndpoint_, fdbMaxConnectRetries, fdbConnectTimeout);
         writeControlStartupMessage();
         eckit::SessionID serverSession = verifyServerStartupResponse();
@@ -112,11 +87,12 @@ void ClientConnection::connect() {
     } catch(eckit::TooManyRetries& e) {
         if (controlClient_.isConnected()) {
             controlClient_.close();
-            throw ConnectionError(fdbMaxConnectRetries, dataEndpoint_);
-        } else {
-            throw ConnectionError(fdbMaxConnectRetries, controlEndpoint_);
+        //     throw ConnectionError(fdbMaxConnectRetries, dataEndpoint_);
+        // } else {
+        //     throw ConnectionError(fdbMaxConnectRetries, fullyQualifiedControlEndpoint_);
         }
     }
+    return connected_;
 }
 
 void ClientConnection::disconnect() {
@@ -137,9 +113,12 @@ void ClientConnection::disconnect() {
     }
 }
 
-const eckit::net::Endpoint& ClientConnection::controlEndpoint() const { 
+const FdbEndpoint& ClientConnection::controlEndpoint() const {
     return controlEndpoint_;
-} 
+}
+// const eckit::net::Endpoint& ClientConnection::fullyQualifiedControlEndpoint() const { 
+//     return fullyQualifiedControlEndpoint_;
+// } 
 const eckit::net::Endpoint& ClientConnection::dataEndpoint() const { 
     return dataEndpoint_;
 } 
@@ -199,7 +178,6 @@ void ClientConnection::controlWrite(const void* data, size_t length) {
 }
 
 void ClientConnection::controlRead(void* data, size_t length) {
-    //std::cout << "ClientConnection::controlRead " << this << std::endl;
     size_t read = controlClient_.read(data, length);
     if (length != read) {
         std::stringstream ss;
@@ -287,7 +265,7 @@ void ClientConnection::writeControlStartupMessage() {
     eckit::Buffer payload(4096);
     eckit::MemoryStream s(payload);
     s << sessionID_;
-    s << controlEndpoint_;
+    s << eckit::net::Endpoint(controlEndpoint_.hostname(), controlEndpoint_.port());
     s << LibFdb5::instance().remoteProtocolVersion().used();
 
     // TODO: Abstract this dictionary into a RemoteConfiguration object, which
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index 3fe232f87..22c219be6 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -16,12 +16,12 @@
 #include "eckit/container/Queue.h"
 #include "eckit/io/Buffer.h"
 #include "eckit/io/Length.h"
-#include "eckit/net/Endpoint.h"
 #include "eckit/net/TCPClient.h"
 #include "eckit/net/TCPStream.h"
 #include "eckit/runtime/SessionID.h"
 
 #include "fdb5/remote/Messages.h"
+#include "fdb5/remote/FdbEndpoint.h"
 
 namespace eckit {
 
@@ -41,20 +41,20 @@ class ClientConnection : eckit::NonCopyable {
 
 public: // methods
 
-    ClientConnection(const eckit::net::Endpoint& controlEndpoint);
+    ClientConnection(const FdbEndpoint& controlEndpoint);
     virtual ~ClientConnection();
 
     void controlWrite(Client& client, remote::Message msg, uint32_t requestID, std::vector> data={});
     void dataWrite   (Client& client, remote::Message msg, uint32_t requestID, std::vector> data={});
 
-    void connect();
+    bool connect(bool singleAttempt = false);
     void disconnect();
 
     uint32_t generateRequestID();
+    const FdbEndpoint& controlEndpoint() const;
 
 protected: // methods
 
-    const eckit::net::Endpoint& controlEndpoint() const;
     const eckit::net::Endpoint& dataEndpoint() const;
     
     // construct dictionary for protocol negotiation - to be defined in the client class
@@ -84,7 +84,7 @@ class ClientConnection : eckit::NonCopyable {
 
     eckit::SessionID sessionID_; 
 
-    eckit::net::Endpoint controlEndpoint_;
+    FdbEndpoint controlEndpoint_;
     eckit::net::Endpoint dataEndpoint_;
 
     eckit::net::TCPClient controlClient_;
@@ -102,8 +102,7 @@ class ClientConnection : eckit::NonCopyable {
     std::mutex idMutex_;
     uint32_t id_;
 
-    bool connected_;
-    
+    bool connected_; 
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.cc b/src/fdb5/remote/client/ClientConnectionRouter.cc
index 846e0187b..5aa6fc069 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.cc
+++ b/src/fdb5/remote/client/ClientConnectionRouter.cc
@@ -1,33 +1,102 @@
 #include "fdb5/remote/client/ClientConnectionRouter.h"
 
+namespace{
+    
+class ConnectionError : public eckit::Exception {
+public:
+    ConnectionError();
+    ConnectionError(const eckit::net::Endpoint&);
+
+    bool retryOnClient() const override { return true; } 
+};
+
+ConnectionError::ConnectionError() {
+    std::ostringstream s;
+    s << "Unable to create a connection with the FDB server";
+    reason(s.str());
+    eckit::Log::status() << what() << std::endl;
+}
+
+ConnectionError::ConnectionError(const eckit::net::Endpoint& endpoint) {
+    std::ostringstream s;
+    s << "Unable to create a connection with the FDB endpoint " << endpoint;
+    reason(s.str());
+    eckit::Log::status() << what() << std::endl;
+}
+}
 namespace fdb5::remote {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-ClientConnection* ClientConnectionRouter::connection(Client& client) {
-
-    // pick the first endpoint - To be improved with DataStoreStrategies
-    const eckit::net::Endpoint& endpoint = client.controlEndpoint();
+ClientConnection* ClientConnectionRouter::connection(Client& client, const FdbEndpoint& endpoint) {
 
     std::lock_guard lock(connectionMutex_);
-
     auto it = connections_.find(endpoint.hostport());
     if (it != connections_.end()) {
         it->second.clients_.insert(&client);
         return it->second.connection_.get();
     } else {
-        auto it = (connections_.emplace(endpoint.hostport(), Connection(std::unique_ptr(new ClientConnection{endpoint}), client))).first;
-        it->second.connection_->connect();
-        return it->second.connection_.get();
+        ClientConnection* clientConnection = new ClientConnection(endpoint);
+        if (clientConnection->connect()) {
+            auto it = (connections_.emplace(endpoint.hostport(), Connection(std::unique_ptr(clientConnection), client))).first;
+            return it->second.connection_.get();
+        } else {
+            throw ConnectionError(endpoint);
+        }
     }
 }
 
+ClientConnection* ClientConnectionRouter::connection(Client& client, const std::vector& endpoints) {
+
+    std::vector fullEndpoints{endpoints};
+    // for (auto hostport: endpoints) {
+    //     eckit::net::Endpoint e{hostport};
+    //     if (domain.empty()) {
+    //         fullEndpoints.push_back(std::make_pair(hostport, e));
+    //     } else {
+    //         fullEndpoints.push_back(std::make_pair(hostport, eckit::net::Endpoint(e.host()+domain, e.port())));
+    //     }
+    // }
+
+    std::lock_guard lock(connectionMutex_);
+    while (fullEndpoints.size()>0) {
+
+        // select a random endpoint
+        size_t idx = std::rand() % fullEndpoints.size();
+        FdbEndpoint endpoint = fullEndpoints.at(idx);
+
+        // look for the selected endpoint
+        auto it = connections_.find(endpoint.hostport());
+        if (it != connections_.end()) {
+            it->second.clients_.insert(&client);
+            return it->second.connection_.get();
+        }
+        else { // not yet there, trying to connect
+            ClientConnection* clientConnection = new ClientConnection(fullEndpoints.at(idx));
+            if (clientConnection->connect(true)) {
+                auto it = (connections_.emplace(endpoint.hostport(), Connection(std::unique_ptr(clientConnection), client))).first;
+                return it->second.connection_.get();
+            }
+        }
+
+        // unable to connect to "endpoint", remove it and try again
+        if (idx != fullEndpoints.size()-1) { // swap with the last element
+            fullEndpoints[idx] = std::move(fullEndpoints.back());
+        }
+        fullEndpoints.pop_back();
+    }
+
+    // no more available endpoints, we have to give up
+    throw ConnectionError();
+}
+
+
 void ClientConnectionRouter::deregister(Client& client) {
-    const eckit::net::Endpoint& endpoint = client.controlEndpoint();
+    const std::string& endpoint = client.controlEndpoint().hostport();
 
     std::lock_guard lock(connectionMutex_);
 
-    auto it = connections_.find(endpoint.hostport());
+    auto it = connections_.find(endpoint);
     ASSERT(it != connections_.end());
 
     auto clientIt = it->second.clients_.find(&client);
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.h b/src/fdb5/remote/client/ClientConnectionRouter.h
index a663dd359..456ce46c5 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.h
+++ b/src/fdb5/remote/client/ClientConnectionRouter.h
@@ -37,7 +37,8 @@ class ClientConnectionRouter : eckit::NonCopyable {
 
     static ClientConnectionRouter& instance();
 
-    ClientConnection* connection(Client& client);
+    ClientConnection* connection(Client& client, const FdbEndpoint& endpoint);
+    ClientConnection* connection(Client& client, const std::vector& endpoints);
 
     void deregister(Client& client);
 

From d51e189969913f6632e81117e0153d38be372e3e Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 7 Dec 2023 15:42:47 +0000
Subject: [PATCH 043/186] passing archiver ID to minimise the number of
 archival threads

---
 src/fdb5/api/RemoteFDB.cc                     | 51 ++-----------------
 src/fdb5/database/ArchiveVisitor.cc           |  7 +--
 src/fdb5/database/Archiver.cc                 |  7 ++-
 src/fdb5/database/Archiver.h                  |  6 +++
 src/fdb5/database/BaseArchiveVisitor.h        |  4 +-
 src/fdb5/database/Catalogue.h                 |  2 +-
 src/fdb5/database/Store.cc                    |  6 +--
 src/fdb5/database/Store.h                     |  4 +-
 src/fdb5/rados/RadosStore.cc                  |  2 +-
 src/fdb5/rados/RadosStore.h                   |  2 +-
 src/fdb5/remote/FdbEndpoint.h                 | 22 ++++++--
 src/fdb5/remote/RemoteCatalogue.cc            | 21 +++++---
 src/fdb5/remote/RemoteCatalogue.h             |  2 +-
 src/fdb5/remote/RemoteStore.cc                | 23 +++++----
 src/fdb5/remote/RemoteStore.h                 |  2 +-
 src/fdb5/remote/client/Client.cc              |  8 +--
 src/fdb5/remote/client/Client.h               |  2 +-
 src/fdb5/remote/client/ClientConnection.cc    | 21 ++++----
 src/fdb5/remote/client/ClientConnection.h     |  2 +-
 .../remote/client/ClientConnectionRouter.cc   | 20 ++------
 .../remote/client/ClientConnectionRouter.h    |  8 +--
 src/fdb5/remote/server/CatalogueHandler.cc    | 35 +++++--------
 src/fdb5/remote/server/CatalogueHandler.h     |  3 +-
 src/fdb5/remote/server/ServerConnection.cc    | 14 +++++
 src/fdb5/remote/server/ServerConnection.h     | 10 ++--
 src/fdb5/remote/server/StoreHandler.cc        | 47 +++++++++--------
 src/fdb5/remote/server/StoreHandler.h         |  5 +-
 src/fdb5/toc/TocCatalogueWriter.cc            |  2 +-
 src/fdb5/toc/TocCatalogueWriter.h             |  2 +-
 src/fdb5/toc/TocStore.cc                      |  2 +-
 src/fdb5/toc/TocStore.h                       |  2 +-
 31 files changed, 169 insertions(+), 175 deletions(-)

diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index 6a766825e..a1c1a627f 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -87,8 +87,7 @@ RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name
         std::vector stores;
         for (size_t i=0; i() << *this << " - Received Store endpoint: " << storeEndpoint_ << " and master schema: " << std::endl;
     // schema->dump(eckit::Log::debug());
     
-    config_.overrideSchema(controlEndpoint().hostport()+"/schema", schema);
+    config_.overrideSchema(static_cast(controlEndpoint())+"/schema", schema);
 }
 
 // -----------------------------------------------------------------------------------------------------
@@ -184,50 +183,6 @@ ListIterator RemoteFDB::inspect(const metkit::mars::MarsRequest& request) {
     return forwardApiCall(InspectHelper(), request);
 }
 
-
-// ListIterator RemoteFDB::inspect(const metkit::mars::MarsRequest& request) {
-//     doinspect_ = true;
-
-//     // worker that does nothing but exposes the AsyncIterator's queue.
-//     auto async_worker = [this] (Queue& queue) {
-//         inspectqueue_ = &queue;
-//         while(!queue.closed()) std::this_thread::sleep_for(std::chrono::milliseconds(10));
-//     };
-
-//     // construct iterator before contacting remote, because we want to point to the asynciterator's queue
-//     auto iter = new APIAsyncIterator(async_worker); 
-
-//     auto req = FDBToolRequest(request);
-//     Buffer encodeBuffer(4096);
-//     MemoryStream s(encodeBuffer);
-//     s << req;
-//     uint32_t id = controlWriteCheckResponse(Message::Inspect, encodeBuffer, s.position());
-
-//     return APIIterator(iter);
-// }
-
-
-// ListIterator RemoteFDB::list(const FDBToolRequest& request) {
-
-//     dolist_ = true;
-    
-//     // worker that does nothing but exposes the AsyncIterator's queue.
-//     auto async_worker = [this] (Queue& queue) {
-//         listqueue_ = &queue;
-//         while(!queue.closed()) std::this_thread::sleep_for(std::chrono::milliseconds(10));
-//     };
-
-//     // construct iterator before contacting remote, because we want to point to the asynciterator's queue
-//     auto iter = new APIAsyncIterator(async_worker); 
-
-//     Buffer encodeBuffer(4096);
-//     MemoryStream s(encodeBuffer);
-//     s << request;
-
-//     controlWriteCheckResponse(Message::List, encodeBuffer, s.position());
-//     return APIIterator(iter);
-// }
-
 void RemoteFDB::print(std::ostream& s) const {
     s << "RemoteFDB(...)";
 }
diff --git a/src/fdb5/database/ArchiveVisitor.cc b/src/fdb5/database/ArchiveVisitor.cc
index 19f682dde..4dd54adcb 100644
--- a/src/fdb5/database/ArchiveVisitor.cc
+++ b/src/fdb5/database/ArchiveVisitor.cc
@@ -10,13 +10,14 @@
 #include 
 #include "eckit/exception/Exceptions.h"
 
+#include "fdb5/database/Archiver.h"
 #include "fdb5/database/ArchiveVisitor.h"
 #include "fdb5/database/Catalogue.h"
 #include "fdb5/database/Store.h"
 
 namespace {
-void CatalogueCallback(fdb5::CatalogueWriter* catalogue, const fdb5::InspectionKey &key, std::unique_ptr fieldLocation) {
-    catalogue->archive(key, std::move(fieldLocation));
+void CatalogueCallback(uint32_t archiverId, fdb5::CatalogueWriter* catalogue, const fdb5::InspectionKey &key, std::unique_ptr fieldLocation) {
+    catalogue->archive(archiverId, key, std::move(fieldLocation));
 }
 }
 namespace fdb5 {
@@ -34,7 +35,7 @@ bool ArchiveVisitor::selectDatum(const InspectionKey &key, const Key &full) {
     checkMissingKeys(full);
     const Key idxKey = current()->currentIndexKey();
 
-    store()->archive(idxKey, data_, size_, std::bind(&CatalogueCallback, current(), key, std::placeholders::_1));
+    store()->archive(owner_.id(), idxKey, data_, size_, std::bind(&CatalogueCallback, owner_.id(), current(), key, std::placeholders::_1));
     
     return true;
 }
diff --git a/src/fdb5/database/Archiver.cc b/src/fdb5/database/Archiver.cc
index 573e0da67..c08d183b9 100644
--- a/src/fdb5/database/Archiver.cc
+++ b/src/fdb5/database/Archiver.cc
@@ -24,11 +24,15 @@ namespace fdb5 {
 
 //----------------------------------------------------------------------------------------------------------------------
 
+uint32_t Archiver::archiverId_=0;
 
 Archiver::Archiver(const Config& dbConfig) :
     dbConfig_(dbConfig),
     catalogue_(nullptr),
     store_(nullptr) {
+
+    std::lock_guard lock(idMutex_);
+    id_ = ++archiverId_;
 }
 
 Archiver::~Archiver() {
@@ -38,6 +42,8 @@ Archiver::~Archiver() {
     databases_.clear(); //< explicitly delete the DBs before schemas are destroyed
 }
 
+uint32_t id();
+
 void Archiver::archive(const Key &key, const void* data, size_t len) {
     ArchiveVisitor visitor(*this, key, data, len);
     archive(key, visitor);
@@ -47,7 +53,6 @@ void Archiver::archive(const Key &key, BaseArchiveVisitor& visitor) {
 
     visitor.rule(nullptr);
     
-
     dbConfig_.schema().expand(key, visitor);
 
     const Rule* rule = visitor.rule();
diff --git a/src/fdb5/database/Archiver.h b/src/fdb5/database/Archiver.h
index fa12bc192..80c9f9f7d 100644
--- a/src/fdb5/database/Archiver.h
+++ b/src/fdb5/database/Archiver.h
@@ -51,6 +51,8 @@ class Archiver : public eckit::NonCopyable {
 
     virtual ~Archiver();
 
+    uint32_t id() const { return id_; }
+
     void archive(const Key &key, BaseArchiveVisitor& visitor);
     void archive(const Key &key, const void* data, size_t len);
 
@@ -70,6 +72,10 @@ class Archiver : public eckit::NonCopyable {
     void selectDatabase(const Key &key);
 
 private: // members
+    static uint32_t archiverId_;
+
+    std::mutex idMutex_;
+    uint32_t id_;
 
     friend class BaseArchiveVisitor;
 
diff --git a/src/fdb5/database/BaseArchiveVisitor.h b/src/fdb5/database/BaseArchiveVisitor.h
index cec173535..c3d7dd071 100644
--- a/src/fdb5/database/BaseArchiveVisitor.h
+++ b/src/fdb5/database/BaseArchiveVisitor.h
@@ -48,10 +48,12 @@ class BaseArchiveVisitor : public WriteVisitor {
     fdb5::CatalogueWriter* current() const;
     fdb5::Store* store() const;
 
-private: // members
+protected: // members
 
     Archiver &owner_;
 
+private: // members
+
     const Key &dataKey_;
 
     bool checkMissingKeysOnWrite_;
diff --git a/src/fdb5/database/Catalogue.h b/src/fdb5/database/Catalogue.h
index d8f050f74..41eece01b 100644
--- a/src/fdb5/database/Catalogue.h
+++ b/src/fdb5/database/Catalogue.h
@@ -164,7 +164,7 @@ class CatalogueWriter : virtual public Catalogue  {
 
     virtual const Index& currentIndex() = 0;
     virtual const Key currentIndexKey() = 0;
-    virtual void archive(const InspectionKey& key, std::unique_ptr fieldLocation) = 0;
+    virtual void archive(const uint32_t archiverId, const InspectionKey& key, std::unique_ptr fieldLocation) = 0;
     virtual void overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) = 0;
     virtual void index(const InspectionKey& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) = 0;
     virtual void reconsolidate() = 0;
diff --git a/src/fdb5/database/Store.cc b/src/fdb5/database/Store.cc
index 243e57f7b..56edae187 100644
--- a/src/fdb5/database/Store.cc
+++ b/src/fdb5/database/Store.cc
@@ -24,11 +24,11 @@ namespace fdb5 {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-void Store::archive(const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive) {
-    catalogue_archive(archive(key, data, length));
+void Store::archive(const uint32_t archiverId, const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive) {
+    catalogue_archive(archive(archiverId, key, data, length));
 }
 
-std::unique_ptr Store::archive(const Key& key, const void *data, eckit::Length length) {
+std::unique_ptr Store::archive(const uint32_t archiverId, const Key& key, const void *data, eckit::Length length) {
     NOTIMP;
 }
 
diff --git a/src/fdb5/database/Store.h b/src/fdb5/database/Store.h
index ecf3e5695..a96a3db57 100644
--- a/src/fdb5/database/Store.h
+++ b/src/fdb5/database/Store.h
@@ -36,8 +36,8 @@ class Store {
 
     virtual eckit::DataHandle* retrieve(Field& field) const = 0;
 //    virtual void archive(const Key& key, const void *data, eckit::Length length, void(*catalogue_archive)(std::unique_ptr fieldLocation)) = 0;
-    virtual void archive(const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive);
-    virtual std::unique_ptr archive(const Key& key, const void *data, eckit::Length length);
+    virtual void archive(const uint32_t archiverId, const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive);
+    virtual std::unique_ptr archive(const uint32_t archiverId, const Key& key, const void *data, eckit::Length length);
 
     virtual void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit = true) const = 0;
 
diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc
index df2338a48..ef44535cc 100644
--- a/src/fdb5/rados/RadosStore.cc
+++ b/src/fdb5/rados/RadosStore.cc
@@ -53,7 +53,7 @@ eckit::DataHandle* RadosStore::retrieve(Field& field, Key& remapKey) const {
         field.dataHandle(remapKey);
 }
 
-std::unique_ptr RadosStore::archive(const Key& key, const void *data, eckit::Length length) {
+std::unique_ptr RadosStore::archive(const uint32_t, const Key& key, const void *data, eckit::Length length) {
     dirty_ = true;
 
     eckit::PathName dataPath = getDataPath(key);
diff --git a/src/fdb5/rados/RadosStore.h b/src/fdb5/rados/RadosStore.h
index 0059c2a89..4b03c31de 100644
--- a/src/fdb5/rados/RadosStore.h
+++ b/src/fdb5/rados/RadosStore.h
@@ -51,7 +51,7 @@ class RadosStore : public Store {
     bool exists() const override;
 
     eckit::DataHandle* retrieve(Field& field, Key& remapKey) const override;
-    std::unique_ptr archive(const Key& key, const void *data, eckit::Length length) override;
+    std::unique_ptr archive(const uint32_t, const Key& key, const void *data, eckit::Length length) override;
 
     void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override;
 
diff --git a/src/fdb5/remote/FdbEndpoint.h b/src/fdb5/remote/FdbEndpoint.h
index cd2991978..8b8bfa7ee 100644
--- a/src/fdb5/remote/FdbEndpoint.h
+++ b/src/fdb5/remote/FdbEndpoint.h
@@ -24,18 +24,30 @@ class FdbEndpoint : public eckit::net::Endpoint {
 public: 
 
     FdbEndpoint(const std::string& endpoint, const std::string& domain = "") :
-        eckit::net::Endpoint(endpoint), domain_(domain) {}
+        eckit::net::Endpoint(endpoint), fullHostname_(host()+domain) {}
     FdbEndpoint(const std::string& host, int port, const std::string& domain = "") :
-        eckit::net::Endpoint(host, port), domain_(domain) {}
+        eckit::net::Endpoint(host, port), fullHostname_(host+domain) {}
     FdbEndpoint(const eckit::net::Endpoint& endpoint, const std::string& domain = "") :
-        eckit::net::Endpoint(endpoint), domain_(domain) {}
+        eckit::net::Endpoint(endpoint), fullHostname_(endpoint.host()+domain) {}
+
+    ~FdbEndpoint() override {}
+
+    const std::string& hostname() const override { return fullHostname_; }
 
-    std::string hostname() const override { return host_+domain_; }
 
 private:
-    std::string domain_;
+    std::string fullHostname_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
 
 } // namespace fdb5::remote
+
+template <>
+struct std::hash
+{
+    std::size_t operator()(const fdb5::remote::FdbEndpoint& endpoint) const noexcept {
+        const std::string& e = endpoint;
+        return std::hash{}(e);
+    }
+};
\ No newline at end of file
diff --git a/src/fdb5/remote/RemoteCatalogue.cc b/src/fdb5/remote/RemoteCatalogue.cc
index 76dd6e318..c059e3ff5 100644
--- a/src/fdb5/remote/RemoteCatalogue.cc
+++ b/src/fdb5/remote/RemoteCatalogue.cc
@@ -13,9 +13,11 @@
 #include "eckit/serialisation/MemoryStream.h"
 
 #include "fdb5/LibFdb5.h"
+#include "fdb5/remote/FdbEndpoint.h"
 #include "fdb5/remote/RemoteCatalogue.h"
 #include "fdb5/remote/RemoteEngine.h"
 
+#include 
 
 using namespace eckit;
 namespace fdb5::remote {
@@ -47,7 +49,7 @@ class RemoteCatalogueArchiver {
 
 public: // methods
 
-    static RemoteCatalogueArchiver* get(const eckit::net::Endpoint& endpoint);
+    static RemoteCatalogueArchiver* get(uint64_t archiverID);
 
     bool valid() { return archiveFuture_.valid(); }
     bool dirty() { return dirty_; }
@@ -75,16 +77,16 @@ class RemoteCatalogueArchiver {
 
 };
 
-RemoteCatalogueArchiver* RemoteCatalogueArchiver::get(const eckit::net::Endpoint& endpoint) {
+RemoteCatalogueArchiver* RemoteCatalogueArchiver::get(uint64_t archiverID) {
 
-    static std::map> archivers_;
+    static std::unordered_map> archivers_;
     static std::mutex getArchiverMutex_;
 
     std::lock_guard lock(getArchiverMutex_);
 
-    auto it = archivers_.find(endpoint.hostport());
+    auto it = archivers_.find(archiverID);
     if (it == archivers_.end()) {
-        it = archivers_.emplace(endpoint.hostport(), new RemoteCatalogueArchiver()).first;
+        it = archivers_.emplace(archiverID, new RemoteCatalogueArchiver()).first;
     }
     return it->second.get();
 }
@@ -241,12 +243,15 @@ void RemoteCatalogue::sendArchiveData(uint32_t id, const Key& key, std::unique_p
     dataWrite(Message::Blob, id, payloads);
 }
 
-void RemoteCatalogue::archive(const InspectionKey& key, std::unique_ptr fieldLocation) {
+void RemoteCatalogue::archive(const uint32_t archiverID, const InspectionKey& key, std::unique_ptr fieldLocation) {
 
     // if there is no archiving thread active, then start one.
     // n.b. reset the archiveQueue_ after a potential flush() cycle.
     if (!archiver_) {
-        archiver_ = RemoteCatalogueArchiver::get(controlEndpoint());
+        uint64_t archiverName = std::hash()(controlEndpoint());
+        archiverName = archiverName << 32;
+        archiverName += archiverID;
+        archiver_ = RemoteCatalogueArchiver::get(archiverName);
     }
     uint32_t id = connection_.generateRequestID();
     if (!archiver_->valid()) {
@@ -323,8 +328,8 @@ void RemoteCatalogue::loadSchema() {
         
         uint32_t id = generateRequestID();
         eckit::Buffer buf = controlWriteReadResponse(Message::Schema, id, keyBuffer, keyStream.position());
+
         eckit::MemoryStream s(buf);
-//        eckit::net::Endpoint storeEndpoint_{s};
         schema_.reset(eckit::Reanimator::reanimate(s));
     }
 }
diff --git a/src/fdb5/remote/RemoteCatalogue.h b/src/fdb5/remote/RemoteCatalogue.h
index 044eb6632..392a6dd12 100644
--- a/src/fdb5/remote/RemoteCatalogue.h
+++ b/src/fdb5/remote/RemoteCatalogue.h
@@ -24,7 +24,7 @@ class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public C
 
     // From CatalogueWriter
     const Index& currentIndex() override;
-    void archive(const InspectionKey& key, std::unique_ptr fieldLocation) override;
+    void archive(const uint32_t archiverId, const InspectionKey& key, std::unique_ptr fieldLocation) override;
     void overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) override;
     void index(const InspectionKey& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) override;
     void reconsolidate() override;
diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index 6cc876127..f9bba2fa0 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -22,10 +22,13 @@
 #include "fdb5/LibFdb5.h"
 #include "fdb5/rules/Rule.h"
 #include "fdb5/database/FieldLocation.h"
+#include "fdb5/remote/FdbEndpoint.h"
 #include "fdb5/remote/RemoteStore.h"
 #include "fdb5/remote/RemoteFieldLocation.h"
 #include "fdb5/io/FDBFileHandle.h"
 
+#include 
+
 using namespace eckit;
 
 //----------------------------------------------------------------------------------------------------------------------
@@ -58,7 +61,7 @@ class RemoteStoreArchiver {
 
 public: // methods
 
-    static RemoteStoreArchiver* get(const eckit::net::Endpoint& endpoint);
+    static RemoteStoreArchiver* get(uint64_t archiverID);
 
     bool valid() { return archiveFuture_.valid(); }
     bool dirty() { return dirty_; }
@@ -84,19 +87,18 @@ class RemoteStoreArchiver {
     size_t maxArchiveQueueLength_;
     std::unique_ptr archiveQueue_;
     std::future archiveFuture_;
-
 };
 
-RemoteStoreArchiver* RemoteStoreArchiver::get(const eckit::net::Endpoint& endpoint) {
+RemoteStoreArchiver* RemoteStoreArchiver::get(uint64_t archiverID) {
 
-    static std::map> archivers_;
+    static std::unordered_map> archivers_;
     static std::mutex getArchiverMutex_;
 
     std::lock_guard lock(getArchiverMutex_);
 
-    auto it = archivers_.find(endpoint.hostport());
+    auto it = archivers_.find(archiverID);
     if (it == archivers_.end()) {
-        it = archivers_.emplace(endpoint.hostport(), new RemoteStoreArchiver()).first;
+        it = archivers_.emplace(archiverID, new RemoteStoreArchiver()).first;
     }
     return it->second.get();
 }
@@ -375,7 +377,7 @@ RemoteStore::RemoteStore(const Key& dbKey, const Config& config) :
     local_ = false;
     if (config.has("localStores")) {
         for (const std::string& localStore : config.getStringVector("localStores")) {
-            if (localStore == controlEndpoint().hostport()) {
+            if (eckit::net::Endpoint(localStore) == controlEndpoint()) {
                 local_ = true;
                 break;
             }
@@ -418,12 +420,15 @@ eckit::DataHandle* RemoteStore::retrieve(Field& field) const {
     return field.dataHandle();
 }
 
-void RemoteStore::archive(const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive) {
+void RemoteStore::archive(const uint32_t archiverID, const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive) {
 
     // if there is no archiving thread active, then start one.
     // n.b. reset the archiveQueue_ after a potential flush() cycle.
     if (!archiver_) {
-        archiver_ = RemoteStoreArchiver::get(controlEndpoint());
+        uint64_t archiverName = std::hash()(controlEndpoint());
+        archiverName = archiverName << 32;
+        archiverName += archiverID;
+        archiver_ = RemoteStoreArchiver::get(archiverName);
     }
     uint32_t id = connection_.generateRequestID();
     if (!archiver_->valid()) {
diff --git a/src/fdb5/remote/RemoteStore.h b/src/fdb5/remote/RemoteStore.h
index 79884dbe8..3d80cd4af 100644
--- a/src/fdb5/remote/RemoteStore.h
+++ b/src/fdb5/remote/RemoteStore.h
@@ -70,7 +70,7 @@ class RemoteStore : public Store, public Client {
     bool exists() const override;
 
     eckit::DataHandle* retrieve(Field& field) const override;
-    void archive(const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive) override;
+    void archive(const uint32_t archiverId, const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive) override;
 
     void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override;
 
diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index cea1ddf45..f5972ac5e 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -87,9 +87,9 @@ void Client::controlWriteCheckResponse(Message msg, uint32_t requestID, const vo
     blockingRequestId_=requestID;
 
     if (payloadLength) {
-        connection_.controlWrite(*this, msg, blockingRequestId_, std::vector>{{payload, payloadLength}});
+        connection_.controlWrite(*this, 0, msg, blockingRequestId_, std::vector>{{payload, payloadLength}});
     } else {
-        connection_.controlWrite(*this, msg, blockingRequestId_);
+        connection_.controlWrite(*this, 0, msg, blockingRequestId_);
     }
 
     f.get();
@@ -106,9 +106,9 @@ eckit::Buffer Client::controlWriteReadResponse(Message msg, uint32_t requestID,
     blockingRequestId_=requestID;
 
     if (payloadLength) {
-        connection_.controlWrite(*this, msg, blockingRequestId_, std::vector>{{payload, payloadLength}});
+        connection_.controlWrite(*this, 0, msg, blockingRequestId_, std::vector>{{payload, payloadLength}});
     } else {
-        connection_.controlWrite(*this, msg, blockingRequestId_);
+        connection_.controlWrite(*this, 0, msg, blockingRequestId_);
     }
 
     eckit::Buffer buf = f.get();
diff --git a/src/fdb5/remote/client/Client.h b/src/fdb5/remote/client/Client.h
index 37abc64f3..b2af9e38b 100644
--- a/src/fdb5/remote/client/Client.h
+++ b/src/fdb5/remote/client/Client.h
@@ -27,7 +27,7 @@ namespace fdb5::remote {
 class RemoteFDBException : public eckit::RemoteException {
 public:
     RemoteFDBException(const std::string& msg, const eckit::net::Endpoint& endpoint):
-        eckit::RemoteException(msg, endpoint.host()+":"+std::to_string(endpoint.port())) {}
+        eckit::RemoteException(msg, endpoint) {}
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index 58da92e8f..34eec663a 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -44,7 +44,7 @@ class TCPException : public eckit::Exception {
 ClientConnection::ClientConnection(const FdbEndpoint& controlEndpoint):
     controlEndpoint_(controlEndpoint),
     id_(1), connected_(false) {
-        eckit::Log::debug() << "ClientConnection::ClientConnection() controlEndpoint: " << controlEndpoint.hostport() << std::endl;
+        eckit::Log::debug() << "ClientConnection::ClientConnection() controlEndpoint: " << controlEndpoint << std::endl;
     }
 
 
@@ -66,12 +66,12 @@ bool ClientConnection::connect(bool singleAttempt) {
         return connected_;
     }
 
-    static int fdbMaxConnectRetries = singleAttempt ? 1 : eckit::Resource("fdbMaxConnectRetries", 3);
-    static int fdbConnectTimeout = eckit::Resource("fdbConnectTimeout", singleAttempt?2:5); // 0 = No timeout 
+    int fdbMaxConnectRetries = (singleAttempt ? 1 : eckit::Resource("fdbMaxConnectRetries", 3));
+    int fdbConnectTimeout = eckit::Resource("fdbConnectTimeout", (singleAttempt ? 2 : 5)); // 0 = No timeout 
 
     try {
         // Connect to server, and check that the server is happy on the response 
-        eckit::Log::debug() << "Connecting to host: " << controlEndpoint_.hostport() << std::endl;
+        eckit::Log::debug() << "Connecting to host: " << controlEndpoint_ << std::endl;
         controlClient_.connect(controlEndpoint_, fdbMaxConnectRetries, fdbConnectTimeout);
         writeControlStartupMessage();
         eckit::SessionID serverSession = verifyServerStartupResponse();
@@ -84,6 +84,7 @@ bool ClientConnection::connect(bool singleAttempt) {
         // And the connections are set up. Let everything start up!
         listeningThread_ = std::thread([this] { listeningThreadLoop(); });
         connected_ = true;
+        return true;
     } catch(eckit::TooManyRetries& e) {
         if (controlClient_.isConnected()) {
             controlClient_.close();
@@ -92,7 +93,7 @@ bool ClientConnection::connect(bool singleAttempt) {
         //     throw ConnectionError(fdbMaxConnectRetries, fullyQualifiedControlEndpoint_);
         }
     }
-    return connected_;
+    return false;
 }
 
 void ClientConnection::disconnect() {
@@ -139,13 +140,13 @@ void ClientConnection::addRequest(Client& client, uint32_t requestID) {
     requests_[requestID] = &client;
 }
 
-void ClientConnection::controlWrite(Client& client, Message msg, uint32_t requestID, std::vector> data) {
+void ClientConnection::controlWrite(Client& client, uint32_t clientID, Message msg, uint32_t requestID, std::vector> data) {
 
     if (requestID) {
         addRequest(client, requestID);
     }
 
-    controlWrite(msg, client.id(), requestID, data);
+    controlWrite(msg, clientID ? clientID : client.id(), requestID, data);
 }
 
 void ClientConnection::controlWrite(Message msg, uint32_t clientID, uint32_t requestID, std::vector> data) {
@@ -155,8 +156,8 @@ void ClientConnection::controlWrite(Message msg, uint32_t clientID, uint32_t req
         ASSERT(d.first);
         payloadLength += d.second;
     }
-    eckit::Log::debug() << "ClientConnection::controlWrite [endpoint=" << controlEndpoint_.hostport() <<
-        ",message=" << ((int) msg) << ",requestID=" << requestID << ",data=" << data.size() << ",payload=" << payloadLength << "]" << std::endl;
+    eckit::Log::debug() << "ClientConnection::controlWrite [endpoint=" << controlEndpoint_ <<
+        ",message=" << ((int) msg) << ",clientID=" << clientID << ",requestID=" << requestID << ",data=" << data.size() << ",payload=" << payloadLength << "]" << std::endl;
 
     MessageHeader message(msg, clientID, requestID, payloadLength);
 
@@ -210,7 +211,7 @@ void ClientConnection::dataWrite(remote::Message msg, uint32_t clientID, uint32_
     }
     MessageHeader message(msg, clientID, requestID, payloadLength);
 
-    eckit::Log::debug() << "ClientConnection::dataWrite [endpoint=" << dataEndpoint_.hostport() <<
+    eckit::Log::debug() << "ClientConnection::dataWrite [endpoint=" << dataEndpoint_ <<
         ",message=" << ((int) msg) << ",requestID=" << requestID << ",data=" << data.size() << ",payload=" << payloadLength << "]" << std::endl;
 
     std::lock_guard lock(dataMutex_);
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index 22c219be6..40ee9d4c2 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -44,7 +44,7 @@ class ClientConnection : eckit::NonCopyable {
     ClientConnection(const FdbEndpoint& controlEndpoint);
     virtual ~ClientConnection();
 
-    void controlWrite(Client& client, remote::Message msg, uint32_t requestID, std::vector> data={});
+    void controlWrite(Client& client, uint32_t clientID, remote::Message msg, uint32_t requestID, std::vector> data={});
     void dataWrite   (Client& client, remote::Message msg, uint32_t requestID, std::vector> data={});
 
     bool connect(bool singleAttempt = false);
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.cc b/src/fdb5/remote/client/ClientConnectionRouter.cc
index 5aa6fc069..b51691f2d 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.cc
+++ b/src/fdb5/remote/client/ClientConnectionRouter.cc
@@ -31,14 +31,14 @@ namespace fdb5::remote {
 ClientConnection* ClientConnectionRouter::connection(Client& client, const FdbEndpoint& endpoint) {
 
     std::lock_guard lock(connectionMutex_);
-    auto it = connections_.find(endpoint.hostport());
+    auto it = connections_.find(endpoint);
     if (it != connections_.end()) {
         it->second.clients_.insert(&client);
         return it->second.connection_.get();
     } else {
         ClientConnection* clientConnection = new ClientConnection(endpoint);
         if (clientConnection->connect()) {
-            auto it = (connections_.emplace(endpoint.hostport(), Connection(std::unique_ptr(clientConnection), client))).first;
+            auto it = (connections_.emplace(endpoint, Connection(std::unique_ptr(clientConnection), client))).first;
             return it->second.connection_.get();
         } else {
             throw ConnectionError(endpoint);
@@ -49,14 +49,6 @@ ClientConnection* ClientConnectionRouter::connection(Client& client, const FdbEn
 ClientConnection* ClientConnectionRouter::connection(Client& client, const std::vector& endpoints) {
 
     std::vector fullEndpoints{endpoints};
-    // for (auto hostport: endpoints) {
-    //     eckit::net::Endpoint e{hostport};
-    //     if (domain.empty()) {
-    //         fullEndpoints.push_back(std::make_pair(hostport, e));
-    //     } else {
-    //         fullEndpoints.push_back(std::make_pair(hostport, eckit::net::Endpoint(e.host()+domain, e.port())));
-    //     }
-    // }
 
     std::lock_guard lock(connectionMutex_);
     while (fullEndpoints.size()>0) {
@@ -66,7 +58,7 @@ ClientConnection* ClientConnectionRouter::connection(Client& client, const std::
         FdbEndpoint endpoint = fullEndpoints.at(idx);
 
         // look for the selected endpoint
-        auto it = connections_.find(endpoint.hostport());
+        auto it = connections_.find(endpoint);
         if (it != connections_.end()) {
             it->second.clients_.insert(&client);
             return it->second.connection_.get();
@@ -74,7 +66,7 @@ ClientConnection* ClientConnectionRouter::connection(Client& client, const std::
         else { // not yet there, trying to connect
             ClientConnection* clientConnection = new ClientConnection(fullEndpoints.at(idx));
             if (clientConnection->connect(true)) {
-                auto it = (connections_.emplace(endpoint.hostport(), Connection(std::unique_ptr(clientConnection), client))).first;
+                auto it = (connections_.emplace(endpoint, Connection(std::unique_ptr(clientConnection), client))).first;
                 return it->second.connection_.get();
             }
         }
@@ -92,11 +84,9 @@ ClientConnection* ClientConnectionRouter::connection(Client& client, const std::
 
 
 void ClientConnectionRouter::deregister(Client& client) {
-    const std::string& endpoint = client.controlEndpoint().hostport();
-
     std::lock_guard lock(connectionMutex_);
 
-    auto it = connections_.find(endpoint);
+    auto it = connections_.find(client.controlEndpoint());
     ASSERT(it != connections_.end());
 
     auto clientIt = it->second.clients_.find(&client);
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.h b/src/fdb5/remote/client/ClientConnectionRouter.h
index 456ce46c5..02380cbc5 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.h
+++ b/src/fdb5/remote/client/ClientConnectionRouter.h
@@ -10,11 +10,11 @@
 
 #pragma once
 
-#include 
-#include 
-
 #include "fdb5/remote/client/Client.h"
 #include "fdb5/remote/client/ClientConnection.h"
+#include "fdb5/remote/FdbEndpoint.h"
+
+#include 
 
 namespace fdb5::remote {
 
@@ -49,7 +49,7 @@ class ClientConnectionRouter : eckit::NonCopyable {
     std::mutex connectionMutex_;
 
     // endpoint -> connection
-    std::map connections_;
+    std::unordered_map connections_;
 };
 
 }
\ No newline at end of file
diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc
index 12a5a2cdf..cb29d730b 100644
--- a/src/fdb5/remote/server/CatalogueHandler.cc
+++ b/src/fdb5/remote/server/CatalogueHandler.cc
@@ -290,37 +290,30 @@ void CatalogueHandler::flush(const MessageHeader& hdr) {
     size_t numArchived;
     s >> numArchived;
 
-    ASSERT(numArchived == 0 || archiveFuture_.valid());
+    std::future& archive = archiveFuture_[hdr.clientID];
+    // std::cout << "numArchived: " << numArchived << " | archiveFuture_.valid: " << archive.valid() << std::endl;
 
-    if (archiveFuture_.valid()) {
+    ASSERT(numArchived == 0 || archive.valid());
+
+    if (archive.valid()) {
         // Ensure that the expected number of fields have been written, and that the
         // archive worker processes have been cleanly wound up.
-        size_t n = archiveFuture_.get();
-        ASSERT(numArchived == n); // XXX Currently this will fail if there is more than one database in request.
+        size_t n = archive.get();
+        ASSERT(numArchived == n);
 
         // Do the actual flush!
         Log::info() << "Flushing" << std::endl;
         Log::status() << "Flushing" << std::endl;
-
-        for (auto it = catalogues_.begin(); it != catalogues_.end(); it++) {
-            it->second->flush();
-        }
+        catalogues_[hdr.clientID]->flush();
+        // for (auto it = catalogues_.begin(); it != catalogues_.end(); it++) {
+        //     it->second->flush();
+        // }
     }
     Log::info() << "Flush complete" << std::endl;
     Log::status() << "Flush complete" << std::endl;
 }
 
-void CatalogueHandler::archive(const MessageHeader& hdr) {
-    // NOTE identical to RemoteCatalogueWriter::archive()
-
-    if(!archiveFuture_.valid()) {
-        Log::debug() << "CatalogueHandler::archive start threadloop" << std::endl;
-        // Start archive worker thread
-        archiveFuture_ = std::async(std::launch::async, [this] { return archiveThreadLoop(); });
-    }
-}
-
-size_t CatalogueHandler::archiveThreadLoop() {
+size_t CatalogueHandler::archiveThreadLoop(uint32_t archiverID) {
     size_t totalArchived = 0;
 
     // Create a worker that will do the actual archiving
@@ -329,7 +322,7 @@ size_t CatalogueHandler::archiveThreadLoop() {
     eckit::Queue> queue(queueSize);
 
 
-    std::future worker = std::async(std::launch::async, [this, &queue] {
+    std::future worker = std::async(std::launch::async, [this, &queue, archiverID] {
         size_t totalArchived = 0;
         std::pair elem = std::make_pair(0, Buffer{0});
 
@@ -349,7 +342,7 @@ size_t CatalogueHandler::archiveThreadLoop() {
                     << idxKey << key << " and location.uri" << location->uri() << std::endl;
         
                 cat.selectIndex(idxKey);
-                cat.archive(key, std::move(location));
+                cat.archive(archiverID, key, std::move(location));
                 totalArchived += 1;
             }
         }
diff --git a/src/fdb5/remote/server/CatalogueHandler.h b/src/fdb5/remote/server/CatalogueHandler.h
index a8750291c..ba6ee6a17 100644
--- a/src/fdb5/remote/server/CatalogueHandler.h
+++ b/src/fdb5/remote/server/CatalogueHandler.h
@@ -30,7 +30,6 @@ class CatalogueHandler : public ServerConnection {
 
     void read(const MessageHeader& hdr);
     void flush(const MessageHeader& hdr);
-    void archive(const MessageHeader& hdr);
     void list(const MessageHeader& hdr);
     void inspect(const MessageHeader& hdr);
     void schema(const MessageHeader& hdr);
@@ -39,7 +38,7 @@ class CatalogueHandler : public ServerConnection {
     CatalogueWriter& catalogue(uint32_t id);
     CatalogueWriter& catalogue(uint32_t id, const Key& dbKey);
 
-    size_t archiveThreadLoop();
+    size_t archiveThreadLoop(uint32_t archiverID) override;
 
     // API functionality
     template 
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index 1b6e3f5f0..5e1fb6693 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -276,6 +276,20 @@ void ServerConnection::socketRead(void* data, size_t length, eckit::net::TCPSock
     }
 }
 
+void ServerConnection::archive(const MessageHeader& hdr) {
+
+    ASSERT(hdr.payloadSize == 0);
+
+    auto archive = archiveFuture_.find(hdr.clientID);
+
+    // Ensure that we aren't already running a catalogue/store
+    if(archive == archiveFuture_.end() || !archive->second.valid()) {
+        // Start archive worker thread
+        uint32_t archiverID = hdr.clientID;
+        archiveFuture_[hdr.clientID] = std::async(std::launch::async, [this, archiverID] { return archiveThreadLoop(archiverID); });
+    }
+}
+
 void ServerConnection::dataWrite(Message msg, uint32_t clientID, uint32_t requestID, const void* payload, uint32_t payloadLength) {
 
     eckit::Log::debug() << "ServerConnection::dataWrite [message="<< static_cast(msg) << ",requestID=" << requestID << ",payloadLength=" << payloadLength << "]" << std::endl;
diff --git a/src/fdb5/remote/server/ServerConnection.h b/src/fdb5/remote/server/ServerConnection.h
index a31efa8be..eb5bea25b 100644
--- a/src/fdb5/remote/server/ServerConnection.h
+++ b/src/fdb5/remote/server/ServerConnection.h
@@ -82,6 +82,9 @@ class ServerConnection : private eckit::NonCopyable {
     void tidyWorkers();
     void waitForWorkers();
 
+    void archive(const MessageHeader& hdr);
+    virtual size_t archiveThreadLoop(uint32_t archiverID) = 0;
+
 private:
 
     void controlWrite(Message msg, uint32_t clientID, uint32_t requestID, const void* payload = nullptr, uint32_t payloadLength = 0);
@@ -106,11 +109,12 @@ class ServerConnection : private eckit::NonCopyable {
     eckit::LocalConfiguration agreedConf_;
     std::mutex controlWriteMutex_;
     std::mutex dataWriteMutex_;
-    std::map> workerThreads_;
     std::thread readLocationWorker_;
     
-    std::future archiveFuture_;
-
+    // archiverID --> future ??????
+    std::map> workerThreads_;
+    // archiverID --> archiveFuture
+    std::unordered_map> archiveFuture_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/server/StoreHandler.cc b/src/fdb5/remote/server/StoreHandler.cc
index 0f8fc3785..c9bb37b03 100644
--- a/src/fdb5/remote/server/StoreHandler.cc
+++ b/src/fdb5/remote/server/StoreHandler.cc
@@ -201,17 +201,18 @@ void StoreHandler::writeToParent(const uint32_t clientID, const uint32_t request
     }
 }
 
-void StoreHandler::archive(const MessageHeader& hdr) {
+// void StoreHandler::archive(const MessageHeader& hdr) {
 
-    ASSERT(hdr.payloadSize == 0);
+//     ASSERT(hdr.payloadSize == 0);
 
-    // Ensure that we aren't already running a store()
-    if(!archiveFuture_.valid()) {
+//     auto archive = archiveFuture_.find(hdr.clientID());
 
-        // Start archive worker thread
-        archiveFuture_ = std::async(std::launch::async, [this] { return archiveThreadLoop(); });
-    }
-}
+//     // Ensure that we aren't already running a store()
+//     if(archive == archiveFuture_.end() || !archive->second.valid())
+//         // Start archive worker thread
+//         archiveFuture_[hdr.clientID()] = std::async(std::launch::async, [this] { return archiveThreadLoop(); });
+//     }
+// }
 
 Store& StoreHandler::store(uint32_t id, Key dbKey) {
     auto it = stores_.find(id);
@@ -223,7 +224,7 @@ Store& StoreHandler::store(uint32_t id, Key dbKey) {
 }
 
 // A helper function to make archiveThreadLoop a bit cleaner
-void StoreHandler::archiveBlobPayload(const uint32_t clientID, const uint32_t requestID, const void* data, size_t length) {
+void StoreHandler::archiveBlobPayload(const uint32_t archiverID, const uint32_t clientID, const uint32_t requestID, const void* data, size_t length) {
     MemoryStream s(data, length);
 
     fdb5::Key dbKey(s);
@@ -237,7 +238,7 @@ void StoreHandler::archiveBlobPayload(const uint32_t clientID, const uint32_t re
     
     Store& ss = store(clientID, dbKey);
 
-    auto futureLocation = ss.archive(idxKey, charData + s.position(), length - s.position());
+    auto futureLocation = ss.archive(archiverID, idxKey, charData + s.position(), length - s.position());
     Log::status() << "Archiving done: " << ss_key.str() << std::endl;
     
     auto loc = futureLocation.get();
@@ -262,7 +263,7 @@ struct archiveElem {
         clientID(clientID), requestID(requestID), payload(std::move(payload)), multiblob(multiblob) {}
 };
 
-size_t StoreHandler::archiveThreadLoop() {
+size_t StoreHandler::archiveThreadLoop(uint32_t archiverID) {
     size_t totalArchived = 0;
 
     // Create a worker that will do the actual archiving
@@ -270,7 +271,7 @@ size_t StoreHandler::archiveThreadLoop() {
     static size_t queueSize(eckit::Resource("fdbServerMaxQueueSize", 32));
     eckit::Queue queue(queueSize);
 
-    std::future worker = std::async(std::launch::async, [this, &queue] {
+    std::future worker = std::async(std::launch::async, [this, &queue, archiverID] {
         size_t totalArchived = 0;
 
         archiveElem elem;
@@ -299,13 +300,13 @@ size_t StoreHandler::archiveThreadLoop() {
                         ASSERT(*e == EndMarker);
                         charData += sizeof(EndMarker);
 
-                        archiveBlobPayload(elem.clientID, elem.requestID, payloadData, hdr->payloadSize);
+                        archiveBlobPayload(archiverID, elem.clientID, elem.requestID, payloadData, hdr->payloadSize);
                         totalArchived += 1;
                     }
                 }
                 else {
                     // Handle single blob
-                    archiveBlobPayload(elem.clientID, elem.requestID, elem.payload.data(), elem.payload.size());
+                    archiveBlobPayload(archiverID, elem.clientID, elem.requestID, elem.payload.data(), elem.payload.size());
                     totalArchived += 1;
                 }
             }
@@ -398,22 +399,24 @@ void StoreHandler::flush(const MessageHeader& hdr) {
     size_t numArchived;
     s >> numArchived;
 
-    ASSERT(numArchived == 0 || archiveFuture_.valid());
+    std::future& archive = archiveFuture_[hdr.clientID];
+    // std::cout << "numArchived: " << numArchived << " | archiveFuture_.valid: " << archive.valid() << std::endl;
+
+    ASSERT(numArchived == 0 || archive.valid());
 
-    if (archiveFuture_.valid()) {
+    if (archive.valid()) {
         // Ensure that the expected number of fields have been written, and that the
         // archive worker processes have been cleanly wound up.
-        size_t n = archiveFuture_.get();
+        size_t n = archive.get();
         ASSERT(numArchived == n);
 
         // Do the actual flush!
         Log::info() << "Flushing" << std::endl;
         Log::status() << "Flushing" << std::endl;
-
-        for (auto it = stores_.begin(); it != stores_.end(); it++) {
-            it->second->flush();
-        }
-
+        stores_[hdr.clientID]->flush();
+        // for (auto it = stores_.begin(); it != stores_.end(); it++) {
+        //     it->second->flush();
+        // }
     }
 
     Log::info() << "Flush complete" << std::endl;
diff --git a/src/fdb5/remote/server/StoreHandler.h b/src/fdb5/remote/server/StoreHandler.h
index 10c77885c..81cf43677 100644
--- a/src/fdb5/remote/server/StoreHandler.h
+++ b/src/fdb5/remote/server/StoreHandler.h
@@ -30,9 +30,8 @@ class StoreHandler : public ServerConnection {
     void readLocationThreadLoop();
     void writeToParent(const uint32_t clientID, const uint32_t requestID, std::unique_ptr dh);
 
-    void archive(const MessageHeader& hdr);
-    size_t archiveThreadLoop();
-    void archiveBlobPayload(const uint32_t clientID, const uint32_t requestID, const void* data, size_t length);
+    size_t archiveThreadLoop(uint32_t archiverID) override;
+    void archiveBlobPayload(const uint32_t archiverID, const uint32_t clientID, const uint32_t requestID, const void* data, size_t length);
 
     void flush(const MessageHeader& hdr);
 
diff --git a/src/fdb5/toc/TocCatalogueWriter.cc b/src/fdb5/toc/TocCatalogueWriter.cc
index 21e8bed4e..47cc32026 100644
--- a/src/fdb5/toc/TocCatalogueWriter.cc
+++ b/src/fdb5/toc/TocCatalogueWriter.cc
@@ -300,7 +300,7 @@ bool TocCatalogueWriter::enabled(const ControlIdentifier& controlIdentifier) con
     return TocCatalogue::enabled(controlIdentifier);
 }
 
-void TocCatalogueWriter::archive(const InspectionKey& key, std::unique_ptr fieldLocation) {
+void TocCatalogueWriter::archive(const uint32_t, const InspectionKey& key, std::unique_ptr fieldLocation) {
     dirty_ = true;
 
     if (current_.null()) {
diff --git a/src/fdb5/toc/TocCatalogueWriter.h b/src/fdb5/toc/TocCatalogueWriter.h
index 268282517..3c2ddb81f 100644
--- a/src/fdb5/toc/TocCatalogueWriter.h
+++ b/src/fdb5/toc/TocCatalogueWriter.h
@@ -71,7 +71,7 @@ class TocCatalogueWriter : public TocCatalogue, public CatalogueWriter {
     void clean() override;
     void close() override;
 
-    void archive(const InspectionKey& key, std::unique_ptr fieldLocation) override;
+    void archive(const uint32_t archiverId, const InspectionKey& key, std::unique_ptr fieldLocation) override;
     void reconsolidateIndexesAndTocs();
 
     virtual void print( std::ostream &out ) const override;
diff --git a/src/fdb5/toc/TocStore.cc b/src/fdb5/toc/TocStore.cc
index edd1426c3..589db9e9b 100644
--- a/src/fdb5/toc/TocStore.cc
+++ b/src/fdb5/toc/TocStore.cc
@@ -52,7 +52,7 @@ eckit::DataHandle* TocStore::retrieve(Field& field) const {
     return field.dataHandle();
 }
 
-std::unique_ptr TocStore::archive(const Key& key, const void *data, eckit::Length length) {
+std::unique_ptr TocStore::archive(const uint32_t, const Key& key, const void *data, eckit::Length length) {
     
     dirty_ = true;
 
diff --git a/src/fdb5/toc/TocStore.h b/src/fdb5/toc/TocStore.h
index 30d7be60f..5b1652989 100644
--- a/src/fdb5/toc/TocStore.h
+++ b/src/fdb5/toc/TocStore.h
@@ -58,7 +58,7 @@ class TocStore : public Store, public TocCommon {
     bool exists() const override;
 
     eckit::DataHandle* retrieve(Field& field) const override;
-    std::unique_ptr archive(const Key& key, const void *data, eckit::Length length) override;
+    std::unique_ptr archive(const uint32_t archiverId, const Key& key, const void *data, eckit::Length length) override;
 
     void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override;
 

From d9937fa15d0b5125df2bb236840f9d6671ba955b Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 7 Dec 2023 15:44:02 +0000
Subject: [PATCH 044/186] version bump

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 484ca723f..57785b6ea 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.92
+5.11.93

From 1da73765842eff2853f37b0bb77abaeb64fff37f Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 7 Dec 2023 16:11:17 +0000
Subject: [PATCH 045/186] fix dependency

---
 src/fdb5/remote/server/ServerConnection.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/fdb5/remote/server/ServerConnection.h b/src/fdb5/remote/server/ServerConnection.h
index eb5bea25b..91930f7a6 100644
--- a/src/fdb5/remote/server/ServerConnection.h
+++ b/src/fdb5/remote/server/ServerConnection.h
@@ -19,6 +19,7 @@
 
 #include 
 #include 
+#include 
 
 #include "eckit/io/Buffer.h"
 #include "eckit/io/DataHandle.h"

From 8170786997071b31cede2b570bf17023bb668692 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 7 Dec 2023 17:00:51 +0000
Subject: [PATCH 046/186] Message::Store

---
 src/fdb5/remote/Messages.h                 | 1 +
 src/fdb5/remote/RemoteCatalogue.cc         | 1 -
 src/fdb5/remote/RemoteStore.cc             | 5 ++---
 src/fdb5/remote/client/ClientConnection.cc | 8 ++++----
 src/fdb5/remote/client/ClientConnection.h  | 2 +-
 src/fdb5/remote/server/StoreHandler.cc     | 4 ++--
 6 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/src/fdb5/remote/Messages.h b/src/fdb5/remote/Messages.h
index 675900f58..3547fff67 100644
--- a/src/fdb5/remote/Messages.h
+++ b/src/fdb5/remote/Messages.h
@@ -59,6 +59,7 @@ enum class Message : uint16_t {
     Inspect,
     Read,
     Move,
+    Store,
     Schema,
 
     // Responses
diff --git a/src/fdb5/remote/RemoteCatalogue.cc b/src/fdb5/remote/RemoteCatalogue.cc
index c059e3ff5..4f868fbd1 100644
--- a/src/fdb5/remote/RemoteCatalogue.cc
+++ b/src/fdb5/remote/RemoteCatalogue.cc
@@ -258,7 +258,6 @@ void RemoteCatalogue::archive(const uint32_t archiverID, const InspectionKey& ke
         archiver_->start();
         ASSERT(archiver_->valid());
 
-        // std::cout << "Catalogue - controlWriteCheckResponse(Message::Archive)" << std::endl;
         controlWriteCheckResponse(Message::Archive, id);
     }
     // eckit::Log::debug() << " RemoteCatalogue::archive - adding to queue [id=" << id << ",key=" << key << ",fieldLocation=" << fieldLocation->uri() << "]" << std::endl;
diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index f9bba2fa0..88ddfb81b 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -435,8 +435,7 @@ void RemoteStore::archive(const uint32_t archiverID, const Key& key, const void
         archiver_->start();
         ASSERT(archiver_->valid());
 
-        // std::cout << "controlWriteCheckResponse(Message::Archive)" << std::endl;
-        controlWriteCheckResponse(Message::Archive, id);
+        controlWriteCheckResponse(Message::Store, id);
     }
     locations_[id] = catalogue_archive;
     archiver_->emplace(id, this, key, data, length);
@@ -517,7 +516,7 @@ bool RemoteStore::handle(Message message, uint32_t requestID, eckit::Buffer&& pa
 
     switch (message) {
     
-        case Message::Archive: {
+        case Message::Store: {
             auto it = locations_.find(requestID);
             if (it != locations_.end()) {
                 MemoryStream s(payload);
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index 34eec663a..4583fea92 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -134,15 +134,13 @@ eckit::LocalConfiguration ClientConnection::availableFunctionality() const {
 // -----------------------------------------------------------------------------------------------------
 
 void ClientConnection::addRequest(Client& client, uint32_t requestID) {
-    ASSERT(requestID);
-
-    std::lock_guard lock(requestMutex_);
     requests_[requestID] = &client;
 }
 
 void ClientConnection::controlWrite(Client& client, uint32_t clientID, Message msg, uint32_t requestID, std::vector> data) {
 
     if (requestID) {
+        std::lock_guard lock(requestMutex_);
         addRequest(client, requestID);
     }
 
@@ -191,6 +189,7 @@ void ClientConnection::controlRead(void* data, size_t length) {
 void ClientConnection::dataWrite(Client& client, remote::Message msg, uint32_t requestID, std::vector> data) {
 
     if (requestID) {
+        std::lock_guard lock(requestMutex_);
         auto it = requests_.find(requestID);
         if (it != requests_.end()) {
             ASSERT(it->second == &client);
@@ -354,7 +353,8 @@ void ClientConnection::listeningThreadLoop() {
                 auto it = requests_.find(hdr.requestID);
                 if (it == requests_.end()) {
                     std::stringstream ss;
-                    ss << "ERROR: Unexpected answer to requestID recieved (" << hdr.requestID << "). ABORTING";
+                    ss << "ERROR: Received [clientID="<< hdr.clientID << ",requestID="<< hdr.requestID << ",message=" << ((int) hdr.message) << ",payload=" << hdr.payloadSize << "]" << std::endl;
+                    ss << "Unexpected answer to requestID recieved (" << hdr.requestID << "). ABORTING";
                     eckit::Log::status() << ss.str() << std::endl;
                     eckit::Log::error() << "Retrieving... " << ss.str() << std::endl;
                     throw eckit::SeriousBug(ss.str(), Here());
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index 40ee9d4c2..25c7014db 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -90,7 +90,7 @@ class ClientConnection : eckit::NonCopyable {
     eckit::net::TCPClient controlClient_;
     eckit::net::TCPClient dataClient_;
 
-    std::map requests_;
+    std::map requests_;
 
     std::thread listeningThread_;
     
diff --git a/src/fdb5/remote/server/StoreHandler.cc b/src/fdb5/remote/server/StoreHandler.cc
index c9bb37b03..6e2ba2683 100644
--- a/src/fdb5/remote/server/StoreHandler.cc
+++ b/src/fdb5/remote/server/StoreHandler.cc
@@ -99,7 +99,7 @@ void StoreHandler::handle() {
                     flush(hdr);
                     break;
 
-                case Message::Archive:
+                case Message::Store:
                     archive(hdr);
                     break;
 
@@ -248,7 +248,7 @@ void StoreHandler::archiveBlobPayload(const uint32_t archiverID, const uint32_t
     eckit::Buffer locBuffer(16 * 1024);
     MemoryStream locStream(locBuffer);
     locStream << (*loc);
-    dataWrite(Message::Archive, clientID, requestID, locBuffer.data(), locStream.position());
+    dataWrite(Message::Store, clientID, requestID, locBuffer.data(), locStream.position());
 }
 
 struct archiveElem {

From ecf3f432fecd2c7bfa2c3e73b7aaa60835c4462c Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 7 Dec 2023 17:25:42 +0000
Subject: [PATCH 047/186] wip

---
 src/fdb5/remote/client/Client.cc           | 8 ++++----
 src/fdb5/remote/client/ClientConnection.cc | 5 ++++-
 src/fdb5/remote/client/ClientConnection.h  | 6 ++----
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index f5972ac5e..7f9d53354 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -87,9 +87,9 @@ void Client::controlWriteCheckResponse(Message msg, uint32_t requestID, const vo
     blockingRequestId_=requestID;
 
     if (payloadLength) {
-        connection_.controlWrite(*this, 0, msg, blockingRequestId_, std::vector>{{payload, payloadLength}});
+        connection_.controlWrite(*this, msg, blockingRequestId_, 0, std::vector>{{payload, payloadLength}});
     } else {
-        connection_.controlWrite(*this, 0, msg, blockingRequestId_);
+        connection_.controlWrite(*this, msg, blockingRequestId_);
     }
 
     f.get();
@@ -106,9 +106,9 @@ eckit::Buffer Client::controlWriteReadResponse(Message msg, uint32_t requestID,
     blockingRequestId_=requestID;
 
     if (payloadLength) {
-        connection_.controlWrite(*this, 0, msg, blockingRequestId_, std::vector>{{payload, payloadLength}});
+        connection_.controlWrite(*this, msg, blockingRequestId_, 0, std::vector>{{payload, payloadLength}});
     } else {
-        connection_.controlWrite(*this, 0, msg, blockingRequestId_);
+        connection_.controlWrite(*this, msg, blockingRequestId_);
     }
 
     eckit::Buffer buf = f.get();
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index 4583fea92..f2e12ad54 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -137,10 +137,13 @@ void ClientConnection::addRequest(Client& client, uint32_t requestID) {
     requests_[requestID] = &client;
 }
 
-void ClientConnection::controlWrite(Client& client, uint32_t clientID, Message msg, uint32_t requestID, std::vector> data) {
+void ClientConnection::controlWrite(Client& client, Message msg, uint32_t requestID, uint32_t clientID, std::vector> data) {
 
     if (requestID) {
         std::lock_guard lock(requestMutex_);
+        auto it = requests_.find(requestID);
+        ASSERT(it == requests_.end());
+        
         addRequest(client, requestID);
     }
 
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index 25c7014db..6dee6d748 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -44,7 +44,7 @@ class ClientConnection : eckit::NonCopyable {
     ClientConnection(const FdbEndpoint& controlEndpoint);
     virtual ~ClientConnection();
 
-    void controlWrite(Client& client, uint32_t clientID, remote::Message msg, uint32_t requestID, std::vector> data={});
+    void controlWrite(Client& client, remote::Message msg, uint32_t requestID, uint32_t clientID=0, std::vector> data={});
     void dataWrite   (Client& client, remote::Message msg, uint32_t requestID, std::vector> data={});
 
     bool connect(bool singleAttempt = false);
@@ -53,15 +53,13 @@ class ClientConnection : eckit::NonCopyable {
     uint32_t generateRequestID();
     const FdbEndpoint& controlEndpoint() const;
 
-protected: // methods
+private: // methods
 
     const eckit::net::Endpoint& dataEndpoint() const;
     
     // construct dictionary for protocol negotiation - to be defined in the client class
     virtual eckit::LocalConfiguration availableFunctionality() const;
 
-private: // methods
-
     void controlWrite(Message msg, uint32_t clientID, uint32_t requestID = 0, std::vector> data = {});
     void controlWrite(const void* data, size_t length);
     void controlRead (      void* data, size_t length);

From 9473df204477fecee953b90613e41facf61ffbc8 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 7 Dec 2023 17:56:56 +0000
Subject: [PATCH 048/186] wip

---
 src/fdb5/remote/client/ClientConnection.cc | 32 +++++++++-------------
 src/fdb5/remote/client/ClientConnection.h  |  4 +--
 2 files changed, 14 insertions(+), 22 deletions(-)

diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index f2e12ad54..cff7dd9ff 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -133,18 +133,14 @@ eckit::LocalConfiguration ClientConnection::availableFunctionality() const {
 
 // -----------------------------------------------------------------------------------------------------
 
-void ClientConnection::addRequest(Client& client, uint32_t requestID) {
-    requests_[requestID] = &client;
-}
-
 void ClientConnection::controlWrite(Client& client, Message msg, uint32_t requestID, uint32_t clientID, std::vector> data) {
 
-    if (requestID) {
+    {
         std::lock_guard lock(requestMutex_);
-        auto it = requests_.find(requestID);
-        ASSERT(it == requests_.end());
-        
-        addRequest(client, requestID);
+        auto it = clients_.find(client.id());
+        if (it == clients_.end()) {
+            clients_[client.id()] = &client;
+        }
     }
 
     controlWrite(msg, clientID ? clientID : client.id(), requestID, data);
@@ -191,13 +187,11 @@ void ClientConnection::controlRead(void* data, size_t length) {
 
 void ClientConnection::dataWrite(Client& client, remote::Message msg, uint32_t requestID, std::vector> data) {
 
-    if (requestID) {
+    {
         std::lock_guard lock(requestMutex_);
-        auto it = requests_.find(requestID);
-        if (it != requests_.end()) {
-            ASSERT(it->second == &client);
-        } else {
-            addRequest(client, requestID);
+        auto it = clients_.find(client.id());
+        if (it == clients_.end()) {
+            clients_[client.id()] = &client;
         }
     }
 
@@ -351,13 +345,13 @@ void ClientConnection::listeningThreadLoop() {
                 return;
             }
 
-            if (hdr.requestID) {
+            if (hdr.clientID) {
                 bool handled = false;
-                auto it = requests_.find(hdr.requestID);
-                if (it == requests_.end()) {
+                auto it = clients_.find(hdr.clientID);
+                if (it == clients_.end()) {
                     std::stringstream ss;
                     ss << "ERROR: Received [clientID="<< hdr.clientID << ",requestID="<< hdr.requestID << ",message=" << ((int) hdr.message) << ",payload=" << hdr.payloadSize << "]" << std::endl;
-                    ss << "Unexpected answer to requestID recieved (" << hdr.requestID << "). ABORTING";
+                    ss << "Unexpected answer for clientID recieved (" << hdr.clientID << "). ABORTING";
                     eckit::Log::status() << ss.str() << std::endl;
                     eckit::Log::error() << "Retrieving... " << ss.str() << std::endl;
                     throw eckit::SeriousBug(ss.str(), Here());
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index 6dee6d748..7116f728e 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -67,8 +67,6 @@ class ClientConnection : eckit::NonCopyable {
     void dataWrite   (const void* data, size_t length);
     void dataRead    (      void* data, size_t length);
  
-    void addRequest(Client& client, uint32_t requestID);
-
     void writeControlStartupMessage();
     void writeDataStartupMessage(const eckit::SessionID& serverSession);
 
@@ -88,7 +86,7 @@ class ClientConnection : eckit::NonCopyable {
     eckit::net::TCPClient controlClient_;
     eckit::net::TCPClient dataClient_;
 
-    std::map requests_;
+    std::map clients_;
 
     std::thread listeningThread_;
     

From e094c0fd05fecf21aaa5a941838bd5ac2cc9e1b2 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 8 Dec 2023 13:26:50 +0000
Subject: [PATCH 049/186] cleanup

---
 src/fdb5/remote/RemoteStore.cc             | 2 +-
 src/fdb5/remote/client/ClientConnection.cc | 4 ++--
 src/fdb5/remote/client/ClientConnection.h  | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index 88ddfb81b..0689277e5 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -474,7 +474,7 @@ void RemoteStore::remove(const Key& key) const {
 }
 
 void RemoteStore::print(std::ostream &out) const {
-    out << "RemoteStore(host=" << /*controlEndpoint() << ", data=" << dataEndpoint() << */ ")";
+    out << "RemoteStore(host=" << controlEndpoint() /* << ", data=" << dataEndpoint() */ << ")";
 }
 
 bool RemoteStore::handle(Message message, uint32_t requestID) {
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index cff7dd9ff..3c98f0658 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -133,7 +133,7 @@ eckit::LocalConfiguration ClientConnection::availableFunctionality() const {
 
 // -----------------------------------------------------------------------------------------------------
 
-void ClientConnection::controlWrite(Client& client, Message msg, uint32_t requestID, uint32_t clientID, std::vector> data) {
+void ClientConnection::controlWrite(Client& client, Message msg, uint32_t requestID, uint32_t archiverID, std::vector> data) {
 
     {
         std::lock_guard lock(requestMutex_);
@@ -143,7 +143,7 @@ void ClientConnection::controlWrite(Client& client, Message msg, uint32_t reques
         }
     }
 
-    controlWrite(msg, clientID ? clientID : client.id(), requestID, data);
+    controlWrite(msg, archiverID ? archiverID : client.id(), requestID, data);
 }
 
 void ClientConnection::controlWrite(Message msg, uint32_t clientID, uint32_t requestID, std::vector> data) {
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index 7116f728e..dc7a7e2bb 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -44,7 +44,7 @@ class ClientConnection : eckit::NonCopyable {
     ClientConnection(const FdbEndpoint& controlEndpoint);
     virtual ~ClientConnection();
 
-    void controlWrite(Client& client, remote::Message msg, uint32_t requestID, uint32_t clientID=0, std::vector> data={});
+    void controlWrite(Client& client, remote::Message msg, uint32_t requestID, uint32_t archiverID=0, std::vector> data={});
     void dataWrite   (Client& client, remote::Message msg, uint32_t requestID, std::vector> data={});
 
     bool connect(bool singleAttempt = false);

From ed0ad67f2f922789ac4d4b145415ab68f8feef21 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 12 Dec 2023 21:10:24 +0000
Subject: [PATCH 050/186] fdb remote multiple networks

---
 src/fdb5/CMakeLists.txt                       |  1 -
 src/fdb5/api/RemoteFDB.cc                     | 78 +++++++---------
 src/fdb5/api/RemoteFDB.h                      | 10 +--
 src/fdb5/remote/FdbEndpoint.h                 | 53 -----------
 src/fdb5/remote/RemoteCatalogue.cc            | 23 +++--
 src/fdb5/remote/RemoteStore.cc                | 46 ++++------
 src/fdb5/remote/RemoteStore.h                 |  2 -
 src/fdb5/remote/client/Client.cc              |  6 +-
 src/fdb5/remote/client/Client.h               |  7 +-
 src/fdb5/remote/client/ClientConnection.cc    |  8 +-
 src/fdb5/remote/client/ClientConnection.h     | 10 ++-
 .../remote/client/ClientConnectionRouter.cc   | 12 +--
 .../remote/client/ClientConnectionRouter.h    |  7 +-
 src/fdb5/remote/server/CatalogueHandler.cc    | 89 ++++++++++++++-----
 src/fdb5/remote/server/CatalogueHandler.h     | 10 ++-
 15 files changed, 168 insertions(+), 194 deletions(-)
 delete mode 100644 src/fdb5/remote/FdbEndpoint.h

diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt
index 647f926e6..a9bd1380a 100644
--- a/src/fdb5/CMakeLists.txt
+++ b/src/fdb5/CMakeLists.txt
@@ -241,7 +241,6 @@ if(HAVE_FDB_REMOTE)
         remote/Messages.cc
         remote/AvailablePortList.cc
         remote/AvailablePortList.h
-        remote/FdbEndpoint.h
         remote/FdbServer.h
         remote/FdbServer.cc
     )
diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index a1c1a627f..613d4ee67 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -13,7 +13,6 @@
 
 #include "fdb5/remote/client/ClientConnectionRouter.h"
 #include "fdb5/remote/RemoteFieldLocation.h"
-#include "fdb5/remote/FdbEndpoint.h"
 
 using namespace fdb5::remote;
 using namespace eckit;
@@ -45,20 +44,12 @@ struct InspectHelper : BaseAPIHelper() << std::endl;
 
         if (elem.location().uri().scheme() == "fdb") {
-            if (fdb->remoteDomain().empty()) {
-                return elem;
-            }
-            fdb5::remote::FdbEndpoint endpoint{elem.location().uri().host(), elem.location().uri().port(), fdb->remoteDomain()};
-            std::shared_ptr remoteLocation = fdb5::remote::RemoteFieldLocation(endpoint, static_cast(elem.location())).make_shared();
-
-            // eckit::Log::debug() << "InspectHelper::valueFromStream - reconstructed location: ";
-            // remoteLocation->dump(eckit::Log::debug());
-            // eckit::Log::debug() << std::endl;
+            eckit::net::Endpoint endpoint{elem.location().uri().host(), elem.location().uri().port()};
 
+            std::shared_ptr remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(endpoint), static_cast(elem.location())).make_shared();
             return fdb5::ListElement(elem.key(), remoteLocation, elem.timestamp());
         }
-
-        std::shared_ptr remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->localStoreEndpoint(), elem.location()).make_shared();
+        std::shared_ptr remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(""), elem.location()).make_shared();
         return fdb5::ListElement(elem.key(), remoteLocation, elem.timestamp());
     }
 };
@@ -66,51 +57,50 @@ struct InspectHelper : BaseAPIHelper 0);
-
-    return localStores_.at(std::rand() % localStores_.size());
+const eckit::net::Endpoint& RemoteFDB::storeEndpoint(const std::string& endpoint) const {
+    auto it = stores_.find(endpoint);
+    ASSERT(it != stores_.end());
+    return it->second.at(std::rand() % it->second.size());
 }
 
 RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name):
     LocalFDB(config, name),
-    Client(remote::FdbEndpoint(config.getString("host"), config.getInt("port"), config.getString("remoteDomain", ""))) {
-
-    remoteDomain_ = config.getString("remoteDomain", "");
+    Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port")), "") {
 
     uint32_t id = generateRequestID();
     eckit::Buffer buf = controlWriteReadResponse(remote::Message::Schema, id);
     eckit::MemoryStream s(buf);
     size_t numStores;
     s >> numStores;
-    if (numStores > 0) {
-        std::vector stores;
-        for (size_t i=0; i 0);
+
+    std::vector stores;
+    std::vector defaultEndpoints;
+
+    for (size_t i=0; i> store;
+        size_t numAliases;
+        s >> numAliases;
+        std::vector aliases;
+        if (numAliases == 0) {
+            stores.push_back(store);
+            defaultEndpoints.push_back(store);
+            Log::debug() << "store endpoint: " << store << " default endpoint: " << store << std::endl;
+        } else {
+            for (size_t j=0; j() << "store endpoint: " << endpoint << " default endpoint: " << store << std::endl;
+            }
         }
-    } else {
-        ASSERT(config_.has("stores"));
+        stores_.emplace(store, aliases);
     }
 
-    // local stores
-    s >> numStores;
-    if (numStores > 0) {
-        std::vector localStores;
-        for (size_t i=0; i::reanimate(s);
 
diff --git a/src/fdb5/api/RemoteFDB.h b/src/fdb5/api/RemoteFDB.h
index b97edde65..0c2345f58 100644
--- a/src/fdb5/api/RemoteFDB.h
+++ b/src/fdb5/api/RemoteFDB.h
@@ -61,9 +61,7 @@ class RemoteFDB : public LocalFDB, public remote::Client {
 
     MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) override { NOTIMP; }
 
-    const eckit::net::Endpoint& localStoreEndpoint() const;
-
-    std::string remoteDomain() { return remoteDomain_; }
+    const eckit::net::Endpoint& storeEndpoint(const std::string& endpoint) const;
 
 private: // methods
 
@@ -82,17 +80,15 @@ class RemoteFDB : public LocalFDB, public remote::Client {
 
 private: // members
 
-    std::string remoteDomain_;
-
     std::unique_ptr archiver_;
-    std::vector localStores_;
+    std::map> stores_;
 
     // Where do we put received messages
     // @note This is a map of requestID:MessageQueue. At the point that a request is
     // complete, errored or otherwise killed, it needs to be removed from the map.
     // The shared_ptr allows this removal to be asynchronous with the actual task
     // cleaning up and returning to the client.
-    std::map> messageQueues_;
+    std::unordered_map> messageQueues_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/FdbEndpoint.h b/src/fdb5/remote/FdbEndpoint.h
deleted file mode 100644
index 8b8bfa7ee..000000000
--- a/src/fdb5/remote/FdbEndpoint.h
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/// @author Emanuele Danovaro
-/// @date   December 2023
-
-#pragma once
-
-#include "eckit/net/Endpoint.h"
-
-namespace fdb5::remote {
-
-//----------------------------------------------------------------------------------------------------------------------
-
-class FdbEndpoint : public eckit::net::Endpoint {
-
-public: 
-
-    FdbEndpoint(const std::string& endpoint, const std::string& domain = "") :
-        eckit::net::Endpoint(endpoint), fullHostname_(host()+domain) {}
-    FdbEndpoint(const std::string& host, int port, const std::string& domain = "") :
-        eckit::net::Endpoint(host, port), fullHostname_(host+domain) {}
-    FdbEndpoint(const eckit::net::Endpoint& endpoint, const std::string& domain = "") :
-        eckit::net::Endpoint(endpoint), fullHostname_(endpoint.host()+domain) {}
-
-    ~FdbEndpoint() override {}
-
-    const std::string& hostname() const override { return fullHostname_; }
-
-
-private:
-    std::string fullHostname_;
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace fdb5::remote
-
-template <>
-struct std::hash
-{
-    std::size_t operator()(const fdb5::remote::FdbEndpoint& endpoint) const noexcept {
-        const std::string& e = endpoint;
-        return std::hash{}(e);
-    }
-};
\ No newline at end of file
diff --git a/src/fdb5/remote/RemoteCatalogue.cc b/src/fdb5/remote/RemoteCatalogue.cc
index 4f868fbd1..229a0b0d2 100644
--- a/src/fdb5/remote/RemoteCatalogue.cc
+++ b/src/fdb5/remote/RemoteCatalogue.cc
@@ -13,7 +13,6 @@
 #include "eckit/serialisation/MemoryStream.h"
 
 #include "fdb5/LibFdb5.h"
-#include "fdb5/remote/FdbEndpoint.h"
 #include "fdb5/remote/RemoteCatalogue.h"
 #include "fdb5/remote/RemoteEngine.h"
 
@@ -194,26 +193,26 @@ FDBStats RemoteCatalogueArchiver::archiveThreadLoop() {
 }
 
 
-std::string host(const eckit::Configuration& config) {
-    std::string host = config.getString("host");
-    std::string remoteDomain = config.getString("remoteDomain", "");
-    if (remoteDomain.empty()) {
-        return host;
-    }
-    return host+remoteDomain;
-}
+// std::string host(const eckit::Configuration& config) {
+//     std::string host = config.getString("host");
+//     std::string remoteDomain = config.getString("remoteDomain", "");
+//     if (remoteDomain.empty()) {
+//         return host;
+//     }
+//     return host+remoteDomain;
+// }
 
 
 RemoteCatalogue::RemoteCatalogue(const Key& key, const Config& config):
     CatalogueImpl(key, ControlIdentifiers(), config), // xxx what are control identifiers? Setting empty here...
-    Client(eckit::net::Endpoint(host(config), config.getInt("port"))),
+    Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port")), ""),
     config_(config), schema_(nullptr), archiver_(nullptr) {
 
     loadSchema();
 }
 
 RemoteCatalogue::RemoteCatalogue(const eckit::URI& uri, const Config& config):
-    Client(eckit::net::Endpoint(host(config), config.getInt("port"))), config_(config), schema_(nullptr), archiver_(nullptr)
+    Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port")), ""), config_(config), schema_(nullptr), archiver_(nullptr)
     {
         NOTIMP;
     }
@@ -248,7 +247,7 @@ void RemoteCatalogue::archive(const uint32_t archiverID, const InspectionKey& ke
     // if there is no archiving thread active, then start one.
     // n.b. reset the archiveQueue_ after a potential flush() cycle.
     if (!archiver_) {
-        uint64_t archiverName = std::hash()(controlEndpoint());
+        uint64_t archiverName = std::hash()(controlEndpoint());
         archiverName = archiverName << 32;
         archiverName += archiverID;
         archiver_ = RemoteCatalogueArchiver::get(archiverName);
diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index 0689277e5..65d720b46 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -22,7 +22,6 @@
 #include "fdb5/LibFdb5.h"
 #include "fdb5/rules/Rule.h"
 #include "fdb5/database/FieldLocation.h"
-#include "fdb5/remote/FdbEndpoint.h"
 #include "fdb5/remote/RemoteStore.h"
 #include "fdb5/remote/RemoteFieldLocation.h"
 #include "fdb5/io/FDBFileHandle.h"
@@ -353,40 +352,34 @@ class FDBRemoteDataHandle : public DataHandle {
 //     return eckit::net::Endpoint(stores.at(std::rand() % stores.size()));
 // }
 
-std::vector storeEndpoint(const Config& config) {
-    if (config.has("storeHost") && config.has("storePort")) {
-        return std::vector{FdbEndpoint{config.getString("storeHost"), config.getInt("storePort"), config.getString("remoteDomain", "")}};
-    }
+std::vector> storeEndpoints(const Config& config) {
+    // if (config.has("storeHost") && config.has("storePort")) {
+    //     return std::vector{eckit::net::Endpoint{config.getString("storeHost"), config.getInt("storePort"), config.getString("remoteDomain", "")}};
+    // }
     ASSERT(config.has("stores"));
-    std::vector stores;
-    for (auto s : config.getStringVector("stores")) {
-        stores.push_back(FdbEndpoint{s, config.getString("remoteDomain", "")});
+    ASSERT(config.has("storesDefaultEndpoints"));
+    std::vector stores = config.getStringVector("stores");
+    std::vector defaultEndpoints = config.getStringVector("storesDefaultEndpoints");
+
+    ASSERT(stores.size() == defaultEndpoints.size());
+    std::vector> out;
+    for (size_t i=0; i("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
     archiver_(nullptr) {
-
-    local_ = false;
-    if (config.has("localStores")) {
-        for (const std::string& localStore : config.getStringVector("localStores")) {
-            if (eckit::net::Endpoint(localStore) == controlEndpoint()) {
-                local_ = true;
-                break;
-            }
-        }
-    }
 }
 
 RemoteStore::RemoteStore(const eckit::URI& uri, const Config& config) :
-    Client(eckit::net::Endpoint(uri.hostport())),
+    Client(eckit::net::Endpoint(uri.hostport()), uri.hostport()),
     dbKey_(Key()), config_(config),
     retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
     archiver_(nullptr) {
@@ -425,7 +418,7 @@ void RemoteStore::archive(const uint32_t archiverID, const Key& key, const void
     // if there is no archiving thread active, then start one.
     // n.b. reset the archiveQueue_ after a potential flush() cycle.
     if (!archiver_) {
-        uint64_t archiverName = std::hash()(controlEndpoint());
+        uint64_t archiverName = std::hash()(controlEndpoint());
         archiverName = archiverName << 32;
         archiverName += archiverID;
         archiver_ = RemoteStoreArchiver::get(archiverName);
@@ -521,13 +514,10 @@ bool RemoteStore::handle(Message message, uint32_t requestID, eckit::Buffer&& pa
             if (it != locations_.end()) {
                 MemoryStream s(payload);
                 std::unique_ptr location(eckit::Reanimator::reanimate(s));
-                if (local_) {
-                    // std::cout << "LOCAL: " << location->uri().asRawString() << std::endl;
+                if (defaultEndpoint().empty()) {
                     it->second(std::move(location));
                 } else {
-                    eckit::net::Endpoint remoteEndpoint{controlEndpoint().host(), controlEndpoint().port()};
-                    std::unique_ptr remoteLocation = std::unique_ptr(new RemoteFieldLocation(remoteEndpoint, *location));
-                    // std::cout << "REMOTE: " << remoteLocation->uri().asRawString() << std::endl;
+                    std::unique_ptr remoteLocation = std::unique_ptr(new RemoteFieldLocation(eckit::net::Endpoint{defaultEndpoint()}, *location));
                     it->second(std::move(remoteLocation));
                 }
             }
diff --git a/src/fdb5/remote/RemoteStore.h b/src/fdb5/remote/RemoteStore.h
index 3d80cd4af..bdf88b9fc 100644
--- a/src/fdb5/remote/RemoteStore.h
+++ b/src/fdb5/remote/RemoteStore.h
@@ -102,8 +102,6 @@ class RemoteStore : public Store, public Client {
 
     // not owning
     RemoteStoreArchiver* archiver_;
-
-    bool local_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index 7f9d53354..a5058e211 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -31,8 +31,8 @@ uint32_t Client::clientId_=0;
 //     id_ = ++clientId_;
 // }
 
-Client::Client(const FdbEndpoint& endpoint) :
-    connection_(*(ClientConnectionRouter::instance().connection(*this, endpoint))),
+Client::Client(const eckit::net::Endpoint& endpoint, const std::string& defaultEndpoint) :
+    connection_(*(ClientConnectionRouter::instance().connection(*this, endpoint, defaultEndpoint))),
     blockingRequestId_(0) {
 
     std::lock_guard lock(idMutex_);
@@ -40,7 +40,7 @@ Client::Client(const FdbEndpoint& endpoint) :
 }
 
 
-Client::Client(const std::vector& endpoints) :
+Client::Client(const std::vector>& endpoints) :
     connection_(*(ClientConnectionRouter::instance().connection(*this, endpoints))),
     blockingRequestId_(0) {
 
diff --git a/src/fdb5/remote/client/Client.h b/src/fdb5/remote/client/Client.h
index b2af9e38b..023bef0d5 100644
--- a/src/fdb5/remote/client/Client.h
+++ b/src/fdb5/remote/client/Client.h
@@ -34,12 +34,13 @@ class RemoteFDBException : public eckit::RemoteException {
 
 class Client : eckit::NonCopyable {
 public:
-    Client(const FdbEndpoint& endpoint);
-    Client(const std::vector& endpoints);
+    Client(const eckit::net::Endpoint& endpoint, const std::string& defaultEndpoint);
+    Client(const std::vector>& endpoints);
     ~Client();
 
     uint32_t id() { return id_; }
-    const FdbEndpoint& controlEndpoint() const { return connection_.controlEndpoint(); }
+    const eckit::net::Endpoint& controlEndpoint() const { return connection_.controlEndpoint(); }
+    const std::string& defaultEndpoint() const { return connection_.defaultEndpoint(); }
     // const eckit::net::Endpoint& fullyQualifiedControlEndpoint() const { return connection_.controlEndpoint(); }
 
     uint32_t generateRequestID() { return connection_.generateRequestID(); }
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index 3c98f0658..3300945a4 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -41,13 +41,11 @@ class TCPException : public eckit::Exception {
 //----------------------------------------------------------------------------------------------------------------------
 
 
-ClientConnection::ClientConnection(const FdbEndpoint& controlEndpoint):
-    controlEndpoint_(controlEndpoint),
-    id_(1), connected_(false) {
+ClientConnection::ClientConnection(const eckit::net::Endpoint& controlEndpoint, const std::string& defaultEndpoint):
+    controlEndpoint_(controlEndpoint), defaultEndpoint_(defaultEndpoint), id_(1), connected_(false) {
         eckit::Log::debug() << "ClientConnection::ClientConnection() controlEndpoint: " << controlEndpoint << std::endl;
     }
 
-
 ClientConnection::~ClientConnection() {
     disconnect();
 }
@@ -114,7 +112,7 @@ void ClientConnection::disconnect() {
     }
 }
 
-const FdbEndpoint& ClientConnection::controlEndpoint() const {
+const eckit::net::Endpoint& ClientConnection::controlEndpoint() const {
     return controlEndpoint_;
 }
 // const eckit::net::Endpoint& ClientConnection::fullyQualifiedControlEndpoint() const { 
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index dc7a7e2bb..463b9df94 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -21,7 +21,6 @@
 #include "eckit/runtime/SessionID.h"
 
 #include "fdb5/remote/Messages.h"
-#include "fdb5/remote/FdbEndpoint.h"
 
 namespace eckit {
 
@@ -41,7 +40,7 @@ class ClientConnection : eckit::NonCopyable {
 
 public: // methods
 
-    ClientConnection(const FdbEndpoint& controlEndpoint);
+    ClientConnection(const eckit::net::Endpoint& controlEndpoint, const std::string& defaultEndpoint);
     virtual ~ClientConnection();
 
     void controlWrite(Client& client, remote::Message msg, uint32_t requestID, uint32_t archiverID=0, std::vector> data={});
@@ -51,7 +50,8 @@ class ClientConnection : eckit::NonCopyable {
     void disconnect();
 
     uint32_t generateRequestID();
-    const FdbEndpoint& controlEndpoint() const;
+    const eckit::net::Endpoint& controlEndpoint() const;
+    const std::string& defaultEndpoint() const { return defaultEndpoint_; }
 
 private: // methods
 
@@ -80,9 +80,11 @@ class ClientConnection : eckit::NonCopyable {
 
     eckit::SessionID sessionID_; 
 
-    FdbEndpoint controlEndpoint_;
+    eckit::net::Endpoint controlEndpoint_;
     eckit::net::Endpoint dataEndpoint_;
 
+    std::string defaultEndpoint_;
+
     eckit::net::TCPClient controlClient_;
     eckit::net::TCPClient dataClient_;
 
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.cc b/src/fdb5/remote/client/ClientConnectionRouter.cc
index b51691f2d..d1f6e7a50 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.cc
+++ b/src/fdb5/remote/client/ClientConnectionRouter.cc
@@ -28,7 +28,7 @@ namespace fdb5::remote {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-ClientConnection* ClientConnectionRouter::connection(Client& client, const FdbEndpoint& endpoint) {
+ClientConnection* ClientConnectionRouter::connection(Client& client, const eckit::net::Endpoint& endpoint, const std::string& defaultEndpoint) {
 
     std::lock_guard lock(connectionMutex_);
     auto it = connections_.find(endpoint);
@@ -36,7 +36,7 @@ ClientConnection* ClientConnectionRouter::connection(Client& client, const FdbEn
         it->second.clients_.insert(&client);
         return it->second.connection_.get();
     } else {
-        ClientConnection* clientConnection = new ClientConnection(endpoint);
+        ClientConnection* clientConnection = new ClientConnection(endpoint, defaultEndpoint);
         if (clientConnection->connect()) {
             auto it = (connections_.emplace(endpoint, Connection(std::unique_ptr(clientConnection), client))).first;
             return it->second.connection_.get();
@@ -46,16 +46,16 @@ ClientConnection* ClientConnectionRouter::connection(Client& client, const FdbEn
     }
 }
 
-ClientConnection* ClientConnectionRouter::connection(Client& client, const std::vector& endpoints) {
+ClientConnection* ClientConnectionRouter::connection(Client& client, const std::vector>& endpoints) {
 
-    std::vector fullEndpoints{endpoints};
+    std::vector> fullEndpoints{endpoints};
 
     std::lock_guard lock(connectionMutex_);
     while (fullEndpoints.size()>0) {
 
         // select a random endpoint
         size_t idx = std::rand() % fullEndpoints.size();
-        FdbEndpoint endpoint = fullEndpoints.at(idx);
+        eckit::net::Endpoint endpoint = fullEndpoints.at(idx).first;
 
         // look for the selected endpoint
         auto it = connections_.find(endpoint);
@@ -64,7 +64,7 @@ ClientConnection* ClientConnectionRouter::connection(Client& client, const std::
             return it->second.connection_.get();
         }
         else { // not yet there, trying to connect
-            ClientConnection* clientConnection = new ClientConnection(fullEndpoints.at(idx));
+            ClientConnection* clientConnection = new ClientConnection(fullEndpoints.at(idx).first, fullEndpoints.at(idx).second);
             if (clientConnection->connect(true)) {
                 auto it = (connections_.emplace(endpoint, Connection(std::unique_ptr(clientConnection), client))).first;
                 return it->second.connection_.get();
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.h b/src/fdb5/remote/client/ClientConnectionRouter.h
index 02380cbc5..9898bee37 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.h
+++ b/src/fdb5/remote/client/ClientConnectionRouter.h
@@ -12,7 +12,6 @@
 
 #include "fdb5/remote/client/Client.h"
 #include "fdb5/remote/client/ClientConnection.h"
-#include "fdb5/remote/FdbEndpoint.h"
 
 #include 
 
@@ -37,8 +36,8 @@ class ClientConnectionRouter : eckit::NonCopyable {
 
     static ClientConnectionRouter& instance();
 
-    ClientConnection* connection(Client& client, const FdbEndpoint& endpoint);
-    ClientConnection* connection(Client& client, const std::vector& endpoints);
+    ClientConnection* connection(Client& client, const eckit::net::Endpoint& endpoint, const std::string& defaultEndpoint);
+    ClientConnection* connection(Client& client, const std::vector>& endpoints);
 
     void deregister(Client& client);
 
@@ -49,7 +48,7 @@ class ClientConnectionRouter : eckit::NonCopyable {
     std::mutex connectionMutex_;
 
     // endpoint -> connection
-    std::unordered_map connections_;
+    std::unordered_map connections_;
 };
 
 }
\ No newline at end of file
diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc
index cb29d730b..bd6b3b5ef 100644
--- a/src/fdb5/remote/server/CatalogueHandler.cc
+++ b/src/fdb5/remote/server/CatalogueHandler.cc
@@ -9,6 +9,7 @@
  */
 
 #include "eckit/config/Resource.h"
+#include "eckit/net/NetMask.h"
 #include "eckit/serialisation/MemoryStream.h"
 
 #include "fdb5/LibFdb5.h"
@@ -30,7 +31,9 @@ namespace fdb5::remote {
 // ***************************************************************************************
 
 CatalogueHandler::CatalogueHandler(eckit::net::TCPSocket& socket, const Config& config):
-    ServerConnection(socket, config) {}
+    ServerConnection(socket, config) {
+
+}
 
 CatalogueHandler::~CatalogueHandler() {}
 
@@ -50,37 +53,81 @@ void CatalogueHandler::initialiseConnections() {
     socketRead(&tail, sizeof(tail), controlSocket_);
     ASSERT(tail == EndMarker);
 
-    std::vector stores;
-    std::vector localStores;
 
-    if (::getenv("FDB_STORE_HOST") && ::getenv("FDB_STORE_PORT")) {
-        // override the configuration
-        stores.push_back(net::Endpoint(::getenv("FDB_STORE_HOST"), std::stoi(::getenv("FDB_STORE_PORT"))));
-    }
-    else {
-        std::vector endpoints = config_.getStringVector("stores");
-        for (const std::string& endpoint: endpoints) {
-            stores.push_back(eckit::net::Endpoint(endpoint));
+    eckit::net::IPAddress address{controlSocket_.remoteAddr()};
+
+    std::string network = "";
+    if (config_.has("networks")) {
+        for (const auto& net: config_.getSubConfigurations("networks")) {
+            if (net.has("name") && net.has("netmask")) {
+                eckit::net::NetMask netmask{net.getString("netmask")};
+                if (netmask.contains(address)) {
+                    network = net.getString("name");
+                    break;
+                }
+            }
         }
-        if (config_.has("localStores")) {
-            std::vector endpoints = config_.getStringVector("localStores");
-            for (const std::string& endpoint: endpoints) {
-                localStores.push_back(eckit::net::Endpoint(endpoint));
+    }
+
+    Log::debug() << "Client " << address << " from network '" << network << "'" << std::endl;
+
+    ASSERT(config_.has("stores"));
+    std::map> stores;
+    // std::vector> stores;
+    for (const auto& store: config_.getSubConfigurations("stores")) {
+        ASSERT(store.has("default"));
+        std::string defaultEndpoint{store.getString("default")}; // can be an empty string, in case of local store
+        std::string networkEndpoint{defaultEndpoint};
+        if (!network.empty()) {
+            if (store.has(network)) {
+                networkEndpoint = store.getString(network);
             }
         }
+        auto it = stores.find(defaultEndpoint);
+        if (it == stores.end()) {
+            stores.emplace(defaultEndpoint, std::vector{networkEndpoint});
+        } else {
+            it->second.push_back(eckit::net::Endpoint{networkEndpoint});
+        }
     }
+    // std::string networkName{""};
+    // auto remoteAddress = eckit::net::IPAddress(controlSocket_.remoteAddr());
+    // for (const auto& net : networks_) {
+    //     if (net.second.contains(remoteAddress)) {
+    //         networkName = net.first;
+    //     }
+    // }
+
+    // std::vector stores;
+    // std::vector localStores;
+
+    // if (::getenv("FDB_STORE_HOST") && ::getenv("FDB_STORE_PORT")) {
+    //     // override the configuration
+    //     stores.push_back(net::Endpoint(::getenv("FDB_STORE_HOST"), std::stoi(::getenv("FDB_STORE_PORT"))));
+    // }
+    // else {
+    //     std::vector endpoints = config_.getStringVector("stores");
+    //     for (const std::string& endpoint: endpoints) {
+    //         stores.push_back(eckit::net::Endpoint(endpoint));
+    //     }
+    //     if (config_.has("localStores")) {
+    //         std::vector endpoints = config_.getStringVector("localStores");
+    //         for (const std::string& endpoint: endpoints) {
+    //             localStores.push_back(eckit::net::Endpoint(endpoint));
+    //         }
+    //     }
+    // }
 
     {
         Buffer startupBuffer(1024*1024);
         MemoryStream s(startupBuffer);
 
         s << stores.size();
-        for (const eckit::net::Endpoint& endpoint : stores) {
-            s << endpoint;
-        }
-        s << localStores.size();
-        for (const eckit::net::Endpoint& endpoint : localStores) {
-            s << endpoint;
+        for (const auto& store : stores) {
+            s << store.first << store.second.size();
+            for (const auto& ee: store.second) {
+                s << ee;
+            }
         }
         s << config_.schema();
 
diff --git a/src/fdb5/remote/server/CatalogueHandler.h b/src/fdb5/remote/server/CatalogueHandler.h
index ba6ee6a17..f33acd854 100644
--- a/src/fdb5/remote/server/CatalogueHandler.h
+++ b/src/fdb5/remote/server/CatalogueHandler.h
@@ -14,6 +14,13 @@
 #include "fdb5/api/FDB.h"
 
 namespace fdb5::remote {
+
+// class StoreEndpoint : public eckit::net::Endpoint {
+// public:
+//     StoreEndpoint(const std::string& endpoint) : eckit::net::Endpoint(endpoint) {}
+//     std::map aliases_;
+// };
+
 //----------------------------------------------------------------------------------------------------------------------
 class CatalogueHandler : public ServerConnection {
 public:  // methods
@@ -46,7 +53,8 @@ class CatalogueHandler : public ServerConnection {
 
 private:  // member
 
-   std::map> catalogues_;
+    // clientID --> Catalogue
+    std::map> catalogues_;
 
     FDB fdb_;
 };

From 4d030a9a17120e6aab4871de4d4b7054ad72f2b1 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 12 Dec 2023 21:35:28 +0000
Subject: [PATCH 051/186] missing include

---
 src/fdb5/api/RemoteFDB.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/fdb5/api/RemoteFDB.h b/src/fdb5/api/RemoteFDB.h
index 0c2345f58..03757341c 100644
--- a/src/fdb5/api/RemoteFDB.h
+++ b/src/fdb5/api/RemoteFDB.h
@@ -25,6 +25,8 @@
 #include "fdb5/api/LocalFDB.h"
 #include "fdb5/remote/client/Client.h"
 
+#include 
+
 namespace fdb5 {
 
 //----------------------------------------------------------------------------------------------------------------------

From 49a81e379b973803fd166377c0570e11c395ae2b Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 12 Dec 2023 22:39:13 +0000
Subject: [PATCH 052/186] version bump

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 57785b6ea..1a9c0c37d 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.93
+5.11.94

From 830fedca8505ce2922b0bf11a2d955bebac844e3 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 15 Dec 2023 11:45:27 +0000
Subject: [PATCH 053/186] wip

---
 src/fdb5/remote/client/ClientConnection.cc | 104 ++++++++++++---------
 src/fdb5/remote/client/ClientConnection.h  |   4 +-
 2 files changed, 61 insertions(+), 47 deletions(-)

diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index 3300945a4..ed9aa69d7 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -100,8 +100,11 @@ void ClientConnection::disconnect() {
 
     if (connected_) {
 
-        // Send termination message
-        controlWrite(Message::Exit, 0);
+        for (auto c: clients_) {
+            // Send termination message
+            controlWrite(Message::Exit, 0);
+
+        }
 
         listeningThread_.join();
 
@@ -332,6 +335,10 @@ void ClientConnection::listeningThreadLoop() {
 
         while (true) {
 
+            if (clients_.empty()) {
+                return;
+            }
+
             dataRead(&hdr, sizeof(hdr));
 
             eckit::Log::debug() << "ClientConnection::listeningThreadLoop - got [message=" << ((int) hdr.message) << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
@@ -340,56 +347,63 @@ void ClientConnection::listeningThreadLoop() {
             ASSERT(hdr.version == CurrentVersion);
 
             if (hdr.message == Message::Exit) {
-                return;
-            }
-
-            if (hdr.clientID) {
-                bool handled = false;
-                auto it = clients_.find(hdr.clientID);
-                if (it == clients_.end()) {
-                    std::stringstream ss;
-                    ss << "ERROR: Received [clientID="<< hdr.clientID << ",requestID="<< hdr.requestID << ",message=" << ((int) hdr.message) << ",payload=" << hdr.payloadSize << "]" << std::endl;
-                    ss << "Unexpected answer for clientID recieved (" << hdr.clientID << "). ABORTING";
-                    eckit::Log::status() << ss.str() << std::endl;
-                    eckit::Log::error() << "Retrieving... " << ss.str() << std::endl;
-                    throw eckit::SeriousBug(ss.str(), Here());
-
-                    ASSERT(false); // todo report the error
+                if (hdr.clientID) {
+                    auto it = clients_.find(hdr.clientID);
+                    if (it == clients_.end()) {
+                        clients_.erase(it);
+                    }
+                }
+                if (clients_.empty()) {
+                    return;
                 }
+            } else {
+                if (hdr.clientID) {
+                    bool handled = false;
+                    auto it = clients_.find(hdr.clientID);
+                    if (it == clients_.end()) {
+                        std::stringstream ss;
+                        ss << "ERROR: Received [clientID="<< hdr.clientID << ",requestID="<< hdr.requestID << ",message=" << ((int) hdr.message) << ",payload=" << hdr.payloadSize << "]" << std::endl;
+                        ss << "Unexpected answer for clientID recieved (" << hdr.clientID << "). ABORTING";
+                        eckit::Log::status() << ss.str() << std::endl;
+                        eckit::Log::error() << "Retrieving... " << ss.str() << std::endl;
+                        throw eckit::SeriousBug(ss.str(), Here());
+
+                        ASSERT(false); // todo report the error
+                    }
 
-                if (hdr.payloadSize == 0) {
-                    if (it->second->blockingRequestId() == hdr.requestID) {
-                        ASSERT(hdr.message == Message::Received);
-                        handled = it->second->response(hdr.requestID);
-                    } else {
-                        handled = it->second->handle(hdr.message, hdr.requestID);
+                    if (hdr.payloadSize == 0) {
+                        if (it->second->blockingRequestId() == hdr.requestID) {
+                            ASSERT(hdr.message == Message::Received);
+                            handled = it->second->response(hdr.requestID);
+                        } else {
+                            handled = it->second->handle(hdr.message, hdr.requestID);
+                        }
                     }
-                }
-                else {
-                    eckit::Buffer payload{hdr.payloadSize};
-                    dataRead(payload, hdr.payloadSize);
-
-                    if (it->second->blockingRequestId() == hdr.requestID) {
-                        handled = it->second->response(hdr.requestID, std::move(payload));
-                    } else {
-                        handled = it->second->handle(hdr.message, hdr.requestID, std::move(payload));
+                    else {
+                        eckit::Buffer payload{hdr.payloadSize};
+                        dataRead(payload, hdr.payloadSize);
+
+                        if (it->second->blockingRequestId() == hdr.requestID) {
+                            handled = it->second->response(hdr.requestID, std::move(payload));
+                        } else {
+                            handled = it->second->handle(hdr.message, hdr.requestID, std::move(payload));
+                        }
                     }
-                }
 
-                if (!handled) {
-                    std::stringstream ss;
-                    ss << "ERROR: Unexpected message recieved (" << static_cast(hdr.message) << "). ABORTING";
-                    eckit::Log::status() << ss.str() << std::endl;
-                    eckit::Log::error() << "Client Retrieving... " << ss.str() << std::endl;
-                    throw eckit::SeriousBug(ss.str(), Here());
-                }
-            } else {
-                if (hdr.payloadSize) {
-                    eckit::Buffer payload{hdr.payloadSize};
-                    dataRead(payload, hdr.payloadSize);
+                    if (!handled) {
+                        std::stringstream ss;
+                        ss << "ERROR: Unexpected message recieved (" << static_cast(hdr.message) << "). ABORTING";
+                        eckit::Log::status() << ss.str() << std::endl;
+                        eckit::Log::error() << "Client Retrieving... " << ss.str() << std::endl;
+                        throw eckit::SeriousBug(ss.str(), Here());
+                    }
+                } else {
+                    if (hdr.payloadSize) {
+                        eckit::Buffer payload{hdr.payloadSize};
+                        dataRead(payload, hdr.payloadSize);
+                    }
                 }
             }
-
             // Ensure we have consumed exactly the correct amount from the socket.
             dataRead(&tail, sizeof(tail));
             ASSERT(tail == EndMarker);
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index 463b9df94..12f8e0c3e 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -41,7 +41,7 @@ class ClientConnection : eckit::NonCopyable {
 public: // methods
 
     ClientConnection(const eckit::net::Endpoint& controlEndpoint, const std::string& defaultEndpoint);
-    virtual ~ClientConnection();
+    ~ClientConnection();
 
     void controlWrite(Client& client, remote::Message msg, uint32_t requestID, uint32_t archiverID=0, std::vector> data={});
     void dataWrite   (Client& client, remote::Message msg, uint32_t requestID, std::vector> data={});
@@ -58,7 +58,7 @@ class ClientConnection : eckit::NonCopyable {
     const eckit::net::Endpoint& dataEndpoint() const;
     
     // construct dictionary for protocol negotiation - to be defined in the client class
-    virtual eckit::LocalConfiguration availableFunctionality() const;
+    eckit::LocalConfiguration availableFunctionality() const;
 
     void controlWrite(Message msg, uint32_t clientID, uint32_t requestID = 0, std::vector> data = {});
     void controlWrite(const void* data, size_t length);

From 6fb345f287e9ab5ebbbb31b146b6dffbb590e96e Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 15 Dec 2023 12:01:46 +0000
Subject: [PATCH 054/186] wip

---
 src/fdb5/remote/client/ClientConnection.cc | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index ed9aa69d7..633dc151f 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -100,11 +100,11 @@ void ClientConnection::disconnect() {
 
     if (connected_) {
 
+        // Send termination message
         for (auto c: clients_) {
-            // Send termination message
-            controlWrite(Message::Exit, 0);
-
+            controlWrite(Message::Exit, c.first);
         }
+        controlWrite(Message::Exit, 0);
 
         listeningThread_.join();
 
@@ -335,10 +335,6 @@ void ClientConnection::listeningThreadLoop() {
 
         while (true) {
 
-            if (clients_.empty()) {
-                return;
-            }
-
             dataRead(&hdr, sizeof(hdr));
 
             eckit::Log::debug() << "ClientConnection::listeningThreadLoop - got [message=" << ((int) hdr.message) << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;

From 9438332d354d97cbf7f945d329d8f7543d1214e6 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 15 Dec 2023 12:08:47 +0000
Subject: [PATCH 055/186] wip

---
 src/fdb5/remote/client/ClientConnection.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index 633dc151f..b457940cd 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -42,7 +42,7 @@ class TCPException : public eckit::Exception {
 
 
 ClientConnection::ClientConnection(const eckit::net::Endpoint& controlEndpoint, const std::string& defaultEndpoint):
-    controlEndpoint_(controlEndpoint), defaultEndpoint_(defaultEndpoint), id_(1), connected_(false) {
+    controlEndpoint_(controlEndpoint), defaultEndpoint_(defaultEndpoint), id_(1), exit_(false), connected_(false) {
         eckit::Log::debug() << "ClientConnection::ClientConnection() controlEndpoint: " << controlEndpoint << std::endl;
     }
 
@@ -100,6 +100,7 @@ void ClientConnection::disconnect() {
 
     if (connected_) {
 
+        exit_ = true;
         // Send termination message
         for (auto c: clients_) {
             controlWrite(Message::Exit, c.first);
@@ -230,7 +231,7 @@ void ClientConnection::dataWrite(const void* data, size_t length) {
 
 void ClientConnection::dataRead(void* data, size_t length) {
     size_t read = dataClient_.read(data, length);
-    if (length != read) {
+    if (!exit_ && length != read) {
         std::stringstream ss;
         ss << "Read error. Expected " << length << " bytes, read " << read;
         throw TCPException(ss.str(), Here());

From 25cb57858136e4fd4fb7ab848bafe4f3d66272c5 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 15 Dec 2023 14:09:36 +0000
Subject: [PATCH 056/186] fix disconnection

---
 src/fdb5/remote/client/Client.h            | 4 ++--
 src/fdb5/remote/client/ClientConnection.cc | 3 ++-
 src/fdb5/remote/client/ClientConnection.h  | 1 +
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/fdb5/remote/client/Client.h b/src/fdb5/remote/client/Client.h
index 023bef0d5..c4f8bb04b 100644
--- a/src/fdb5/remote/client/Client.h
+++ b/src/fdb5/remote/client/Client.h
@@ -52,8 +52,8 @@ class Client : eckit::NonCopyable {
     void dataWrite(remote::Message msg, uint32_t requestID, std::vector> data={});
     
     // handlers for incoming messages - to be defined in the client class
-    virtual bool handle(Message message, uint32_t requestID) = 0;
-    virtual bool handle(Message message, uint32_t requestID, /*eckit::net::Endpoint endpoint,*/ eckit::Buffer&& payload) = 0;
+    virtual bool handle(Message message, uint32_t requestID) { return true; }
+    virtual bool handle(Message message, uint32_t requestID, /*eckit::net::Endpoint endpoint,*/ eckit::Buffer&& payload) { return true; }
     virtual void handleException(std::exception_ptr e) = 0;
 
     bool response(uint32_t requestID);
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index b457940cd..08098a5f1 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -334,7 +334,7 @@ void ClientConnection::listeningThreadLoop() {
         MessageHeader hdr;
         eckit::FixedString<4> tail;
 
-        while (true) {
+        while (!exit_) {
 
             dataRead(&hdr, sizeof(hdr));
 
@@ -351,6 +351,7 @@ void ClientConnection::listeningThreadLoop() {
                     }
                 }
                 if (clients_.empty()) {
+                    exit_ = true;
                     return;
                 }
             } else {
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index 12f8e0c3e..f0c45ea7a 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -100,6 +100,7 @@ class ClientConnection : eckit::NonCopyable {
     std::mutex idMutex_;
     uint32_t id_;
 
+    bool exit_;
     bool connected_; 
 };
 

From 3a239194da97013f7b27c642c198efd562908681 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 15 Dec 2023 14:10:29 +0000
Subject: [PATCH 057/186] version bump

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 1a9c0c37d..a98bad358 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.94
+5.11.95

From 49a23658e2e139e32ef74eab5dfce6879a7d68f8 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 19 Dec 2023 00:02:17 +0100
Subject: [PATCH 058/186] initial implementation of
 FDBRemoteDataHandle::estimate

---
 src/fdb5/remote/RemoteStore.cc | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index 65d720b46..f2ae64b25 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -225,10 +225,11 @@ class FDBRemoteDataHandle : public DataHandle {
 
 public: // methods
 
-    FDBRemoteDataHandle(uint32_t requestID,
+    FDBRemoteDataHandle(uint32_t requestID, Length estimate,
                         RemoteStore::MessageQueue& queue,
                         const net::Endpoint& remoteEndpoint) :
         requestID_(requestID),
+        estimate_(estimate),
         queue_(queue),
         remoteEndpoint_(remoteEndpoint),
         pos_(0),
@@ -309,7 +310,7 @@ class FDBRemoteDataHandle : public DataHandle {
     }
 
     Length estimate() override {
-        return 0;
+        return estimate_;
     }
 
     Offset position() override {
@@ -319,6 +320,7 @@ class FDBRemoteDataHandle : public DataHandle {
 private: // members
 
     uint32_t requestID_;
+    Length estimate_;
     RemoteStore::MessageQueue& queue_;
     net::Endpoint remoteEndpoint_;
     size_t pos_;
@@ -604,7 +606,7 @@ eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation, c
     uint32_t id = connection_.generateRequestID();
     controlWriteCheckResponse(fdb5::remote::Message::Read, id, encodeBuffer, s.position());
 
-    return new FDBRemoteDataHandle(id, retrieveMessageQueue_, controlEndpoint());
+    return new FDBRemoteDataHandle(id, fieldLocation.length(), retrieveMessageQueue_, controlEndpoint());
 }
 
 RemoteStore& RemoteStore::get(const eckit::URI& uri) {

From b1ae0351b85598bd07fd3d1e0cdb5ed5ddf4a4da Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 19 Dec 2023 00:02:55 +0100
Subject: [PATCH 059/186] version bump

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index a98bad358..de3def843 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.95
+5.11.96

From f2b725110ad51056ad990bb20702bf2d3b6e7cf1 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Sun, 24 Dec 2023 23:25:18 +0100
Subject: [PATCH 060/186] ClientConnection cleanup

---
 src/fdb5/api/RemoteFDB.cc                     | 22 ++++++-
 src/fdb5/remote/client/Client.cc              | 15 +++--
 src/fdb5/remote/client/Client.h               | 16 ++---
 src/fdb5/remote/client/ClientConnection.cc    | 61 ++++++++++---------
 src/fdb5/remote/client/ClientConnection.h     | 10 ++-
 .../remote/client/ClientConnectionRouter.cc   | 37 +++++------
 .../remote/client/ClientConnectionRouter.h    | 21 ++-----
 src/fdb5/remote/server/CatalogueHandler.cc    |  1 -
 src/fdb5/remote/server/ServerConnection.cc    |  4 +-
 src/fdb5/remote/server/StoreHandler.cc        |  1 -
 10 files changed, 97 insertions(+), 91 deletions(-)

diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index 613d4ee67..c2c09467f 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -32,7 +32,27 @@ struct BaseAPIHelper {
     static ValueType valueFromStream(eckit::Stream& s, fdb5::RemoteFDB* fdb) { return ValueType(s); }
 };
 
-using ListHelper = BaseAPIHelper;
+// using ListHelper = BaseAPIHelper;
+
+struct ListHelper : BaseAPIHelper {
+
+    static fdb5::ListElement valueFromStream(eckit::Stream& s, fdb5::RemoteFDB* fdb) {
+        fdb5::ListElement elem(s);
+
+        eckit::Log::debug() << "ListHelper::valueFromStream - original location: ";
+        elem.location().dump(eckit::Log::debug());
+        eckit::Log::debug() << std::endl;
+
+        if (elem.location().uri().scheme() == "fdb") {
+            eckit::net::Endpoint endpoint{elem.location().uri().host(), elem.location().uri().port()};
+
+            std::shared_ptr remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(endpoint), static_cast(elem.location())).make_shared();
+            return fdb5::ListElement(elem.key(), remoteLocation, elem.timestamp());
+        }
+        std::shared_ptr remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(""), elem.location()).make_shared();
+        return fdb5::ListElement(elem.key(), remoteLocation, elem.timestamp());
+    }
+};
 
 struct InspectHelper : BaseAPIHelper {
 
diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index a5058e211..7a9333bff 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -32,24 +32,28 @@ uint32_t Client::clientId_=0;
 // }
 
 Client::Client(const eckit::net::Endpoint& endpoint, const std::string& defaultEndpoint) :
-    connection_(*(ClientConnectionRouter::instance().connection(*this, endpoint, defaultEndpoint))),
+    connection_(ClientConnectionRouter::instance().connection(endpoint, defaultEndpoint)),
     blockingRequestId_(0) {
-
+    
     std::lock_guard lock(idMutex_);
     id_ = ++clientId_;
+
+    connection_.add(*this);
 }
 
 
 Client::Client(const std::vector>& endpoints) :
-    connection_(*(ClientConnectionRouter::instance().connection(*this, endpoints))),
+    connection_(ClientConnectionRouter::instance().connection(endpoints)),
     blockingRequestId_(0) {
 
     std::lock_guard lock(idMutex_);
     id_ = ++clientId_;
+
+    connection_.add(*this);
 }
 
 Client::~Client() {
-    ClientConnectionRouter::instance().deregister(*this);
+    ASSERT(connection_.remove(id_));
 }
 
 bool Client::response(uint32_t requestID) {
@@ -79,6 +83,7 @@ bool Client::response(uint32_t requestID, eckit::Buffer&& payload) {
 void Client::controlWriteCheckResponse(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
 
     ASSERT(requestID);
+    ASSERT(!(!payloadLength ^ !payload));
     std::lock_guard lock(blockingRequestMutex_);
     
     promise_ = {};
@@ -98,6 +103,8 @@ void Client::controlWriteCheckResponse(Message msg, uint32_t requestID, const vo
 
 eckit::Buffer Client::controlWriteReadResponse(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
 
+    ASSERT(requestID);
+    ASSERT(!(!payloadLength ^ !payload));
     std::lock_guard lock(blockingRequestMutex_);
 
     payloadPromise_ = {};
diff --git a/src/fdb5/remote/client/Client.h b/src/fdb5/remote/client/Client.h
index c4f8bb04b..7b052a78a 100644
--- a/src/fdb5/remote/client/Client.h
+++ b/src/fdb5/remote/client/Client.h
@@ -38,10 +38,9 @@ class Client : eckit::NonCopyable {
     Client(const std::vector>& endpoints);
     ~Client();
 
-    uint32_t id() { return id_; }
+    uint32_t id() const { return id_; }
     const eckit::net::Endpoint& controlEndpoint() const { return connection_.controlEndpoint(); }
     const std::string& defaultEndpoint() const { return connection_.defaultEndpoint(); }
-    // const eckit::net::Endpoint& fullyQualifiedControlEndpoint() const { return connection_.controlEndpoint(); }
 
     uint32_t generateRequestID() { return connection_.generateRequestID(); }
 
@@ -52,8 +51,8 @@ class Client : eckit::NonCopyable {
     void dataWrite(remote::Message msg, uint32_t requestID, std::vector> data={});
     
     // handlers for incoming messages - to be defined in the client class
-    virtual bool handle(Message message, uint32_t requestID) { return true; }
-    virtual bool handle(Message message, uint32_t requestID, /*eckit::net::Endpoint endpoint,*/ eckit::Buffer&& payload) { return true; }
+    virtual bool handle(Message message, uint32_t requestID) = 0;
+    virtual bool handle(Message message, uint32_t requestID, /*eckit::net::Endpoint endpoint,*/ eckit::Buffer&& payload) = 0;
     virtual void handleException(std::exception_ptr e) = 0;
 
     bool response(uint32_t requestID);
@@ -62,24 +61,17 @@ class Client : eckit::NonCopyable {
 
 protected:
     
-    // std::vector endpoints_;
-    // std::string domain_;
-
-    // eckit::net::Endpoint endpoint_;
-    // eckit::net::Endpoint fullyQualifiedEndpoint_;
     ClientConnection& connection_;
 
 private:
+    inline static std::mutex idMutex_;
     static uint32_t clientId_;
-
-    std::mutex idMutex_;
     uint32_t id_;
 
     std::mutex blockingRequestMutex_;
     uint32_t blockingRequestId_;
     std::promise promise_;
     std::promise payloadPromise_;
-
 };
 
 }
\ No newline at end of file
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index 08098a5f1..7bef20622 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -46,6 +46,30 @@ ClientConnection::ClientConnection(const eckit::net::Endpoint& controlEndpoint,
         eckit::Log::debug() << "ClientConnection::ClientConnection() controlEndpoint: " << controlEndpoint << std::endl;
     }
 
+void ClientConnection::add(Client& client) {
+    std::lock_guard lock(clientsMutex_);
+    auto it = clients_.find(client.id());
+    ASSERT(it == clients_.end());
+
+    clients_[client.id()] = &client;
+}
+
+
+bool ClientConnection::remove(uint32_t clientID) {
+    std::lock_guard lock(clientsMutex_);
+    auto it = clients_.find(clientID);
+    bool exist = it != clients_.end();
+
+    if (exist) {
+        clients_.erase(it);
+        if (clients_.empty()) {
+            exit_ = true;
+            ClientConnectionRouter::instance().deregister(*this);
+        }
+    }
+    return exist;
+}
+
 ClientConnection::~ClientConnection() {
     disconnect();
 }
@@ -58,7 +82,7 @@ uint32_t ClientConnection::generateRequestID() {
 
 bool ClientConnection::connect(bool singleAttempt) {
 
-    std::lock_guard lock(requestMutex_);
+//    std::lock_guard lock(requestMutex_);
     if (connected_) {
         eckit::Log::warning() << "ClientConnection::connect() called when already connected" << std::endl;
         return connected_;
@@ -71,6 +95,7 @@ bool ClientConnection::connect(bool singleAttempt) {
         // Connect to server, and check that the server is happy on the response 
         eckit::Log::debug() << "Connecting to host: " << controlEndpoint_ << std::endl;
         controlClient_.connect(controlEndpoint_, fdbMaxConnectRetries, fdbConnectTimeout);
+
         writeControlStartupMessage();
         eckit::SessionID serverSession = verifyServerStartupResponse();
 
@@ -86,9 +111,6 @@ bool ClientConnection::connect(bool singleAttempt) {
     } catch(eckit::TooManyRetries& e) {
         if (controlClient_.isConnected()) {
             controlClient_.close();
-        //     throw ConnectionError(fdbMaxConnectRetries, dataEndpoint_);
-        // } else {
-        //     throw ConnectionError(fdbMaxConnectRetries, fullyQualifiedControlEndpoint_);
         }
     }
     return false;
@@ -96,10 +118,9 @@ bool ClientConnection::connect(bool singleAttempt) {
 
 void ClientConnection::disconnect() {
 
-    std::lock_guard lock(requestMutex_);
+//    std::lock_guard lock(requestMutex_);
 
     if (connected_) {
-
         exit_ = true;
         // Send termination message
         for (auto c: clients_) {
@@ -137,13 +158,8 @@ eckit::LocalConfiguration ClientConnection::availableFunctionality() const {
 
 void ClientConnection::controlWrite(Client& client, Message msg, uint32_t requestID, uint32_t archiverID, std::vector> data) {
 
-    {
-        std::lock_guard lock(requestMutex_);
-        auto it = clients_.find(client.id());
-        if (it == clients_.end()) {
-            clients_[client.id()] = &client;
-        }
-    }
+    auto it = clients_.find(client.id());
+    ASSERT(it != clients_.end());
 
     controlWrite(msg, archiverID ? archiverID : client.id(), requestID, data);
 }
@@ -189,13 +205,8 @@ void ClientConnection::controlRead(void* data, size_t length) {
 
 void ClientConnection::dataWrite(Client& client, remote::Message msg, uint32_t requestID, std::vector> data) {
 
-    {
-        std::lock_guard lock(requestMutex_);
-        auto it = clients_.find(client.id());
-        if (it == clients_.end()) {
-            clients_[client.id()] = &client;
-        }
-    }
+    auto it = clients_.find(client.id());
+    ASSERT(it != clients_.end());
 
     dataWrite(msg, client.id(), requestID, data);
 }
@@ -272,7 +283,6 @@ void ClientConnection::writeControlStartupMessage() {
     //       essentially JSON) over the wire for flexibility.
     s << availableFunctionality().get();
 
-    //std::cout << "writeControlStartupMessage" << std::endl;
     controlWrite(Message::Startup, 0, 0, std::vector>{{payload, s.position()}});
 }
 
@@ -345,14 +355,7 @@ void ClientConnection::listeningThreadLoop() {
 
             if (hdr.message == Message::Exit) {
                 if (hdr.clientID) {
-                    auto it = clients_.find(hdr.clientID);
-                    if (it == clients_.end()) {
-                        clients_.erase(it);
-                    }
-                }
-                if (clients_.empty()) {
-                    exit_ = true;
-                    return;
+                    remove(hdr.clientID);
                 }
             } else {
                 if (hdr.clientID) {
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index f0c45ea7a..6554b1f5c 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -31,6 +31,7 @@ class Buffer;
 namespace fdb5::remote {
 
 class Client;
+class ClientConnectionRouter;
 
 //----------------------------------------------------------------------------------------------------------------------
 
@@ -40,12 +41,14 @@ class ClientConnection : eckit::NonCopyable {
 
 public: // methods
 
-    ClientConnection(const eckit::net::Endpoint& controlEndpoint, const std::string& defaultEndpoint);
     ~ClientConnection();
 
     void controlWrite(Client& client, remote::Message msg, uint32_t requestID, uint32_t archiverID=0, std::vector> data={});
     void dataWrite   (Client& client, remote::Message msg, uint32_t requestID, std::vector> data={});
 
+    void add(Client& client);
+    bool remove(uint32_t clientID);
+
     bool connect(bool singleAttempt = false);
     void disconnect();
 
@@ -55,6 +58,10 @@ class ClientConnection : eckit::NonCopyable {
 
 private: // methods
 
+    friend class ClientConnectionRouter;
+
+    ClientConnection(const eckit::net::Endpoint& controlEndpoint, const std::string& defaultEndpoint);
+
     const eckit::net::Endpoint& dataEndpoint() const;
     
     // construct dictionary for protocol negotiation - to be defined in the client class
@@ -88,6 +95,7 @@ class ClientConnection : eckit::NonCopyable {
     eckit::net::TCPClient controlClient_;
     eckit::net::TCPClient dataClient_;
 
+    std::mutex clientsMutex_;
     std::map clients_;
 
     std::thread listeningThread_;
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.cc b/src/fdb5/remote/client/ClientConnectionRouter.cc
index d1f6e7a50..f689d959e 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.cc
+++ b/src/fdb5/remote/client/ClientConnectionRouter.cc
@@ -28,25 +28,26 @@ namespace fdb5::remote {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-ClientConnection* ClientConnectionRouter::connection(Client& client, const eckit::net::Endpoint& endpoint, const std::string& defaultEndpoint) {
+ClientConnection& ClientConnectionRouter::connection(const eckit::net::Endpoint& endpoint, const std::string& defaultEndpoint) {
 
     std::lock_guard lock(connectionMutex_);
+
     auto it = connections_.find(endpoint);
     if (it != connections_.end()) {
-        it->second.clients_.insert(&client);
-        return it->second.connection_.get();
+        return *(it->second);
     } else {
-        ClientConnection* clientConnection = new ClientConnection(endpoint, defaultEndpoint);
+        ClientConnection* clientConnection = new ClientConnection{endpoint, defaultEndpoint};
         if (clientConnection->connect()) {
-            auto it = (connections_.emplace(endpoint, Connection(std::unique_ptr(clientConnection), client))).first;
-            return it->second.connection_.get();
+            auto it = (connections_.emplace(endpoint, std::unique_ptr(clientConnection))).first;
+            return *(it->second);
         } else {
+            delete clientConnection;
             throw ConnectionError(endpoint);
         }
     }
 }
 
-ClientConnection* ClientConnectionRouter::connection(Client& client, const std::vector>& endpoints) {
+ClientConnection& ClientConnectionRouter::connection(const std::vector>& endpoints) {
 
     std::vector> fullEndpoints{endpoints};
 
@@ -60,14 +61,13 @@ ClientConnection* ClientConnectionRouter::connection(Client& client, const std::
         // look for the selected endpoint
         auto it = connections_.find(endpoint);
         if (it != connections_.end()) {
-            it->second.clients_.insert(&client);
-            return it->second.connection_.get();
+            return *(it->second);
         }
         else { // not yet there, trying to connect
-            ClientConnection* clientConnection = new ClientConnection(fullEndpoints.at(idx).first, fullEndpoints.at(idx).second);
+             std::unique_ptr clientConnection =  std::unique_ptr(new ClientConnection{endpoint, fullEndpoints.at(idx).second});
             if (clientConnection->connect(true)) {
-                auto it = (connections_.emplace(endpoint, Connection(std::unique_ptr(clientConnection), client))).first;
-                return it->second.connection_.get();
+                auto it = (connections_.emplace(endpoint, std::move(clientConnection))).first;
+                return *(it->second);
             }
         }
 
@@ -82,18 +82,11 @@ ClientConnection* ClientConnectionRouter::connection(Client& client, const std::
     throw ConnectionError();
 }
 
+void ClientConnectionRouter::deregister(ClientConnection& connection) {
 
-void ClientConnectionRouter::deregister(Client& client) {
     std::lock_guard lock(connectionMutex_);
-
-    auto it = connections_.find(client.controlEndpoint());
-    ASSERT(it != connections_.end());
-
-    auto clientIt = it->second.clients_.find(&client);
-    ASSERT(clientIt != it->second.clients_.end());
-
-    it->second.clients_.erase(clientIt);
-    if (it->second.clients_.empty()) {
+    auto it = connections_.find(connection.controlEndpoint());
+    if (it != connections_.end()) {
         connections_.erase(it);
     }
 }
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.h b/src/fdb5/remote/client/ClientConnectionRouter.h
index 9898bee37..ea7e1755f 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.h
+++ b/src/fdb5/remote/client/ClientConnectionRouter.h
@@ -19,36 +19,23 @@ namespace fdb5::remote {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-class Connection {
-public:
-
-    Connection(std::unique_ptr connection, Client& clients)
-        : connection_(std::move(connection)), clients_(std::set{&clients}) {}
-
-    std::unique_ptr connection_;
-    std::set clients_;
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
 class ClientConnectionRouter : eckit::NonCopyable {
 public:
 
     static ClientConnectionRouter& instance();
 
-    ClientConnection* connection(Client& client, const eckit::net::Endpoint& endpoint, const std::string& defaultEndpoint);
-    ClientConnection* connection(Client& client, const std::vector>& endpoints);
+    ClientConnection& connection(const eckit::net::Endpoint& endpoint, const std::string& defaultEndpoint);
+    ClientConnection& connection(const std::vector>& endpoints);
 
-    void deregister(Client& client);
+    void deregister(ClientConnection& connection);
 
 private:
 
     ClientConnectionRouter() {} ///< private constructor only used by singleton
 
     std::mutex connectionMutex_;
-
     // endpoint -> connection
-    std::unordered_map connections_;
+    std::unordered_map> connections_;
 };
 
 }
\ No newline at end of file
diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc
index bd6b3b5ef..cc0744131 100644
--- a/src/fdb5/remote/server/CatalogueHandler.cc
+++ b/src/fdb5/remote/server/CatalogueHandler.cc
@@ -338,7 +338,6 @@ void CatalogueHandler::flush(const MessageHeader& hdr) {
     s >> numArchived;
 
     std::future& archive = archiveFuture_[hdr.clientID];
-    // std::cout << "numArchived: " << numArchived << " | archiveFuture_.valid: " << archive.valid() << std::endl;
 
     ASSERT(numArchived == 0 || archive.valid());
 
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index 5e1fb6693..bf2eb15ec 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -99,8 +99,6 @@ void ServerConnection::initialiseConnections() {
     ASSERT(hdr.message == Message::Startup);
     ASSERT(hdr.requestID == 0);
 
-    // std::cout << "ServerConnection::initialiseConnections() - payload "<< hdr.payloadSize << std::endl;
-
     eckit::Buffer payload1 = receivePayload(hdr, controlSocket_);
     eckit::FixedString<4> tail;
     socketRead(&tail, sizeof(tail), controlSocket_);
@@ -267,7 +265,7 @@ void ServerConnection::controlWrite(const void* data, size_t length) {
 }
 
 void ServerConnection::socketRead(void* data, size_t length, eckit::net::TCPSocket& socket) {
-    // std::cout << "ServerConnection::socketRead " << socket.remoteHost() << ":" << socket.remotePort() << "  length: " << length << std::endl;
+
     size_t read = socket.read(data, length);
     if (length != read) {
         std::stringstream ss;
diff --git a/src/fdb5/remote/server/StoreHandler.cc b/src/fdb5/remote/server/StoreHandler.cc
index 6e2ba2683..2f88a535e 100644
--- a/src/fdb5/remote/server/StoreHandler.cc
+++ b/src/fdb5/remote/server/StoreHandler.cc
@@ -400,7 +400,6 @@ void StoreHandler::flush(const MessageHeader& hdr) {
     s >> numArchived;
 
     std::future& archive = archiveFuture_[hdr.clientID];
-    // std::cout << "numArchived: " << numArchived << " | archiveFuture_.valid: " << archive.valid() << std::endl;
 
     ASSERT(numArchived == 0 || archive.valid());
 

From 5284b992da969ab8cd2fa0d79700a6a476aaf103 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 2 Jan 2024 10:58:35 +0000
Subject: [PATCH 061/186] tag

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index de3def843..6d0019e0f 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.96
+5.11.97

From 44370ca44c23923d543d08a00eec4b89d695cc97 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 4 Jan 2024 14:25:43 +0000
Subject: [PATCH 062/186] improved error report

---
 src/fdb5/api/RemoteFDB.cc | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index c2c09467f..208b3675b 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -79,7 +79,18 @@ namespace fdb5 {
 
 const eckit::net::Endpoint& RemoteFDB::storeEndpoint(const std::string& endpoint) const {
     auto it = stores_.find(endpoint);
-    ASSERT(it != stores_.end());
+    if (it == stores_.end()) {
+        std::stringstream ss;
+        ss << "Unable to find a matching endpoint. Looking for " << endpoint << std::endl;
+        ss << "Available endpoints:";
+        for (auto s : stores_) {
+            ss << std::endl << s.first << " - ";
+            for (auto e: s.second) {
+                ss << e << ", ";
+            }
+        }
+        throw eckit::SeriousBug(ss.str());
+    }
     return it->second.at(std::rand() % it->second.size());
 }
 

From 476a1eccf5fd7ee9c9faec889d9f57a07c15b881 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 11 Jan 2024 15:35:32 +0000
Subject: [PATCH 063/186] refactored store endpoint configuration

---
 VERSION                                    |  2 +-
 src/fdb5/api/RemoteFDB.cc                  | 95 ++++++++++++++--------
 src/fdb5/api/RemoteFDB.h                   |  8 +-
 src/fdb5/remote/RemoteStore.cc             | 38 ++-------
 src/fdb5/remote/server/CatalogueHandler.cc | 68 ++++++----------
 src/fdb5/toc/Root.h                        |  2 +-
 src/fdb5/tools/fdb-copy.cc                 |  7 --
 src/fdb5/types/TypeClimateDaily.cc         |  3 +-
 8 files changed, 100 insertions(+), 123 deletions(-)

diff --git a/VERSION b/VERSION
index 6d0019e0f..70ea29e34 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.97
+5.11.99
diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index 208b3675b..100e23a14 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -38,18 +38,19 @@ struct ListHelper : BaseAPIHelper() << "ListHelper::valueFromStream - original location: ";
         elem.location().dump(eckit::Log::debug());
         eckit::Log::debug() << std::endl;
 
         if (elem.location().uri().scheme() == "fdb") {
-            eckit::net::Endpoint endpoint{elem.location().uri().host(), elem.location().uri().port()};
+            eckit::net::Endpoint fieldLocationEndpoint{elem.location().uri().host(), elem.location().uri().port()};
 
-            std::shared_ptr remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(endpoint), static_cast(elem.location())).make_shared();
+            std::shared_ptr remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(fieldLocationEndpoint), static_cast(elem.location())).make_shared();
             return fdb5::ListElement(elem.key(), remoteLocation, elem.timestamp());
         }
-        std::shared_ptr remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(""), elem.location()).make_shared();
+        std::shared_ptr remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(), elem.location()).make_shared();
         return fdb5::ListElement(elem.key(), remoteLocation, elem.timestamp());
     }
 };
@@ -64,34 +65,48 @@ struct InspectHelper : BaseAPIHelper() << std::endl;
 
         if (elem.location().uri().scheme() == "fdb") {
-            eckit::net::Endpoint endpoint{elem.location().uri().host(), elem.location().uri().port()};
+            eckit::net::Endpoint fieldLocationEndpoint{elem.location().uri().host(), elem.location().uri().port()};
 
-            std::shared_ptr remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(endpoint), static_cast(elem.location())).make_shared();
+            std::shared_ptr remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(fieldLocationEndpoint), static_cast(elem.location())).make_shared();
             return fdb5::ListElement(elem.key(), remoteLocation, elem.timestamp());
         }
-        std::shared_ptr remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(""), elem.location()).make_shared();
+        std::shared_ptr remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(), elem.location()).make_shared();
         return fdb5::ListElement(elem.key(), remoteLocation, elem.timestamp());
     }
 };
+
+class FDBEndpoint {
+public:
+    eckit::net::Endpoint fieldLocationEndpoint_;
+    bool localFieldLocation_;
+
+    FDBEndpoint(const eckit::net::Endpoint& fieldLocationEndpoint, bool localFieldLocation) :
+        fieldLocationEndpoint_(fieldLocationEndpoint), localFieldLocation_(localFieldLocation) {}
+};
+
 }
 
 namespace fdb5 {
 
-const eckit::net::Endpoint& RemoteFDB::storeEndpoint(const std::string& endpoint) const {
-    auto it = stores_.find(endpoint);
-    if (it == stores_.end()) {
+const eckit::net::Endpoint& RemoteFDB::storeEndpoint() const {
+    if (storesLocalFields_.empty()) {
+        throw eckit::SeriousBug("Unable to find a store to serve local data");
+    }
+    return storesLocalFields_.at(std::rand() % storesLocalFields_.size());
+}
+const eckit::net::Endpoint& RemoteFDB::storeEndpoint(const eckit::net::Endpoint& fieldLocationEndpoint) const {
+    // looking for an alias for the given endpoint
+    auto it = storesReadMapping_.find(fieldLocationEndpoint);
+    if (it == storesReadMapping_.end()) {
         std::stringstream ss;
-        ss << "Unable to find a matching endpoint. Looking for " << endpoint << std::endl;
-        ss << "Available endpoints:";
-        for (auto s : stores_) {
-            ss << std::endl << s.first << " - ";
-            for (auto e: s.second) {
-                ss << e << ", ";
-            }
+        ss << "Unable to find a matching endpoint. Looking for " << fieldLocationEndpoint << std::endl;
+        ss << "Available endpoints:" << std::endl;
+        for (auto s : storesReadMapping_) {
+            ss << s.first << " --> " << s.second << std::endl;
         }
         throw eckit::SeriousBug(ss.str());
     }
-    return it->second.at(std::rand() % it->second.size());
+    return it->second;
 }
 
 RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name):
@@ -105,9 +120,11 @@ RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name
     s >> numStores;
     ASSERT(numStores > 0);
 
-    std::vector stores;
-    std::vector defaultEndpoints;
+    std::unordered_set localFields;
 
+    std::vector stores;
+    std::vector fieldLocationEndpoints;
+    
     for (size_t i=0; i> store;
@@ -115,29 +132,39 @@ RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name
         s >> numAliases;
         std::vector aliases;
         if (numAliases == 0) {
-            stores.push_back(store);
-            defaultEndpoints.push_back(store);
-            Log::debug() << "store endpoint: " << store << " default endpoint: " << store << std::endl;
+            eckit::net::Endpoint storeEndpoint{store};
+            storesReadMapping_[storeEndpoint] = storeEndpoint;
+            Log::debug() << "store endpoint: " << storeEndpoint << " default data location endpoint: " << storeEndpoint << std::endl;
         } else {
             for (size_t j=0; j() << "store endpoint: " << endpoint << " default endpoint: " << store << std::endl;
+                eckit::net::Endpoint alias(s);
+                if (store.empty()) {
+                    storesLocalFields_.push_back(alias);
+                    localFields.emplace(alias);
+                } else {
+                    eckit::net::Endpoint fieldLocationEndpoint{store};
+                    storesReadMapping_[fieldLocationEndpoint] = alias;
+                }
+                Log::debug() << "store endpoint: " << alias << " default data location endpoint: " << store << std::endl;
             }
         }
-        stores_.emplace(store, aliases);
     }
-
-    config_.set("stores", stores);
-    config_.set("storesDefaultEndpoints", defaultEndpoints);
+    for(const auto& s: storesReadMapping_) {
+        if (localFields.find(s.second) == localFields.end()) {
+            storesArchiveMapping_.push_back(std::make_pair(s.second, s.first));
+            stores.push_back(s.second);
+            fieldLocationEndpoints.push_back(s.first);
+        }
+    }
+    for (const auto& s: storesLocalFields_) {
+        stores.push_back(s);
+        fieldLocationEndpoints.push_back("");
+    }
 
     fdb5::Schema* schema = eckit::Reanimator::reanimate(s);
 
-    // eckit::Log::debug() << *this << " - Received Store endpoint: " << storeEndpoint_ << " and master schema: " << std::endl;
-    // schema->dump(eckit::Log::debug());
-    
+    config_.set("stores", stores);
+    config_.set("fieldLocationEndpoints", fieldLocationEndpoints);
     config_.overrideSchema(static_cast(controlEndpoint())+"/schema", schema);
 }
 
diff --git a/src/fdb5/api/RemoteFDB.h b/src/fdb5/api/RemoteFDB.h
index 03757341c..7b1e15728 100644
--- a/src/fdb5/api/RemoteFDB.h
+++ b/src/fdb5/api/RemoteFDB.h
@@ -63,7 +63,8 @@ class RemoteFDB : public LocalFDB, public remote::Client {
 
     MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) override { NOTIMP; }
 
-    const eckit::net::Endpoint& storeEndpoint(const std::string& endpoint) const;
+    const eckit::net::Endpoint& storeEndpoint() const;
+    const eckit::net::Endpoint& storeEndpoint(const eckit::net::Endpoint& fieldLocationEndpoint) const;
 
 private: // methods
 
@@ -82,8 +83,9 @@ class RemoteFDB : public LocalFDB, public remote::Client {
 
 private: // members
 
-    std::unique_ptr archiver_;
-    std::map> stores_;
+    std::unordered_map storesReadMapping_;
+    std::vector> storesArchiveMapping_;
+    std::vector storesLocalFields_;
 
     // Where do we put received messages
     // @note This is a map of requestID:MessageQueue. At the point that a request is
diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index f2ae64b25..3e25f3d82 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -329,44 +329,17 @@ class FDBRemoteDataHandle : public DataHandle {
     bool complete_;
 };
 
-// std::vector writeEndpoints(const Config& config) {
-//     if (config.has("storeHost") && config.has("storePort")) {
-//         return std::vector{eckit::net::Endpoint(config.getString("storeHost"), config.getInt("storePort"))};
-//     }
-//     ASSERT(config.has("stores"));
-//     std::vector stores = config.getSubConfigurations("stores");
-//     for (const auto& root : stores) {
-//         spaceRoots.emplace_back(
-//             Root(
-//                 root.getString("path"),
-//                 root.getString("name", ""),
-//                 root.getBool("list", visit),
-//                 root.getBool("retrieve", visit),
-//                 root.getBool("archive", writable),
-//                 root.getBool("wipe", writable)
-//             )
-//         );
-//     }
-
-//     .getStringVector("stores");
-
-//     std::srand(std::time(nullptr));
-//     return eckit::net::Endpoint(stores.at(std::rand() % stores.size()));
-// }
-
 std::vector> storeEndpoints(const Config& config) {
-    // if (config.has("storeHost") && config.has("storePort")) {
-    //     return std::vector{eckit::net::Endpoint{config.getString("storeHost"), config.getInt("storePort"), config.getString("remoteDomain", "")}};
-    // }
+
     ASSERT(config.has("stores"));
-    ASSERT(config.has("storesDefaultEndpoints"));
+    ASSERT(config.has("fieldLocationEndpoints"));
     std::vector stores = config.getStringVector("stores");
-    std::vector defaultEndpoints = config.getStringVector("storesDefaultEndpoints");
+    std::vector fieldLocationEndpoints = config.getStringVector("fieldLocationEndpoints");
 
-    ASSERT(stores.size() == defaultEndpoints.size());
+    ASSERT(stores.size() == fieldLocationEndpoints.size());
     std::vector> out;
     for (size_t i=0; i() << "Client " << address << " from network '" << network << "'" << std::endl;
+    Log::debug() << "Client " << clientIPaddress << " from network '" << clientNetwork << "'" << std::endl;
 
     ASSERT(config_.has("stores"));
     std::map> stores;
     // std::vector> stores;
-    for (const auto& store: config_.getSubConfigurations("stores")) {
-        ASSERT(store.has("default"));
-        std::string defaultEndpoint{store.getString("default")}; // can be an empty string, in case of local store
-        std::string networkEndpoint{defaultEndpoint};
-        if (!network.empty()) {
-            if (store.has(network)) {
-                networkEndpoint = store.getString(network);
+    for (const auto& configStore: config_.getSubConfigurations("stores")) {
+        ASSERT(configStore.has("default"));
+        eckit::net::Endpoint fieldLocationEndpoint{configStore.getString("default")};
+        eckit::net::Endpoint storeEndpoint{fieldLocationEndpoint};
+        if (!clientNetwork.empty()) {
+            if (configStore.has(clientNetwork)) {
+                storeEndpoint = eckit::net::Endpoint{configStore.getString(clientNetwork)};
             }
         }
-        auto it = stores.find(defaultEndpoint);
+
+        auto it = stores.find(fieldLocationEndpoint);
         if (it == stores.end()) {
-            stores.emplace(defaultEndpoint, std::vector{networkEndpoint});
+            stores.emplace(fieldLocationEndpoint, std::vector{storeEndpoint});
         } else {
-            it->second.push_back(eckit::net::Endpoint{networkEndpoint});
+            it->second.push_back(storeEndpoint);
+        }
+
+        if (configStore.getBool("serveLocalData", false)) {
+            it = stores.find("");
+            if (it == stores.end()) {
+                stores.emplace("", std::vector{storeEndpoint});
+            } else {
+                it->second.push_back(storeEndpoint);
+            }
         }
     }
-    // std::string networkName{""};
-    // auto remoteAddress = eckit::net::IPAddress(controlSocket_.remoteAddr());
-    // for (const auto& net : networks_) {
-    //     if (net.second.contains(remoteAddress)) {
-    //         networkName = net.first;
-    //     }
-    // }
-
-    // std::vector stores;
-    // std::vector localStores;
-
-    // if (::getenv("FDB_STORE_HOST") && ::getenv("FDB_STORE_PORT")) {
-    //     // override the configuration
-    //     stores.push_back(net::Endpoint(::getenv("FDB_STORE_HOST"), std::stoi(::getenv("FDB_STORE_PORT"))));
-    // }
-    // else {
-    //     std::vector endpoints = config_.getStringVector("stores");
-    //     for (const std::string& endpoint: endpoints) {
-    //         stores.push_back(eckit::net::Endpoint(endpoint));
-    //     }
-    //     if (config_.has("localStores")) {
-    //         std::vector endpoints = config_.getStringVector("localStores");
-    //         for (const std::string& endpoint: endpoints) {
-    //             localStores.push_back(eckit::net::Endpoint(endpoint));
-    //         }
-    //     }
-    // }
 
     {
         Buffer startupBuffer(1024*1024);
diff --git a/src/fdb5/toc/Root.h b/src/fdb5/toc/Root.h
index c263230bf..3a4f1b958 100644
--- a/src/fdb5/toc/Root.h
+++ b/src/fdb5/toc/Root.h
@@ -47,7 +47,7 @@ class Root  {
     const eckit::PathName &path() const;
 
     /// Root exists in the filesystem, use this check to avoid errors when accessing
-    /// This result is cached at construction
+    /// This result is computed with a lazy approach and then memoized
     bool exists() const;
 
     bool enabled(const ControlIdentifier& controlIdentifier) const {
diff --git a/src/fdb5/tools/fdb-copy.cc b/src/fdb5/tools/fdb-copy.cc
index b6047b323..a604d75a5 100644
--- a/src/fdb5/tools/fdb-copy.cc
+++ b/src/fdb5/tools/fdb-copy.cc
@@ -51,8 +51,6 @@ static std::vector readRequest(const CmdArgs& args) {
 
     for (size_t i = 0; i < args.count(); ++i) {
 
-        // std::cout << "Reading " << args(i) << std::endl;
-
         std::ifstream in(args(i).c_str());
         if (in.bad()) {
             std::ostringstream msg;
@@ -98,12 +96,7 @@ void FDBCopy::execute(const CmdArgs& args) {
 
     std::vector requests = readRequest(args);
 
-    // std::cout << "REQUESTS: " << std::endl;
-    // for (auto r : requests)
-    //     std::cout << r << std::endl;
-
     // Evaluate the requests to obtain data handles
-
     const bool sort = args.getBool("sort", false);
     fdb5::HandleGatherer handles(sort);
 
diff --git a/src/fdb5/types/TypeClimateDaily.cc b/src/fdb5/types/TypeClimateDaily.cc
index 279c50394..55b46c184 100644
--- a/src/fdb5/types/TypeClimateDaily.cc
+++ b/src/fdb5/types/TypeClimateDaily.cc
@@ -90,9 +90,8 @@ void TypeClimateDaily::getValues(const metkit::mars::MarsRequest &request,
 bool TypeClimateDaily::match(const std::string&,
                    const std::string& value1,
                    const std::string& value2) const {
-    // std::cout << value1 << " vs. " << value2 << " are " << (month(value1) == month(value2)) << std::endl;
 
-    return month(value1) == month(value2);
+  return month(value1) == month(value2);
 }
 
 void TypeClimateDaily::print(std::ostream &out) const {

From 2ff5a35d477b9e8ef310581275a1a470561cdc29 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 8 Feb 2024 09:29:27 +0100
Subject: [PATCH 064/186] multi-archiver

---
 src/fdb5/CMakeLists.txt                    |   2 +
 src/fdb5/api/RemoteFDB.cc                  |  10 +-
 src/fdb5/database/ArchiveVisitor.cc        |   6 +-
 src/fdb5/database/Archiver.cc              |  10 +-
 src/fdb5/database/Archiver.h               |   7 +-
 src/fdb5/database/Catalogue.h              |   2 +-
 src/fdb5/database/InspectionKey.cc         |   8 +-
 src/fdb5/database/InspectionKey.h          |   2 +-
 src/fdb5/database/Store.cc                 |   6 +-
 src/fdb5/database/Store.h                  |   4 +-
 src/fdb5/remote/Connection.cc              | 148 ++++++++++
 src/fdb5/remote/Connection.h               |  71 +++++
 src/fdb5/remote/Messages.cc                |  56 ++++
 src/fdb5/remote/Messages.h                 |  47 ++--
 src/fdb5/remote/RemoteCatalogue.cc         |  27 +-
 src/fdb5/remote/RemoteCatalogue.h          |   2 +-
 src/fdb5/remote/RemoteStore.cc             |  28 +-
 src/fdb5/remote/RemoteStore.h              |   2 +-
 src/fdb5/remote/client/Client.cc           |  10 +-
 src/fdb5/remote/client/Client.h            |   3 +-
 src/fdb5/remote/client/ClientConnection.cc | 304 ++++++++++++---------
 src/fdb5/remote/client/ClientConnection.h  |  32 ++-
 src/fdb5/remote/server/CatalogueHandler.cc |  87 +++---
 src/fdb5/remote/server/CatalogueHandler.h  |   5 +-
 src/fdb5/remote/server/ServerConnection.cc |  11 +-
 src/fdb5/remote/server/ServerConnection.h  |   8 +-
 src/fdb5/remote/server/StoreHandler.cc     |  30 +-
 src/fdb5/remote/server/StoreHandler.h      |   4 +-
 src/fdb5/toc/TocCatalogueWriter.cc         |   2 +-
 src/fdb5/toc/TocCatalogueWriter.h          |   2 +-
 src/fdb5/toc/TocStore.cc                   |   2 +-
 src/fdb5/toc/TocStore.h                    |   2 +-
 src/fdb5/tools/fdb-write.cc                |  20 +-
 33 files changed, 643 insertions(+), 317 deletions(-)
 create mode 100644 src/fdb5/remote/Connection.cc
 create mode 100644 src/fdb5/remote/Connection.h

diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt
index a9bd1380a..4bfae8bdd 100644
--- a/src/fdb5/CMakeLists.txt
+++ b/src/fdb5/CMakeLists.txt
@@ -220,6 +220,8 @@ if(HAVE_FDB_REMOTE)
         remote/RemoteCatalogue.cc
         # remote/RemoteEngine.h
         # remote/RemoteEngine.cc
+        remote/Connection.h
+        remote/Connection.cc
 
         remote/client/Client.h
         remote/client/Client.cc
diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index 100e23a14..12c77b374 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -114,7 +114,7 @@ RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name
     Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port")), "") {
 
     uint32_t id = generateRequestID();
-    eckit::Buffer buf = controlWriteReadResponse(remote::Message::Schema, id);
+    eckit::Buffer buf = controlWriteReadResponse(remote::Message::Stores, id);
     eckit::MemoryStream s(buf);
     size_t numStores;
     s >> numStores;
@@ -161,7 +161,11 @@ RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name
         fieldLocationEndpoints.push_back("");
     }
 
-    fdb5::Schema* schema = eckit::Reanimator::reanimate(s);
+    id = generateRequestID();
+    buf = controlWriteReadResponse(remote::Message::Schema, id);
+    eckit::MemoryStream s2(buf);
+
+    fdb5::Schema* schema = eckit::Reanimator::reanimate(s2);
 
     config_.set("stores", stores);
     config_.set("fieldLocationEndpoints", fieldLocationEndpoints);
@@ -199,7 +203,7 @@ auto RemoteFDB::forwardApiCall(const HelperClass& helper, const FDBToolRequest&
     s << request;
     helper.encodeExtra(s);
 
-    controlWriteCheckResponse(HelperClass::message(), id, encodeBuffer, s.position());
+    controlWriteCheckResponse(HelperClass::message(), id, true, encodeBuffer, s.position());
 
     // Return an AsyncIterator to allow the messages to be retrieved in the API
 
diff --git a/src/fdb5/database/ArchiveVisitor.cc b/src/fdb5/database/ArchiveVisitor.cc
index 4dd54adcb..4f71be421 100644
--- a/src/fdb5/database/ArchiveVisitor.cc
+++ b/src/fdb5/database/ArchiveVisitor.cc
@@ -16,8 +16,8 @@
 #include "fdb5/database/Store.h"
 
 namespace {
-void CatalogueCallback(uint32_t archiverId, fdb5::CatalogueWriter* catalogue, const fdb5::InspectionKey &key, std::unique_ptr fieldLocation) {
-    catalogue->archive(archiverId, key, std::move(fieldLocation));
+void CatalogueCallback(fdb5::CatalogueWriter* catalogue, const fdb5::InspectionKey &key, std::unique_ptr fieldLocation) {
+    catalogue->archive(key, std::move(fieldLocation));
 }
 }
 namespace fdb5 {
@@ -35,7 +35,7 @@ bool ArchiveVisitor::selectDatum(const InspectionKey &key, const Key &full) {
     checkMissingKeys(full);
     const Key idxKey = current()->currentIndexKey();
 
-    store()->archive(owner_.id(), idxKey, data_, size_, std::bind(&CatalogueCallback, owner_.id(), current(), key, std::placeholders::_1));
+    store()->archive(idxKey, data_, size_, std::bind(&CatalogueCallback, current(), key, std::placeholders::_1));
     
     return true;
 }
diff --git a/src/fdb5/database/Archiver.cc b/src/fdb5/database/Archiver.cc
index c08d183b9..f1632e4cb 100644
--- a/src/fdb5/database/Archiver.cc
+++ b/src/fdb5/database/Archiver.cc
@@ -24,16 +24,10 @@ namespace fdb5 {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-uint32_t Archiver::archiverId_=0;
-
 Archiver::Archiver(const Config& dbConfig) :
     dbConfig_(dbConfig),
     catalogue_(nullptr),
-    store_(nullptr) {
-
-    std::lock_guard lock(idMutex_);
-    id_ = ++archiverId_;
-}
+    store_(nullptr) {}
 
 Archiver::~Archiver() {
 
@@ -42,8 +36,6 @@ Archiver::~Archiver() {
     databases_.clear(); //< explicitly delete the DBs before schemas are destroyed
 }
 
-uint32_t id();
-
 void Archiver::archive(const Key &key, const void* data, size_t len) {
     ArchiveVisitor visitor(*this, key, data, len);
     archive(key, visitor);
diff --git a/src/fdb5/database/Archiver.h b/src/fdb5/database/Archiver.h
index 80c9f9f7d..0beab4267 100644
--- a/src/fdb5/database/Archiver.h
+++ b/src/fdb5/database/Archiver.h
@@ -51,7 +51,7 @@ class Archiver : public eckit::NonCopyable {
 
     virtual ~Archiver();
 
-    uint32_t id() const { return id_; }
+    // uint32_t id() const { return id_; }
 
     void archive(const Key &key, BaseArchiveVisitor& visitor);
     void archive(const Key &key, const void* data, size_t len);
@@ -72,11 +72,6 @@ class Archiver : public eckit::NonCopyable {
     void selectDatabase(const Key &key);
 
 private: // members
-    static uint32_t archiverId_;
-
-    std::mutex idMutex_;
-    uint32_t id_;
-
     friend class BaseArchiveVisitor;
 
     Config dbConfig_;
diff --git a/src/fdb5/database/Catalogue.h b/src/fdb5/database/Catalogue.h
index 41eece01b..d8f050f74 100644
--- a/src/fdb5/database/Catalogue.h
+++ b/src/fdb5/database/Catalogue.h
@@ -164,7 +164,7 @@ class CatalogueWriter : virtual public Catalogue  {
 
     virtual const Index& currentIndex() = 0;
     virtual const Key currentIndexKey() = 0;
-    virtual void archive(const uint32_t archiverId, const InspectionKey& key, std::unique_ptr fieldLocation) = 0;
+    virtual void archive(const InspectionKey& key, std::unique_ptr fieldLocation) = 0;
     virtual void overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) = 0;
     virtual void index(const InspectionKey& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) = 0;
     virtual void reconsolidate() = 0;
diff --git a/src/fdb5/database/InspectionKey.cc b/src/fdb5/database/InspectionKey.cc
index a8c43ad32..d23698d40 100644
--- a/src/fdb5/database/InspectionKey.cc
+++ b/src/fdb5/database/InspectionKey.cc
@@ -88,10 +88,10 @@ InspectionKey::InspectionKey(const eckit::StringDict &keys) :
     }
 }
 
-// InspectionKey::InspectionKey(eckit::Stream& s) :
-//     rule_(nullptr) {
-//     decode(s);
-// }
+InspectionKey::InspectionKey(eckit::Stream& s) :
+    rule_(nullptr) {
+    decode(s);
+}
 
 Key InspectionKey::canonical() const {
     Key key;
diff --git a/src/fdb5/database/InspectionKey.h b/src/fdb5/database/InspectionKey.h
index 53ce7e878..da801d440 100644
--- a/src/fdb5/database/InspectionKey.h
+++ b/src/fdb5/database/InspectionKey.h
@@ -33,7 +33,7 @@ class InspectionKey : public Key {
     InspectionKey();
 
     explicit InspectionKey(const Key& other);
-    // explicit InspectionKey(eckit::Stream &);
+    explicit InspectionKey(eckit::Stream &);
     explicit InspectionKey(const std::string &request);
     explicit InspectionKey(const std::string &keys, const Rule* rule);
 
diff --git a/src/fdb5/database/Store.cc b/src/fdb5/database/Store.cc
index 56edae187..243e57f7b 100644
--- a/src/fdb5/database/Store.cc
+++ b/src/fdb5/database/Store.cc
@@ -24,11 +24,11 @@ namespace fdb5 {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-void Store::archive(const uint32_t archiverId, const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive) {
-    catalogue_archive(archive(archiverId, key, data, length));
+void Store::archive(const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive) {
+    catalogue_archive(archive(key, data, length));
 }
 
-std::unique_ptr Store::archive(const uint32_t archiverId, const Key& key, const void *data, eckit::Length length) {
+std::unique_ptr Store::archive(const Key& key, const void *data, eckit::Length length) {
     NOTIMP;
 }
 
diff --git a/src/fdb5/database/Store.h b/src/fdb5/database/Store.h
index a96a3db57..ecf3e5695 100644
--- a/src/fdb5/database/Store.h
+++ b/src/fdb5/database/Store.h
@@ -36,8 +36,8 @@ class Store {
 
     virtual eckit::DataHandle* retrieve(Field& field) const = 0;
 //    virtual void archive(const Key& key, const void *data, eckit::Length length, void(*catalogue_archive)(std::unique_ptr fieldLocation)) = 0;
-    virtual void archive(const uint32_t archiverId, const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive);
-    virtual std::unique_ptr archive(const uint32_t archiverId, const Key& key, const void *data, eckit::Length length);
+    virtual void archive(const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive);
+    virtual std::unique_ptr archive(const Key& key, const void *data, eckit::Length length);
 
     virtual void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit = true) const = 0;
 
diff --git a/src/fdb5/remote/Connection.cc b/src/fdb5/remote/Connection.cc
new file mode 100644
index 000000000..fc3a4ff24
--- /dev/null
+++ b/src/fdb5/remote/Connection.cc
@@ -0,0 +1,148 @@
+#include "eckit/io/Buffer.h"
+#include "eckit/log/Log.h"
+#include "eckit/os/BackTrace.h"
+
+#include "fdb5/LibFdb5.h"
+// #include "fdb5/remote/Handler.h"
+#include "fdb5/remote/Connection.h"
+
+namespace fdb5::remote {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+namespace{
+
+class TCPException : public eckit::Exception {
+public:
+    TCPException(const std::string& msg, const eckit::CodeLocation& here) :
+        eckit::Exception(std::string("TCPException: ") + msg, here) {
+
+        eckit::Log::error() << "TCP Exception; backtrace(): " << std::endl;
+        eckit::Log::error() << eckit::BackTrace::dump() << std::endl;
+    }
+};
+
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+
+void Connection::writeUnsafe(bool control, const void* data, size_t length) {
+    size_t written = 0;
+    if (control || single_) {
+        written = controlSocket().write(data, length);
+    } else {
+        written = dataSocket().write(data, length);
+    }
+    if (length != written) {
+        std::stringstream ss;
+        ss << "Write error. Expected " << length << " bytes, wrote " << written;
+        throw TCPException(ss.str(), Here());
+    }
+}
+
+void Connection::readUnsafe(bool control, void* data, size_t length) {
+    size_t read = 0;
+    if (control || single_) {
+        read = controlSocket().read(data, length);
+    } else {
+        read = dataSocket().read(data, length);
+    }
+    if (!exit_ && length != read) {
+        std::stringstream ss;
+        ss << "Read error. Expected " << length << " bytes, read " << read;
+        throw TCPException(ss.str(), Here());
+    }
+}
+
+eckit::Buffer Connection::read(bool control, MessageHeader& hdr) {
+    eckit::FixedString<4> tail;
+
+    readUnsafe(control, &hdr, sizeof(hdr));
+
+    std::cout << "READ [" <<  "endpoint=" <<  ((control || single_) ? controlSocket() : dataSocket()).remotePort() << ",message=" << hdr.message << ",clientID=" << hdr.clientID() << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
+
+    ASSERT(hdr.marker == StartMarker);
+    ASSERT(hdr.version == CurrentVersion);
+    ASSERT(hdr.control() == control);
+
+    eckit::Buffer payload{hdr.payloadSize};
+    if (hdr.payloadSize > 0) {
+        readUnsafe(control, payload, hdr.payloadSize);
+    }
+    // Ensure we have consumed exactly the correct amount from the socket.
+    readUnsafe(control, &tail, sizeof(tail));
+    ASSERT(tail == EndMarker);
+
+    if (hdr.message == Message::Error) {
+        std::cout << "ERROR while reading: ";
+
+        char msg[hdr.payloadSize+1];
+        if (hdr.payloadSize) {
+            char msg[hdr.payloadSize+1];
+            std::cout << static_cast(payload.data());
+        }
+        std::cout << std::endl;
+    }
+
+    return payload;
+}
+
+// void Connection::write(remote::Message msg, bool control, const Handler& clientID, uint32_t requestID, const void* data, uint32_t length) {
+//     write(msg, control, clientID.clientId(), requestID, std::vector>{{data, length}});
+// }
+// void Connection::write(remote::Message msg, bool control, const Handler& clientID, uint32_t requestID, std::vector> data) {
+//     write(msg, control, clientID.clientId(), requestID, data);
+// }
+void Connection::write(remote::Message msg, bool control, uint32_t clientID, uint32_t requestID, const void* data, uint32_t length) {
+    write(msg, control, clientID, requestID, std::vector>{{data, length}});
+}
+void Connection::write(remote::Message msg, bool control, uint32_t clientID, uint32_t requestID, std::vector> data) {
+
+    uint32_t payloadLength = 0;
+    for (auto d: data) {
+        ASSERT(d.first);
+        payloadLength += d.second;
+    }
+
+    MessageHeader message{msg, control, clientID, requestID, payloadLength};
+
+    std::cout << "WRITE [" << "endpoint=" << ((control || single_) ? controlSocket() : dataSocket()).remotePort() <<
+    ",message=" << message.message << ",clientID=" << message.clientID() << ",requestID=" << message.requestID << ",payload=" << message.payloadSize << "]" << std::endl;
+
+    eckit::Log::debug() << "Connection::write [endpoint=" << ((control || single_) ? controlSocket() : dataSocket()) <<
+        ",message=" << msg << ",clientID=" << message.clientID() << ",requestID=" << requestID << ",data=" << data.size() << ",payload=" << payloadLength << "]" << std::endl;
+
+    std::lock_guard lock((control || single_) ? controlMutex_ : dataMutex_);
+    writeUnsafe(control, &message, sizeof(message));
+    for (auto d: data) {
+        writeUnsafe(control, d.first, d.second);
+    }
+    writeUnsafe(control, &EndMarker, sizeof(EndMarker));
+}
+
+// void Connection::writeControl(remote::Message msg, uint32_t clientID, uint32_t requestID, const void* data, uint32_t length) {
+//     write(msg, true, clientID, requestID, std::vector>{{data, length}});
+// }
+// void Connection::writeControl(remote::Message msg, uint32_t clientID, uint32_t requestID, std::vector> data) {
+//     write(msg, true, clientID, requestID, data);
+// }
+// void Connection::writeData(remote::Message msg, uint32_t clientID, uint32_t requestID, const void* data, uint32_t length) {
+//     write(msg, false, clientID, requestID, std::vector>{{data, length}});
+// }
+// void Connection::writeData(remote::Message msg, uint32_t clientID, uint32_t requestID, std::vector> data) {
+//     write(msg, false, clientID, requestID, data);
+// }
+void Connection::error(const std::string& msg, uint32_t clientID, uint32_t requestID) {
+    write(Message::Error, true, clientID, requestID, std::vector>{{msg.c_str(), msg.length()}});
+}
+// void Connection::error(const std::string& msg, const Handler& clientID, uint32_t requestID) {
+//     write(Message::Error, true, clientID.clientId(), requestID, std::vector>{{msg.c_str(), msg.length()}});
+// }
+eckit::Buffer Connection::readControl(MessageHeader& hdr) {
+    return read(true, hdr);
+}
+eckit::Buffer Connection::readData(MessageHeader& hdr) {
+    return read(false, hdr);
+}
+
+}  // namespace fdb5::remote
diff --git a/src/fdb5/remote/Connection.h b/src/fdb5/remote/Connection.h
new file mode 100644
index 000000000..2b4fc0ad4
--- /dev/null
+++ b/src/fdb5/remote/Connection.h
@@ -0,0 +1,71 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+#pragma once
+
+#include "eckit/net/TCPSocket.h"
+
+#include "fdb5/remote/Messages.h"
+
+namespace eckit {
+
+class Buffer;
+
+}
+
+namespace fdb5::remote {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+class Connection : eckit::NonCopyable {
+
+public: // methods
+    Connection() : single_(false), exit_(false) {}
+    virtual ~Connection() {}
+
+    void write(Message msg, bool control, uint32_t clientID, uint32_t requestID, const void* data, uint32_t length);
+    void write(Message msg, bool control, uint32_t clientID, uint32_t requestID, std::vector> data = {});
+    // void write(Message msg, bool control, const Handler& clientID, uint32_t requestID, const void* data, uint32_t length);
+    // void write(Message msg, bool control, const Handler& clientID, uint32_t requestID, std::vector> data = {});
+    // void writeControl(Message msg, uint32_t clientID, uint32_t requestID, const void* data, uint32_t length);
+    // void writeControl(Message msg, uint32_t clientID, uint32_t requestID, std::vector> data = {});
+    // void writeData(Message msg, uint32_t clientID, uint32_t requestID, const void* data, uint32_t length);
+    // void writeData(Message msg, uint32_t clientID, uint32_t requestID, std::vector> data = {});
+
+    void error(const std::string& msg, uint32_t clientID, uint32_t requestID);
+    // void error(const std::string& msg, const Handler& clientID, uint32_t requestID);
+
+    eckit::Buffer readControl(MessageHeader& hdr);
+    eckit::Buffer readData(MessageHeader& hdr);
+
+private: // methods
+
+    eckit::Buffer read(bool control, MessageHeader& hdr);
+
+    void writeUnsafe(bool control, const void* data, size_t length);
+    void readUnsafe(bool control, void* data, size_t length);
+
+    virtual eckit::net::TCPSocket& controlSocket() = 0;
+    virtual eckit::net::TCPSocket& dataSocket() = 0;
+
+protected: // members
+
+    bool single_;
+    bool exit_;
+
+private: // members
+
+    std::mutex controlMutex_;
+    std::mutex dataMutex_;
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
+}  // namespace fdb5::remote
\ No newline at end of file
diff --git a/src/fdb5/remote/Messages.cc b/src/fdb5/remote/Messages.cc
index 0a31b5636..e30fc1ced 100644
--- a/src/fdb5/remote/Messages.cc
+++ b/src/fdb5/remote/Messages.cc
@@ -15,6 +15,7 @@
 
 #include "fdb5/remote/Messages.h"
 
+
 //#include "eckit/serialisation/Stream.h"
 
 using namespace eckit;
@@ -25,6 +26,61 @@ namespace remote {
 
 //----------------------------------------------------------------------------------------------------------------------
 
+
+std::ostream& operator<<(std::ostream& s, const Message& m) {
+    switch (m) {
+        case Message::None: s << "None"; break;
+        case Message::Exit: s << "Exit"; break;
+        case Message::Startup: s << "Startup"; break;
+        case Message::Error: s << "Error"; break;
+        case Message::Stores: s << "Stores"; break;
+        case Message::Schema: s << "Schema"; break;
+
+    // API calls to forward
+        case Message::Flush: s << "Flush"; break;
+        case Message::Archive: s << "Archive"; break;
+        case Message::Retrieve: s << "Retrieve"; break;
+        case Message::List: s << "List"; break;
+        case Message::Dump: s << "Dump"; break;
+        case Message::Status: s << "Status"; break;
+        case Message::Wipe: s << "Wipe"; break;
+        case Message::Purge: s << "Purge"; break;
+        case Message::Stats: s << "Stats"; break;
+        case Message::Control: s << "Control"; break;
+        case Message::Inspect: s << "Inspect"; break;
+        case Message::Read: s << "Read"; break;
+        case Message::Move: s << "Move"; break;
+        case Message::Store: s << "Store"; break;
+
+    // Responses
+        case Message::Received: s << "Received"; break;
+        case Message::Complete: s << "Complete"; break;
+
+    // Data communication
+        case Message::Blob: s << "Blob"; break;
+        case Message::MultiBlob: s << "MultiBlob"; break;
+    }
+    s << "(" << ((int) m) << ")";
+    return s;
+}
+
+// MessageHeader::MessageHeader(Message message, bool control, const Handler& clientID, uint32_t requestID, uint32_t payloadSize) :
+//     marker(StartMarker),
+//     version(CurrentVersion),
+//     message(message),
+//     clientID_((clientID.clientId()<<1) + (control ? 1 : 0)),
+//     requestID(requestID),
+//     payloadSize(payloadSize) {}
+
+
+MessageHeader::MessageHeader(Message message, bool control, uint32_t clientID, uint32_t requestID, uint32_t payloadSize) :
+    marker(StartMarker),
+    version(CurrentVersion),
+    message(message),
+    clientID_((clientID<<1) + (control ? 1 : 0)),
+    requestID(requestID),
+    payloadSize(payloadSize) {}
+
 //ClassSpec MessageHeader::classSpec_ = {&MessageHeader::classSpec(), "MessageHeader",};
 //Reanimator MessageHeader::reanimator_;
 //
diff --git a/src/fdb5/remote/Messages.h b/src/fdb5/remote/Messages.h
index 3547fff67..90cc02e99 100644
--- a/src/fdb5/remote/Messages.h
+++ b/src/fdb5/remote/Messages.h
@@ -18,10 +18,11 @@
 
 #pragma once
 
-#include "eckit/types/FixedString.h"
-#include "eckit/serialisation/Streamable.h"
-
 #include 
+#include 
+
+#include "eckit/types/FixedString.h"
+// #include "fdb5/remote/Handler.h"
 
 namespace eckit {
     class Stream;
@@ -34,8 +35,7 @@ namespace fdb5::remote {
 const static eckit::FixedString<4> StartMarker {"SFDB"};
 const static eckit::FixedString<4> EndMarker {"EFDB"};
 
-constexpr uint16_t CurrentVersion = 11;
-
+constexpr uint16_t CurrentVersion = 12;
 
 enum class Message : uint16_t {
 
@@ -44,6 +44,8 @@ enum class Message : uint16_t {
     Exit,
     Startup,
     Error,
+    Stores,
+    Schema,
 
     // API calls to forward
     Flush = 100,
@@ -60,7 +62,6 @@ enum class Message : uint16_t {
     Read,
     Move,
     Store,
-    Schema,
 
     // Responses
     Received = 200,
@@ -68,46 +69,52 @@ enum class Message : uint16_t {
 
     // Data communication
     Blob = 300,
-    MultiBlob,
+    MultiBlob
+
 };
 
+std::ostream& operator<<(std::ostream& s, const Message& m);
 
 // Header used for all messages
-
-struct MessageHeader {
+class MessageHeader {
 
 public: // methods
 
     MessageHeader() :
         version(CurrentVersion),
         message(Message::None),
-        clientID(0),
+        clientID_(0),
         requestID(0),
         payloadSize(0) {}
 
-    MessageHeader(Message message, uint32_t clientID, uint32_t requestID, uint32_t payloadSize=0) :
-        marker(StartMarker),
-        version(CurrentVersion),
-        message(message),
-        clientID(clientID),
-        requestID(requestID),
-        payloadSize(payloadSize) {}
+    // MessageHeader(Message message, bool control, const Handler& clientID, uint32_t requestID, uint32_t payloadSize=0);
+
+    MessageHeader(Message message, bool control, uint32_t clientID, uint32_t requestID, uint32_t payloadSize);
+    
+    bool control() const {
+        return ((clientID_ & 0x00000001) == 1);
+    }
+    uint32_t clientID() const {
+        return (clientID_>>1);
+    }
+
+public:
 
     eckit::FixedString<4> marker;   // 4 bytes  --> 4
 
     uint16_t version;               // 2 bytes  --> 6
 
     Message message;                // 2 bytes  --> 8
-
-    uint32_t clientID;              // 4 bytes  --> 12
+ 
+    uint32_t clientID_;             // 4 bytes  --> 12
 
     uint32_t requestID;             // 4 bytes  --> 16
 
     uint32_t payloadSize;           // 4 bytes  --> 20
 
     eckit::FixedString<16> hash;    // 16 bytes --> 36
-};
 
+};
 
 //----------------------------------------------------------------------------------------------------------------------
 
diff --git a/src/fdb5/remote/RemoteCatalogue.cc b/src/fdb5/remote/RemoteCatalogue.cc
index 229a0b0d2..895f7625f 100644
--- a/src/fdb5/remote/RemoteCatalogue.cc
+++ b/src/fdb5/remote/RemoteCatalogue.cc
@@ -48,7 +48,7 @@ class RemoteCatalogueArchiver {
 
 public: // methods
 
-    static RemoteCatalogueArchiver* get(uint64_t archiverID);
+    static RemoteCatalogueArchiver* get();
 
     bool valid() { return archiveFuture_.valid(); }
     bool dirty() { return dirty_; }
@@ -76,18 +76,14 @@ class RemoteCatalogueArchiver {
 
 };
 
-RemoteCatalogueArchiver* RemoteCatalogueArchiver::get(uint64_t archiverID) {
+RemoteCatalogueArchiver* RemoteCatalogueArchiver::get() {
 
-    static std::unordered_map> archivers_;
-    static std::mutex getArchiverMutex_;
+    static std::unique_ptr archiver_;
 
-    std::lock_guard lock(getArchiverMutex_);
-
-    auto it = archivers_.find(archiverID);
-    if (it == archivers_.end()) {
-        it = archivers_.emplace(archiverID, new RemoteCatalogueArchiver()).first;
+    if (!archiver_) {
+        archiver_.reset(new RemoteCatalogueArchiver());
     }
-    return it->second.get();
+    return archiver_.get();
 }
 
 void RemoteCatalogueArchiver::start() {
@@ -148,7 +144,7 @@ FDBStats RemoteCatalogueArchiver::flush(RemoteCatalogue* catalogue) {
     eckit::Log::debug() << " RemoteCatalogue::flush - flushing " << numArchive << " fields" << std::endl;
     uint32_t id = catalogue->generateRequestID();
     // The flush call is blocking
-    catalogue->controlWriteCheckResponse(Message::Flush, id, sendBuf, s.position());
+    catalogue->controlWriteCheckResponse(Message::Flush, id, false, sendBuf, s.position());
 
     dirty_ = false;
 
@@ -242,22 +238,19 @@ void RemoteCatalogue::sendArchiveData(uint32_t id, const Key& key, std::unique_p
     dataWrite(Message::Blob, id, payloads);
 }
 
-void RemoteCatalogue::archive(const uint32_t archiverID, const InspectionKey& key, std::unique_ptr fieldLocation) {
+void RemoteCatalogue::archive(const InspectionKey& key, std::unique_ptr fieldLocation) {
 
     // if there is no archiving thread active, then start one.
     // n.b. reset the archiveQueue_ after a potential flush() cycle.
     if (!archiver_) {
-        uint64_t archiverName = std::hash()(controlEndpoint());
-        archiverName = archiverName << 32;
-        archiverName += archiverID;
-        archiver_ = RemoteCatalogueArchiver::get(archiverName);
+        archiver_ = RemoteCatalogueArchiver::get();
     }
     uint32_t id = connection_.generateRequestID();
     if (!archiver_->valid()) {
         archiver_->start();
         ASSERT(archiver_->valid());
 
-        controlWriteCheckResponse(Message::Archive, id);
+        controlWriteCheckResponse(Message::Archive, id, true);
     }
     // eckit::Log::debug() << " RemoteCatalogue::archive - adding to queue [id=" << id << ",key=" << key << ",fieldLocation=" << fieldLocation->uri() << "]" << std::endl;
     archiver_->emplace(id, this, key, std::move(fieldLocation));
diff --git a/src/fdb5/remote/RemoteCatalogue.h b/src/fdb5/remote/RemoteCatalogue.h
index 392a6dd12..044eb6632 100644
--- a/src/fdb5/remote/RemoteCatalogue.h
+++ b/src/fdb5/remote/RemoteCatalogue.h
@@ -24,7 +24,7 @@ class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public C
 
     // From CatalogueWriter
     const Index& currentIndex() override;
-    void archive(const uint32_t archiverId, const InspectionKey& key, std::unique_ptr fieldLocation) override;
+    void archive(const InspectionKey& key, std::unique_ptr fieldLocation) override;
     void overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) override;
     void index(const InspectionKey& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) override;
     void reconsolidate() override;
diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index 3e25f3d82..bd3d7c4da 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -60,7 +60,7 @@ class RemoteStoreArchiver {
 
 public: // methods
 
-    static RemoteStoreArchiver* get(uint64_t archiverID);
+    static RemoteStoreArchiver* get();
 
     bool valid() { return archiveFuture_.valid(); }
     bool dirty() { return dirty_; }
@@ -88,18 +88,15 @@ class RemoteStoreArchiver {
     std::future archiveFuture_;
 };
 
-RemoteStoreArchiver* RemoteStoreArchiver::get(uint64_t archiverID) {
+RemoteStoreArchiver* RemoteStoreArchiver::get() {
 
-    static std::unordered_map> archivers_;
+    static std::unique_ptr archiver_;
     static std::mutex getArchiverMutex_;
 
-    std::lock_guard lock(getArchiverMutex_);
-
-    auto it = archivers_.find(archiverID);
-    if (it == archivers_.end()) {
-        it = archivers_.emplace(archiverID, new RemoteStoreArchiver()).first;
+    if (!archiver_) {
+        archiver_.reset(new RemoteStoreArchiver());
     }
-    return it->second.get();
+    return archiver_.get();
 }
 
 void RemoteStoreArchiver::start() {
@@ -160,7 +157,7 @@ FDBStats RemoteStoreArchiver::flush(RemoteStore* store) {
     eckit::Log::debug() << " RemoteStoreArchiver::flush - flushing " << numArchive << " fields" << std::endl;
     // The flush call is blocking
     uint32_t id = store->generateRequestID();
-    store->controlWriteCheckResponse(Message::Flush, id, sendBuf, s.position());
+    store->controlWriteCheckResponse(Message::Flush, id, false, sendBuf, s.position());
 
     dirty_ = false;
 
@@ -389,22 +386,19 @@ eckit::DataHandle* RemoteStore::retrieve(Field& field) const {
     return field.dataHandle();
 }
 
-void RemoteStore::archive(const uint32_t archiverID, const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive) {
+void RemoteStore::archive(const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive) {
 
     // if there is no archiving thread active, then start one.
     // n.b. reset the archiveQueue_ after a potential flush() cycle.
     if (!archiver_) {
-        uint64_t archiverName = std::hash()(controlEndpoint());
-        archiverName = archiverName << 32;
-        archiverName += archiverID;
-        archiver_ = RemoteStoreArchiver::get(archiverName);
+        archiver_ = RemoteStoreArchiver::get();
     }
     uint32_t id = connection_.generateRequestID();
     if (!archiver_->valid()) {
         archiver_->start();
         ASSERT(archiver_->valid());
 
-        controlWriteCheckResponse(Message::Store, id);
+        controlWriteCheckResponse(Message::Store, id, true);
     }
     locations_[id] = catalogue_archive;
     archiver_->emplace(id, this, key, data, length);
@@ -578,7 +572,7 @@ eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation, c
     s << remapKey;
 
     uint32_t id = connection_.generateRequestID();
-    controlWriteCheckResponse(fdb5::remote::Message::Read, id, encodeBuffer, s.position());
+    controlWriteCheckResponse(fdb5::remote::Message::Read, id, true, encodeBuffer, s.position());
 
     return new FDBRemoteDataHandle(id, fieldLocation.length(), retrieveMessageQueue_, controlEndpoint());
 }
diff --git a/src/fdb5/remote/RemoteStore.h b/src/fdb5/remote/RemoteStore.h
index bdf88b9fc..b08a63ecc 100644
--- a/src/fdb5/remote/RemoteStore.h
+++ b/src/fdb5/remote/RemoteStore.h
@@ -70,7 +70,7 @@ class RemoteStore : public Store, public Client {
     bool exists() const override;
 
     eckit::DataHandle* retrieve(Field& field) const override;
-    void archive(const uint32_t archiverId, const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive) override;
+    void archive(const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive) override;
 
     void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override;
 
diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index 7a9333bff..4ce54f746 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -80,7 +80,7 @@ bool Client::response(uint32_t requestID, eckit::Buffer&& payload) {
 //     return id;
 // }
 
-void Client::controlWriteCheckResponse(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
+void Client::controlWriteCheckResponse(Message msg, uint32_t requestID, bool dataListener, const void* payload, uint32_t payloadLength) {
 
     ASSERT(requestID);
     ASSERT(!(!payloadLength ^ !payload));
@@ -92,9 +92,9 @@ void Client::controlWriteCheckResponse(Message msg, uint32_t requestID, const vo
     blockingRequestId_=requestID;
 
     if (payloadLength) {
-        connection_.controlWrite(*this, msg, blockingRequestId_, 0, std::vector>{{payload, payloadLength}});
+        connection_.controlWrite(*this, msg, blockingRequestId_, dataListener, std::vector>{{payload, payloadLength}});
     } else {
-        connection_.controlWrite(*this, msg, blockingRequestId_);
+        connection_.controlWrite(*this, msg, blockingRequestId_, dataListener);
     }
 
     f.get();
@@ -113,9 +113,9 @@ eckit::Buffer Client::controlWriteReadResponse(Message msg, uint32_t requestID,
     blockingRequestId_=requestID;
 
     if (payloadLength) {
-        connection_.controlWrite(*this, msg, blockingRequestId_, 0, std::vector>{{payload, payloadLength}});
+        connection_.controlWrite(*this, msg, blockingRequestId_, false, std::vector>{{payload, payloadLength}});
     } else {
-        connection_.controlWrite(*this, msg, blockingRequestId_);
+        connection_.controlWrite(*this, msg, blockingRequestId_, false);
     }
 
     eckit::Buffer buf = f.get();
diff --git a/src/fdb5/remote/client/Client.h b/src/fdb5/remote/client/Client.h
index 7b052a78a..b5c80269a 100644
--- a/src/fdb5/remote/client/Client.h
+++ b/src/fdb5/remote/client/Client.h
@@ -38,6 +38,7 @@ class Client : eckit::NonCopyable {
     Client(const std::vector>& endpoints);
     ~Client();
 
+    uint32_t clientId() const { return id_; }
     uint32_t id() const { return id_; }
     const eckit::net::Endpoint& controlEndpoint() const { return connection_.controlEndpoint(); }
     const std::string& defaultEndpoint() const { return connection_.defaultEndpoint(); }
@@ -45,7 +46,7 @@ class Client : eckit::NonCopyable {
     uint32_t generateRequestID() { return connection_.generateRequestID(); }
 
     // blocking requests
-    void          controlWriteCheckResponse(Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
+    void controlWriteCheckResponse(Message msg, uint32_t requestID, bool dataListener, const void* payload=nullptr, uint32_t payloadLength=0);
     eckit::Buffer controlWriteReadResponse (Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
 
     void dataWrite(remote::Message msg, uint32_t requestID, std::vector> data={});
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index 7bef20622..20c847a8a 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -124,9 +124,9 @@ void ClientConnection::disconnect() {
         exit_ = true;
         // Send termination message
         for (auto c: clients_) {
-            controlWrite(Message::Exit, c.first);
+            Connection::write(Message::Exit, true, c.first, 0);
         }
-        controlWrite(Message::Exit, 0);
+        Connection::write(Message::Exit, true, 0, 0);
 
         listeningThread_.join();
 
@@ -143,9 +143,9 @@ const eckit::net::Endpoint& ClientConnection::controlEndpoint() const {
 // const eckit::net::Endpoint& ClientConnection::fullyQualifiedControlEndpoint() const { 
 //     return fullyQualifiedControlEndpoint_;
 // } 
-const eckit::net::Endpoint& ClientConnection::dataEndpoint() const { 
-    return dataEndpoint_;
-} 
+// const eckit::net::Endpoint& ClientConnection::dataEndpoint() const { 
+//     return dataEndpoint_;
+// } 
 
 eckit::LocalConfiguration ClientConnection::availableFunctionality() const {
     eckit::LocalConfiguration conf;
@@ -156,119 +156,167 @@ eckit::LocalConfiguration ClientConnection::availableFunctionality() const {
 
 // -----------------------------------------------------------------------------------------------------
 
-void ClientConnection::controlWrite(Client& client, Message msg, uint32_t requestID, uint32_t archiverID, std::vector> data) {
-
-    auto it = clients_.find(client.id());
-    ASSERT(it != clients_.end());
-
-    controlWrite(msg, archiverID ? archiverID : client.id(), requestID, data);
-}
-
-void ClientConnection::controlWrite(Message msg, uint32_t clientID, uint32_t requestID, std::vector> data) {
-
-    uint32_t payloadLength = 0;
-    for (auto d: data) {
-        ASSERT(d.first);
-        payloadLength += d.second;
-    }
-    eckit::Log::debug() << "ClientConnection::controlWrite [endpoint=" << controlEndpoint_ <<
-        ",message=" << ((int) msg) << ",clientID=" << clientID << ",requestID=" << requestID << ",data=" << data.size() << ",payload=" << payloadLength << "]" << std::endl;
-
-    MessageHeader message(msg, clientID, requestID, payloadLength);
-
-    std::lock_guard lock(controlMutex_);
-    controlWrite(&message, sizeof(message));
-    for (auto d: data) {
-        controlWrite(d.first, d.second);
-    }
-    controlWrite(&EndMarker, sizeof(EndMarker));
-}
-
-void ClientConnection::controlWrite(const void* data, size_t length) {
-    size_t written = controlClient_.write(data, length);
-    if (length != written) {
-        std::stringstream ss;
-        ss << "Write error. Expected " << length << " bytes, wrote " << written;
-        throw TCPException(ss.str(), Here());
-    }
-}
-
-void ClientConnection::controlRead(void* data, size_t length) {
-    size_t read = controlClient_.read(data, length);
-    if (length != read) {
-        std::stringstream ss;
-        ss << "Read error. Expected " << length << " bytes, read " << read;
-        throw TCPException(ss.str(), Here());
+void ClientConnection::controlWrite(Client& client, Message msg, uint32_t requestID, bool dataListener, std::vector> data) {
+    auto it = clients_.find(client.clientId());
+    if (it == clients_.end()) {
+        std::cout << "Existing clients: ";
+        for (auto& c : clients_) {
+            std::cout << c.first << "  ";
+        }
+        std::cout << std::endl << "Searching client: " << client.clientId() << std::endl;
     }
+    ASSERT(it != clients_.end());
+    // controlListeners_++;
+    // if (dataListener) {
+    //     dataListeners_++;
+    // }
+    // std::cout << controlEndpoint_ << " SENT msg: " << msg << "  controlListeners: " << controlListeners_ << "  dataListeners: " << dataListeners_ << std::endl;
+    Connection::write(msg, true, client.clientId(), requestID, data);
 }
 
+// void ClientConnection::controlWrite(Client& client, Message msg, uint32_t requestID, uint32_t archiverID, std::vector> data) {
+
+//     auto it = clients_.find(client.id());
+//     ASSERT(it != clients_.end());
+
+//     controlWrite(msg, archiverID ? archiverID : client.id(), requestID, data);
+// }
+
+// void ClientConnection::controlWrite(Message msg, uint32_t clientID, uint32_t requestID, std::vector> data) {
+
+//     uint32_t payloadLength = 0;
+//     for (auto d: data) {
+//         ASSERT(d.first);
+//         payloadLength += d.second;
+//     }
+//     eckit::Log::debug() << "ClientConnection::controlWrite [endpoint=" << controlEndpoint_ <<
+//         ",message=" << ((int) msg) << ",clientID=" << clientID << ",requestID=" << requestID << ",data=" << data.size() << ",payload=" << payloadLength << "]" << std::endl;
+
+//     MessageHeader message(msg, clientID, requestID, payloadLength);
+
+//     std::lock_guard lock(controlMutex_);
+//     controlWrite(&message, sizeof(message));
+//     for (auto d: data) {
+//         controlWrite(d.first, d.second);
+//     }
+//     controlWrite(&EndMarker, sizeof(EndMarker));
+// }
+
+// void ClientConnection::controlWrite(const void* data, size_t length) {
+//     size_t written = controlClient_.write(data, length);
+//     if (length != written) {
+//         std::stringstream ss;
+//         ss << "Write error. Expected " << length << " bytes, wrote " << written;
+//         throw TCPException(ss.str(), Here());
+//     }
+// }
+
+// void ClientConnection::controlRead(void* data, size_t length) {
+//     size_t read = controlClient_.read(data, length);
+//     if (length != read) {
+//         std::stringstream ss;
+//         ss << "Read error. Expected " << length << " bytes, read " << read;
+//         throw TCPException(ss.str(), Here());
+//     }
+// }
 
 void ClientConnection::dataWrite(Client& client, remote::Message msg, uint32_t requestID, std::vector> data) {
 
-    auto it = clients_.find(client.id());
+    auto it = clients_.find(client.clientId());
+    // if (it == clients_.end()) {
+    //     std::cout << "Existing clients: ";
+    //     for (auto& c : clients_) {
+    //         std::cout << c.first << "' ";
+    //     }
+    //     std::cout << std::endl << "Searching client: " << client.clientId() << std::endl;
+    // }
     ASSERT(it != clients_.end());
-
-    dataWrite(msg, client.id(), requestID, data);
+    Connection::write(msg, false, client.clientId(), requestID, data);
 }
 
-void ClientConnection::dataWrite(remote::Message msg, uint32_t clientID, uint32_t requestID, std::vector> data) {
 
-    uint32_t payloadLength = 0;
-    for (auto d: data) {
-        ASSERT(d.first);
-        payloadLength += d.second;
-    }
-    MessageHeader message(msg, clientID, requestID, payloadLength);
+// void ClientConnection::dataWrite(Client& client, remote::Message msg, uint32_t requestID, std::vector> data) {
+
+//     auto it = clients_.find(client.id());
+//     ASSERT(it != clients_.end());
+
+//     dataWrite(msg, client.id(), requestID, data);
+// }
+
+// void ClientConnection::dataWrite(remote::Message msg, uint32_t clientID, uint32_t requestID, std::vector> data) {
+
+//     uint32_t payloadLength = 0;
+//     for (auto d: data) {
+//         ASSERT(d.first);
+//         payloadLength += d.second;
+//     }
+//     MessageHeader message(msg, clientID, requestID, payloadLength);
+
+//     eckit::Log::debug() << "ClientConnection::dataWrite [endpoint=" << dataEndpoint_ <<
+//         ",message=" << ((int) msg) << ",requestID=" << requestID << ",data=" << data.size() << ",payload=" << payloadLength << "]" << std::endl;
+
+//     std::lock_guard lock(dataMutex_);
+//     dataWrite(&message, sizeof(message));
+//     for (auto d: data) {
+//         dataWrite(d.first, d.second);
+//     }
+//     dataWrite(&EndMarker, sizeof(EndMarker));
+// }
+
+// void ClientConnection::dataWrite(const void* data, size_t length) {
+//     size_t written = dataClient_.write(data, length);
+//     if (length != written) {
+//         std::stringstream ss;
+//         ss << "Write error. Expected " << length << " bytes, wrote " << written;
+//         throw TCPException(ss.str(), Here());
+//     }
+// }
+
+// void ClientConnection::dataRead(void* data, size_t length) {
+//     size_t read = dataClient_.read(data, length);
+//     if (!exit_ && length != read) {
+//         std::stringstream ss;
+//         ss << "Read error. Expected " << length << " bytes, read " << read;
+//         throw TCPException(ss.str(), Here());
+//     }
+// }
+
+void ClientConnection::handleError(const MessageHeader& hdr, eckit::Buffer buffer) {
+    ASSERT(hdr.marker == StartMarker);
+    ASSERT(hdr.version == CurrentVersion);
 
-    eckit::Log::debug() << "ClientConnection::dataWrite [endpoint=" << dataEndpoint_ <<
-        ",message=" << ((int) msg) << ",requestID=" << requestID << ",data=" << data.size() << ",payload=" << payloadLength << "]" << std::endl;
+    if (hdr.message == Message::Error) {
+        ASSERT(hdr.payloadSize > 9);
+        std::string what(buffer.size()+1, ' ');
+        buffer.copy(what.c_str(), buffer.size());
+        what[buffer.size()] = 0; // Just in case
 
-    std::lock_guard lock(dataMutex_);
-    dataWrite(&message, sizeof(message));
-    for (auto d: data) {
-        dataWrite(d.first, d.second);
+        throw RemoteFDBException(what, controlEndpoint_);
     }
-    dataWrite(&EndMarker, sizeof(EndMarker));
 }
 
-void ClientConnection::dataWrite(const void* data, size_t length) {
-    size_t written = dataClient_.write(data, length);
-    if (length != written) {
-        std::stringstream ss;
-        ss << "Write error. Expected " << length << " bytes, wrote " << written;
-        throw TCPException(ss.str(), Here());
-    }
-}
 
-void ClientConnection::dataRead(void* data, size_t length) {
-    size_t read = dataClient_.read(data, length);
-    if (!exit_ && length != read) {
-        std::stringstream ss;
-        ss << "Read error. Expected " << length << " bytes, read " << read;
-        throw TCPException(ss.str(), Here());
-    }
-}
 
-void ClientConnection::handleError(const MessageHeader& hdr) {
+// void ClientConnection::handleError(const MessageHeader& hdr) {
 
-    ASSERT(hdr.marker == StartMarker);
-    ASSERT(hdr.version == CurrentVersion);
+//     ASSERT(hdr.marker == StartMarker);
+//     ASSERT(hdr.version == CurrentVersion);
 
-    if (hdr.message == Message::Error) {
-        ASSERT(hdr.payloadSize > 9);
+//     if (hdr.message == Message::Error) {
+//         ASSERT(hdr.payloadSize > 9);
 
-        std::string what(hdr.payloadSize, ' ');
-        dataRead(&what[0], hdr.payloadSize);
-        what[hdr.payloadSize] = 0; // Just in case
+//         std::string what(hdr.payloadSize, ' ');
+//         dataRead(&what[0], hdr.payloadSize);
+//         what[hdr.payloadSize] = 0; // Just in case
 
-        try {
-            eckit::FixedString<4> tail;
-            dataRead(&tail, sizeof(tail));
-        } catch (...) {}
+//         try {
+//             eckit::FixedString<4> tail;
+//             dataRead(&tail, sizeof(tail));
+//         } catch (...) {}
 
-        throw RemoteFDBException(what, controlEndpoint_);
-    }
-}
+//         throw RemoteFDBException(what, controlEndpoint_);
+//     }
+// }
 
 void ClientConnection::writeControlStartupMessage() {
 
@@ -283,7 +331,8 @@ void ClientConnection::writeControlStartupMessage() {
     //       essentially JSON) over the wire for flexibility.
     s << availableFunctionality().get();
 
-    controlWrite(Message::Startup, 0, 0, std::vector>{{payload, s.position()}});
+    // controlWrite(Message::Startup, 0, 0, std::vector>{{payload, s.position()}});
+    Connection::write(Message::Startup, true, 0, 0, payload, s.position());
 }
 
 void ClientConnection::writeDataStartupMessage(const eckit::SessionID& serverSession) {
@@ -294,24 +343,26 @@ void ClientConnection::writeDataStartupMessage(const eckit::SessionID& serverSes
     s << sessionID_;
     s << serverSession;
 
-    dataWrite(Message::Startup, 0, 0, std::vector>{{payload, s.position()}});
+    // dataWrite(Message::Startup, 0, 0, std::vector>{{payload, s.position()}});
+    Connection::write(Message::Startup, false, 0, 0, payload, s.position());
 }
 
 eckit::SessionID ClientConnection::verifyServerStartupResponse() {
 
     MessageHeader hdr;
-    controlRead(&hdr, sizeof(hdr));
+    eckit::Buffer payload = Connection::readControl(hdr);
+    // controlRead(&hdr, sizeof(hdr));
 
-    ASSERT(hdr.marker == StartMarker);
-    ASSERT(hdr.version == CurrentVersion);
-    ASSERT(hdr.message == Message::Startup);
+    // ASSERT(hdr.marker == StartMarker);
+    // ASSERT(hdr.version == CurrentVersion);
+    // ASSERT(hdr.message == Message::Startup);
     ASSERT(hdr.requestID == 0);
 
-    eckit::Buffer payload(hdr.payloadSize);
-    eckit::FixedString<4> tail;
-    controlRead(payload, hdr.payloadSize);
-    controlRead(&tail, sizeof(tail));
-    ASSERT(tail == EndMarker);
+    // eckit::Buffer payload(hdr.payloadSize);
+    // eckit::FixedString<4> tail;
+    // controlRead(payload, hdr.payloadSize);
+    // controlRead(&tail, sizeof(tail));
+    // ASSERT(tail == EndMarker);
 
     eckit::MemoryStream s(payload);
     eckit::SessionID clientSession(s);
@@ -333,6 +384,15 @@ eckit::SessionID ClientConnection::verifyServerStartupResponse() {
            << sessionID_ << " != " << clientSession;
         throw eckit::BadValue(ss.str(), Here());
     }
+    if (serverFunctionality.has("NumberOfConnections") && serverFunctionality.getInt("NumberOfConnections")==1) {
+        single_ = true;
+    }
+
+    if (single_ && !(dataEndpoint_ == controlEndpoint_)) {
+        eckit::Log::warning() << "Returned control interface does not match. "
+                       << dataEndpoint_ << " /= "
+                       << controlEndpoint_ << std::endl;
+    }
 
     return serverSession;
 }
@@ -346,25 +406,22 @@ void ClientConnection::listeningThreadLoop() {
 
         while (!exit_) {
 
-            dataRead(&hdr, sizeof(hdr));
+            eckit::Buffer payload = Connection::readData(hdr);
 
-            eckit::Log::debug() << "ClientConnection::listeningThreadLoop - got [message=" << ((int) hdr.message) << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
-
-            ASSERT(hdr.marker == StartMarker);
-            ASSERT(hdr.version == CurrentVersion);
+            eckit::Log::debug() << "ClientConnection::listeningThreadLoop - got [message=" << hdr.message << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
 
             if (hdr.message == Message::Exit) {
-                if (hdr.clientID) {
-                    remove(hdr.clientID);
+                if (hdr.clientID()) {
+                    remove(hdr.clientID());
                 }
             } else {
-                if (hdr.clientID) {
+                if (hdr.clientID()) {
                     bool handled = false;
-                    auto it = clients_.find(hdr.clientID);
+                    auto it = clients_.find(hdr.clientID());
                     if (it == clients_.end()) {
                         std::stringstream ss;
-                        ss << "ERROR: Received [clientID="<< hdr.clientID << ",requestID="<< hdr.requestID << ",message=" << ((int) hdr.message) << ",payload=" << hdr.payloadSize << "]" << std::endl;
-                        ss << "Unexpected answer for clientID recieved (" << hdr.clientID << "). ABORTING";
+                        ss << "ERROR: Received [clientID="<< hdr.clientID() << ",requestID="<< hdr.requestID << ",message=" << hdr.message << ",payload=" << hdr.payloadSize << "]" << std::endl;
+                        ss << "Unexpected answer for clientID recieved (" << hdr.clientID() << "). ABORTING";
                         eckit::Log::status() << ss.str() << std::endl;
                         eckit::Log::error() << "Retrieving... " << ss.str() << std::endl;
                         throw eckit::SeriousBug(ss.str(), Here());
@@ -381,9 +438,6 @@ void ClientConnection::listeningThreadLoop() {
                         }
                     }
                     else {
-                        eckit::Buffer payload{hdr.payloadSize};
-                        dataRead(payload, hdr.payloadSize);
-
                         if (it->second->blockingRequestId() == hdr.requestID) {
                             handled = it->second->response(hdr.requestID, std::move(payload));
                         } else {
@@ -398,16 +452,8 @@ void ClientConnection::listeningThreadLoop() {
                         eckit::Log::error() << "Client Retrieving... " << ss.str() << std::endl;
                         throw eckit::SeriousBug(ss.str(), Here());
                     }
-                } else {
-                    if (hdr.payloadSize) {
-                        eckit::Buffer payload{hdr.payloadSize};
-                        dataRead(payload, hdr.payloadSize);
-                    }
                 }
             }
-            // Ensure we have consumed exactly the correct amount from the socket.
-            dataRead(&tail, sizeof(tail));
-            ASSERT(tail == EndMarker);
         }
 
     // We don't want to let exceptions escape inside a worker thread.
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index 6554b1f5c..bfaec3890 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -21,6 +21,7 @@
 #include "eckit/runtime/SessionID.h"
 
 #include "fdb5/remote/Messages.h"
+#include "fdb5/remote/Connection.h"
 
 namespace eckit {
 
@@ -35,7 +36,7 @@ class ClientConnectionRouter;
 
 //----------------------------------------------------------------------------------------------------------------------
 
-class ClientConnection : eckit::NonCopyable {
+class ClientConnection : protected Connection {
 
 public: // types
 
@@ -43,8 +44,8 @@ class ClientConnection : eckit::NonCopyable {
 
     ~ClientConnection();
 
-    void controlWrite(Client& client, remote::Message msg, uint32_t requestID, uint32_t archiverID=0, std::vector> data={});
-    void dataWrite   (Client& client, remote::Message msg, uint32_t requestID, std::vector> data={});
+    void controlWrite(Client& client, Message msg, uint32_t requestID, bool dataListener, std::vector> data={});
+    void dataWrite(Client& client, remote::Message msg, uint32_t requestID, std::vector> data={});
 
     void add(Client& client);
     bool remove(uint32_t clientID);
@@ -62,27 +63,30 @@ class ClientConnection : eckit::NonCopyable {
 
     ClientConnection(const eckit::net::Endpoint& controlEndpoint, const std::string& defaultEndpoint);
 
-    const eckit::net::Endpoint& dataEndpoint() const;
-    
+    // const eckit::net::Endpoint& dataEndpoint() const;
+
     // construct dictionary for protocol negotiation - to be defined in the client class
     eckit::LocalConfiguration availableFunctionality() const;
 
-    void controlWrite(Message msg, uint32_t clientID, uint32_t requestID = 0, std::vector> data = {});
-    void controlWrite(const void* data, size_t length);
-    void controlRead (      void* data, size_t length);
-    void dataWrite   (Message msg, uint32_t clientID, uint32_t requestID = 0, std::vector> data = {});
-    void dataWrite   (const void* data, size_t length);
-    void dataRead    (      void* data, size_t length);
+    // void controlWrite(Message msg, uint32_t clientID, uint32_t requestID = 0, std::vector> data = {});
+    // void controlWrite(const void* data, size_t length);
+    // void controlRead (      void* data, size_t length);
+    // void dataWrite   (Message msg, uint32_t clientID, uint32_t requestID = 0, std::vector> data = {});
+    // void dataWrite   (const void* data, size_t length);
+    // void dataRead    (      void* data, size_t length);
  
     void writeControlStartupMessage();
     void writeDataStartupMessage(const eckit::SessionID& serverSession);
 
     eckit::SessionID verifyServerStartupResponse();
 
-    void handleError(const MessageHeader& hdr);
+    void handleError(const MessageHeader& hdr, eckit::Buffer buffer);
 
     void listeningThreadLoop();
 
+    eckit::net::TCPSocket& controlSocket() override { return controlClient_; }
+    eckit::net::TCPSocket& dataSocket() override { return dataClient_; }
+
 private: // members
 
     eckit::SessionID sessionID_; 
@@ -101,8 +105,8 @@ class ClientConnection : eckit::NonCopyable {
     std::thread listeningThread_;
     
     std::mutex requestMutex_;
-    std::mutex controlMutex_;
-    std::mutex dataMutex_;
+    // std::mutex controlMutex_;
+    // std::mutex dataMutex_;
 
     // requestID
     std::mutex idMutex_;
diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc
index 83b961711..cf81886ac 100644
--- a/src/fdb5/remote/server/CatalogueHandler.cc
+++ b/src/fdb5/remote/server/CatalogueHandler.cc
@@ -39,19 +39,9 @@ CatalogueHandler::~CatalogueHandler() {}
 
 void CatalogueHandler::initialiseConnections() {
     ServerConnection::initialiseConnections();
+}
 
-    MessageHeader hdr;
-    eckit::FixedString<4> tail;
-
-    socketRead(&hdr, sizeof(hdr), controlSocket_);
-
-    ASSERT(hdr.marker == StartMarker);
-    ASSERT(hdr.version == CurrentVersion);
-    ASSERT(hdr.message == Message::Schema);
-
-    // Ensure we have consumed exactly the correct amount from the socket.
-    socketRead(&tail, sizeof(tail), controlSocket_);
-    ASSERT(tail == EndMarker);
+void CatalogueHandler::stores(const MessageHeader& hdr) {
 
     eckit::net::IPAddress clientIPaddress{controlSocket_.remoteAddr()};
 
@@ -101,7 +91,7 @@ void CatalogueHandler::initialiseConnections() {
     }
 
     {
-        Buffer startupBuffer(1024*1024);
+        Buffer startupBuffer(16*1024);
         MemoryStream s(startupBuffer);
 
         s << stores.size();
@@ -111,13 +101,12 @@ void CatalogueHandler::initialiseConnections() {
                 s << ee;
             }
         }
-        s << config_.schema();
+        // s << config_.schema();
 
-        dataWrite(Message::Received, hdr.clientID, hdr.requestID, startupBuffer.data(), s.position());
+        dataWrite(Message::Received, hdr.clientID(), hdr.requestID, startupBuffer.data(), s.position());
     }
 }
 
-
 // API forwarding logic, adapted from original remoteHandler
 // Used for Inspect and List
 // ***************************************************************************************
@@ -180,19 +169,19 @@ void CatalogueHandler::forwardApiCall(const MessageHeader& hdr) {
                 typename decltype(iterator)::value_type elem;
                 while (iterator.next(elem)) {
                     auto encoded(helper.encode(elem, *this));
-                    dataWrite(Message::Blob, hdr.clientID, hdr.requestID, encoded.buf, encoded.position);
+                    dataWrite(Message::Blob, hdr.clientID(), hdr.requestID, encoded.buf, encoded.position);
                 }
-                dataWrite(Message::Complete, hdr.clientID, hdr.requestID);
+                dataWrite(Message::Complete, hdr.clientID(), hdr.requestID);
             }
             catch (std::exception& e) {
                 // n.b. more general than eckit::Exception
                 std::string what(e.what());
-                dataWrite(Message::Error, hdr.clientID, hdr.requestID, what.c_str(), what.length());
+                dataWrite(Message::Error, hdr.clientID(), hdr.requestID, what.c_str(), what.length());
             }
             catch (...) {
                 // We really don't want to std::terminate the thread
                 std::string what("Caught unexpected, unknown exception in worker");
-                dataWrite(Message::Error, hdr.clientID, hdr.requestID, what.c_str(), what.length());
+                dataWrite(Message::Error, hdr.clientID(), hdr.requestID, what.c_str(), what.length());
             }
         }));
 }
@@ -213,7 +202,7 @@ void CatalogueHandler::handle() {
 
         ASSERT(hdr.marker == StartMarker);
         ASSERT(hdr.version == CurrentVersion);
-        Log::debug() << "CatalogueHandler - got message " << ((int) hdr.message) << " with request ID: " << hdr.requestID << std::endl;
+        Log::debug() << "CatalogueHandler - got message " << hdr.message << " with request ID: " << hdr.requestID << std::endl;
 
         bool ack = false;
 
@@ -268,6 +257,11 @@ void CatalogueHandler::handle() {
                     archive(hdr);
                     break;
 
+                case Message::Stores:
+                    stores(hdr);
+                    ack = true;
+                    break;
+
                 case Message::Schema:
                     schema(hdr);
                     ack = true;
@@ -289,17 +283,17 @@ void CatalogueHandler::handle() {
 
             if (!ack) {
                 // Acknowledge receipt of command
-                dataWrite(Message::Received, hdr.clientID, hdr.requestID);
+                dataWrite(Message::Received, hdr.clientID(), hdr.requestID);
             }
         }
         catch (std::exception& e) {
             // n.b. more general than eckit::Exception
             std::string what(e.what());
-            dataWrite(Message::Error, hdr.clientID, hdr.requestID, what.c_str(), what.length());
+            dataWrite(Message::Error, hdr.clientID(), hdr.requestID, what.c_str(), what.length());
         }
         catch (...) {
             std::string what("Caught unexpected and unknown error");
-            dataWrite(Message::Error, hdr.clientID, hdr.requestID, what.c_str(), what.length());
+            dataWrite(Message::Error, hdr.clientID(), hdr.requestID, what.c_str(), what.length());
         }
     }
 }
@@ -319,7 +313,7 @@ void CatalogueHandler::flush(const MessageHeader& hdr) {
     size_t numArchived;
     s >> numArchived;
 
-    std::future& archive = archiveFuture_[hdr.clientID];
+    std::future& archive = archiveFuture_;
 
     ASSERT(numArchived == 0 || archive.valid());
 
@@ -332,7 +326,7 @@ void CatalogueHandler::flush(const MessageHeader& hdr) {
         // Do the actual flush!
         Log::info() << "Flushing" << std::endl;
         Log::status() << "Flushing" << std::endl;
-        catalogues_[hdr.clientID]->flush();
+        catalogues_[hdr.clientID()]->flush();
         // for (auto it = catalogues_.begin(); it != catalogues_.end(); it++) {
         //     it->second->flush();
         // }
@@ -341,7 +335,7 @@ void CatalogueHandler::flush(const MessageHeader& hdr) {
     Log::status() << "Flush complete" << std::endl;
 }
 
-size_t CatalogueHandler::archiveThreadLoop(uint32_t archiverID) {
+size_t CatalogueHandler::archiveThreadLoop() {
     size_t totalArchived = 0;
 
     // Create a worker that will do the actual archiving
@@ -350,27 +344,29 @@ size_t CatalogueHandler::archiveThreadLoop(uint32_t archiverID) {
     eckit::Queue> queue(queueSize);
 
 
-    std::future worker = std::async(std::launch::async, [this, &queue, archiverID] {
+    std::future worker = std::async(std::launch::async, [this, &queue] {
         size_t totalArchived = 0;
         std::pair elem = std::make_pair(0, Buffer{0});
 
         try {
             long queuelen;
             while ((queuelen = queue.pop(elem)) != -1) {
+
                 uint32_t clientID = elem.first;
                 // Key key = elem.second.first;
                 eckit::Buffer payload = std::move(elem.second);
                 MemoryStream s(payload);
                 Key idxKey(s);
-                InspectionKey key; // xxx no stream constructor for inspection key?
-                s >> key;
+                InspectionKey key(s);
                 std::unique_ptr location(eckit::Reanimator::reanimate(s));
+                std::cout << "ARCHIVING location [clientID=" << clientID << ",key=" << idxKey <<  key << ",location=" << *location << std::endl;
+
                 CatalogueWriter& cat = catalogue(clientID);
                 Log::debug() << "CatalogueHandler::archiveThreadLoop message has clientID: " << clientID << " key: " << cat.key()
                     << idxKey << key << " and location.uri" << location->uri() << std::endl;
         
                 cat.selectIndex(idxKey);
-                cat.archive(archiverID, key, std::move(location));
+                cat.archive(key, std::move(location));
                 totalArchived += 1;
             }
         }
@@ -411,7 +407,7 @@ size_t CatalogueHandler::archiveThreadLoop(uint32_t archiverID) {
             socketRead(&tail, sizeof(tail), dataSocket_);
             ASSERT(tail == EndMarker);
 
-            size_t queuelen = queue.emplace(hdr.clientID, std::move(payload));
+            size_t queuelen = queue.emplace(hdr.clientID(), std::move(payload));
         }
 
         // Trigger cleanup of the workers
@@ -455,22 +451,28 @@ void CatalogueHandler::inspect(const MessageHeader& hdr) {
 
 void CatalogueHandler::schema(const MessageHeader& hdr) {
 
-    // 1. Read dbkey to select catalogue
-    Buffer payload(receivePayload(hdr, controlSocket_));
-    MemoryStream s(payload);
-    Key dbKey(s);
-    
-    // 2. Get catalogue
-    Catalogue& cat = catalogue(hdr.clientID, dbKey);
-    const Schema& schema = cat.schema();
     eckit::Buffer schemaBuffer(1024*1024);
     eckit::MemoryStream stream(schemaBuffer);
-    stream << schema;
 
-    dataWrite(Message::Received, hdr.clientID, hdr.requestID, schemaBuffer.data(), stream.position());
+    if (hdr.payloadSize == 0) { // client requesting the top-level schema
+        stream << config_.schema();
+    } else {
+        // 1. Read dbkey to select catalogue
+        Buffer payload(receivePayload(hdr, controlSocket_));
+        MemoryStream s(payload);
+        Key dbKey(s);
+    
+        // 2. Get catalogue
+        Catalogue& cat = catalogue(hdr.clientID(), dbKey);
+        const Schema& schema = cat.schema();
+        stream << schema;
+    }
+
+    dataWrite(Message::Received, hdr.clientID(), hdr.requestID, schemaBuffer.data(), stream.position());
 }
 
 CatalogueWriter& CatalogueHandler::catalogue(uint32_t id) {
+    std::lock_guard lock(cataloguesMutex_);
     auto it = catalogues_.find(id);
     if (it == catalogues_.end()) {
         std::string what("Requested unknown catalogue id: " + std::to_string(id));
@@ -482,6 +484,7 @@ CatalogueWriter& CatalogueHandler::catalogue(uint32_t id) {
 }
 
 CatalogueWriter& CatalogueHandler::catalogue(uint32_t id, const Key& dbKey) {
+    std::lock_guard lock(cataloguesMutex_);
     return *(catalogues_[id] = CatalogueWriterFactory::instance().build(dbKey, config_));
 }
 
diff --git a/src/fdb5/remote/server/CatalogueHandler.h b/src/fdb5/remote/server/CatalogueHandler.h
index f33acd854..5ad086399 100644
--- a/src/fdb5/remote/server/CatalogueHandler.h
+++ b/src/fdb5/remote/server/CatalogueHandler.h
@@ -40,12 +40,13 @@ class CatalogueHandler : public ServerConnection {
     void list(const MessageHeader& hdr);
     void inspect(const MessageHeader& hdr);
     void schema(const MessageHeader& hdr);
+    void stores(const MessageHeader& hdr);
     
 
     CatalogueWriter& catalogue(uint32_t id);
     CatalogueWriter& catalogue(uint32_t id, const Key& dbKey);
 
-    size_t archiveThreadLoop(uint32_t archiverID) override;
+    size_t archiveThreadLoop() override;
 
     // API functionality
     template 
@@ -54,6 +55,8 @@ class CatalogueHandler : public ServerConnection {
 private:  // member
 
     // clientID --> Catalogue
+    std::mutex cataloguesMutex_;
+
     std::map> catalogues_;
 
     FDB fdb_;
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index bf2eb15ec..0bd5c0e1a 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -246,7 +246,7 @@ int ServerConnection::selectDataPort() {
 void ServerConnection::controlWrite(Message msg, uint32_t clientID, uint32_t requestID, const void* payload, uint32_t payloadLength) {
     ASSERT((payload == nullptr) == (payloadLength == 0));
 
-    MessageHeader message(msg, clientID, requestID, payloadLength);
+    MessageHeader message(msg, true, clientID, requestID, payloadLength);
     std::lock_guard lock(controlWriteMutex_);
     controlWrite(&message, sizeof(message));
     if (payload) {
@@ -278,13 +278,10 @@ void ServerConnection::archive(const MessageHeader& hdr) {
 
     ASSERT(hdr.payloadSize == 0);
 
-    auto archive = archiveFuture_.find(hdr.clientID);
-
     // Ensure that we aren't already running a catalogue/store
-    if(archive == archiveFuture_.end() || !archive->second.valid()) {
+    if (!archiveFuture_.valid()) {
         // Start archive worker thread
-        uint32_t archiverID = hdr.clientID;
-        archiveFuture_[hdr.clientID] = std::async(std::launch::async, [this, archiverID] { return archiveThreadLoop(archiverID); });
+        archiveFuture_ = std::async(std::launch::async, [this] { return archiveThreadLoop(); });
     }
 }
 
@@ -294,7 +291,7 @@ void ServerConnection::dataWrite(Message msg, uint32_t clientID, uint32_t reques
 
     ASSERT((payload == nullptr) == (payloadLength == 0));
 
-    MessageHeader message(msg, clientID, requestID, payloadLength);
+    MessageHeader message(msg, false, clientID, requestID, payloadLength);
 
     std::lock_guard lock(dataWriteMutex_);
     dataWriteUnsafe(&message, sizeof(message));
diff --git a/src/fdb5/remote/server/ServerConnection.h b/src/fdb5/remote/server/ServerConnection.h
index 91930f7a6..1f963ff45 100644
--- a/src/fdb5/remote/server/ServerConnection.h
+++ b/src/fdb5/remote/server/ServerConnection.h
@@ -33,7 +33,7 @@
 
 namespace fdb5::remote {
 
-struct MessageHeader;
+class MessageHeader;
 
 //----------------------------------------------------------------------------------------------------------------------
 // Base class for CatalogueHandler and StoreHandler
@@ -84,7 +84,7 @@ class ServerConnection : private eckit::NonCopyable {
     void waitForWorkers();
 
     void archive(const MessageHeader& hdr);
-    virtual size_t archiveThreadLoop(uint32_t archiverID) = 0;
+    virtual size_t archiveThreadLoop() = 0;
 
 private:
 
@@ -112,10 +112,8 @@ class ServerConnection : private eckit::NonCopyable {
     std::mutex dataWriteMutex_;
     std::thread readLocationWorker_;
     
-    // archiverID --> future ??????
     std::map> workerThreads_;
-    // archiverID --> archiveFuture
-    std::unordered_map> archiveFuture_;
+    std::future archiveFuture_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/server/StoreHandler.cc b/src/fdb5/remote/server/StoreHandler.cc
index 2f88a535e..281c3e6b0 100644
--- a/src/fdb5/remote/server/StoreHandler.cc
+++ b/src/fdb5/remote/server/StoreHandler.cc
@@ -50,7 +50,7 @@ void StoreHandler::handle() {
 
         ASSERT(hdr.marker == StartMarker);
         ASSERT(hdr.version == CurrentVersion);
-        Log::debug() << "StoreHandler - got message " << ((int) hdr.message) << " with request ID: " << hdr.requestID << std::endl;
+        Log::debug() << "StoreHandler - got message " << hdr.message << " with request ID: " << hdr.requestID << std::endl;
 
         try {
             switch (hdr.message) {
@@ -118,16 +118,16 @@ void StoreHandler::handle() {
             ASSERT(tail == EndMarker);
 
             // Acknowledge receipt of command
-            dataWrite(Message::Received, hdr.clientID, hdr.requestID);
+            dataWrite(Message::Received, hdr.clientID(), hdr.requestID);
         }
         catch (std::exception& e) {
             // n.b. more general than eckit::Exception
             std::string what(e.what());
-            dataWrite(Message::Error, hdr.clientID, hdr.requestID, what.c_str(), what.length());
+            dataWrite(Message::Error, hdr.clientID(), hdr.requestID, what.c_str(), what.length());
         }
         catch (...) {
             std::string what("Caught unexpected and unknown error");
-            dataWrite(Message::Error, hdr.clientID, hdr.requestID, what.c_str(), what.length());
+            dataWrite(Message::Error, hdr.clientID(), hdr.requestID, what.c_str(), what.length());
         }
     }
 }
@@ -148,7 +148,7 @@ void StoreHandler::read(const MessageHeader& hdr) {
     std::unique_ptr dh;
     dh.reset(location->dataHandle());
 
-    readLocationQueue_.emplace(readLocationElem(hdr.clientID, hdr.requestID, std::move(dh)));
+    readLocationQueue_.emplace(readLocationElem(hdr.clientID(), hdr.requestID, std::move(dh)));
 }
 
 void StoreHandler::readLocationThreadLoop() {
@@ -224,7 +224,7 @@ Store& StoreHandler::store(uint32_t id, Key dbKey) {
 }
 
 // A helper function to make archiveThreadLoop a bit cleaner
-void StoreHandler::archiveBlobPayload(const uint32_t archiverID, const uint32_t clientID, const uint32_t requestID, const void* data, size_t length) {
+void StoreHandler::archiveBlobPayload(const uint32_t clientID, const uint32_t requestID, const void* data, size_t length) {
     MemoryStream s(data, length);
 
     fdb5::Key dbKey(s);
@@ -238,7 +238,7 @@ void StoreHandler::archiveBlobPayload(const uint32_t archiverID, const uint32_t
     
     Store& ss = store(clientID, dbKey);
 
-    auto futureLocation = ss.archive(archiverID, idxKey, charData + s.position(), length - s.position());
+    auto futureLocation = ss.archive(idxKey, charData + s.position(), length - s.position());
     Log::status() << "Archiving done: " << ss_key.str() << std::endl;
     
     auto loc = futureLocation.get();
@@ -263,7 +263,7 @@ struct archiveElem {
         clientID(clientID), requestID(requestID), payload(std::move(payload)), multiblob(multiblob) {}
 };
 
-size_t StoreHandler::archiveThreadLoop(uint32_t archiverID) {
+size_t StoreHandler::archiveThreadLoop() {
     size_t totalArchived = 0;
 
     // Create a worker that will do the actual archiving
@@ -271,7 +271,7 @@ size_t StoreHandler::archiveThreadLoop(uint32_t archiverID) {
     static size_t queueSize(eckit::Resource("fdbServerMaxQueueSize", 32));
     eckit::Queue queue(queueSize);
 
-    std::future worker = std::async(std::launch::async, [this, &queue, archiverID] {
+    std::future worker = std::async(std::launch::async, [this, &queue] {
         size_t totalArchived = 0;
 
         archiveElem elem;
@@ -289,7 +289,7 @@ size_t StoreHandler::archiveThreadLoop(uint32_t archiverID) {
                         ASSERT(hdr->marker == StartMarker);
                         ASSERT(hdr->version == CurrentVersion);
                         ASSERT(hdr->message == Message::Blob);
-                        ASSERT(hdr->clientID == elem.clientID);
+                        ASSERT(hdr->clientID() == elem.clientID);
                         ASSERT(hdr->requestID == elem.requestID);
                         charData += sizeof(MessageHeader);
 
@@ -300,13 +300,13 @@ size_t StoreHandler::archiveThreadLoop(uint32_t archiverID) {
                         ASSERT(*e == EndMarker);
                         charData += sizeof(EndMarker);
 
-                        archiveBlobPayload(archiverID, elem.clientID, elem.requestID, payloadData, hdr->payloadSize);
+                        archiveBlobPayload(elem.clientID, elem.requestID, payloadData, hdr->payloadSize);
                         totalArchived += 1;
                     }
                 }
                 else {
                     // Handle single blob
-                    archiveBlobPayload(archiverID, elem.clientID, elem.requestID, elem.payload.data(), elem.payload.size());
+                    archiveBlobPayload(elem.clientID, elem.requestID, elem.payload.data(), elem.payload.size());
                     totalArchived += 1;
                 }
             }
@@ -353,7 +353,7 @@ size_t StoreHandler::archiveThreadLoop(uint32_t archiverID) {
 
             size_t sz = payload.size();
             Log::debug() << "Queueing data: " << sz << std::endl;
-            size_t queuelen = queue.emplace(archiveElem(hdr.clientID, hdr.requestID, std::move(payload), hdr.message == Message::MultiBlob));
+            size_t queuelen = queue.emplace(archiveElem(hdr.clientID(), hdr.requestID, std::move(payload), hdr.message == Message::MultiBlob));
             Log::status() << "Queued data (" << queuelen << ", size=" << sz << ")" << std::endl;
             ;
             Log::debug() << "Queued data (" << queuelen << ", size=" << sz << ")"
@@ -399,7 +399,7 @@ void StoreHandler::flush(const MessageHeader& hdr) {
     size_t numArchived;
     s >> numArchived;
 
-    std::future& archive = archiveFuture_[hdr.clientID];
+    std::future& archive = archiveFuture_;
 
     ASSERT(numArchived == 0 || archive.valid());
 
@@ -412,7 +412,7 @@ void StoreHandler::flush(const MessageHeader& hdr) {
         // Do the actual flush!
         Log::info() << "Flushing" << std::endl;
         Log::status() << "Flushing" << std::endl;
-        stores_[hdr.clientID]->flush();
+        stores_[hdr.clientID()]->flush();
         // for (auto it = stores_.begin(); it != stores_.end(); it++) {
         //     it->second->flush();
         // }
diff --git a/src/fdb5/remote/server/StoreHandler.h b/src/fdb5/remote/server/StoreHandler.h
index 81cf43677..10d42bf1c 100644
--- a/src/fdb5/remote/server/StoreHandler.h
+++ b/src/fdb5/remote/server/StoreHandler.h
@@ -30,8 +30,8 @@ class StoreHandler : public ServerConnection {
     void readLocationThreadLoop();
     void writeToParent(const uint32_t clientID, const uint32_t requestID, std::unique_ptr dh);
 
-    size_t archiveThreadLoop(uint32_t archiverID) override;
-    void archiveBlobPayload(const uint32_t archiverID, const uint32_t clientID, const uint32_t requestID, const void* data, size_t length);
+    size_t archiveThreadLoop() override;
+    void archiveBlobPayload(const uint32_t clientID, const uint32_t requestID, const void* data, size_t length);
 
     void flush(const MessageHeader& hdr);
 
diff --git a/src/fdb5/toc/TocCatalogueWriter.cc b/src/fdb5/toc/TocCatalogueWriter.cc
index 47cc32026..21e8bed4e 100644
--- a/src/fdb5/toc/TocCatalogueWriter.cc
+++ b/src/fdb5/toc/TocCatalogueWriter.cc
@@ -300,7 +300,7 @@ bool TocCatalogueWriter::enabled(const ControlIdentifier& controlIdentifier) con
     return TocCatalogue::enabled(controlIdentifier);
 }
 
-void TocCatalogueWriter::archive(const uint32_t, const InspectionKey& key, std::unique_ptr fieldLocation) {
+void TocCatalogueWriter::archive(const InspectionKey& key, std::unique_ptr fieldLocation) {
     dirty_ = true;
 
     if (current_.null()) {
diff --git a/src/fdb5/toc/TocCatalogueWriter.h b/src/fdb5/toc/TocCatalogueWriter.h
index 3c2ddb81f..268282517 100644
--- a/src/fdb5/toc/TocCatalogueWriter.h
+++ b/src/fdb5/toc/TocCatalogueWriter.h
@@ -71,7 +71,7 @@ class TocCatalogueWriter : public TocCatalogue, public CatalogueWriter {
     void clean() override;
     void close() override;
 
-    void archive(const uint32_t archiverId, const InspectionKey& key, std::unique_ptr fieldLocation) override;
+    void archive(const InspectionKey& key, std::unique_ptr fieldLocation) override;
     void reconsolidateIndexesAndTocs();
 
     virtual void print( std::ostream &out ) const override;
diff --git a/src/fdb5/toc/TocStore.cc b/src/fdb5/toc/TocStore.cc
index 589db9e9b..edd1426c3 100644
--- a/src/fdb5/toc/TocStore.cc
+++ b/src/fdb5/toc/TocStore.cc
@@ -52,7 +52,7 @@ eckit::DataHandle* TocStore::retrieve(Field& field) const {
     return field.dataHandle();
 }
 
-std::unique_ptr TocStore::archive(const uint32_t, const Key& key, const void *data, eckit::Length length) {
+std::unique_ptr TocStore::archive(const Key& key, const void *data, eckit::Length length) {
     
     dirty_ = true;
 
diff --git a/src/fdb5/toc/TocStore.h b/src/fdb5/toc/TocStore.h
index 5b1652989..30d7be60f 100644
--- a/src/fdb5/toc/TocStore.h
+++ b/src/fdb5/toc/TocStore.h
@@ -58,7 +58,7 @@ class TocStore : public Store, public TocCommon {
     bool exists() const override;
 
     eckit::DataHandle* retrieve(Field& field) const override;
-    std::unique_ptr archive(const uint32_t archiverId, const Key& key, const void *data, eckit::Length length) override;
+    std::unique_ptr archive(const Key& key, const void *data, eckit::Length length) override;
 
     void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override;
 
diff --git a/src/fdb5/tools/fdb-write.cc b/src/fdb5/tools/fdb-write.cc
index 92549a1be..cabe18041 100644
--- a/src/fdb5/tools/fdb-write.cc
+++ b/src/fdb5/tools/fdb-write.cc
@@ -77,10 +77,15 @@ void FDBWrite::init(const eckit::option::CmdArgs& args)
 
 void FDBWrite::execute(const eckit::option::CmdArgs &args) {
 
-    fdb5::MessageArchiver archiver(fdb5::Key(), false, verbose_, config(args));
+    fdb5::MessageArchiver archiver1(fdb5::Key(), false, verbose_, config(args));
 
-    archiver.filters(filterInclude_, filterExclude_);
-    archiver.modifiers(modifiers_);
+    archiver1.filters(filterInclude_, filterExclude_);
+    archiver1.modifiers(modifiers_);
+
+    fdb5::MessageArchiver archiver2(fdb5::Key(), false, verbose_, config(args));
+
+    archiver2.filters(filterInclude_, filterExclude_);
+    archiver2.modifiers(modifiers_);
 
     for (size_t i = 0; i < args.count(); i++) {
 
@@ -90,8 +95,15 @@ void FDBWrite::execute(const eckit::option::CmdArgs &args) {
 
         std::unique_ptr dh ( path.fileHandle() );
 
-        archiver.archive( *dh );
+        if (i%2==0) {
+            archiver1.archive( *dh );
+        } else {
+            archiver2.archive( *dh );
+        }
     }
+
+    // archiver1.flush();
+    // archiver2.flush();
 }
 
 //----------------------------------------------------------------------------------------------------------------------

From bcd5d9037edbc15240a419f510ff17345f804cfa Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 20 Feb 2024 18:18:39 +0000
Subject: [PATCH 065/186] async data write, unified connection

---
 src/fdb5/api/FDBStats.cc                      |   5 +
 src/fdb5/api/FDBStats.h                       |   3 +
 src/fdb5/api/RemoteFDB.cc                     |   4 +-
 src/fdb5/api/RemoteFDB.h                      |   5 +-
 src/fdb5/database/Archiver.cc                 |   9 +-
 src/fdb5/remote/Connection.cc                 |  18 +-
 src/fdb5/remote/Connection.h                  |  11 +-
 src/fdb5/remote/RemoteCatalogue.cc            | 249 ++------
 src/fdb5/remote/RemoteCatalogue.h             |  16 +-
 src/fdb5/remote/RemoteStore.cc                | 314 +++-------
 src/fdb5/remote/RemoteStore.h                 |  20 +-
 src/fdb5/remote/client/Client.cc              |   8 +-
 src/fdb5/remote/client/Client.h               |   4 +-
 src/fdb5/remote/client/ClientConnection.cc    | 284 ++++-----
 src/fdb5/remote/client/ClientConnection.h     |  16 +-
 .../remote/client/ClientConnectionRouter.cc   |   2 +-
 src/fdb5/remote/server/CatalogueHandler.cc    | 572 ++++++++---------
 src/fdb5/remote/server/CatalogueHandler.h     |  59 +-
 src/fdb5/remote/server/ServerConnection.cc    | 542 ++++++++++++----
 src/fdb5/remote/server/ServerConnection.h     | 107 +++-
 src/fdb5/remote/server/StoreHandler.cc        | 590 +++++++++---------
 src/fdb5/remote/server/StoreHandler.h         |  26 +-
 src/fdb5/tools/fdb-write.cc                   |   5 +-
 23 files changed, 1432 insertions(+), 1437 deletions(-)

diff --git a/src/fdb5/api/FDBStats.cc b/src/fdb5/api/FDBStats.cc
index f7b4d656c..8376eeb68 100644
--- a/src/fdb5/api/FDBStats.cc
+++ b/src/fdb5/api/FDBStats.cc
@@ -29,6 +29,7 @@ namespace fdb5 {
 
 FDBStats::FDBStats() :
     numArchive_(0),
+    numLocation_(0),
     numFlush_(0),
     numRetrieve_(0),
     bytesArchive_(0),
@@ -48,6 +49,7 @@ FDBStats::~FDBStats() {}
 
 FDBStats& FDBStats::operator+=(const FDBStats& rhs) {
     numArchive_ += rhs.numArchive_;
+    numLocation_ += rhs.numLocation_;
     numFlush_ += rhs.numFlush_;
     numRetrieve_ += rhs.numRetrieve_;
     bytesArchive_ += rhs.bytesArchive_;
@@ -81,6 +83,9 @@ void FDBStats::addArchive(size_t length, eckit::Timer& timer, size_t nfields) {
                          << ", total: " << Seconds(elapsedArchive_) << std::endl;
 }
 
+void FDBStats::addLocation(size_t nfields) {
+    numLocation_ += nfields;
+}
 
 void FDBStats::addRetrieve(size_t length, eckit::Timer& timer) {
 
diff --git a/src/fdb5/api/FDBStats.h b/src/fdb5/api/FDBStats.h
index b39fc802d..cf776379e 100644
--- a/src/fdb5/api/FDBStats.h
+++ b/src/fdb5/api/FDBStats.h
@@ -36,9 +36,11 @@ class FDBStats : public eckit::Statistics {
     ~FDBStats();
 
     size_t numArchive() const { return numArchive_; }
+    size_t numLocation() const { return numLocation_; }
     size_t numFlush() const { return numFlush_; }
 
     void addArchive(size_t length, eckit::Timer& timer, size_t nfields=1);
+    void addLocation(size_t nfields=1);
     void addRetrieve(size_t length, eckit::Timer& timer);
     void addFlush(eckit::Timer& timer);
 
@@ -49,6 +51,7 @@ class FDBStats : public eckit::Statistics {
 private: // members
 
     size_t numArchive_;
+    size_t numLocation_;
     size_t numFlush_;
     size_t numRetrieve_;
 
diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index 12c77b374..973fbe39e 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -240,7 +240,7 @@ void RemoteFDB::print(std::ostream& s) const {
 }
 
 // Client
-bool RemoteFDB::handle(remote::Message message, uint32_t requestID) {
+bool RemoteFDB::handle(remote::Message message, bool control, uint32_t requestID) {
     
     switch (message) {
         case fdb5::remote::Message::Complete: {
@@ -273,7 +273,7 @@ bool RemoteFDB::handle(remote::Message message, uint32_t requestID) {
             return false;
     }
 }
-bool RemoteFDB::handle(remote::Message message, uint32_t requestID, eckit::Buffer&& payload) {
+bool RemoteFDB::handle(remote::Message message, bool control, uint32_t requestID, eckit::Buffer&& payload) {
 
     switch (message) {
         case fdb5::remote::Message::Blob: {
diff --git a/src/fdb5/api/RemoteFDB.h b/src/fdb5/api/RemoteFDB.h
index 7b1e15728..9e32120c8 100644
--- a/src/fdb5/api/RemoteFDB.h
+++ b/src/fdb5/api/RemoteFDB.h
@@ -42,6 +42,7 @@ class RemoteFDB : public LocalFDB, public remote::Client {
 public: // method
 
     RemoteFDB(const eckit::Configuration& config, const std::string& name);
+    ~RemoteFDB() {}
 
     ListIterator inspect(const metkit::mars::MarsRequest& request) override;
 
@@ -77,8 +78,8 @@ class RemoteFDB : public LocalFDB, public remote::Client {
 
     // Client
 
-    bool handle(remote::Message message, uint32_t requestID) override;
-    bool handle(remote::Message message, uint32_t requestID, eckit::Buffer&& payload) override;
+    bool handle(remote::Message message, bool control, uint32_t requestID) override;
+    bool handle(remote::Message message, bool control, uint32_t requestID, eckit::Buffer&& payload) override;
     void handleException(std::exception_ptr e) override;
 
 private: // members
diff --git a/src/fdb5/database/Archiver.cc b/src/fdb5/database/Archiver.cc
index f1632e4cb..725d4d901 100644
--- a/src/fdb5/database/Archiver.cc
+++ b/src/fdb5/database/Archiver.cc
@@ -30,10 +30,15 @@ Archiver::Archiver(const Config& dbConfig) :
     store_(nullptr) {}
 
 Archiver::~Archiver() {
-
     flush(); // certify that all sessions are flushed before closing them
 
-    databases_.clear(); //< explicitly delete the DBs before schemas are destroyed
+    // for (auto it = databases_.begin(); it != databases_.end(); it++) {
+    //     databases_.erase(it);
+    // }
+
+//    std::cout << databases_.size();
+
+    // databases_.clear(); //< explicitly delete the DBs before schemas are destroyed
 }
 
 void Archiver::archive(const Key &key, const void* data, size_t len) {
diff --git a/src/fdb5/remote/Connection.cc b/src/fdb5/remote/Connection.cc
index fc3a4ff24..e8669f6d3 100644
--- a/src/fdb5/remote/Connection.cc
+++ b/src/fdb5/remote/Connection.cc
@@ -47,7 +47,7 @@ void Connection::readUnsafe(bool control, void* data, size_t length) {
     } else {
         read = dataSocket().read(data, length);
     }
-    if (!exit_ && length != read) {
+    if (length != read) {
         std::stringstream ss;
         ss << "Read error. Expected " << length << " bytes, read " << read;
         throw TCPException(ss.str(), Here());
@@ -59,7 +59,7 @@ eckit::Buffer Connection::read(bool control, MessageHeader& hdr) {
 
     readUnsafe(control, &hdr, sizeof(hdr));
 
-    std::cout << "READ [" <<  "endpoint=" <<  ((control || single_) ? controlSocket() : dataSocket()).remotePort() << ",message=" << hdr.message << ",clientID=" << hdr.clientID() << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
+    // std::cout << "READ [" <<  "endpoint=" <<  ((control || single_) ? controlSocket() : dataSocket()).remotePort() << ",message=" << hdr.message << ",clientID=" << hdr.clientID() << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
 
     ASSERT(hdr.marker == StartMarker);
     ASSERT(hdr.version == CurrentVersion);
@@ -74,14 +74,14 @@ eckit::Buffer Connection::read(bool control, MessageHeader& hdr) {
     ASSERT(tail == EndMarker);
 
     if (hdr.message == Message::Error) {
-        std::cout << "ERROR while reading: ";
+        // std::cout << "ERROR while reading: ";
 
         char msg[hdr.payloadSize+1];
         if (hdr.payloadSize) {
             char msg[hdr.payloadSize+1];
-            std::cout << static_cast(payload.data());
+            // std::cout << static_cast(payload.data());
         }
-        std::cout << std::endl;
+        // std::cout << std::endl;
     }
 
     return payload;
@@ -106,11 +106,9 @@ void Connection::write(remote::Message msg, bool control, uint32_t clientID, uin
 
     MessageHeader message{msg, control, clientID, requestID, payloadLength};
 
-    std::cout << "WRITE [" << "endpoint=" << ((control || single_) ? controlSocket() : dataSocket()).remotePort() <<
-    ",message=" << message.message << ",clientID=" << message.clientID() << ",requestID=" << message.requestID << ",payload=" << message.payloadSize << "]" << std::endl;
+    // std::cout << "WRITE [" << "endpoint=" << ((control || single_) ? controlSocket() : dataSocket()).remotePort() << ",message=" << message.message << ",clientID=" << message.clientID() << ",requestID=" << message.requestID << ",payload=" << message.payloadSize << "]" << std::endl;
 
-    eckit::Log::debug() << "Connection::write [endpoint=" << ((control || single_) ? controlSocket() : dataSocket()) <<
-        ",message=" << msg << ",clientID=" << message.clientID() << ",requestID=" << requestID << ",data=" << data.size() << ",payload=" << payloadLength << "]" << std::endl;
+    eckit::Log::debug() << "Connection::write [message=" << msg << ",clientID=" << message.clientID() << ",requestID=" << requestID << ",data=" << data.size() << ",payload=" << payloadLength << "]" << std::endl;
 
     std::lock_guard lock((control || single_) ? controlMutex_ : dataMutex_);
     writeUnsafe(control, &message, sizeof(message));
@@ -133,7 +131,7 @@ void Connection::write(remote::Message msg, bool control, uint32_t clientID, uin
 //     write(msg, false, clientID, requestID, data);
 // }
 void Connection::error(const std::string& msg, uint32_t clientID, uint32_t requestID) {
-    write(Message::Error, true, clientID, requestID, std::vector>{{msg.c_str(), msg.length()}});
+    write(Message::Error, false, clientID, requestID, std::vector>{{msg.c_str(), msg.length()}});
 }
 // void Connection::error(const std::string& msg, const Handler& clientID, uint32_t requestID) {
 //     write(Message::Error, true, clientID.clientId(), requestID, std::vector>{{msg.c_str(), msg.length()}});
diff --git a/src/fdb5/remote/Connection.h b/src/fdb5/remote/Connection.h
index 2b4fc0ad4..c82347789 100644
--- a/src/fdb5/remote/Connection.h
+++ b/src/fdb5/remote/Connection.h
@@ -27,20 +27,13 @@ namespace fdb5::remote {
 class Connection : eckit::NonCopyable {
 
 public: // methods
-    Connection() : single_(false), exit_(false) {}
+    Connection() : single_(false) {}
     virtual ~Connection() {}
 
     void write(Message msg, bool control, uint32_t clientID, uint32_t requestID, const void* data, uint32_t length);
     void write(Message msg, bool control, uint32_t clientID, uint32_t requestID, std::vector> data = {});
-    // void write(Message msg, bool control, const Handler& clientID, uint32_t requestID, const void* data, uint32_t length);
-    // void write(Message msg, bool control, const Handler& clientID, uint32_t requestID, std::vector> data = {});
-    // void writeControl(Message msg, uint32_t clientID, uint32_t requestID, const void* data, uint32_t length);
-    // void writeControl(Message msg, uint32_t clientID, uint32_t requestID, std::vector> data = {});
-    // void writeData(Message msg, uint32_t clientID, uint32_t requestID, const void* data, uint32_t length);
-    // void writeData(Message msg, uint32_t clientID, uint32_t requestID, std::vector> data = {});
 
     void error(const std::string& msg, uint32_t clientID, uint32_t requestID);
-    // void error(const std::string& msg, const Handler& clientID, uint32_t requestID);
 
     eckit::Buffer readControl(MessageHeader& hdr);
     eckit::Buffer readData(MessageHeader& hdr);
@@ -58,7 +51,7 @@ class Connection : eckit::NonCopyable {
 protected: // members
 
     bool single_;
-    bool exit_;
+//    bool exit_;
 
 private: // members
 
diff --git a/src/fdb5/remote/RemoteCatalogue.cc b/src/fdb5/remote/RemoteCatalogue.cc
index 895f7625f..31b90f374 100644
--- a/src/fdb5/remote/RemoteCatalogue.cc
+++ b/src/fdb5/remote/RemoteCatalogue.cc
@@ -21,194 +21,16 @@
 using namespace eckit;
 namespace fdb5::remote {
 
-class CatalogueArchivalRequest {
-
-public:
-
-    CatalogueArchivalRequest() :
-        id_(0), catalogue_(nullptr), key_(Key()) {}
-
-    CatalogueArchivalRequest(uint32_t id, RemoteCatalogue* catalogue, const fdb5::Key& key, std::unique_ptr location) :
-        id_(id), catalogue_(catalogue), key_(key), location_(std::move(location)) {}
-
-    std::pair>>  element;
-
-    uint32_t id_;
-    RemoteCatalogue* catalogue_;
-    fdb5::Key key_;
-    std::unique_ptr location_;
-};
-
-
-class RemoteCatalogueArchiver {
-
-public: // types
-
-    using CatalogueArchiveQueue = eckit::Queue;
-
-public: // methods
-
-    static RemoteCatalogueArchiver* get();
-
-    bool valid() { return archiveFuture_.valid(); }
-    bool dirty() { return dirty_; }
-
-    void start();
-    void error(eckit::Buffer payload, const eckit::net::Endpoint& endpoint);
-    
-    void emplace(uint32_t id, RemoteCatalogue* catalogue, const Key& key, std::unique_ptr location);
-    FDBStats flush(RemoteCatalogue* catalogue);
-
-private: // methods
-
-    RemoteCatalogueArchiver() : dirty_(false), maxArchiveQueueLength_(eckit::Resource("fdbRemoteArchiveQueueLength;$FDB_REMOTE_ARCHIVE_QUEUE_LENGTH", 200)) {}
-
-    FDBStats archiveThreadLoop();
-
-private: // members
-
-    bool dirty_;
-
-    std::mutex archiveQueuePtrMutex_;
-    size_t maxArchiveQueueLength_;
-    std::unique_ptr archiveQueue_;
-    std::future archiveFuture_;
-
-};
-
-RemoteCatalogueArchiver* RemoteCatalogueArchiver::get() {
-
-    static std::unique_ptr archiver_;
-
-    if (!archiver_) {
-        archiver_.reset(new RemoteCatalogueArchiver());
-    }
-    return archiver_.get();
-}
-
-void RemoteCatalogueArchiver::start() {
-    // if there is no archiving thread active, then start one.
-    // n.b. reset the archiveQueue_ after a potential flush() cycle.
-    if (!archiveFuture_.valid()) {
-
-        {
-            // Reset the queue after previous done/errors
-            std::lock_guard lock(archiveQueuePtrMutex_);
-            ASSERT(!archiveQueue_);
-            archiveQueue_.reset(new CatalogueArchiveQueue(maxArchiveQueueLength_));
-        }
-
-        archiveFuture_ = std::async(std::launch::async, [this] { return archiveThreadLoop(); });
-    }
-}
-
-void RemoteCatalogueArchiver::error(eckit::Buffer payload, const eckit::net::Endpoint& endpoint) {
-
-    std::lock_guard lock(archiveQueuePtrMutex_);
-
-    if (archiveQueue_) {
-        std::string msg;
-        if (payload.size() > 0) {
-            msg.resize(payload.size(), ' ');
-            payload.copy(&msg[0], payload.size());
-        }
-
-        archiveQueue_->interrupt(std::make_exception_ptr(RemoteFDBException(msg, endpoint)));
-    }
-}
-
-void RemoteCatalogueArchiver::emplace(uint32_t id, RemoteCatalogue* catalogue, const Key& key, std::unique_ptr location) {
-
-    dirty_ = true;
-
-    ASSERT(archiveQueue_);
-    archiveQueue_->emplace(id, catalogue, key, std::move(location));
-}
-
-FDBStats RemoteCatalogueArchiver::flush(RemoteCatalogue* catalogue) {
-
-    std::lock_guard lock(archiveQueuePtrMutex_);
-    ASSERT(archiveQueue_);
-    archiveQueue_->close();
-
-    FDBStats stats = archiveFuture_.get();
-    ASSERT(!archiveQueue_);
-
-    ASSERT(stats.numFlush() == 0);
-    size_t numArchive = stats.numArchive();
-
-    Buffer sendBuf(4096);
-    MemoryStream s(sendBuf);
-    s << numArchive;
-
-    eckit::Log::debug() << " RemoteCatalogue::flush - flushing " << numArchive << " fields" << std::endl;
-    uint32_t id = catalogue->generateRequestID();
-    // The flush call is blocking
-    catalogue->controlWriteCheckResponse(Message::Flush, id, false, sendBuf, s.position());
-
-    dirty_ = false;
-
-    return stats;
-}
-
-FDBStats RemoteCatalogueArchiver::archiveThreadLoop() {
-    FDBStats localStats;
-    eckit::Timer timer;
-
-    CatalogueArchivalRequest element;
-
-    try {
-
-        ASSERT(archiveQueue_);
-        long popped;
-        while ((popped = archiveQueue_->pop(element)) != -1) {
-
-            timer.start();
-            element.catalogue_->sendArchiveData(element.id_, element.key_, std::move(element.location_));
-            timer.stop();
-
-            localStats.addArchive(0, timer, 1);
-        }
-
-        // And note that we are done. (don't time this, as already being blocked
-        // on by the ::flush() routine)
-
-        element.catalogue_->dataWrite(Message::Flush, 0);
-
-        archiveQueue_.reset();
-
-    } catch (...) {
-        archiveQueue_->interrupt(std::current_exception());
-        throw;
-    }
-
-    return localStats;
-
-    // We are inside an async, so don't need to worry about exceptions escaping.
-    // They will be released when flush() is called.
-}
-
-
-// std::string host(const eckit::Configuration& config) {
-//     std::string host = config.getString("host");
-//     std::string remoteDomain = config.getString("remoteDomain", "");
-//     if (remoteDomain.empty()) {
-//         return host;
-//     }
-//     return host+remoteDomain;
-// }
-
-
 RemoteCatalogue::RemoteCatalogue(const Key& key, const Config& config):
     CatalogueImpl(key, ControlIdentifiers(), config), // xxx what are control identifiers? Setting empty here...
     Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port")), ""),
-    config_(config), schema_(nullptr), archiver_(nullptr) {
+    config_(config), schema_(nullptr), numLocations_(0) {
 
     loadSchema();
 }
 
 RemoteCatalogue::RemoteCatalogue(const eckit::URI& uri, const Config& config):
-    Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port")), ""), config_(config), schema_(nullptr), archiver_(nullptr)
+    Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port")), ""), config_(config), schema_(nullptr), numLocations_(0)
     {
         NOTIMP;
     }
@@ -240,20 +62,34 @@ void RemoteCatalogue::sendArchiveData(uint32_t id, const Key& key, std::unique_p
 
 void RemoteCatalogue::archive(const InspectionKey& key, std::unique_ptr fieldLocation) {
 
-    // if there is no archiving thread active, then start one.
-    // n.b. reset the archiveQueue_ after a potential flush() cycle.
-    if (!archiver_) {
-        archiver_ = RemoteCatalogueArchiver::get();
-    }
-    uint32_t id = connection_.generateRequestID();
-    if (!archiver_->valid()) {
-        archiver_->start();
-        ASSERT(archiver_->valid());
+    // eckit::Timer timer;
+    // timer.start();
 
-        controlWriteCheckResponse(Message::Archive, id, true);
+    ASSERT(!key.empty());
+    ASSERT(fieldLocation);
+
+    uint32_t id = connection_.generateRequestID();
+    {
+        std::lock_guard lock(archiveMutex_);
+        if (numLocations_ == 0) { // if this is the first archival request, notify the server
+            controlWriteCheckResponse(Message::Archive, id, true);
+        }
+        numLocations_++;
     }
-    // eckit::Log::debug() << " RemoteCatalogue::archive - adding to queue [id=" << id << ",key=" << key << ",fieldLocation=" << fieldLocation->uri() << "]" << std::endl;
-    archiver_->emplace(id, this, key, std::move(fieldLocation));
+
+    Buffer buffer(8192);
+    MemoryStream stream(buffer);
+    stream << currentIndexKey_;
+    stream << key;
+    stream << *fieldLocation;
+
+    std::vector> payloads;
+    payloads.push_back(std::pair{buffer, stream.position()});
+
+    dataWrite(Message::Blob, id, payloads);
+    // timer.stop();
+
+    // archivalStats_.addArchive(0, timer, 0);
 }
 
 bool RemoteCatalogue::selectIndex(const Key& idxKey) {
@@ -278,18 +114,29 @@ const Schema& RemoteCatalogue::schema() const {
 
 void RemoteCatalogue::flush() {
 
-    Timer timer;
+    // Timer timer;
 
-    timer.start();
+    // timer.start();
 
+    std::lock_guard lock(archiveMutex_);
     // Flush only does anything if there is an ongoing archive();
-    if (archiver_->valid()) {
-        archiver_->flush(this);
-//        internalStats_ += stats;
+    if (numLocations_ > 0) {
+
+        Buffer sendBuf(1024);
+        MemoryStream s(sendBuf);
+        s << numLocations_;
+
+        eckit::Log::debug() << " RemoteCatalogue::flush - flushing " << numLocations_ << " fields" << std::endl;
+
+        // The flush call is blocking
+        uint32_t id = generateRequestID();
+        controlWriteCheckResponse(Message::Flush, id, false, sendBuf, s.position());
+
+        numLocations_ = 0;
     }
 
-    timer.stop();
-//    internalStats_.addFlush(timer);
+    // timer.stop();
+    // internalStats_.addFlush(timer);
 }
 
 void RemoteCatalogue::clean() {NOTIMP;}
@@ -325,12 +172,12 @@ void RemoteCatalogue::loadSchema() {
     }
 }
 
-bool RemoteCatalogue::handle(Message message, uint32_t requestID) {
+bool RemoteCatalogue::handle(Message message, bool control, uint32_t requestID) {
     eckit::Log::debug() << *this << " - Received [message=" << ((uint) message) << ",requestID=" << requestID << "]" << std::endl;
     NOTIMP;
     return false;
 }
-bool RemoteCatalogue::handle(Message message, uint32_t requestID, eckit::Buffer&& payload) {
+bool RemoteCatalogue::handle(Message message, bool control, uint32_t requestID, eckit::Buffer&& payload) {
     eckit::Log::debug() << *this << " - Received [message=" << ((uint) message) << ",requestID=" << requestID << ",payloadSize=" << payload.size() << "]" << std::endl;
     // if (message == Message::Schema) {
     //     eckit::Log::debug() << "RemoteCatalogue::handle received payload size: " << payload.size() << std::endl;
diff --git a/src/fdb5/remote/RemoteCatalogue.h b/src/fdb5/remote/RemoteCatalogue.h
index 044eb6632..d4d15c043 100644
--- a/src/fdb5/remote/RemoteCatalogue.h
+++ b/src/fdb5/remote/RemoteCatalogue.h
@@ -9,7 +9,7 @@
 
 namespace fdb5::remote {
 
-class RemoteCatalogueArchiver;
+// class RemoteCatalogueArchiver;
 //----------------------------------------------------------------------------------------------------------------------
 
 class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public CatalogueImpl, public Client {
@@ -21,7 +21,6 @@ class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public C
 
     ~RemoteCatalogue() {}
 
-
     // From CatalogueWriter
     const Index& currentIndex() override;
     void archive(const InspectionKey& key, std::unique_ptr fieldLocation) override;
@@ -65,13 +64,15 @@ class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public C
 
 protected:
 
+    // FDBStats archivalCompleted();
+
     void loadSchema() override;
 
 private:
     // From Client
     // handlers for incoming messages - to be defined in the client class
-    bool handle(Message message, uint32_t requestID) override;
-    bool handle(Message message, uint32_t requestID, eckit::Buffer&& payload) override;
+    bool handle(Message message, bool control, uint32_t requestID) override;
+    bool handle(Message message, bool control, uint32_t requestID, eckit::Buffer&& payload) override;
     void handleException(std::exception_ptr e) override;
 
 protected:
@@ -84,8 +85,11 @@ class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public C
     Key currentIndexKey_;
     std::unique_ptr schema_;
 
-    // not owning
-    RemoteCatalogueArchiver* archiver_;
+    std::mutex archiveMutex_;
+    size_t numLocations_;
+
+    // // not owning
+    // RemoteCatalogueArchiver* archiver_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index bd3d7c4da..e7c66e2d1 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -34,175 +34,6 @@ using namespace eckit;
 
 namespace fdb5::remote {
 
-class StoreArchivalRequest {
-
-public:
-
-    StoreArchivalRequest() :
-        id_(0), store_(nullptr), key_(Key()), buffer_(Buffer()) {}
-
-    StoreArchivalRequest(uint32_t id, RemoteStore* store, const fdb5::Key& key, const void *data, eckit::Length length) :
-        id_(id), store_(store), key_(key), buffer_(Buffer(reinterpret_cast(data), length)) {}
-
-    uint32_t id_;
-    RemoteStore* store_;
-    fdb5::Key key_;
-    eckit::Buffer buffer_;
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-class RemoteStoreArchiver {
-
-public: // types
-
-    using StoreArchiveQueue = eckit::Queue;
-
-public: // methods
-
-    static RemoteStoreArchiver* get();
-
-    bool valid() { return archiveFuture_.valid(); }
-    bool dirty() { return dirty_; }
-
-    void start();
-    void error(eckit::Buffer&& payload, const eckit::net::Endpoint& endpoint);
-    
-    void emplace(uint32_t id, RemoteStore* store, const Key& key, const void *data, eckit::Length length);
-    FDBStats flush(RemoteStore* store);
-
-private: // methods
-
-    RemoteStoreArchiver() : dirty_(false), maxArchiveQueueLength_(eckit::Resource("fdbRemoteArchiveQueueLength;$FDB_REMOTE_ARCHIVE_QUEUE_LENGTH", 200)),
-    archiveQueue_(nullptr) {}
-
-    FDBStats archiveThreadLoop();
-
-private: // members
-
-    bool dirty_;
-
-    std::mutex archiveQueuePtrMutex_;
-    size_t maxArchiveQueueLength_;
-    std::unique_ptr archiveQueue_;
-    std::future archiveFuture_;
-};
-
-RemoteStoreArchiver* RemoteStoreArchiver::get() {
-
-    static std::unique_ptr archiver_;
-    static std::mutex getArchiverMutex_;
-
-    if (!archiver_) {
-        archiver_.reset(new RemoteStoreArchiver());
-    }
-    return archiver_.get();
-}
-
-void RemoteStoreArchiver::start() {
-    // if there is no archiving thread active, then start one.
-    // n.b. reset the archiveQueue_ after a potential flush() cycle.
-    if (!archiveFuture_.valid()) {
-
-        {
-            // Reset the queue after previous done/errors
-            std::lock_guard lock(archiveQueuePtrMutex_);
-            ASSERT(!archiveQueue_);
-            archiveQueue_.reset(new StoreArchiveQueue(maxArchiveQueueLength_));
-        }
-
-        archiveFuture_ = std::async(std::launch::async, [this] { return archiveThreadLoop(); });
-    }
-}
-
-void RemoteStoreArchiver::error(eckit::Buffer&& payload, const eckit::net::Endpoint& endpoint) {
-
-    std::lock_guard lock(archiveQueuePtrMutex_);
-
-    if (archiveQueue_) {
-        std::string msg;
-        if (payload.size() > 0) {
-            msg.resize(payload.size(), ' ');
-            payload.copy(&msg[0], payload.size());
-        }
-
-        archiveQueue_->interrupt(std::make_exception_ptr(RemoteFDBException(msg, endpoint)));
-    }
-}
-
-void RemoteStoreArchiver::emplace(uint32_t id, RemoteStore* store, const Key& key, const void *data, eckit::Length length) {
-
-    dirty_ = true;
-
-    ASSERT(archiveQueue_);
-    archiveQueue_->emplace(id, store, key, data, length);
-}
-
-FDBStats RemoteStoreArchiver::flush(RemoteStore* store) {
-
-    std::lock_guard lock(archiveQueuePtrMutex_);
-    ASSERT(archiveQueue_);
-    archiveQueue_->close();
-
-    FDBStats stats = archiveFuture_.get();
-    ASSERT(!archiveQueue_);
-
-    ASSERT(stats.numFlush() == 0);
-    size_t numArchive = stats.numArchive();
-
-    Buffer sendBuf(1024);
-    MemoryStream s(sendBuf);
-    s << numArchive;
-
-    eckit::Log::debug() << " RemoteStoreArchiver::flush - flushing " << numArchive << " fields" << std::endl;
-    // The flush call is blocking
-    uint32_t id = store->generateRequestID();
-    store->controlWriteCheckResponse(Message::Flush, id, false, sendBuf, s.position());
-
-    dirty_ = false;
-
-    return stats;
-}
-
-FDBStats RemoteStoreArchiver::archiveThreadLoop() {
-    FDBStats localStats;
-    eckit::Timer timer;
-
-    StoreArchivalRequest element;
-
-    try {
-
-        ASSERT(archiveQueue_);
-        long popped;
-        while ((popped = archiveQueue_->pop(element)) != -1) {
-
-            eckit::Log::debug() << "RemoteStoreArchiver archiving [id="<sendArchiveData(element.id_, element.key_, element.buffer_.data(), element.buffer_.size());
-            timer.stop();
-
-            localStats.addArchive(element.buffer_.size(), timer, 1);
-        }
-
-        // And note that we are done. (don't time this, as already being blocked
-        // on by the ::flush() routine)
-        element.store_->dataWrite(Message::Flush, 0);
-
-        archiveQueue_.reset();
-
-    } catch (...) {
-        archiveQueue_->interrupt(std::current_exception());
-        throw;
-    }
-
-    return localStats;
-
-    // We are inside an async, so don't need to worry about exceptions escaping.
-    // They will be released when flush() is called.
-}
-
-
 // -----------------------------------------------------------------------------------------------------
 
 //
@@ -347,15 +178,14 @@ RemoteStore::RemoteStore(const Key& dbKey, const Config& config) :
     Client(storeEndpoints(config)),
     dbKey_(dbKey), config_(config),
     retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
-    archiver_(nullptr) {
-}
+    dirty_(false), flushRequested_(false) {}
 
 // this is used only in retrieval, with an URI already referring to an accessible Store
 RemoteStore::RemoteStore(const eckit::URI& uri, const Config& config) :
     Client(eckit::net::Endpoint(uri.hostport()), uri.hostport()),
     dbKey_(Key()), config_(config),
     retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
-    archiver_(nullptr) {
+    dirty_(false), flushRequested_(false) { 
 
     // no need to set the local_ flag on the read path
     ASSERT(uri.scheme() == "fdb");
@@ -365,12 +195,13 @@ RemoteStore::~RemoteStore() {
     // If we have launched a thread with an async and we manage to get here, this is
     // an error. n.b. if we don't do something, we will block in the destructor
     // of std::future.
-    if (archiver_ && archiver_->valid()) {
-        Log::error() << "Attempting to destruct DecoupledFDB with active archive thread" << std::endl;
-        eckit::Main::instance().terminate();
-    }
+    // if (archiver_ && archiver_->valid()) {
+    //     Log::error() << "Attempting to destruct DecoupledFDB with active archive thread" << std::endl;
+    //     eckit::Main::instance().terminate();
+    // }
 
-    ASSERT(!archiver_ || !archiver_->dirty());
+    ASSERT(!archivalCompleted_.valid());
+    // ASSERT(!archiver_ || !archiver_->dirty());
 //    disconnect();
 }
 
@@ -388,40 +219,93 @@ eckit::DataHandle* RemoteStore::retrieve(Field& field) const {
 
 void RemoteStore::archive(const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive) {
 
-    // if there is no archiving thread active, then start one.
-    // n.b. reset the archiveQueue_ after a potential flush() cycle.
-    if (!archiver_) {
-        archiver_ = RemoteStoreArchiver::get();
-    }
-    uint32_t id = connection_.generateRequestID();
-    if (!archiver_->valid()) {
-        archiver_->start();
-        ASSERT(archiver_->valid());
+    eckit::Timer timer;
+    timer.start();
+
+    ASSERT(!key.empty());
+    ASSERT(data);
+    ASSERT(length != 0);
 
-        controlWriteCheckResponse(Message::Store, id, true);
+    uint32_t id = connection_.generateRequestID();
+    {
+        std::lock_guard lock(archiveMutex_);
+        if (!dirty_) { // if this is the first archival request, notify the server
+            ASSERT(archivalStats_.numArchive() == 0);
+            ASSERT(!archivalCompleted_.valid());
+            ASSERT(locations_.size() == 0);
+            archivalCompleted_ = fieldLocationsReceived_.get_future();
+
+            controlWriteCheckResponse(Message::Store, id, true);
+            dirty_=true;
+        }
     }
+
     locations_[id] = catalogue_archive;
-    archiver_->emplace(id, this, key, data, length);
+
+    Buffer keyBuffer(4096);
+    MemoryStream keyStream(keyBuffer);
+    keyStream << dbKey_;
+    keyStream << key;
+
+    std::vector> payloads;
+    payloads.push_back(std::pair{keyBuffer, keyStream.position()});
+    payloads.push_back(std::pair{data, length});
+
+    dataWrite(Message::Blob, id, payloads);
+
+    timer.stop();
+
+    archivalStats_.addArchive(length, timer, 1);
 }
 
 bool RemoteStore::open() {
     return true;
 }
 
+FDBStats RemoteStore::archivalCompleted() {
+
+    if (flushRequested_ && (archivalStats_.numArchive() == archivalStats_.numLocation()) && locations_.empty()) {
+        fieldLocationsReceived_.set_value(archivalStats_);
+    }
+
+    FDBStats stats = archivalCompleted_.get();
+
+    ASSERT(locations_.empty());
+    archivalStats_ = FDBStats{};
+    return stats;
+}
+
 void RemoteStore::flush() {
 
     Timer timer;
 
     timer.start();
 
+    flushRequested_ = true;
+
     // Flush only does anything if there is an ongoing archive();
-    if (archiver_->valid()) {
-        archiver_->flush(this);
-//        internalStats_ += stats;
+    std::lock_guard lock(archiveMutex_);
+    if (archivalCompleted_.valid()) {
+
+        // wait for archival completion (received all fieldLocations)
+        FDBStats stats = archivalCompleted();
+
+        if (stats.numArchive() > 0) {
+            Buffer sendBuf(1024);
+            MemoryStream s(sendBuf);
+            s << stats.numArchive();
+
+            eckit::Log::debug() << " RemoteStore::flush - flushing " << stats.numArchive() << " fields" << std::endl;
+            // The flush call is blocking
+            uint32_t id = generateRequestID();
+            controlWriteCheckResponse(Message::Flush, id, false, sendBuf, s.position());
+        }
+        dirty_ = false;
     }
 
     timer.stop();
-//    internalStats_.addFlush(timer);
+    flushRequested_ = false;
+    internalStats_.addFlush(timer);
 }
 
 void RemoteStore::close() {
@@ -440,7 +324,7 @@ void RemoteStore::print(std::ostream &out) const {
     out << "RemoteStore(host=" << controlEndpoint() /* << ", data=" << dataEndpoint() */ << ")";
 }
 
-bool RemoteStore::handle(Message message, uint32_t requestID) {
+bool RemoteStore::handle(Message message, bool control, uint32_t requestID) {
 
     switch (message) {  
         case Message::Complete: {
@@ -475,12 +359,13 @@ bool RemoteStore::handle(Message message, uint32_t requestID) {
             return false;
     }
 }
-bool RemoteStore::handle(Message message, uint32_t requestID, eckit::Buffer&& payload) {
+bool RemoteStore::handle(Message message, bool control, uint32_t requestID, eckit::Buffer&& payload) {
 
     switch (message) {
     
-        case Message::Store: {
+        case Message::Store: { // received a Field location from the remote store, can forward to the archiver for the indexing
             auto it = locations_.find(requestID);
+            ASSERT(it != locations_.end());
             if (it != locations_.end()) {
                 MemoryStream s(payload);
                 std::unique_ptr location(eckit::Reanimator::reanimate(s));
@@ -490,8 +375,16 @@ bool RemoteStore::handle(Message message, uint32_t requestID, eckit::Buffer&& pa
                     std::unique_ptr remoteLocation = std::unique_ptr(new RemoteFieldLocation(eckit::net::Endpoint{defaultEndpoint()}, *location));
                     it->second(std::move(remoteLocation));
                 }
+
+                locations_.erase(it);
+                archivalStats_.addLocation();
+
+                if (flushRequested_ && (archivalStats_.numArchive() == archivalStats_.numLocation()) && locations_.empty()) {
+                    fieldLocationsReceived_.set_value(archivalStats_);
+                }
+                return true;
             }
-            return true;
+            return false;
         }
         case Message::Blob: {
             auto it = messageQueues_.find(requestID);
@@ -518,7 +411,7 @@ bool RemoteStore::handle(Message message, uint32_t requestID, eckit::Buffer&& pa
             } else {
                 auto it = locations_.find(requestID);
                 if (it != locations_.end()) {
-                    archiver_->error(std::move(payload), controlEndpoint());
+//                    archiver_->error(std::move(payload), controlEndpoint());
                 } else {
                     retrieveMessageQueue_.emplace(message, std::move(payload));
                 }
@@ -533,33 +426,6 @@ void RemoteStore::handleException(std::exception_ptr e) {
     Log::error() << "RemoteStore::handleException " << std::endl;
 }
 
-void RemoteStore::flush(FDBStats& internalStats) {
-    // Flush only does anything if there is an ongoing archive();
-    if (! archiver_->valid()) return;
-
-    internalStats += archiver_->flush(this);
-}
-
-void RemoteStore::sendArchiveData(uint32_t id, const Key& key, const void* data, size_t length) {
-    
-    ASSERT(!dbKey_.empty());
-    ASSERT(!key.empty());
-
-    ASSERT(data);
-    ASSERT(length != 0);
-
-    Buffer keyBuffer(4096);
-    MemoryStream keyStream(keyBuffer);
-    keyStream << dbKey_;
-    keyStream << key;
-
-    std::vector> payloads;
-    payloads.push_back(std::pair{keyBuffer, keyStream.position()});
-    payloads.push_back(std::pair{data, length});
-
-    dataWrite(Message::Blob, id, payloads);
-}
-
 eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation) {
     return dataHandle(fieldLocation, Key());
 }
diff --git a/src/fdb5/remote/RemoteStore.h b/src/fdb5/remote/RemoteStore.h
index b08a63ecc..d8a36be71 100644
--- a/src/fdb5/remote/RemoteStore.h
+++ b/src/fdb5/remote/RemoteStore.h
@@ -22,7 +22,7 @@
 
 namespace fdb5::remote {
 
-class RemoteStoreArchiver;
+// class RemoteStoreArchiver;
 
 //----------------------------------------------------------------------------------------------------------------------
 
@@ -61,7 +61,6 @@ class RemoteStore : public Store, public Client {
 
    const Config& config() { return config_; }
    
-    void sendArchiveData(uint32_t id, const Key& key, const void* data, size_t length);
 
 protected: // methods
 
@@ -78,11 +77,13 @@ class RemoteStore : public Store, public Client {
 
 private: // methods
 
+    FDBStats archivalCompleted();
+
     void flush(FDBStats& stats);
 
     // handlers for incoming messages - to be defined in the client class
-    bool handle(Message message, uint32_t requestID) override;
-    bool handle(Message message, uint32_t requestID, eckit::Buffer&& payload) override;
+    bool handle(Message message, bool control, uint32_t requestID) override;
+    bool handle(Message message, bool control, uint32_t requestID, eckit::Buffer&& payload) override;
     void handleException(std::exception_ptr e) override;
 
 private: // members
@@ -91,7 +92,6 @@ class RemoteStore : public Store, public Client {
 
     const Config& config_;
 
-    std::map fieldLocation)>> locations_;
 
     // @note This is a map of requestID:MessageQueue. At the point that a request is
     // complete, errored or otherwise killed, it needs to be removed from the map.
@@ -100,8 +100,14 @@ class RemoteStore : public Store, public Client {
     std::map> messageQueues_;
     MessageQueue retrieveMessageQueue_;
 
-    // not owning
-    RemoteStoreArchiver* archiver_;
+    std::map fieldLocation)>> locations_;
+    FDBStats internalStats_;
+    FDBStats archivalStats_;
+    std::promise fieldLocationsReceived_;
+    std::future archivalCompleted_;
+    std::mutex archiveMutex_;
+    bool dirty_;
+    bool flushRequested_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index 4ce54f746..50345425a 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -53,7 +53,7 @@ Client::Client(const std::vector>&
 }
 
 Client::~Client() {
-    ASSERT(connection_.remove(id_));
+    connection_.remove(id_);
 }
 
 bool Client::response(uint32_t requestID) {
@@ -74,12 +74,6 @@ bool Client::response(uint32_t requestID, eckit::Buffer&& payload) {
     return true;
 }
 
-// uint32_t Client::controlWriteCheckResponse(Message msg, const void* payload, uint32_t payloadLength) {
-//     uint32_t id = connection_.generateRequestID();
-//     controlWriteCheckResponse(msg, id, payload, payloadLength);
-//     return id;
-// }
-
 void Client::controlWriteCheckResponse(Message msg, uint32_t requestID, bool dataListener, const void* payload, uint32_t payloadLength) {
 
     ASSERT(requestID);
diff --git a/src/fdb5/remote/client/Client.h b/src/fdb5/remote/client/Client.h
index b5c80269a..2f5b353a8 100644
--- a/src/fdb5/remote/client/Client.h
+++ b/src/fdb5/remote/client/Client.h
@@ -52,8 +52,8 @@ class Client : eckit::NonCopyable {
     void dataWrite(remote::Message msg, uint32_t requestID, std::vector> data={});
     
     // handlers for incoming messages - to be defined in the client class
-    virtual bool handle(Message message, uint32_t requestID) = 0;
-    virtual bool handle(Message message, uint32_t requestID, /*eckit::net::Endpoint endpoint,*/ eckit::Buffer&& payload) = 0;
+    virtual bool handle(Message message, bool control, uint32_t requestID) = 0;
+    virtual bool handle(Message message, bool control, uint32_t requestID, eckit::Buffer&& payload) = 0;
     virtual void handleException(std::exception_ptr e) = 0;
 
     bool response(uint32_t requestID);
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index 20c847a8a..e4077f9be 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -40,11 +40,28 @@ class TCPException : public eckit::Exception {
 
 //----------------------------------------------------------------------------------------------------------------------
 
+class DataWriteRequest {
+
+public:
+
+    DataWriteRequest() :
+        client_(nullptr), msg_(Message::None), id_(0), data_(eckit::Buffer(0)) {}
+
+    DataWriteRequest(Client* client, Message msg, uint32_t id, eckit::Buffer&& data) :
+        client_(client), msg_(msg), id_(id), data_(std::move(data)) {}
+
+    Client* client_;
+    Message msg_;
+    uint32_t id_;
+    eckit::Buffer data_;
+};
+
 
 ClientConnection::ClientConnection(const eckit::net::Endpoint& controlEndpoint, const std::string& defaultEndpoint):
-    controlEndpoint_(controlEndpoint), defaultEndpoint_(defaultEndpoint), id_(1), exit_(false), connected_(false) {
-        eckit::Log::debug() << "ClientConnection::ClientConnection() controlEndpoint: " << controlEndpoint << std::endl;
-    }
+    controlEndpoint_(controlEndpoint), defaultEndpoint_(defaultEndpoint), id_(1), connected_(false), dataWriteQueue_(nullptr) {
+
+    eckit::Log::debug() << "ClientConnection::ClientConnection() controlEndpoint: " << controlEndpoint << std::endl;
+}
 
 void ClientConnection::add(Client& client) {
     std::lock_guard lock(clientsMutex_);
@@ -54,23 +71,33 @@ void ClientConnection::add(Client& client) {
     clients_[client.id()] = &client;
 }
 
-
 bool ClientConnection::remove(uint32_t clientID) {
-    std::lock_guard lock(clientsMutex_);
-    auto it = clients_.find(clientID);
-    bool exist = it != clients_.end();
-
-    if (exist) {
-        clients_.erase(it);
-        if (clients_.empty()) {
-            exit_ = true;
-            ClientConnectionRouter::instance().deregister(*this);
+    if (clientID > 0) {
+
+        //std::lock_guard lock(clientsMutex_);
+        auto it = clients_.find(clientID);
+
+        if (it != clients_.end()) {
+            Connection::write(Message::Exit, true, clientID, 0);
+            // TODO make the data connection dying automatically, when there are no more async writes
+            Connection::write(Message::Exit, false, clientID, 0);
+
+            clients_.erase(it);
         }
     }
-    return exist;
+
+    if (clients_.empty()) {
+        ClientConnectionRouter::instance().deregister(*this);
+    }
+
+    return clients_.empty();
 }
 
 ClientConnection::~ClientConnection() {
+    if (dataWriteQueue_) {
+        dataWriteQueue_->close();
+    }
+
     disconnect();
 }
 
@@ -106,6 +133,7 @@ bool ClientConnection::connect(bool singleAttempt) {
 
         // And the connections are set up. Let everything start up!
         listeningThread_ = std::thread([this] { listeningThreadLoop(); });
+        //listeningThread_.detach();
         connected_ = true;
         return true;
     } catch(eckit::TooManyRetries& e) {
@@ -118,15 +146,8 @@ bool ClientConnection::connect(bool singleAttempt) {
 
 void ClientConnection::disconnect() {
 
-//    std::lock_guard lock(requestMutex_);
-
+    ASSERT(clients_.empty());
     if (connected_) {
-        exit_ = true;
-        // Send termination message
-        for (auto c: clients_) {
-            Connection::write(Message::Exit, true, c.first, 0);
-        }
-        Connection::write(Message::Exit, true, 0, 0);
 
         listeningThread_.join();
 
@@ -140,12 +161,6 @@ void ClientConnection::disconnect() {
 const eckit::net::Endpoint& ClientConnection::controlEndpoint() const {
     return controlEndpoint_;
 }
-// const eckit::net::Endpoint& ClientConnection::fullyQualifiedControlEndpoint() const { 
-//     return fullyQualifiedControlEndpoint_;
-// } 
-// const eckit::net::Endpoint& ClientConnection::dataEndpoint() const { 
-//     return dataEndpoint_;
-// } 
 
 eckit::LocalConfiguration ClientConnection::availableFunctionality() const {
     eckit::LocalConfiguration conf;
@@ -158,128 +173,75 @@ eckit::LocalConfiguration ClientConnection::availableFunctionality() const {
 
 void ClientConnection::controlWrite(Client& client, Message msg, uint32_t requestID, bool dataListener, std::vector> data) {
     auto it = clients_.find(client.clientId());
-    if (it == clients_.end()) {
-        std::cout << "Existing clients: ";
-        for (auto& c : clients_) {
-            std::cout << c.first << "  ";
-        }
-        std::cout << std::endl << "Searching client: " << client.clientId() << std::endl;
-    }
     ASSERT(it != clients_.end());
-    // controlListeners_++;
-    // if (dataListener) {
-    //     dataListeners_++;
-    // }
-    // std::cout << controlEndpoint_ << " SENT msg: " << msg << "  controlListeners: " << controlListeners_ << "  dataListeners: " << dataListeners_ << std::endl;
+
     Connection::write(msg, true, client.clientId(), requestID, data);
 }
 
-// void ClientConnection::controlWrite(Client& client, Message msg, uint32_t requestID, uint32_t archiverID, std::vector> data) {
-
-//     auto it = clients_.find(client.id());
-//     ASSERT(it != clients_.end());
-
-//     controlWrite(msg, archiverID ? archiverID : client.id(), requestID, data);
-// }
-
-// void ClientConnection::controlWrite(Message msg, uint32_t clientID, uint32_t requestID, std::vector> data) {
-
-//     uint32_t payloadLength = 0;
-//     for (auto d: data) {
-//         ASSERT(d.first);
-//         payloadLength += d.second;
-//     }
-//     eckit::Log::debug() << "ClientConnection::controlWrite [endpoint=" << controlEndpoint_ <<
-//         ",message=" << ((int) msg) << ",clientID=" << clientID << ",requestID=" << requestID << ",data=" << data.size() << ",payload=" << payloadLength << "]" << std::endl;
-
-//     MessageHeader message(msg, clientID, requestID, payloadLength);
-
-//     std::lock_guard lock(controlMutex_);
-//     controlWrite(&message, sizeof(message));
-//     for (auto d: data) {
-//         controlWrite(d.first, d.second);
-//     }
-//     controlWrite(&EndMarker, sizeof(EndMarker));
-// }
-
-// void ClientConnection::controlWrite(const void* data, size_t length) {
-//     size_t written = controlClient_.write(data, length);
-//     if (length != written) {
-//         std::stringstream ss;
-//         ss << "Write error. Expected " << length << " bytes, wrote " << written;
-//         throw TCPException(ss.str(), Here());
-//     }
-// }
-
-// void ClientConnection::controlRead(void* data, size_t length) {
-//     size_t read = controlClient_.read(data, length);
-//     if (length != read) {
-//         std::stringstream ss;
-//         ss << "Read error. Expected " << length << " bytes, read " << read;
-//         throw TCPException(ss.str(), Here());
-//     }
-// }
+void ClientConnection::dataWrite(DataWriteRequest& r) {
+    Connection::write(r.msg_, false, r.client_->clientId(), r.id_, r.data_.data(), r.data_.size());
+}
 
 void ClientConnection::dataWrite(Client& client, remote::Message msg, uint32_t requestID, std::vector> data) {
 
+    static size_t maxQueueLength = eckit::Resource("fdbDataWriteQueueLength;$FDB_DATA_WRITE_QUEUE_LENGTH", 200);
     auto it = clients_.find(client.clientId());
-    // if (it == clients_.end()) {
-    //     std::cout << "Existing clients: ";
-    //     for (auto& c : clients_) {
-    //         std::cout << c.first << "' ";
-    //     }
-    //     std::cout << std::endl << "Searching client: " << client.clientId() << std::endl;
-    // }
     ASSERT(it != clients_.end());
-    Connection::write(msg, false, client.clientId(), requestID, data);
+
+    if (!dataWriteFuture_.valid()) {
+
+        {
+            // Reset the queue after previous done/errors
+            std::lock_guard lock(dataWriteQueueMutex_);
+            ASSERT(!dataWriteQueue_);
+
+            dataWriteQueue_.reset(new eckit::Queue{maxQueueLength});
+        }
+
+        dataWriteFuture_ = std::async(std::launch::async, [this] { return dataWriteThreadLoop(); });
+    }
+
+    uint32_t payloadLength = 0;
+    for (auto d: data) {
+        ASSERT(d.first);
+        payloadLength += d.second;
+    }
+
+    eckit::Buffer buffer{payloadLength};
+    uint32_t offset = 0;
+    for (auto d: data) {
+        buffer.copy(d.first, d.second, offset);
+        offset += d.second;
+    }
+
+    dataWriteQueue_->emplace(&client, msg, requestID, std::move(buffer));
+//    Connection::write(msg, false, client.clientId(), requestID, data);
 }
 
 
-// void ClientConnection::dataWrite(Client& client, remote::Message msg, uint32_t requestID, std::vector> data) {
-
-//     auto it = clients_.find(client.id());
-//     ASSERT(it != clients_.end());
-
-//     dataWrite(msg, client.id(), requestID, data);
-// }
-
-// void ClientConnection::dataWrite(remote::Message msg, uint32_t clientID, uint32_t requestID, std::vector> data) {
-
-//     uint32_t payloadLength = 0;
-//     for (auto d: data) {
-//         ASSERT(d.first);
-//         payloadLength += d.second;
-//     }
-//     MessageHeader message(msg, clientID, requestID, payloadLength);
-
-//     eckit::Log::debug() << "ClientConnection::dataWrite [endpoint=" << dataEndpoint_ <<
-//         ",message=" << ((int) msg) << ",requestID=" << requestID << ",data=" << data.size() << ",payload=" << payloadLength << "]" << std::endl;
-
-//     std::lock_guard lock(dataMutex_);
-//     dataWrite(&message, sizeof(message));
-//     for (auto d: data) {
-//         dataWrite(d.first, d.second);
-//     }
-//     dataWrite(&EndMarker, sizeof(EndMarker));
-// }
-
-// void ClientConnection::dataWrite(const void* data, size_t length) {
-//     size_t written = dataClient_.write(data, length);
-//     if (length != written) {
-//         std::stringstream ss;
-//         ss << "Write error. Expected " << length << " bytes, wrote " << written;
-//         throw TCPException(ss.str(), Here());
-//     }
-// }
-
-// void ClientConnection::dataRead(void* data, size_t length) {
-//     size_t read = dataClient_.read(data, length);
-//     if (!exit_ && length != read) {
-//         std::stringstream ss;
-//         ss << "Read error. Expected " << length << " bytes, read " << read;
-//         throw TCPException(ss.str(), Here());
-//     }
-// }
+void ClientConnection::dataWriteThreadLoop() {
+
+    eckit::Timer timer;
+    DataWriteRequest element;
+
+    try {
+
+        ASSERT(dataWriteQueue_);
+        while (dataWriteQueue_->pop(element) != -1) {
+
+            dataWrite(element);
+        }
+
+        dataWriteQueue_.reset();
+
+    } catch (...) {
+        dataWriteQueue_->interrupt(std::current_exception());
+        throw;
+    }
+
+    // We are inside an async, so don't need to worry about exceptions escaping.
+    // They will be released when flush() is called.
+}
 
 void ClientConnection::handleError(const MessageHeader& hdr, eckit::Buffer buffer) {
     ASSERT(hdr.marker == StartMarker);
@@ -296,28 +258,6 @@ void ClientConnection::handleError(const MessageHeader& hdr, eckit::Buffer buffe
 }
 
 
-
-// void ClientConnection::handleError(const MessageHeader& hdr) {
-
-//     ASSERT(hdr.marker == StartMarker);
-//     ASSERT(hdr.version == CurrentVersion);
-
-//     if (hdr.message == Message::Error) {
-//         ASSERT(hdr.payloadSize > 9);
-
-//         std::string what(hdr.payloadSize, ' ');
-//         dataRead(&what[0], hdr.payloadSize);
-//         what[hdr.payloadSize] = 0; // Just in case
-
-//         try {
-//             eckit::FixedString<4> tail;
-//             dataRead(&tail, sizeof(tail));
-//         } catch (...) {}
-
-//         throw RemoteFDBException(what, controlEndpoint_);
-//     }
-// }
-
 void ClientConnection::writeControlStartupMessage() {
 
     eckit::Buffer payload(4096);
@@ -351,19 +291,9 @@ eckit::SessionID ClientConnection::verifyServerStartupResponse() {
 
     MessageHeader hdr;
     eckit::Buffer payload = Connection::readControl(hdr);
-    // controlRead(&hdr, sizeof(hdr));
 
-    // ASSERT(hdr.marker == StartMarker);
-    // ASSERT(hdr.version == CurrentVersion);
-    // ASSERT(hdr.message == Message::Startup);
     ASSERT(hdr.requestID == 0);
 
-    // eckit::Buffer payload(hdr.payloadSize);
-    // eckit::FixedString<4> tail;
-    // controlRead(payload, hdr.payloadSize);
-    // controlRead(&tail, sizeof(tail));
-    // ASSERT(tail == EndMarker);
-
     eckit::MemoryStream s(payload);
     eckit::SessionID clientSession(s);
     eckit::SessionID serverSession(s);
@@ -404,15 +334,16 @@ void ClientConnection::listeningThreadLoop() {
         MessageHeader hdr;
         eckit::FixedString<4> tail;
 
-        while (!exit_) {
+        while (true) {
 
             eckit::Buffer payload = Connection::readData(hdr);
 
             eckit::Log::debug() << "ClientConnection::listeningThreadLoop - got [message=" << hdr.message << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
 
             if (hdr.message == Message::Exit) {
-                if (hdr.clientID()) {
-                    remove(hdr.clientID());
+
+                if (clients_.empty()) {
+                    return;
                 }
             } else {
                 if (hdr.clientID()) {
@@ -434,20 +365,20 @@ void ClientConnection::listeningThreadLoop() {
                             ASSERT(hdr.message == Message::Received);
                             handled = it->second->response(hdr.requestID);
                         } else {
-                            handled = it->second->handle(hdr.message, hdr.requestID);
+                            handled = it->second->handle(hdr.message, hdr.control(), hdr.requestID);
                         }
                     }
                     else {
                         if (it->second->blockingRequestId() == hdr.requestID) {
                             handled = it->second->response(hdr.requestID, std::move(payload));
                         } else {
-                            handled = it->second->handle(hdr.message, hdr.requestID, std::move(payload));
+                            handled = it->second->handle(hdr.message, hdr.control(), hdr.requestID, std::move(payload));
                         }
                     }
 
                     if (!handled) {
                         std::stringstream ss;
-                        ss << "ERROR: Unexpected message recieved (" << static_cast(hdr.message) << "). ABORTING";
+                        ss << "ERROR: Unexpected message recieved (" << hdr.message << "). ABORTING";
                         eckit::Log::status() << ss.str() << std::endl;
                         eckit::Log::error() << "Client Retrieving... " << ss.str() << std::endl;
                         throw eckit::SeriousBug(ss.str(), Here());
@@ -462,6 +393,7 @@ void ClientConnection::listeningThreadLoop() {
     } catch (...) {
 //        ClientConnectionRouter::instance().handleException(std::current_exception());
     }
+    // ClientConnectionRouter::instance().deregister(*this);
 }
 
 }  // namespace fdb5::remote
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index bfaec3890..15ea9c564 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -11,6 +11,7 @@
 #pragma once
 
 #include 
+#include 
 
 #include "eckit/config/LocalConfiguration.h"
 #include "eckit/container/Queue.h"
@@ -33,6 +34,7 @@ namespace fdb5::remote {
 
 class Client;
 class ClientConnectionRouter;
+class DataWriteRequest;
 
 //----------------------------------------------------------------------------------------------------------------------
 
@@ -42,7 +44,7 @@ class ClientConnection : protected Connection {
 
 public: // methods
 
-    ~ClientConnection();
+    virtual ~ClientConnection();
 
     void controlWrite(Client& client, Message msg, uint32_t requestID, bool dataListener, std::vector> data={});
     void dataWrite(Client& client, remote::Message msg, uint32_t requestID, std::vector> data={});
@@ -59,6 +61,10 @@ class ClientConnection : protected Connection {
 
 private: // methods
 
+//    bool remove(uint32_t clientID);
+
+    void dataWrite(DataWriteRequest& dataWriteRequest);
+
     friend class ClientConnectionRouter;
 
     ClientConnection(const eckit::net::Endpoint& controlEndpoint, const std::string& defaultEndpoint);
@@ -83,6 +89,7 @@ class ClientConnection : protected Connection {
     void handleError(const MessageHeader& hdr, eckit::Buffer buffer);
 
     void listeningThreadLoop();
+    void dataWriteThreadLoop();
 
     eckit::net::TCPSocket& controlSocket() override { return controlClient_; }
     eckit::net::TCPSocket& dataSocket() override { return dataClient_; }
@@ -105,15 +112,16 @@ class ClientConnection : protected Connection {
     std::thread listeningThread_;
     
     std::mutex requestMutex_;
-    // std::mutex controlMutex_;
-    // std::mutex dataMutex_;
 
     // requestID
     std::mutex idMutex_;
     uint32_t id_;
 
-    bool exit_;
     bool connected_; 
+
+    std::mutex dataWriteQueueMutex_;
+    std::unique_ptr> dataWriteQueue_;
+    std::future dataWriteFuture_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.cc b/src/fdb5/remote/client/ClientConnectionRouter.cc
index f689d959e..ea1e62c5c 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.cc
+++ b/src/fdb5/remote/client/ClientConnectionRouter.cc
@@ -84,7 +84,7 @@ ClientConnection& ClientConnectionRouter::connection(const std::vector lock(connectionMutex_);
+    // std::lock_guard lock(connectionMutex_);
     auto it = connections_.find(connection.controlEndpoint());
     if (it != connections_.end()) {
         connections_.erase(it);
diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc
index cf81886ac..178fa19a3 100644
--- a/src/fdb5/remote/server/CatalogueHandler.cc
+++ b/src/fdb5/remote/server/CatalogueHandler.cc
@@ -13,7 +13,6 @@
 #include "eckit/serialisation/MemoryStream.h"
 
 #include "fdb5/LibFdb5.h"
-#include "fdb5/api/FDBFactory.h"
 #include "fdb5/api/helpers/FDBToolRequest.h"
 #include "fdb5/remote/server/CatalogueHandler.h"
 
@@ -31,82 +30,142 @@ namespace fdb5::remote {
 // ***************************************************************************************
 
 CatalogueHandler::CatalogueHandler(eckit::net::TCPSocket& socket, const Config& config):
-    ServerConnection(socket, config) {
-
-}
+    ServerConnection(socket, config) {}
 
 CatalogueHandler::~CatalogueHandler() {}
 
-void CatalogueHandler::initialiseConnections() {
-    ServerConnection::initialiseConnections();
-}
-
-void CatalogueHandler::stores(const MessageHeader& hdr) {
+Handled CatalogueHandler::handleControl(Message message, uint32_t clientID, uint32_t requestID) {
 
-    eckit::net::IPAddress clientIPaddress{controlSocket_.remoteAddr()};
-
-    std::string clientNetwork = "";
-    if (config_.has("networks")) {
-        for (const auto& net: config_.getSubConfigurations("networks")) {
-            if (net.has("name") && net.has("netmask")) {
-                eckit::net::NetMask netmask{net.getString("netmask")};
-                if (netmask.contains(clientIPaddress)) {
-                    clientNetwork = net.getString("name");
-                    break;
-                }
+    try {
+        switch (message) {
+            case Message::Schema: // request top-level schema
+                schema(clientID, requestID, eckit::Buffer(0));
+                return Handled::Replied;
+
+            case Message::Stores: // request the list of FDB stores and the corresponging endpoints
+                stores(clientID, requestID);
+                return Handled::Replied;
+
+
+            case Message::Archive: // notification that the client is starting to send data locations for archival
+                archiver();
+                return Handled::YesAddArchiveListener;
+
+            case Message::Flush: // notification that the client has sent all data locations for archival
+                flush(clientID, requestID, eckit::Buffer{0});
+                return Handled::Yes;
+
+            default: {
+                std::stringstream ss;
+                ss << "ERROR: Unexpected message recieved (" << message << "). ABORTING";
+                Log::status() << ss.str() << std::endl;
+                Log::error() << ss.str() << std::endl;
+                throw SeriousBug(ss.str(), Here());
             }
         }
     }
-
-    Log::debug() << "Client " << clientIPaddress << " from network '" << clientNetwork << "'" << std::endl;
-
-    ASSERT(config_.has("stores"));
-    std::map> stores;
-    // std::vector> stores;
-    for (const auto& configStore: config_.getSubConfigurations("stores")) {
-        ASSERT(configStore.has("default"));
-        eckit::net::Endpoint fieldLocationEndpoint{configStore.getString("default")};
-        eckit::net::Endpoint storeEndpoint{fieldLocationEndpoint};
-        if (!clientNetwork.empty()) {
-            if (configStore.has(clientNetwork)) {
-                storeEndpoint = eckit::net::Endpoint{configStore.getString(clientNetwork)};
-            }
-        }
-
-        auto it = stores.find(fieldLocationEndpoint);
-        if (it == stores.end()) {
-            stores.emplace(fieldLocationEndpoint, std::vector{storeEndpoint});
-        } else {
-            it->second.push_back(storeEndpoint);
-        }
-
-        if (configStore.getBool("serveLocalData", false)) {
-            it = stores.find("");
-            if (it == stores.end()) {
-                stores.emplace("", std::vector{storeEndpoint});
-            } else {
-                it->second.push_back(storeEndpoint);
-            }
-        }
+    catch (std::exception& e) {
+        // n.b. more general than eckit::Exception
+        error(e.what(), clientID, requestID);
+    }
+    catch (...) {
+        error("Caught unexpected and unknown error", clientID, requestID);
     }
+    return Handled::No;
+}
 
-    {
-        Buffer startupBuffer(16*1024);
-        MemoryStream s(startupBuffer);
+Handled CatalogueHandler::handleControl(Message message, uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) {
 
-        s << stores.size();
-        for (const auto& store : stores) {
-            s << store.first << store.second.size();
-            for (const auto& ee: store.second) {
-                s << ee;
+    try {
+        switch (message) {
+
+            case Message::Schema: // request catalogue schema
+                schema(clientID, requestID, std::move(payload));
+                return Handled::Replied;
+
+            case Message::List: // list request. Location are sent aynchronously over the data connection
+                list(clientID, requestID, std::move(payload));
+                return Handled::Yes;
+
+            case Message::Inspect: // inspect request. Location are sent aynchronously over the data connection
+                inspect(clientID, requestID, std::move(payload));
+                return Handled::Yes;
+
+            case Message::Flush: // flush catalogue
+                flush(clientID, requestID, std::move(payload));
+                return Handled::Yes;
+
+            default: {
+                std::stringstream ss;
+                ss << "ERROR: Unexpected message recieved (" << message << "). ABORTING";
+                Log::status() << ss.str() << std::endl;
+                Log::error() << ss.str() << std::endl;
+                throw SeriousBug(ss.str(), Here());
             }
         }
-        // s << config_.schema();
-
-        dataWrite(Message::Received, hdr.clientID(), hdr.requestID, startupBuffer.data(), s.position());
     }
+    catch (std::exception& e) {
+        // n.b. more general than eckit::Exception
+        error(e.what(), clientID, requestID);
+    }
+    catch (...) {
+        error("Caught unexpected and unknown error", clientID, requestID);
+    }
+    return Handled::No;
 }
 
+// Handled CatalogueHandler::handleData(Message message, uint32_t clientID, uint32_t requestID) {
+//     try {
+//         switch (message) {
+//             case Message::Flush: // notification that the client has sent all data locations for archival
+//                 return Handled::YesRemoveArchiveListener; // ????
+//
+//             default: {
+//                 std::stringstream ss;
+//                 ss << "ERROR: Unexpected message recieved (" << message << "). ABORTING";
+//                 Log::status() << ss.str() << std::endl;
+//                 Log::error() << ss.str() << std::endl;
+//                 throw SeriousBug(ss.str(), Here());
+//             }
+//         }
+//     }
+//     catch (std::exception& e) {
+//         // n.b. more general than eckit::Exception
+//         error(e.what(), clientID, requestID);
+//     }
+//     catch (...) {
+//         error("Caught unexpected and unknown error", clientID, requestID);
+//     }
+//     return Handled::No;
+// }
+
+// Handled CatalogueHandler::handleData(Message message, uint32_t clientID, uint32_t requestID, /*eckit::net::Endpoint endpoint,*/ eckit::Buffer&& payload) {
+//     try {
+//         switch (message) {
+//             case Message::Blob:
+//             case Message::MultiBlob:
+//                 queue(message, clientID, requestID, std::move(payload));
+//                 return Handled::Yes;
+//
+//             default: {
+//                 std::stringstream ss;
+//                 ss << "ERROR: Unexpected message recieved (" << message << "). ABORTING";
+//                 Log::status() << ss.str() << std::endl;
+//                 Log::error() << ss.str() << std::endl;
+//                 throw SeriousBug(ss.str(), Here());
+//             }
+//         }
+//     }
+//     catch (std::exception& e) {
+//         // n.b. more general than eckit::Exception
+//         error(e.what(), clientID, requestID);
+//     }
+//     catch (...) {
+//         error("Caught unexpected and unknown error", clientID, requestID);
+//     }
+//     return Handled::No;
+// }
+
 // API forwarding logic, adapted from original remoteHandler
 // Used for Inspect and List
 // ***************************************************************************************
@@ -148,10 +207,9 @@ struct InspectHelper : public BaseHelper {
 };
 
 template 
-void CatalogueHandler::forwardApiCall(const MessageHeader& hdr) {
+void CatalogueHandler::forwardApiCall(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) {
     HelperClass helper;
 
-    Buffer payload(receivePayload(hdr, controlSocket_));
     MemoryStream s(payload);
 
     FDBToolRequest request(s);
@@ -159,333 +217,209 @@ void CatalogueHandler::forwardApiCall(const MessageHeader& hdr) {
 
     // Construct worker thread to feed responses back to client
 
-    ASSERT(workerThreads_.find(hdr.requestID) == workerThreads_.end());
+    ASSERT(workerThreads_.find(requestID) == workerThreads_.end());
 
     workerThreads_.emplace(
-        hdr.requestID, std::async(std::launch::async, [request, hdr, helper, this]() {
+        requestID, std::async(std::launch::async, [request, clientID, requestID, helper, this]() {
 
             try {
                 auto iterator = helper.apiCall(fdb_, request);
                 typename decltype(iterator)::value_type elem;
                 while (iterator.next(elem)) {
                     auto encoded(helper.encode(elem, *this));
-                    dataWrite(Message::Blob, hdr.clientID(), hdr.requestID, encoded.buf, encoded.position);
+                    write(Message::Blob, false, clientID, requestID, std::vector>{{encoded.buf, encoded.position}});
                 }
-                dataWrite(Message::Complete, hdr.clientID(), hdr.requestID);
+                write(Message::Complete, false, clientID, requestID);
             }
             catch (std::exception& e) {
                 // n.b. more general than eckit::Exception
-                std::string what(e.what());
-                dataWrite(Message::Error, hdr.clientID(), hdr.requestID, what.c_str(), what.length());
+                error(e.what(), clientID, requestID);
             }
             catch (...) {
                 // We really don't want to std::terminate the thread
-                std::string what("Caught unexpected, unknown exception in worker");
-                dataWrite(Message::Error, hdr.clientID(), hdr.requestID, what.c_str(), what.length());
+                error("Caught unexpected, unknown exception in worker", clientID, requestID);
             }
         }));
 }
 
-void CatalogueHandler::handle() {
-    initialiseConnections();
-
-    Log::info() << "CatalogueServer started ..." << std::endl;
-
-    MessageHeader hdr;
-    eckit::FixedString<4> tail;
-
-    // listen loop
-    while (true) {
-        tidyWorkers();
-
-        socketRead(&hdr, sizeof(hdr), controlSocket_);
-
-        ASSERT(hdr.marker == StartMarker);
-        ASSERT(hdr.version == CurrentVersion);
-        Log::debug() << "CatalogueHandler - got message " << hdr.message << " with request ID: " << hdr.requestID << std::endl;
-
-        bool ack = false;
-
-        try {
-            switch (hdr.message) {
-                case Message::Exit:
-                    Log::status() << "Exiting" << std::endl;
-                    Log::info() << "Exiting" << std::endl;
-                    return;
-
-                case Message::List:
-                    list(hdr);
-                    break;
-
-                case Message::Dump:
-                    NOTIMP;
-                    break;
-
-                case Message::Purge:
-                    NOTIMP;
-                    break;
-
-                case Message::Stats:
-                    NOTIMP;
-                    break;
-
-                case Message::Status:
-                    NOTIMP;
-                    break;
+void CatalogueHandler::list(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) {
+    forwardApiCall(clientID, requestID, std::move(payload));
+}
 
-                case Message::Wipe:
-                    NOTIMP;
-                    break;
+void CatalogueHandler::inspect(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) {
+    forwardApiCall(clientID, requestID, std::move(payload));
+}
 
-                case Message::Control:
-                    NOTIMP;
-                    break;
+void CatalogueHandler::schema(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) {
 
-                case Message::Inspect:
-                    inspect(hdr);
-                    break;
+    eckit::Buffer schemaBuffer(256*1024);
+    eckit::MemoryStream stream(schemaBuffer);
 
-                case Message::Read: 
-                    read(hdr);
-                    break;
+    if (payload.size() == 0) { // client requesting the top-level schema
+        stream << config_.schema();
+    } else {
+        // 1. Read dbkey to select catalogue
+        MemoryStream s(payload);
+        Key dbKey(s);
+    
+        // 2. Get catalogue
+        Catalogue& cat = catalogue(clientID, dbKey);
+        const Schema& schema = cat.schema();
+        stream << schema;
+    }
 
-                case Message::Flush:
-                    flush(hdr);
-                    break;
+    write(Message::Received, false, clientID, requestID, schemaBuffer.data(), stream.position());
+}
 
-                case Message::Archive:
-                    archive(hdr);
-                    break;
+void CatalogueHandler::stores(uint32_t clientID, uint32_t requestID) {
 
-                case Message::Stores:
-                    stores(hdr);
-                    ack = true;
-                    break;
+    eckit::net::IPAddress clientIPaddress{controlSocket_.remoteAddr()};
 
-                case Message::Schema:
-                    schema(hdr);
-                    ack = true;
+    std::string clientNetwork = "";
+    if (config_.has("networks")) {
+        for (const auto& net: config_.getSubConfigurations("networks")) {
+            if (net.has("name") && net.has("netmask")) {
+                eckit::net::NetMask netmask{net.getString("netmask")};
+                if (netmask.contains(clientIPaddress)) {
+                    clientNetwork = net.getString("name");
                     break;
-
-                default: {
-                    std::stringstream ss;
-                    ss << "ERROR: Unexpected message recieved (" << static_cast(hdr.message)
-                       << "). ABORTING";
-                    Log::status() << ss.str() << std::endl;
-                    Log::error() << "Catalogue Retrieving... " << ss.str() << std::endl;
-                    throw SeriousBug(ss.str(), Here());
                 }
             }
+        }
+    }
 
-            // Ensure we have consumed exactly the correct amount from the socket.
-            socketRead(&tail, sizeof(tail), controlSocket_);
-            ASSERT(tail == EndMarker);
+    Log::debug() << "Client " << clientIPaddress << " from network '" << clientNetwork << "'" << std::endl;
 
-            if (!ack) {
-                // Acknowledge receipt of command
-                dataWrite(Message::Received, hdr.clientID(), hdr.requestID);
+    ASSERT(config_.has("stores"));
+    std::map> stores;
+    // std::vector> stores;
+    for (const auto& configStore: config_.getSubConfigurations("stores")) {
+        ASSERT(configStore.has("default"));
+        eckit::net::Endpoint fieldLocationEndpoint{configStore.getString("default")};
+        eckit::net::Endpoint storeEndpoint{fieldLocationEndpoint};
+        if (!clientNetwork.empty()) {
+            if (configStore.has(clientNetwork)) {
+                storeEndpoint = eckit::net::Endpoint{configStore.getString(clientNetwork)};
             }
         }
-        catch (std::exception& e) {
-            // n.b. more general than eckit::Exception
-            std::string what(e.what());
-            dataWrite(Message::Error, hdr.clientID(), hdr.requestID, what.c_str(), what.length());
+
+        auto it = stores.find(fieldLocationEndpoint);
+        if (it == stores.end()) {
+            stores.emplace(fieldLocationEndpoint, std::vector{storeEndpoint});
+        } else {
+            it->second.push_back(storeEndpoint);
         }
-        catch (...) {
-            std::string what("Caught unexpected and unknown error");
-            dataWrite(Message::Error, hdr.clientID(), hdr.requestID, what.c_str(), what.length());
+
+        if (configStore.getBool("serveLocalData", false)) {
+            it = stores.find("");
+            if (it == stores.end()) {
+                stores.emplace("", std::vector{storeEndpoint});
+            } else {
+                it->second.push_back(storeEndpoint);
+            }
         }
     }
-}
-
-void CatalogueHandler::index(const MessageHeader& hdr) {
-    NOTIMP;
-}
 
-void CatalogueHandler::read(const MessageHeader& hdr) {
-    NOTIMP;
-}
-
-void CatalogueHandler::flush(const MessageHeader& hdr) {
-    Buffer payload(receivePayload(hdr, controlSocket_));
-    MemoryStream s(payload);
-
-    size_t numArchived;
-    s >> numArchived;
-
-    std::future& archive = archiveFuture_;
-
-    ASSERT(numArchived == 0 || archive.valid());
-
-    if (archive.valid()) {
-        // Ensure that the expected number of fields have been written, and that the
-        // archive worker processes have been cleanly wound up.
-        size_t n = archive.get();
-        ASSERT(numArchived == n);
+    {
+        Buffer startupBuffer(16*1024);
+        MemoryStream s(startupBuffer);
 
-        // Do the actual flush!
-        Log::info() << "Flushing" << std::endl;
-        Log::status() << "Flushing" << std::endl;
-        catalogues_[hdr.clientID()]->flush();
-        // for (auto it = catalogues_.begin(); it != catalogues_.end(); it++) {
-        //     it->second->flush();
-        // }
+        s << stores.size();
+        for (const auto& store : stores) {
+            s << store.first << store.second.size();
+            for (const auto& ee: store.second) {
+                s << ee;
+            }
+        }
+        write(Message::Received, false, clientID, requestID, startupBuffer.data(), s.position());
     }
-    Log::info() << "Flush complete" << std::endl;
-    Log::status() << "Flush complete" << std::endl;
 }
 
-size_t CatalogueHandler::archiveThreadLoop() {
-    size_t totalArchived = 0;
-
-    // Create a worker that will do the actual archiving
 
-    static size_t queueSize(eckit::Resource("fdbServerMaxQueueSize", 32));
-    eckit::Queue> queue(queueSize);
 
 
-    std::future worker = std::async(std::launch::async, [this, &queue] {
-        size_t totalArchived = 0;
-        std::pair elem = std::make_pair(0, Buffer{0});
 
-        try {
-            long queuelen;
-            while ((queuelen = queue.pop(elem)) != -1) {
 
-                uint32_t clientID = elem.first;
-                // Key key = elem.second.first;
-                eckit::Buffer payload = std::move(elem.second);
-                MemoryStream s(payload);
-                Key idxKey(s);
-                InspectionKey key(s);
-                std::unique_ptr location(eckit::Reanimator::reanimate(s));
-                std::cout << "ARCHIVING location [clientID=" << clientID << ",key=" << idxKey <<  key << ",location=" << *location << std::endl;
 
-                CatalogueWriter& cat = catalogue(clientID);
-                Log::debug() << "CatalogueHandler::archiveThreadLoop message has clientID: " << clientID << " key: " << cat.key()
-                    << idxKey << key << " and location.uri" << location->uri() << std::endl;
-        
-                cat.selectIndex(idxKey);
-                cat.archive(key, std::move(location));
-                totalArchived += 1;
-            }
-        }
-        catch (...) {
-            // Ensure exception propagates across the queue back to the parent thread.
-            queue.interrupt(std::current_exception());
-            throw;
-        }
+void CatalogueHandler::flush(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) {
+    
+    size_t numArchived = 0;
+    
+    if (payload.size() > 0) {
+        MemoryStream s(payload);
+        s >> numArchived;
+    }
 
-        return totalArchived;
-    });
+    auto it = catalogues_.find(clientID);
+    ASSERT(it != catalogues_.end());
 
-    try {
-        // The archive loop is the only thing that can listen on the data socket,
-        // so we don't need to to anything clever here.
+    it->second.locationsExpected = numArchived;
+    if (it->second.locationsArchived < numArchived) {
+        it->second.archivalCompleted.get();
+    }
 
-        // n.b. we also don't need to lock on read. We are the only thing that reads.
+    it->second.catalogue->flush();
 
-        while (true) {
-            MessageHeader hdr;
-            Log::debug() << "CatalogueHandler::archiveThreadLoop awaiting message from client..." << std::endl;
-            socketRead(&hdr, sizeof(hdr), dataSocket_);
-            Log::debug() << "CatalogueHandler::archiveThreadLoop got a message from client!" << std::endl;
+    Log::info() << "Flush complete" << std::endl;
+    Log::status() << "Flush complete" << std::endl;
+}
 
-            ASSERT(hdr.marker == StartMarker);
-            ASSERT(hdr.version == CurrentVersion);
-            //ASSERT(hdr.requestID == id);
+// A helper function to make archiveThreadLoop a bit cleaner
+void CatalogueHandler::archiveBlob(const uint32_t clientID, const uint32_t requestID, const void* data, size_t length) {
 
-            // Have we been told that we are done yet?
-            if (hdr.message == Message::Flush)
-                break;
+    MemoryStream s(data, length);
 
-            ASSERT(hdr.message == Message::Blob); // no multiblobs for catalogue
+    // fdb5::Key dbKey(s);
+    fdb5::Key idxKey(s);
+    fdb5::InspectionKey key;
+    s >> key; // xxx no stream constructor for inspection key?
 
-            Buffer payload(receivePayload(hdr, dataSocket_));
+    std::unique_ptr location(eckit::Reanimator::reanimate(s));
 
-            eckit::FixedString<4> tail;
-            socketRead(&tail, sizeof(tail), dataSocket_);
-            ASSERT(tail == EndMarker);
+    Log::debug() << "CatalogueHandler::archiveBlob key: " << idxKey << key << "  location: " << location->uri() << std::endl;
 
-            size_t queuelen = queue.emplace(hdr.clientID(), std::move(payload));
+    std::map::iterator it;
+    {
+        std::lock_guard lock(cataloguesMutex_);
+        it = catalogues_.find(clientID);
+        if (it == catalogues_.end()) {
+            std::string what("Requested unknown catalogue id: " + std::to_string(clientID));
+            error(what, 0, 0);
+            throw;
         }
-
-        // Trigger cleanup of the workers
-        queue.close();
-
-        // Complete reading the Flush instruction
-
-        eckit::FixedString<4> tail;
-        socketRead(&tail, sizeof(tail), dataSocket_);
-        ASSERT(tail == EndMarker);
-
-        // Ensure worker is done
-
-        ASSERT(worker.valid());
-        totalArchived = worker.get();  // n.b. use of async, get() propagates any exceptions.
     }
-    catch (std::exception& e) {
-        // n.b. more general than eckit::Exception
-        std::string what(e.what());
-        dataWrite(Message::Error, 0, 0, what.c_str(), what.length());
-        queue.interrupt(std::current_exception());
-        throw;
-    }
-    catch (...) {
-        std::string what("Caught unexpected, unknown exception in retrieve worker");
-        dataWrite(Message::Error, 0, 0, what.c_str(), what.length());
-        queue.interrupt(std::current_exception());
-        throw;
-    }
-
-    return totalArchived;
-}
 
-void CatalogueHandler::list(const MessageHeader& hdr) {
-    forwardApiCall(hdr);
-}
-
-void CatalogueHandler::inspect(const MessageHeader& hdr) {
-    forwardApiCall(hdr);
+    it->second.catalogue->selectIndex(idxKey);
+    it->second.catalogue->archive(key, std::move(location));
+    it->second.locationsArchived++;
+    if (it->second.locationsExpected > 0 && it->second.locationsExpected == it->second.locationsArchived) {
+        it->second.fieldLocationsReceived.set_value(it->second.locationsExpected);
+    }
 }
 
-void CatalogueHandler::schema(const MessageHeader& hdr) {
-
-    eckit::Buffer schemaBuffer(1024*1024);
-    eckit::MemoryStream stream(schemaBuffer);
-
-    if (hdr.payloadSize == 0) { // client requesting the top-level schema
-        stream << config_.schema();
-    } else {
-        // 1. Read dbkey to select catalogue
-        Buffer payload(receivePayload(hdr, controlSocket_));
-        MemoryStream s(payload);
-        Key dbKey(s);
+bool CatalogueHandler::remove(uint32_t clientID) {
     
-        // 2. Get catalogue
-        Catalogue& cat = catalogue(hdr.clientID(), dbKey);
-        const Schema& schema = cat.schema();
-        stream << schema;
-    }
-
-    dataWrite(Message::Received, hdr.clientID(), hdr.requestID, schemaBuffer.data(), stream.position());
-}
+    std::stringstream ss;
+    ss << "remove " << clientID << " from " << catalogues_.size() << " clients - ids:";
+    for (auto it = catalogues_.begin(); it != catalogues_.end(); it++)
+        ss << "  " << it->first;
+    ss << std::endl;
+    eckit::Log::info() << ss.str();
 
-CatalogueWriter& CatalogueHandler::catalogue(uint32_t id) {
     std::lock_guard lock(cataloguesMutex_);
-    auto it = catalogues_.find(id);
-    if (it == catalogues_.end()) {
-        std::string what("Requested unknown catalogue id: " + std::to_string(id));
-        dataWrite(Message::Error, 0, 0, what.c_str(), what.length());
-        throw;
+
+    auto it = catalogues_.find(clientID);
+    if (it != catalogues_.end()) {
+        catalogues_.erase(it);
     }
 
-    return *(it->second);
+    return catalogues_.empty();
 }
 
+
 CatalogueWriter& CatalogueHandler::catalogue(uint32_t id, const Key& dbKey) {
     std::lock_guard lock(cataloguesMutex_);
-    return *(catalogues_[id] = CatalogueWriterFactory::instance().build(dbKey, config_));
+    return *((catalogues_.emplace(id, CatalogueArchiver(dbKey, config_)).first)->second.catalogue);
 }
 
 }  // namespace fdb5::remote
diff --git a/src/fdb5/remote/server/CatalogueHandler.h b/src/fdb5/remote/server/CatalogueHandler.h
index 5ad086399..1af1f6298 100644
--- a/src/fdb5/remote/server/CatalogueHandler.h
+++ b/src/fdb5/remote/server/CatalogueHandler.h
@@ -15,11 +15,21 @@
 
 namespace fdb5::remote {
 
-// class StoreEndpoint : public eckit::net::Endpoint {
-// public:
-//     StoreEndpoint(const std::string& endpoint) : eckit::net::Endpoint(endpoint) {}
-//     std::map aliases_;
-// };
+//----------------------------------------------------------------------------------------------------------------------
+
+struct CatalogueArchiver {
+    CatalogueArchiver(const Key& dbKey, const Config& config) :
+        catalogue(CatalogueWriterFactory::instance().build(dbKey, config)), locationsExpected(-1), locationsArchived(0) {
+        archivalCompleted = fieldLocationsReceived.get_future();
+    }
+
+    std::unique_ptr catalogue;
+    int32_t locationsExpected;
+    int32_t locationsArchived;
+
+    std::promise fieldLocationsReceived;
+    std::future archivalCompleted;
+};
 
 //----------------------------------------------------------------------------------------------------------------------
 class CatalogueHandler : public ServerConnection {
@@ -28,36 +38,35 @@ class CatalogueHandler : public ServerConnection {
     CatalogueHandler(eckit::net::TCPSocket& socket, const Config& config);
     ~CatalogueHandler();
 
-    void handle() override;
-
 private:  // methods
 
-    void initialiseConnections() override;
-    void index(const MessageHeader& hdr);
+    Handled handleControl(Message message, uint32_t clientID, uint32_t requestID) override;
+    Handled handleControl(Message message, uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) override;
+    // Handled handleData(Message message, uint32_t clientID, uint32_t requestID) override;
+    // Handled handleData(Message message, uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) override;
 
-    void read(const MessageHeader& hdr);
-    void flush(const MessageHeader& hdr);
-    void list(const MessageHeader& hdr);
-    void inspect(const MessageHeader& hdr);
-    void schema(const MessageHeader& hdr);
-    void stores(const MessageHeader& hdr);
-    
+    // API functionality
+    template 
+    void forwardApiCall(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload);
 
-    CatalogueWriter& catalogue(uint32_t id);
-    CatalogueWriter& catalogue(uint32_t id, const Key& dbKey);
+    void flush(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload);
+    void list(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload);
+    void inspect(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload);
+    void schema(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload);
+    void stores(uint32_t clientID, uint32_t requestID);
 
-    size_t archiveThreadLoop() override;
+    void archiveBlob(const uint32_t clientID, const uint32_t requestID, const void* data, size_t length) override;
 
-    // API functionality
-    template 
-    void forwardApiCall(const MessageHeader& hdr);
+    bool remove(uint32_t clientID) override;
+
+    // CatalogueWriter& catalogue(uint32_t catalogueID);
+    CatalogueWriter& catalogue(uint32_t catalogueID, const Key& dbKey);
 
 private:  // member
 
-    // clientID --> Catalogue
     std::mutex cataloguesMutex_;
-
-    std::map> catalogues_;
+    // clientID --> 
+    std::map catalogues_;
 
     FDB fdb_;
 };
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index 0bd5c0e1a..01fbeafb7 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -21,6 +21,7 @@
 #include "eckit/runtime/Main.h"
 #include "eckit/runtime/SessionID.h"
 #include "eckit/serialisation/MemoryStream.h"
+#include "eckit/log/Log.h"
 
 #include "fdb5/LibFdb5.h"
 #include "fdb5/fdb5_version.h"
@@ -61,49 +62,122 @@ std::vector intersection(const eckit::LocalConfiguration& c1, const eckit::
 
 } // namespace
 
+
+//size_t ServerConnection::queueSize_ = eckit::Resource("fdbServerMaxQueueSize", 32);
+
 ServerConnection::ServerConnection(eckit::net::TCPSocket& socket, const Config& config) :
-        config_(config),
-        controlSocket_(socket),
-        dataSocket_(selectDataPort()),
+        Connection(), config_(config),
         dataListenHostname_(config.getString("dataListenHostname", "")),
-        readLocationQueue_(eckit::Resource("fdbRetrieveQueueSize", 10000)) {
-            eckit::Log::debug() << "ServerConnection::ServerConnection initialized" << std::endl;
-    }
+        readLocationQueue_(eckit::Resource("fdbRetrieveQueueSize", 10000)), 
+        archiveQueue_(eckit::Resource("fdbServerMaxQueueSize", 32)),
+        controlSocket_(socket), dataSocket_(nullptr), dataListener_(0) {
+
+    eckit::Log::debug() << "ServerConnection::ServerConnection initialized" << std::endl;
+}
 
 ServerConnection::~ServerConnection() {
     // We don't want to die before the worker threads are cleaned up
     waitForWorkers();
 
     // And notify the client that we are done.
-    eckit::Log::info() << "Sending exit message to client" << std::endl;
-    dataWrite(Message::Exit, 0, 0);
+//     eckit::Log::info() << "Sending exit message to client" << std::endl;
+// //    write(Message::Exit, true, 0, 0);
+//     write(Message::Exit, false, 0, 0);
     eckit::Log::info() << "Done" << std::endl;
 }
 
+
+Handled ServerConnection::handleData(Message message, uint32_t clientID, uint32_t requestID) {
+    try {
+        switch (message) {
+            case Message::Flush: // notification that the client has sent all data for archival
+                return Handled::YesRemoveArchiveListener; // ????
+
+            default: {
+                std::stringstream ss;
+                ss << "ERROR: Unexpected message recieved (" << message << "). ABORTING";
+                eckit::Log::status() << ss.str() << std::endl;
+                eckit::Log::error() << ss.str() << std::endl;
+                throw eckit::SeriousBug(ss.str(), Here());
+            }
+        }
+    }
+    catch (std::exception& e) {
+        // n.b. more general than eckit::Exception
+        error(e.what(), clientID, requestID);
+    }
+    catch (...) {
+        error("Caught unexpected and unknown error", clientID, requestID);
+    }
+    return Handled::No;
+}
+
+Handled ServerConnection::handleData(Message message, uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) {
+    try {
+        switch (message) {
+            case Message::Blob:
+            case Message::MultiBlob:
+                queue(message, clientID, requestID, std::move(payload));
+                return Handled::Yes;
+
+            default: {
+                std::stringstream ss;
+                ss << "ERROR: Unexpected message recieved (" << message << "). ABORTING";
+                eckit::Log::status() << ss.str() << std::endl;
+                eckit::Log::error() << ss.str() << std::endl;
+                throw eckit::SeriousBug(ss.str(), Here());
+            }
+        }
+    }
+    catch (std::exception& e) {
+        // n.b. more general than eckit::Exception
+        error(e.what(), clientID, requestID);
+    }
+    catch (...) {
+        error("Caught unexpected and unknown error", clientID, requestID);
+    }
+    return Handled::No;
+}
+
+// ServerConnection::ServerConnection(eckit::net::TCPSocket& socket, const Config& config) :
+//         config_(config),
+//         controlSocket_(socket),
+//         dataSocket_(selectDataPort()),
+//         dataListenHostname_(config.getString("dataListenHostname", "")),
+//         readLocationQueue_(eckit::Resource("fdbRetrieveQueueSize", 10000)) {
+//             eckit::Log::debug() << "ServerConnection::ServerConnection initialized" << std::endl;
+//     }
+
+// ServerConnection::~ServerConnection() {
+//     // We don't want to die before the worker threads are cleaned up
+//     waitForWorkers();
+
+//     // And notify the client that we are done.
+//     eckit::Log::info() << "Sending exit message to client" << std::endl;
+//     dataWrite(Message::Exit, 0, 0);
+//     eckit::Log::info() << "Done" << std::endl;
+// }
+
 eckit::LocalConfiguration ServerConnection::availableFunctionality() const {
     eckit::LocalConfiguration conf;
 //    Add to the configuration all the components that require to be versioned, as in the following example, with a vector of supported version numbers
     std::vector remoteFieldLocationVersions = {1};
     conf.set("RemoteFieldLocation", remoteFieldLocationVersions);
+    std::vector numberOfConnections = {2};
+    conf.set("NumberOfConnections", numberOfConnections);
     return conf;
 }
 
+
 void ServerConnection::initialiseConnections() {
     // Read the startup message from the client. Check that it all checks out.
 
     MessageHeader hdr;
-    socketRead(&hdr, sizeof(hdr), controlSocket_);
+    eckit::Buffer payload1 = readControl(hdr);
 
-    ASSERT(hdr.marker == StartMarker);
-    ASSERT(hdr.version == CurrentVersion);
     ASSERT(hdr.message == Message::Startup);
     ASSERT(hdr.requestID == 0);
 
-    eckit::Buffer payload1 = receivePayload(hdr, controlSocket_);
-    eckit::FixedString<4> tail;
-    socketRead(&tail, sizeof(tail), controlSocket_);
-    ASSERT(tail == EndMarker);
-
     eckit::MemoryStream s1(payload1);
     eckit::SessionID clientSession(s1);
     eckit::net::Endpoint endpointFromClient(s1);
@@ -128,12 +202,40 @@ void ServerConnection::initialiseConnections() {
         eckit::LocalConfiguration clientAvailableFunctionality(s1);
         eckit::LocalConfiguration serverConf = availableFunctionality();
         agreedConf_ = eckit::LocalConfiguration();
+        bool compatibleProtocol = true;
 
-        // agree on a common functionality by intersecting server and client version numbers
+        
         std::vector rflCommon = intersection(clientAvailableFunctionality, serverConf, "RemoteFieldLocation");
         if (rflCommon.size() > 0) {
             eckit::Log::debug() << "Protocol negotiation - RemoteFieldLocation version " << rflCommon.back() << std::endl;
             agreedConf_.set("RemoteFieldLocation", rflCommon.back());
+        } else {
+            compatibleProtocol = false;
+        }
+
+        if (!clientAvailableFunctionality.has("NumberOfConnections")) { // set the default
+            std::vector conn = {2};
+            clientAvailableFunctionality.set("NumberOfConnections", conn);
+        }
+        // agree on a common functionality by intersecting server and client version numbers
+        std::vector ncCommon = intersection(clientAvailableFunctionality, serverConf, "NumberOfConnections");
+        if (ncCommon.size() > 0) {
+            int ncSelected = 2;
+
+            if (ncCommon.size() == 1) {
+                ncSelected = ncCommon.at(0);
+            } else {
+                ncSelected = ncCommon.back();
+                if (clientAvailableFunctionality.has("PreferSingleConnection")) {
+                    if ( std::find(ncCommon.begin(), ncCommon.end(), (clientAvailableFunctionality.getBool("PreferSingleConnection") ? 1 : 2)) != ncCommon.end() ) {
+                        ncSelected = (clientAvailableFunctionality.getBool("PreferSingleConnection") ? 1 : 2);
+                    }
+                }
+            }
+
+            eckit::Log::debug() << "Protocol negotiation - NumberOfConnections " << ncSelected << std::endl;
+            agreedConf_.set("NumberOfConnections", ncSelected);
+            single_ = (ncSelected == 1);
         }
         else {
             std::stringstream ss;
@@ -150,8 +252,13 @@ void ServerConnection::initialiseConnections() {
     //               server has multiple, then we use that on, whilst retaining
     //               the capacity in the protocol for the server to make a choice.
 
-    int dataport = dataSocket_.localPort();
-    eckit::net::Endpoint dataEndpoint(endpointFromClient.hostname(), dataport);
+    eckit::net::Endpoint dataEndpoint;
+    if (single_) {
+        dataEndpoint = endpointFromClient;
+    } else {
+        dataSocket_.reset(new eckit::net::EphemeralTCPServer(selectDataPort()));
+        dataEndpoint = eckit::net::Endpoint{endpointFromClient.hostname(), dataSocket_->localPort()};
+    }
 
     eckit::Log::info() << "Sending data endpoint to client: " << dataEndpoint << std::endl;
     {
@@ -166,70 +273,48 @@ void ServerConnection::initialiseConnections() {
 
         eckit::Log::debug() << "Protocol negotiation - configuration: " << agreedConf_ <>{{startupBuffer.data(), s.position()}});
     }
 
 
     if (!errorMsg.empty()) {
-        controlWrite(Message::Error, 0, 0, errorMsg.c_str(), errorMsg.length());
+        error(errorMsg, hdr.clientID(), hdr.requestID);
         return;
     }
 
-    dataSocket_.accept();
+    if (!single_) {
+        dataSocket_->accept();
 
-    // Check the response from the client.
-    // Ensure that the hostname matches the original hostname, and that
-    // it returns the details we sent it
-    // IE check that we are connected to the correct client!
+        // Check the response from the client.
+        // Ensure that the hostname matches the original hostname, and that
+        // it returns the details we sent it
+        // IE check that we are connected to the correct client!
 
-    MessageHeader dataHdr;
-    socketRead(&dataHdr, sizeof(dataHdr), dataSocket_);
+        MessageHeader dataHdr;
+        eckit::Buffer payload2 = readData(dataHdr);
 
-    ASSERT(dataHdr.marker == StartMarker);
-    ASSERT(dataHdr.version == CurrentVersion);
-    ASSERT(dataHdr.message == Message::Startup);
-    ASSERT(dataHdr.requestID == 0);
+        ASSERT(dataHdr.version == CurrentVersion);
+        ASSERT(dataHdr.message == Message::Startup);
+        ASSERT(dataHdr.requestID == 0);
 
-    eckit::Buffer payload2 = receivePayload(dataHdr, dataSocket_);
-    socketRead(&tail, sizeof(tail), dataSocket_);
-    ASSERT(tail == EndMarker);
+        eckit::MemoryStream s2(payload2);
+        eckit::SessionID clientSession2(s2);
+        eckit::SessionID serverSession(s2);
 
-    eckit::MemoryStream s2(payload2);
-    eckit::SessionID clientSession2(s2);
-    eckit::SessionID serverSession(s2);
-
-    if (clientSession != clientSession2) {
-        std::stringstream ss;
-        ss << "Client session IDs do not match: " << clientSession << " != " << clientSession2;
-        throw eckit::BadValue(ss.str(), Here());
-    }
+        if (clientSession != clientSession2) {
+            std::stringstream ss;
+            ss << "Client session IDs do not match: " << clientSession << " != " << clientSession2;
+            throw eckit::BadValue(ss.str(), Here());
+        }
 
-    if (serverSession != sessionID_) {
-        std::stringstream ss;
-        ss << "Session IDs do not match: " << serverSession << " != " << sessionID_;
-        throw eckit::BadValue(ss.str(), Here());
+        if (serverSession != sessionID_) {
+            std::stringstream ss;
+            ss << "Session IDs do not match: " << serverSession << " != " << sessionID_;
+            throw eckit::BadValue(ss.str(), Here());
+        }
     }
 }
 
-// Handler& ServerConnection::handler(uint32_t id) {
-//     auto it = handlers_.find(id);
-//     ASSERT(it != handlers_.end());
-
-//     return *it->second;
-// }
-
-// Handler& ServerConnection::handler(uint32_t id, eckit::Buffer payload) {
-
-//     ASSERT(handlers_.find(id) == handlers_.end());
-
-
-
-//     auto it = handlers_.find(id);
-//     ASSERT(it != handlers_.end());
-
-//     return *it->second;
-// }
-
 int ServerConnection::selectDataPort() {
     eckit::Log::info() << "SelectDataPort: " << std::endl;
     eckit::Log::info() << config_ << std::endl;
@@ -243,80 +328,282 @@ int ServerConnection::selectDataPort() {
     return 0;
 }
 
-void ServerConnection::controlWrite(Message msg, uint32_t clientID, uint32_t requestID, const void* payload, uint32_t payloadLength) {
-    ASSERT((payload == nullptr) == (payloadLength == 0));
-
-    MessageHeader message(msg, true, clientID, requestID, payloadLength);
-    std::lock_guard lock(controlWriteMutex_);
-    controlWrite(&message, sizeof(message));
-    if (payload) {
-        controlWrite(payload, payloadLength);
-    }
-    controlWrite(&EndMarker, sizeof(EndMarker));
-}
+size_t ServerConnection::archiveThreadLoop() {
+    size_t totalArchived = 0;
 
-void ServerConnection::controlWrite(const void* data, size_t length) {
-    size_t written = controlSocket_.write(data, length);
-    if (length != written) {
-        std::stringstream ss;
-        ss << "Write error. Expected " << length << " bytes, wrote " << written;
-        throw TCPException(ss.str(), Here());
-    }
-}
+    ArchiveElem elem;
 
-void ServerConnection::socketRead(void* data, size_t length, eckit::net::TCPSocket& socket) {
+    try {
+        long queuelen;
+        while ((queuelen = archiveQueue_.pop(elem)) != -1) {
+            if (elem.multiblob_) {
+                // Handle MultiBlob
+
+                const char* firstData = static_cast(elem.payload_.data());  // For pointer arithmetic
+                const char* charData = firstData;
+                while (size_t(charData - firstData) < elem.payload_.size()) {
+                    const MessageHeader* hdr = static_cast(static_cast(charData));
+                    ASSERT(hdr->message == Message::Blob);
+                    ASSERT(hdr->clientID() == elem.clientID_);
+                    ASSERT(hdr->requestID == elem.requestID_);
+                    charData += sizeof(MessageHeader);
+
+                    const void* payloadData = charData;
+                    charData += hdr->payloadSize;
+
+                    const decltype(EndMarker)* e = static_cast(static_cast(charData));
+                    ASSERT(*e == EndMarker);
+                    charData += sizeof(EndMarker);
+
+                    archiveBlob(elem.clientID_, elem.requestID_, payloadData, hdr->payloadSize);
+                    totalArchived += 1;
+                }
+            }
+            else {
+                // Handle single blob
+                archiveBlob(elem.clientID_, elem.requestID_, elem.payload_.data(), elem.payload_.size());
+                totalArchived += 1;
+            }
+        }
 
-    size_t read = socket.read(data, length);
-    if (length != read) {
-        std::stringstream ss;
-        ss << "Read error. Expected " << length << " bytes, read " << read;
-        throw TCPException(ss.str(), Here());
+        // eckit::Buffer buffer(1024);
+        // eckit::MemoryStream stream(buffer);
+        // stream << totalArchived;
+        
+        // write(Message::Complete, false, 0, 0, buffer, stream.position());
+    }
+    catch (...) {
+        // Ensure exception propagates across the queue back to the parent thread.
+        archiveQueue_.interrupt(std::current_exception());
+        throw;
     }
+
+    return totalArchived;
 }
 
-void ServerConnection::archive(const MessageHeader& hdr) {
 
-    ASSERT(hdr.payloadSize == 0);
 
-    // Ensure that we aren't already running a catalogue/store
-    if (!archiveFuture_.valid()) {
-        // Start archive worker thread
-        archiveFuture_ = std::async(std::launch::async, [this] { return archiveThreadLoop(); });
-    }
-}
+void ServerConnection::listeningThreadLoopData() {
 
-void ServerConnection::dataWrite(Message msg, uint32_t clientID, uint32_t requestID, const void* payload, uint32_t payloadLength) {
+    MessageHeader hdr;
+    uint32_t archiverID;
+
+    try {
 
-    eckit::Log::debug() << "ServerConnection::dataWrite [message="<< static_cast(msg) << ",requestID=" << requestID << ",payloadLength=" << payloadLength << "]" << std::endl;
+        // The archive loop is the only thing that can listen on the data socket,
+        // so we don't need to to anything clever here.
+
+        // n.b. we also don't need to lock on read. We are the only thing that reads.
+        while (true) {
+            eckit::Buffer payload = readData(hdr); // READ DATA
+            // eckit::Log::debug() << "ServerConnection::listeningThreadLoopData - got [message=" << hdr.message << ",clientID="<< hdr.clientID() << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
+
+        if (hdr.message == Message::Exit) {
+            if (remove(hdr.clientID())) {
+                eckit::Log::status() << "Exiting" << std::endl;
+                eckit::Log::info() << "Exiting" << std::endl;
+
+                return;
+            }
+        } else {
+            
+            Handled handled;
+            if (payload.size() == 0) {
+                handled = handleData(hdr.message, hdr.clientID(), hdr.requestID);
+            } else {
+                handled = handleData(hdr.message, hdr.clientID(), hdr.requestID, std::move(payload));
+            }
+
+
+        switch (handled)
+        {
+            case Handled::Replied: // nothing to do
+                break;
+            case Handled::YesRemoveArchiveListener:
+                dataListener_--;
+                if (dataListener_ == 0) {
+                    return;
+                    // listeningThreadData.join();
+                }
+                break;
+            case Handled::YesRemoveReadListener:
+                dataListener_--;
+                if (dataListener_ == 0) {
+                    return;
+                    // listeningThreadData.join();
+                }
+                break;
+            case Handled::Yes:
+//                write(Message::Received, false, hdr.clientID(), hdr.requestID);
+                break;
+            case Handled::No:
+            default:
+                std::stringstream ss;
+                ss << "Unable to handle message " << hdr.message << " received on the data connection";
+                error(ss.str(), hdr.clientID(), hdr.requestID);
+        }
+        }
 
-    ASSERT((payload == nullptr) == (payloadLength == 0));
+            // std::cout << "listeningThreadLoopData " << hdr.message << " " << hdr.clientID() << " " << hdr.requestID << " " << ((int) handled) << std::endl;
+            // switch (handled)
+            // {
+            //     case Handled::YesRemoveArchiveListener:
+            //         dataListener_--;
+            //         if (dataListener_ == 0) {
+            //             // queue.close();
+            //             return;
+            //         }
+            //         break;
+            //     case Handled::Yes:
+            //         break;
+            //     case Handled::Replied:
+            //     case Handled::No:
+            //     default:
+            //         std::stringstream ss;
+            //         ss << "Unable to handle message " << hdr.message << " from data connection";
+            //         error(ss.str(), hdr.clientID(), hdr.requestID);
+            // }
+        }
 
-    MessageHeader message(msg, false, clientID, requestID, payloadLength);
+        // // Trigger cleanup of the workers
+        // auto q = archiveQueues_.find(archiverID);
+        // ASSERT(q != archiveQueues_.end());
+        // q->second.close();
 
-    std::lock_guard lock(dataWriteMutex_);
-    dataWriteUnsafe(&message, sizeof(message));
-    if (payload) {
-        dataWriteUnsafe(payload, payloadLength);
+        // auto w = archiveFuture_.find(archiverID);
+        // ASSERT(w != archiveFuture_.end());
+        // // Ensure worker is done
+        // ASSERT(w->second.valid());
+        // totalArchived = worker.get();  // n.b. use of async, get() propagates any exceptions.
+    }
+    catch (std::exception& e) {
+        // n.b. more general than eckit::Exception
+        error(e.what(), hdr.clientID(), hdr.requestID);
+        // auto q = archiveQueues_.find(archiverID);
+        // if(q != archiveQueues_.end()) {
+        //     q->second.interrupt(std::current_exception());
+        // }
+        throw;
+    }
+    catch (...) {
+        error("Caught unexpected, unknown exception in retrieve worker", hdr.clientID(), hdr.requestID);
+        // auto q = archiveQueues_.find(archiverID);
+        // if(q != archiveQueues_.end()) {
+        //     q->second.interrupt(std::current_exception());
+        // }
+        throw;
     }
-    dataWriteUnsafe(&EndMarker, sizeof(EndMarker));
 }
 
-void ServerConnection::dataWriteUnsafe(const void* data, size_t length) {
-    size_t written = dataSocket_.write(data, length);
-    if (length != written) {
-        std::stringstream ss;
-        ss << "Write error. Expected " << length << " bytes, wrote " << written;
-        throw TCPException(ss.str(), Here());
+void ServerConnection::handle() {
+    initialiseConnections();
+ 
+    std::thread listeningThreadData;
+    // if (!single_) {
+    //     listeningThreadData = std::thread([this] { listeningThreadLoopData(); });
+    // }
+
+    MessageHeader hdr;
+
+    // listen loop
+    while (true) {
+        //tidyWorkers();
+
+        eckit::Buffer payload = readControl(hdr); // READ CONTROL
+        eckit::Log::debug() << "ServerConnection::handle - got [message=" << hdr.message << ",clientID="<< hdr.clientID() << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
+
+        if (hdr.message == Message::Exit) {
+            // write(Message::Exit, true, hdr.clientID(), 0);
+            eckit::Log::info() << "Exit " << controlSocket().localPort() << "  client " <<  hdr.clientID() << std::endl;
+            write(Message::Exit, false, hdr.clientID(), 0);
+
+            if (remove(hdr.clientID())) {
+                eckit::Log::status() << "Exiting" << std::endl;
+                eckit::Log::info() << "Exiting" << std::endl;
+
+                return;
+            }
+        } else {
+
+            Handled handled = Handled::No;
+            ASSERT(single_ || hdr.control());
+
+            if (payload.size()) {
+                if (hdr.control()) {
+                    handled = handleControl(hdr.message, hdr.clientID(), hdr.requestID, std::move(payload));
+                } else {
+                    handled = handleData(hdr.message, hdr.clientID(), hdr.requestID, std::move(payload));
+                }
+            } else {
+                if (hdr.control()) {
+                    handled = handleControl(hdr.message, hdr.clientID(), hdr.requestID);
+                } else {
+                    handled = handleData(hdr.message, hdr.clientID(), hdr.requestID);
+                }
+            }
+            
+
+            switch (handled)
+            {
+                case Handled::Replied: // nothing to do
+                    break;
+                // case Handled::YesRemoveArchiveListener:
+                //     dataListener_--;
+                //     if (dataListener_ == 0) {
+                //         //return;
+                //         // listeningThreadData.join();
+                //     }
+                //     break;
+                case Handled::YesAddArchiveListener:
+                    dataListener_++;
+                    if (dataListener_ == 1) {
+                        listeningThreadData = std::thread([this] { listeningThreadLoopData(); });
+                    }
+                    write(Message::Received, false, hdr.clientID(), hdr.requestID);
+                    break;
+                // case Handled::YesRemoveReadListener:
+                //     dataListener_--;
+                //     if (dataListener_ == 0) {
+                //         //return;
+                //         // listeningThreadData.join();
+                //     }
+                //     break;
+                case Handled::YesAddReadListener:
+                    dataListener_++;
+                    if (dataListener_ == 1) {
+                        listeningThreadData = std::thread([this] { listeningThreadLoopData(); });
+                    }
+                    write(Message::Received, false, hdr.clientID(), hdr.requestID);
+                    break;
+                case Handled::Yes:
+                    write(Message::Received, false, hdr.clientID(), hdr.requestID);
+                    break;
+                case Handled::No:
+                default:
+                    std::stringstream ss;
+                    ss << "Unable to handle message " << hdr.message;
+                    error(ss.str(), hdr.clientID(), hdr.requestID);
+            }
+        }
     }
 }
 
-eckit::Buffer ServerConnection::receivePayload(const MessageHeader& hdr, eckit::net::TCPSocket& socket) {
-    eckit::Buffer payload(hdr.payloadSize);
+void ServerConnection::handleException(std::exception_ptr e) {
+    try
+    {
+        if (e)
+            std::rethrow_exception(e);
+    }
+    catch(const std::exception& e)
+    {
+        error(e.what(), 0, 0);
+    }
+}
 
-    ASSERT(hdr.payloadSize > 0);
-    socketRead(payload, hdr.payloadSize, socket);
+void ServerConnection::queue(Message message, uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) {
 
-    return payload;
+    archiveQueue_.emplace(
+        ArchiveElem{clientID, requestID, std::move(payload), message == Message::MultiBlob});
 }
 
 void ServerConnection::tidyWorkers() {
@@ -335,6 +622,21 @@ void ServerConnection::tidyWorkers() {
     }
 }
 
+
+void ServerConnection::archiver() {
+
+    // Ensure that we aren't already running a catalogue/store
+    if (!archiveFuture_.valid()) {
+        // Start archive worker thread
+        archiveFuture_ = std::async(std::launch::async, [this] { return archiveThreadLoop(); });
+    }
+
+    // // Start data reader thread if double connection and we aren't already running it
+    // if (!single_ &&  !dataReader_.valid()) {
+    //     dataReader_ = std::async(std::launch::async, [this] { return listeningThreadLoopData(); });
+    // }
+}
+
 void ServerConnection::waitForWorkers() {
     readLocationQueue_.close();
 
diff --git a/src/fdb5/remote/server/ServerConnection.h b/src/fdb5/remote/server/ServerConnection.h
index 1f963ff45..5338f9c47 100644
--- a/src/fdb5/remote/server/ServerConnection.h
+++ b/src/fdb5/remote/server/ServerConnection.h
@@ -29,11 +29,33 @@
 
 #include "fdb5/config/Config.h"
 #include "fdb5/remote/Messages.h"
-// #include "fdb5/remote/server/Handler.h"
+#include "fdb5/remote/Connection.h"
 
 namespace fdb5::remote {
 
-class MessageHeader;
+enum class Handled {
+    No = 0,
+    Yes,
+    YesAddArchiveListener,
+    YesRemoveArchiveListener,
+    YesAddReadListener,
+    YesRemoveReadListener,
+    Replied,
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
+class Handler {
+
+public:
+
+    virtual Handled handleControl(Message message, uint32_t clientID, uint32_t requestID) = 0;
+    virtual Handled handleControl(Message message, uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) = 0;
+    virtual Handled handleData(Message message, uint32_t clientID, uint32_t requestID) = 0;
+    virtual Handled handleData(Message message, uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) = 0;
+
+    virtual void handleException(std::exception_ptr e) = 0;
+};
 
 //----------------------------------------------------------------------------------------------------------------------
 // Base class for CatalogueHandler and StoreHandler
@@ -49,17 +71,35 @@ struct readLocationElem {
         clientID(clientID), requestID(requestID), readLocation(std::move(readLocation)) {}
 };
 
-class ServerConnection : private eckit::NonCopyable {
+struct ArchiveElem {
+
+    uint32_t clientID_;
+    uint32_t requestID_;
+    eckit::Buffer payload_;
+    bool multiblob_;
+
+    ArchiveElem() : clientID_(0), requestID_(0), payload_(0), multiblob_(false) {}
+
+    ArchiveElem(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload, bool multiblob) :
+        clientID_(clientID), requestID_(requestID), payload_(std::move(payload)), multiblob_(multiblob) {}
+};
+
+class ServerConnection : public Connection, public Handler {
 public:  // methods
     ServerConnection(eckit::net::TCPSocket& socket, const Config& config);
     ~ServerConnection();
 
-    virtual void handle() { NOTIMP; }
+    void initialiseConnections();
+
+    void handle();
 
     std::string host() const { return controlSocket_.localHost(); }
     int port() const { return controlSocket_.localPort(); }
     const eckit::LocalConfiguration& agreedConf() const { return agreedConf_; }
 
+    Handled handleData(Message message, uint32_t clientID, uint32_t requestID) override;
+    Handled handleData(Message message, uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) override;
+
 protected:
 
     // Handler& handler(uint32_t id);
@@ -67,53 +107,68 @@ class ServerConnection : private eckit::NonCopyable {
 
     // socket methods
     int selectDataPort();
-    virtual void initialiseConnections();
+//    virtual void initialiseConnections();
     eckit::LocalConfiguration availableFunctionality() const;
     
-    void socketRead(void* data, size_t length, eckit::net::TCPSocket& socket);
+        // Worker functionality
+    void tidyWorkers();
+    void waitForWorkers();
 
-    // dataWrite is protected using a mutex, as we may have multiple workers.
-    void dataWrite(Message msg, uint32_t clientID, uint32_t requestID, const void* payload = nullptr, uint32_t payloadLength = 0);
-    eckit::Buffer receivePayload(const MessageHeader& hdr, eckit::net::TCPSocket& socket);
+    // archival thread
+    size_t archiveThreadLoop();
+    virtual void archiveBlob(const uint32_t clientID, const uint32_t requestID, const void* data, size_t length) = 0;
 
-    // socket methods
-    void dataWriteUnsafe(const void* data, size_t length);
+    // archival helper methods
+    void archiver();
+    // emplace new ArchiveElem to archival queue
+    void queue(Message message, uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload);
+    // // retrieve archival queue
+    // eckit::Queue& queue();
 
-    // Worker functionality
-    void tidyWorkers();
-    void waitForWorkers();
+    void handleException(std::exception_ptr e) override;
 
-    void archive(const MessageHeader& hdr);
-    virtual size_t archiveThreadLoop() = 0;
+    // void archive(const MessageHeader& hdr);
+    // virtual size_t archiveThreadLoop() = 0;
 
 private:
 
-    void controlWrite(Message msg, uint32_t clientID, uint32_t requestID, const void* payload = nullptr, uint32_t payloadLength = 0);
-    void controlWrite(const void* data, size_t length);
+    // void controlWrite(Message msg, uint32_t clientID, uint32_t requestID, const void* payload = nullptr, uint32_t payloadLength = 0);
+    // void controlWrite(const void* data, size_t length);
+    void listeningThreadLoopData();
 
-    // virtual void read(const MessageHeader& hdr);
-    // virtual void archive(const MessageHeader& hdr);
-    // // virtual void store(const MessageHeader& hdr);
-    // virtual void flush(const MessageHeader& hdr);
+    eckit::net::TCPSocket& controlSocket() override { return controlSocket_; }
+    eckit::net::TCPSocket& dataSocket() override { 
+        ASSERT(dataSocket_);
+        return *dataSocket_;
+    }
 
 protected:
 
+    virtual bool remove(uint32_t clientID) = 0;
+
     // std::map handlers_;
     Config config_;
-    eckit::net::TCPSocket controlSocket_;
-    eckit::net::EphemeralTCPServer dataSocket_;
     std::string dataListenHostname_;
 
     eckit::Queue readLocationQueue_;
 
     eckit::SessionID sessionID_;
     eckit::LocalConfiguration agreedConf_;
-    std::mutex controlWriteMutex_;
-    std::mutex dataWriteMutex_;
+    // std::mutex controlWriteMutex_;
+    // std::mutex dataWriteMutex_;
     std::thread readLocationWorker_;
     
     std::map> workerThreads_;
+    eckit::Queue archiveQueue_;
     std::future archiveFuture_;
+
+    eckit::net::TCPSocket controlSocket_;
+
+private:
+
+    // data connection
+    std::unique_ptr dataSocket_;
+    size_t dataListener_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/server/StoreHandler.cc b/src/fdb5/remote/server/StoreHandler.cc
index 281c3e6b0..cd7c41b8c 100644
--- a/src/fdb5/remote/server/StoreHandler.cc
+++ b/src/fdb5/remote/server/StoreHandler.cc
@@ -18,137 +18,144 @@
 using namespace eckit;
 using metkit::mars::MarsRequest;
 
-namespace fdb5 {
-namespace remote {
+namespace fdb5::remote {
 
 StoreHandler::StoreHandler(eckit::net::TCPSocket& socket, const Config& config):
     ServerConnection(socket, config) {}
 
 StoreHandler::~StoreHandler() {}
 
-void StoreHandler::initialiseConnections() {
-    ServerConnection::initialiseConnections();
+Handled StoreHandler::handleControl(Message message, uint32_t clientID, uint32_t requestID) {
 
-    Log::info() << "StoreHandler::initialiseConnections" << std::endl;
+    try {
+        switch (message) {
+            case Message::Store: // notification that the client is starting to send data for archival
+                archiver();
+                return Handled::YesAddArchiveListener;
+            
+            // case Message::Flush: // notification that the client has sent all data for archival
+            //     flush(clientID, requestID, eckit::Buffer{0});
+            //     return Handled::YesRemoveArchiveListener; // ????
+
+            default: {
+                std::stringstream ss;
+                ss << "ERROR: Unexpected message recieved (" << message << "). ABORTING";
+                Log::status() << ss.str() << std::endl;
+                Log::error() << ss.str() << std::endl;
+                throw SeriousBug(ss.str(), Here());
+            }
+        }
+    }
+    catch (std::exception& e) {
+        // n.b. more general than eckit::Exception
+        error(e.what(), clientID, requestID);
+    }
+    catch (...) {
+        error("Caught unexpected and unknown error", clientID, requestID);
+    }
+    return Handled::No;
 }
 
+Handled StoreHandler::handleControl(Message message, uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) {
 
-void StoreHandler::handle() {
-    initialiseConnections();
-
-
-    Log::info() << "Server started ..." << std::endl;
-
-    MessageHeader hdr;
-    eckit::FixedString<4> tail;
-
-    // listen loop
-    while (true) {
-        tidyWorkers();
-
-        socketRead(&hdr, sizeof(hdr), controlSocket_);
-
-        ASSERT(hdr.marker == StartMarker);
-        ASSERT(hdr.version == CurrentVersion);
-        Log::debug() << "StoreHandler - got message " << hdr.message << " with request ID: " << hdr.requestID << std::endl;
-
-        try {
-            switch (hdr.message) {
-                case Message::Exit:
-                    Log::status() << "Exiting" << std::endl;
-                    Log::info() << "Exiting" << std::endl;
-                    return;
-
-                case Message::List:
-                    NOTIMP;
-                    break;
-
-                case Message::Dump:
-                    NOTIMP;
-                    break;
-
-                case Message::Purge:
-                    NOTIMP;
-                    break;
-
-                case Message::Stats:
-                    NOTIMP;
-                    break;
-
-                case Message::Status:
-                    NOTIMP;
-                    break;
-
-                case Message::Wipe:
-                    NOTIMP;
-                    break;
-
-                case Message::Control:
-                    NOTIMP;
-                    break;
-
-                case Message::Inspect:
-                    NOTIMP;
-                    break;
-
-                case Message::Read: 
-                    read(hdr);
-                    break;
-
-                case Message::Flush:
-                    flush(hdr);
-                    break;
-
-                case Message::Store:
-                    archive(hdr);
-                    break;
-
-                default: {
-                    std::stringstream ss;
-                    ss << "ERROR: Unexpected message recieved (" << static_cast(hdr.message)
-                       << "). ABORTING";
-                    Log::status() << ss.str() << std::endl;
-                    Log::error() << "Store Retrieving... " << ss.str() << std::endl;
-                    throw SeriousBug(ss.str(), Here());
-                }
+    try {
+        switch (message) {
+
+            case Message::Read: // notification that the client is starting to send data location for read
+                read(clientID, requestID, payload);
+                return Handled::YesAddReadListener;
+
+            case Message::Flush: // flush store
+                flush(clientID, requestID, payload);
+                return Handled::Yes;
+
+            default: {
+                std::stringstream ss;
+                ss << "ERROR: Unexpected message recieved (" << message << "). ABORTING";
+                Log::status() << ss.str() << std::endl;
+                Log::error() << ss.str() << std::endl;
+                throw SeriousBug(ss.str(), Here());
             }
-
-            // Ensure we have consumed exactly the correct amount from the socket.
-            socketRead(&tail, sizeof(tail), controlSocket_);
-            ASSERT(tail == EndMarker);
-
-            // Acknowledge receipt of command
-            dataWrite(Message::Received, hdr.clientID(), hdr.requestID);
-        }
-        catch (std::exception& e) {
-            // n.b. more general than eckit::Exception
-            std::string what(e.what());
-            dataWrite(Message::Error, hdr.clientID(), hdr.requestID, what.c_str(), what.length());
-        }
-        catch (...) {
-            std::string what("Caught unexpected and unknown error");
-            dataWrite(Message::Error, hdr.clientID(), hdr.requestID, what.c_str(), what.length());
         }
     }
+    catch (std::exception& e) {
+        // n.b. more general than eckit::Exception
+        error(e.what(), clientID, requestID);
+    }
+    catch (...) {
+        error("Caught unexpected and unknown error", clientID, requestID);
+    }
+    return Handled::No;
 }
 
-void StoreHandler::read(const MessageHeader& hdr) {
+// Handled StoreHandler::handleData(Message message, uint32_t clientID, uint32_t requestID) {
+//     try {
+//         switch (message) {
+//             case Message::Flush: // notification that the client has sent all data for archival
+//                 return Handled::YesRemoveArchiveListener; // ????
+
+//             default: {
+//                 std::stringstream ss;
+//                 ss << "ERROR: Unexpected message recieved (" << message << "). ABORTING";
+//                 Log::status() << ss.str() << std::endl;
+//                 Log::error() << ss.str() << std::endl;
+//                 throw SeriousBug(ss.str(), Here());
+//             }
+//         }
+//     }
+//     catch (std::exception& e) {
+//         // n.b. more general than eckit::Exception
+//         error(e.what(), clientID, requestID);
+//     }
+//     catch (...) {
+//         error("Caught unexpected and unknown error", clientID, requestID);
+//     }
+//     return Handled::No;
+// }
+
+// Handled StoreHandler::handleData(Message message, uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) {
+//     try {
+//         switch (message) {
+//             case Message::Blob:
+//             case Message::MultiBlob:
+//                 queue(message, clientID, requestID, std::move(payload));
+//                 return Handled::Yes;
+
+//             default: {
+//                 std::stringstream ss;
+//                 ss << "ERROR: Unexpected message recieved (" << message << "). ABORTING";
+//                 Log::status() << ss.str() << std::endl;
+//                 Log::error() << ss.str() << std::endl;
+//                 throw SeriousBug(ss.str(), Here());
+//             }
+//         }
+//     }
+//     catch (std::exception& e) {
+//         // n.b. more general than eckit::Exception
+//         error(e.what(), clientID, requestID);
+//     }
+//     catch (...) {
+//         error("Caught unexpected and unknown error", clientID, requestID);
+//     }
+//     return Handled::No;
+// }
+
+void StoreHandler::read(uint32_t clientID, uint32_t requestID, const eckit::Buffer& payload) {
 
     if (!readLocationWorker_.joinable()) {
         readLocationWorker_ = std::thread([this] { readLocationThreadLoop(); });
     }
 
-    Buffer payload(receivePayload(hdr, controlSocket_));
     MemoryStream s(payload);
 
     std::unique_ptr location(eckit::Reanimator::reanimate(s));
 
-    Log::debug() << "Queuing for read: " << hdr.requestID << " " << *location << std::endl;
+    Log::debug() << "Queuing for read: " << requestID << " " << *location << std::endl;
 
     std::unique_ptr dh;
     dh.reset(location->dataHandle());
 
-    readLocationQueue_.emplace(readLocationElem(hdr.clientID(), hdr.requestID, std::move(dh)));
+    readLocationQueue_.emplace(readLocationElem(clientID, requestID, std::move(dh)));
 }
 
 void StoreHandler::readLocationThreadLoop() {
@@ -176,7 +183,7 @@ void StoreHandler::writeToParent(const uint32_t clientID, const uint32_t request
                               << std::endl;
 
         while ((dataRead = dh->read(writeBuffer, writeBuffer.size())) != 0) {
-            dataWrite(Message::Blob, clientID, requestID, writeBuffer, dataRead);
+            write(Message::Blob, false, clientID, requestID, std::vector>{{writeBuffer, dataRead}});
         }
 
         // And when we are done, add a complete message.
@@ -184,47 +191,24 @@ void StoreHandler::writeToParent(const uint32_t clientID, const uint32_t request
         Log::debug() << "Writing retrieve complete message: " << requestID
                               << std::endl;
 
-        dataWrite(Message::Complete, clientID, requestID);
+        write(Message::Complete, false, clientID, requestID);
 
         Log::status() << "Done retrieve: " << requestID << std::endl;
         Log::debug() << "Done retrieve: " << requestID << std::endl;
     }
     catch (std::exception& e) {
         // n.b. more general than eckit::Exception
-        std::string what(e.what());
-        dataWrite(Message::Error, clientID, requestID, what.c_str(), what.length());
+        error(e.what(), clientID, requestID);
     }
     catch (...) {
         // We really don't want to std::terminate the thread
-        std::string what("Caught unexpected, unknown exception in retrieve worker");
-        dataWrite(Message::Error, clientID, requestID, what.c_str(), what.length());
+        error("Caught unexpected , unknown exception in retrieve worker", clientID, requestID);
     }
 }
 
-// void StoreHandler::archive(const MessageHeader& hdr) {
-
-//     ASSERT(hdr.payloadSize == 0);
-
-//     auto archive = archiveFuture_.find(hdr.clientID());
-
-//     // Ensure that we aren't already running a store()
-//     if(archive == archiveFuture_.end() || !archive->second.valid())
-//         // Start archive worker thread
-//         archiveFuture_[hdr.clientID()] = std::async(std::launch::async, [this] { return archiveThreadLoop(); });
-//     }
-// }
 
-Store& StoreHandler::store(uint32_t id, Key dbKey) {
-    auto it = stores_.find(id);
-    if (it != stores_.end()) {
-        return *(it->second);
-    }
-
-    return *(stores_[id] = StoreFactory::instance().build(dbKey, config_));
-}
-
-// A helper function to make archiveThreadLoop a bit cleaner
-void StoreHandler::archiveBlobPayload(const uint32_t clientID, const uint32_t requestID, const void* data, size_t length) {
+void StoreHandler::archiveBlob(const uint32_t clientID, const uint32_t requestID, const void* data, size_t length) {
+    
     MemoryStream s(data, length);
 
     fdb5::Key dbKey(s);
@@ -243,184 +227,228 @@ void StoreHandler::archiveBlobPayload(const uint32_t clientID, const uint32_t re
     
     auto loc = futureLocation.get();
 
-//    ss.flush();
-
-    eckit::Buffer locBuffer(16 * 1024);
-    MemoryStream locStream(locBuffer);
-    locStream << (*loc);
-    dataWrite(Message::Store, clientID, requestID, locBuffer.data(), locStream.position());
+    eckit::Buffer buffer(16 * 1024);
+    MemoryStream stream(buffer);
+//    stream << archiverID;
+    stream << (*loc);
+    Connection::write(Message::Store, false, clientID, requestID, buffer, stream.position());
 }
 
-struct archiveElem {
-    uint32_t clientID;
-    uint32_t requestID;
-    eckit::Buffer payload;
-    bool multiblob;
-
-    archiveElem() : clientID(0), requestID(0), payload(0), multiblob(false) {}
-
-    archiveElem(uint32_t clientID, uint32_t requestID, eckit::Buffer payload, bool multiblob) :
-        clientID(clientID), requestID(requestID), payload(std::move(payload)), multiblob(multiblob) {}
-};
-
-size_t StoreHandler::archiveThreadLoop() {
-    size_t totalArchived = 0;
-
-    // Create a worker that will do the actual archiving
-
-    static size_t queueSize(eckit::Resource("fdbServerMaxQueueSize", 32));
-    eckit::Queue queue(queueSize);
-
-    std::future worker = std::async(std::launch::async, [this, &queue] {
-        size_t totalArchived = 0;
-
-        archiveElem elem;
-
-        try {
-            long queuelen;
-            while ((queuelen = queue.pop(elem)) != -1) {
-                if (elem.multiblob) {
-                    // Handle MultiBlob
-
-                    const char* firstData = static_cast(elem.payload.data());  // For pointer arithmetic
-                    const char* charData = firstData;
-                    while (size_t(charData - firstData) < elem.payload.size()) {
-                        const MessageHeader* hdr = static_cast(static_cast(charData));
-                        ASSERT(hdr->marker == StartMarker);
-                        ASSERT(hdr->version == CurrentVersion);
-                        ASSERT(hdr->message == Message::Blob);
-                        ASSERT(hdr->clientID() == elem.clientID);
-                        ASSERT(hdr->requestID == elem.requestID);
-                        charData += sizeof(MessageHeader);
-
-                        const void* payloadData = charData;
-                        charData += hdr->payloadSize;
-
-                        const decltype(EndMarker)* e = static_cast(static_cast(charData));
-                        ASSERT(*e == EndMarker);
-                        charData += sizeof(EndMarker);
-
-                        archiveBlobPayload(elem.clientID, elem.requestID, payloadData, hdr->payloadSize);
-                        totalArchived += 1;
-                    }
-                }
-                else {
-                    // Handle single blob
-                    archiveBlobPayload(elem.clientID, elem.requestID, elem.payload.data(), elem.payload.size());
-                    totalArchived += 1;
-                }
-            }
+// struct archiveElem {
+//     uint32_t clientID;
+//     uint32_t requestID;
+//     eckit::Buffer payload;
+//     bool multiblob;
+
+//     archiveElem() : clientID(0), requestID(0), payload(0), multiblob(false) {}
+
+//     archiveElem(uint32_t clientID, uint32_t requestID, eckit::Buffer payload, bool multiblob) :
+//         clientID(clientID), requestID(requestID), payload(std::move(payload)), multiblob(multiblob) {}
+// };
+
+// size_t StoreHandler::archiveThreadLoop() {
+//     size_t totalArchived = 0;
+
+//     // Create a worker that will do the actual archiving
+
+//     static size_t queueSize(eckit::Resource("fdbServerMaxQueueSize", 32));
+//     eckit::Queue queue(queueSize);
+
+//     std::future worker = std::async(std::launch::async, [this, &queue] {
+//         size_t totalArchived = 0;
+
+//         archiveElem elem;
+
+//         try {
+//             long queuelen;
+//             while ((queuelen = queue.pop(elem)) != -1) {
+//                 if (elem.multiblob) {
+//                     // Handle MultiBlob
+
+//                     const char* firstData = static_cast(elem.payload.data());  // For pointer arithmetic
+//                     const char* charData = firstData;
+//                     while (size_t(charData - firstData) < elem.payload.size()) {
+//                         const MessageHeader* hdr = static_cast(static_cast(charData));
+//                         ASSERT(hdr->marker == StartMarker);
+//                         ASSERT(hdr->version == CurrentVersion);
+//                         ASSERT(hdr->message == Message::Blob);
+//                         ASSERT(hdr->clientID() == elem.clientID);
+//                         ASSERT(hdr->requestID == elem.requestID);
+//                         charData += sizeof(MessageHeader);
+
+//                         const void* payloadData = charData;
+//                         charData += hdr->payloadSize;
+
+//                         const decltype(EndMarker)* e = static_cast(static_cast(charData));
+//                         ASSERT(*e == EndMarker);
+//                         charData += sizeof(EndMarker);
+
+//                         archiveBlobPayload(elem.clientID, elem.requestID, payloadData, hdr->payloadSize);
+//                         totalArchived += 1;
+//                     }
+//                 }
+//                 else {
+//                     // Handle single blob
+//                     archiveBlobPayload(elem.clientID, elem.requestID, elem.payload.data(), elem.payload.size());
+//                     totalArchived += 1;
+//                 }
+//             }
             
-            // dataWrite(Message::Complete, 0);
-        }
-        catch (...) {
-            // Ensure exception propagates across the queue back to the parent thread.
-            queue.interrupt(std::current_exception());
-            throw;
-        }
+//             // dataWrite(Message::Complete, 0);
+//         }
+//         catch (...) {
+//             // Ensure exception propagates across the queue back to the parent thread.
+//             queue.interrupt(std::current_exception());
+//             throw;
+//         }
 
-        return totalArchived;
-    });
+//         return totalArchived;
+//     });
 
-    try {
-        // The archive loop is the only thing that can listen on the data socket,
-        // so we don't need to to anything clever here.
+//     try {
+//         // The archive loop is the only thing that can listen on the data socket,
+//         // so we don't need to to anything clever here.
 
-        // n.b. we also don't need to lock on read. We are the only thing that reads.
+//         // n.b. we also don't need to lock on read. We are the only thing that reads.
 
-        while (true) {
-            MessageHeader hdr;
-            socketRead(&hdr, sizeof(hdr), dataSocket_);
+//         while (true) {
+//             MessageHeader hdr;
+//             socketRead(&hdr, sizeof(hdr), dataSocket_);
 
-            ASSERT(hdr.marker == StartMarker);
-            ASSERT(hdr.version == CurrentVersion);
-            //ASSERT(hdr.requestID == id);
+//             ASSERT(hdr.marker == StartMarker);
+//             ASSERT(hdr.version == CurrentVersion);
+//             //ASSERT(hdr.requestID == id);
 
-            // Have we been told that we are done yet?
-            if (hdr.message == Message::Flush)
-                break;
+//             // Have we been told that we are done yet?
+//             if (hdr.message == Message::Flush)
+//                 break;
 
-            ASSERT(hdr.message == Message::Blob || hdr.message == Message::MultiBlob);
+//             ASSERT(hdr.message == Message::Blob || hdr.message == Message::MultiBlob);
 
-            Buffer payload(receivePayload(hdr, dataSocket_));
-            MemoryStream s(payload);
+//             Buffer payload(receivePayload(hdr, dataSocket_));
+//             MemoryStream s(payload);
 
-            eckit::FixedString<4> tail;
-            socketRead(&tail, sizeof(tail), dataSocket_);
-            ASSERT(tail == EndMarker);
+//             eckit::FixedString<4> tail;
+//             socketRead(&tail, sizeof(tail), dataSocket_);
+//             ASSERT(tail == EndMarker);
 
-            // Queueing payload
+//             // Queueing payload
 
-            size_t sz = payload.size();
-            Log::debug() << "Queueing data: " << sz << std::endl;
-            size_t queuelen = queue.emplace(archiveElem(hdr.clientID(), hdr.requestID, std::move(payload), hdr.message == Message::MultiBlob));
-            Log::status() << "Queued data (" << queuelen << ", size=" << sz << ")" << std::endl;
-            ;
-            Log::debug() << "Queued data (" << queuelen << ", size=" << sz << ")"
-                                  << std::endl;
-            ;
-        }
+//             size_t sz = payload.size();
+//             Log::debug() << "Queueing data: " << sz << std::endl;
+//             size_t queuelen = queue.emplace(archiveElem(hdr.clientID(), hdr.requestID, std::move(payload), hdr.message == Message::MultiBlob));
+//             Log::status() << "Queued data (" << queuelen << ", size=" << sz << ")" << std::endl;
+//             ;
+//             Log::debug() << "Queued data (" << queuelen << ", size=" << sz << ")"
+//                                   << std::endl;
+//             ;
+//         }
 
-        // Trigger cleanup of the workers
-        queue.close();
+//         // Trigger cleanup of the workers
+//         queue.close();
 
-        // Complete reading the Flush instruction
+//         // Complete reading the Flush instruction
 
-        eckit::FixedString<4> tail;
-        socketRead(&tail, sizeof(tail), dataSocket_);
-        ASSERT(tail == EndMarker);
+//         eckit::FixedString<4> tail;
+//         socketRead(&tail, sizeof(tail), dataSocket_);
+//         ASSERT(tail == EndMarker);
 
-        // Ensure worker is done
+//         // Ensure worker is done
 
-        ASSERT(worker.valid());
-        totalArchived = worker.get();  // n.b. use of async, get() propagates any exceptions.
-    }
-    catch (std::exception& e) {
-        // n.b. more general than eckit::Exception
-        std::string what(e.what());
-        dataWrite(Message::Error, 0, 0, what.c_str(), what.length());
-        queue.interrupt(std::current_exception());
-        throw;
-    }
-    catch (...) {
-        std::string what("Caught unexpected, unknown exception in retrieve worker");
-        dataWrite(Message::Error, 0, 0, what.c_str(), what.length());
-        queue.interrupt(std::current_exception());
-        throw;
+//         ASSERT(worker.valid());
+//         totalArchived = worker.get();  // n.b. use of async, get() propagates any exceptions.
+//     }
+//     catch (std::exception& e) {
+//         // n.b. more general than eckit::Exception
+//         std::string what(e.what());
+//         dataWrite(Message::Error, 0, 0, what.c_str(), what.length());
+//         queue.interrupt(std::current_exception());
+//         throw;
+//     }
+//     catch (...) {
+//         std::string what("Caught unexpected, unknown exception in retrieve worker");
+//         dataWrite(Message::Error, 0, 0, what.c_str(), what.length());
+//         queue.interrupt(std::current_exception());
+//         throw;
+//     }
+
+//     return totalArchived;
+// }
+
+
+void StoreHandler::flush(uint32_t clientID, uint32_t requestID, const eckit::Buffer& payload) {
+
+    size_t numArchived = 0;
+
+
+    if (requestID != 0) {
+        ASSERT(payload.size() > 0);
+        MemoryStream stream(payload);
+        stream >> numArchived;
     }
 
-    return totalArchived;
+    ASSERT(numArchived == 0 || archiveFuture_.valid());
+
+    stores_[clientID]->flush();
+
+    // if (archiveFuture_.valid()) {
+    //     // Ensure that the expected number of fields have been written, and that the
+    //     // archive worker processes have been cleanly wound up.
+    //     size_t n = archiveFuture_.get();
+    //     ASSERT(numArchived == n);
+
+    //     // Do the actual flush!
+    //     Log::info() << "Flushing" << std::endl;
+    //     Log::status() << "Flushing" << std::endl;
+    //     // for (auto it = stores_.begin(); it != stores_.end(); it++) {
+    //     //     it->second->flush();
+    //     // }
+    // }
+
+    Log::info() << "Flush complete" << std::endl;
+    Log::status() << "Flush complete" << std::endl;
 }
 
-void StoreHandler::flush(const MessageHeader& hdr) {
-    Buffer payload(receivePayload(hdr, controlSocket_));
-    MemoryStream s(payload);
+bool StoreHandler::remove(uint32_t clientID) {
 
-    size_t numArchived;
-    s >> numArchived;
+    std::lock_guard lock(storesMutex_);
+    std::stringstream ss;
+    ss << "remove " << clientID << " from " << stores_.size() << " clients - ids:";
+    for (auto it = stores_.begin(); it != stores_.end(); it++)
+        ss << "  " << it->first;
+    ss << std::endl;
+    eckit::Log::info() << ss.str();
 
-    std::future& archive = archiveFuture_;
+    auto it = stores_.find(clientID);
+    if (it != stores_.end()) {
+        stores_.erase(it);
+    }
 
-    ASSERT(numArchived == 0 || archive.valid());
+    return stores_.empty();
+}
 
-    if (archive.valid()) {
-        // Ensure that the expected number of fields have been written, and that the
-        // archive worker processes have been cleanly wound up.
-        size_t n = archive.get();
-        ASSERT(numArchived == n);
+Store& StoreHandler::store(uint32_t clientID) {
 
-        // Do the actual flush!
-        Log::info() << "Flushing" << std::endl;
-        Log::status() << "Flushing" << std::endl;
-        stores_[hdr.clientID()]->flush();
-        // for (auto it = stores_.begin(); it != stores_.end(); it++) {
-        //     it->second->flush();
-        // }
+    std::lock_guard lock(storesMutex_);
+    auto it = stores_.find(clientID);
+    if (it == stores_.end()) {
+        std::string what("Requested Store has not been loaded id: " + std::to_string(clientID));
+        write(Message::Error, false, 0, 0, what.c_str(), what.length());
+        throw;
     }
 
-    Log::info() << "Flush complete" << std::endl;
-    Log::status() << "Flush complete" << std::endl;
+    return *(it->second);
+}
+
+Store& StoreHandler::store(uint32_t clientID, const Key& dbKey) {
+
+    std::lock_guard lock(storesMutex_);
+    auto it = stores_.find(clientID);
+    if (it != stores_.end()) {
+        return *(it->second);
+    }
+
+    return *(stores_[clientID] = StoreFactory::instance().build(dbKey, config_));
 }
 
-}  // namespace remote
-}  // namespace fdb5
+}  // namespace fdb5::remote
diff --git a/src/fdb5/remote/server/StoreHandler.h b/src/fdb5/remote/server/StoreHandler.h
index 10d42bf1c..fff40cf5d 100644
--- a/src/fdb5/remote/server/StoreHandler.h
+++ b/src/fdb5/remote/server/StoreHandler.h
@@ -10,6 +10,7 @@
 
 #pragma once
 
+#include "fdb5/database/Store.h"
 #include "fdb5/remote/server/ServerConnection.h"
 
 namespace fdb5::remote {
@@ -17,29 +18,32 @@ namespace fdb5::remote {
 //----------------------------------------------------------------------------------------------------------------------
 class StoreHandler : public ServerConnection {
 public:  // methods
+
     StoreHandler(eckit::net::TCPSocket& socket, const Config& config);
     ~StoreHandler();
 
-    void handle() override;
-
 private:  // methods
 
-    void read(const MessageHeader& hdr);
+    Handled handleControl(Message message, uint32_t clientID, uint32_t requestID) override;
+    Handled handleControl(Message message, uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) override;
+
+    void flush(uint32_t clientID, uint32_t requestID, const eckit::Buffer& payload);
+    void read(uint32_t clientID, uint32_t requestID, const eckit::Buffer& payload);
+
+    void archiveBlob(const uint32_t clientID, const uint32_t requestID, const void* data, size_t length) override;
 
-    void initialiseConnections() override;
     void readLocationThreadLoop();
     void writeToParent(const uint32_t clientID, const uint32_t requestID, std::unique_ptr dh);
 
-    size_t archiveThreadLoop() override;
-    void archiveBlobPayload(const uint32_t clientID, const uint32_t requestID, const void* data, size_t length);
-
-    void flush(const MessageHeader& hdr);
+    bool remove(uint32_t clientID) override;
 
-    // Store& store(uint32_t id);
-    Store& store(uint32_t id, Key dbKey);
-  //  Store& store(eckit::URI uri);
+    Store& store(uint32_t clientID);
+    Store& store(uint32_t clientID, const Key& dbKey);
 
 private:  // members
+
+    // clientID --> Store
+    std::mutex storesMutex_;
     std::map> stores_;
 };
 
diff --git a/src/fdb5/tools/fdb-write.cc b/src/fdb5/tools/fdb-write.cc
index cabe18041..e038ee021 100644
--- a/src/fdb5/tools/fdb-write.cc
+++ b/src/fdb5/tools/fdb-write.cc
@@ -98,12 +98,13 @@ void FDBWrite::execute(const eckit::option::CmdArgs &args) {
         if (i%2==0) {
             archiver1.archive( *dh );
         } else {
+            std::cout << "adding to archiver 2\n";
             archiver2.archive( *dh );
         }
     }
 
-    // archiver1.flush();
-    // archiver2.flush();
+    archiver1.flush();
+    archiver2.flush();
 }
 
 //----------------------------------------------------------------------------------------------------------------------

From 33b7bea47fdb028f35b95dd21d502bd90a51a839 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 21 Feb 2024 16:59:18 +0000
Subject: [PATCH 066/186] lowercase metadata

---
 src/fdb5/CMakeLists.txt         |  2 ++
 src/fdb5/types/TypeLowercase.cc | 48 ++++++++++++++++++++++++++++++++
 src/fdb5/types/TypeLowercase.h  | 49 +++++++++++++++++++++++++++++++++
 3 files changed, 99 insertions(+)
 create mode 100644 src/fdb5/types/TypeLowercase.cc
 create mode 100644 src/fdb5/types/TypeLowercase.h

diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt
index 8ccf68784..5a56d04d6 100644
--- a/src/fdb5/CMakeLists.txt
+++ b/src/fdb5/CMakeLists.txt
@@ -194,6 +194,8 @@ list( APPEND fdb5_srcs
     types/TypeIgnore.h
     types/TypeInteger.cc
     types/TypeInteger.h
+    types/TypeLowercase.cc
+    types/TypeLowercase.h
     types/TypeMonth.cc
     types/TypeMonth.h
     types/TypeParam.cc
diff --git a/src/fdb5/types/TypeLowercase.cc b/src/fdb5/types/TypeLowercase.cc
new file mode 100644
index 000000000..1db95f10d
--- /dev/null
+++ b/src/fdb5/types/TypeLowercase.cc
@@ -0,0 +1,48 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+#include "fdb5/types/TypesFactory.h"
+#include "fdb5/types/TypeLowercase.h"
+
+#include 
+
+namespace fdb5 {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+TypeLowercase::TypeLowercase(const std::string &name, const std::string &type) :
+    Type(name, type) {
+}
+
+TypeLowercase::~TypeLowercase() {
+}
+
+void TypeLowercase::print(std::ostream &out) const {
+    out << "TypeLowercase[name=" << name_ << "]";
+}
+
+std::string TypeLowercase::tidy(const std::string &keyword, const std::string &value) const {
+    std::string out;
+    out.resize(value.size());
+
+    std::transform(value.begin(), value.end(), out.begin(), [](unsigned char c){ return std::tolower(c); });
+
+    return out;
+}
+
+std::string TypeLowercase::toKey(const std::string &keyword, const std::string &value) const {
+    return tidy(keyword, value);
+}
+
+static TypeBuilder type("Lowercase");
+
+//----------------------------------------------------------------------------------------------------------------------
+
+} // namespace fdb5
diff --git a/src/fdb5/types/TypeLowercase.h b/src/fdb5/types/TypeLowercase.h
new file mode 100644
index 000000000..0ca69954d
--- /dev/null
+++ b/src/fdb5/types/TypeLowercase.h
@@ -0,0 +1,49 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/// @file   TypeLowercase.h
+/// @author Baudouin Raoult
+/// @author Tiago Quintino
+/// @date   April 2016
+
+#ifndef fdb5_TypeDefault_H
+#define fdb5_TypeDefault_H
+
+#include "fdb5/types/Type.h"
+
+namespace fdb5 {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+class TypeLowercase : public Type {
+
+public: // methods
+
+    TypeLowercase(const std::string &name, const std::string &type);
+
+    virtual ~TypeLowercase() override;
+
+    std::string tidy(const std::string &keyword,
+                     const std::string &value) const override;
+
+    std::string toKey(const std::string &keyword,
+                      const std::string &value) const override;
+
+private: // methods
+
+    virtual void print( std::ostream &out ) const override;
+
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
+} // namespace fdb5
+
+#endif

From 786a777628bd73d5bc2b0933325fcd2bc3f6bd66 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 22 Feb 2024 12:26:20 +0000
Subject: [PATCH 067/186] Revert "lowercase metadata"

This reverts commit 33b7bea47fdb028f35b95dd21d502bd90a51a839.
---
 src/fdb5/CMakeLists.txt         |  2 --
 src/fdb5/types/TypeLowercase.cc | 48 --------------------------------
 src/fdb5/types/TypeLowercase.h  | 49 ---------------------------------
 3 files changed, 99 deletions(-)
 delete mode 100644 src/fdb5/types/TypeLowercase.cc
 delete mode 100644 src/fdb5/types/TypeLowercase.h

diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt
index 866be7cc6..4bfae8bdd 100644
--- a/src/fdb5/CMakeLists.txt
+++ b/src/fdb5/CMakeLists.txt
@@ -194,8 +194,6 @@ list( APPEND fdb5_srcs
     types/TypeIgnore.h
     types/TypeInteger.cc
     types/TypeInteger.h
-    types/TypeLowercase.cc
-    types/TypeLowercase.h
     types/TypeMonth.cc
     types/TypeMonth.h
     types/TypeParam.cc
diff --git a/src/fdb5/types/TypeLowercase.cc b/src/fdb5/types/TypeLowercase.cc
deleted file mode 100644
index 1db95f10d..000000000
--- a/src/fdb5/types/TypeLowercase.cc
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-#include "fdb5/types/TypesFactory.h"
-#include "fdb5/types/TypeLowercase.h"
-
-#include 
-
-namespace fdb5 {
-
-//----------------------------------------------------------------------------------------------------------------------
-
-TypeLowercase::TypeLowercase(const std::string &name, const std::string &type) :
-    Type(name, type) {
-}
-
-TypeLowercase::~TypeLowercase() {
-}
-
-void TypeLowercase::print(std::ostream &out) const {
-    out << "TypeLowercase[name=" << name_ << "]";
-}
-
-std::string TypeLowercase::tidy(const std::string &keyword, const std::string &value) const {
-    std::string out;
-    out.resize(value.size());
-
-    std::transform(value.begin(), value.end(), out.begin(), [](unsigned char c){ return std::tolower(c); });
-
-    return out;
-}
-
-std::string TypeLowercase::toKey(const std::string &keyword, const std::string &value) const {
-    return tidy(keyword, value);
-}
-
-static TypeBuilder type("Lowercase");
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace fdb5
diff --git a/src/fdb5/types/TypeLowercase.h b/src/fdb5/types/TypeLowercase.h
deleted file mode 100644
index 0ca69954d..000000000
--- a/src/fdb5/types/TypeLowercase.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/// @file   TypeLowercase.h
-/// @author Baudouin Raoult
-/// @author Tiago Quintino
-/// @date   April 2016
-
-#ifndef fdb5_TypeDefault_H
-#define fdb5_TypeDefault_H
-
-#include "fdb5/types/Type.h"
-
-namespace fdb5 {
-
-//----------------------------------------------------------------------------------------------------------------------
-
-class TypeLowercase : public Type {
-
-public: // methods
-
-    TypeLowercase(const std::string &name, const std::string &type);
-
-    virtual ~TypeLowercase() override;
-
-    std::string tidy(const std::string &keyword,
-                     const std::string &value) const override;
-
-    std::string toKey(const std::string &keyword,
-                      const std::string &value) const override;
-
-private: // methods
-
-    virtual void print( std::ostream &out ) const override;
-
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace fdb5
-
-#endif

From 14d2d9a7d690bc3e74c93bf30aeb4391446b051d Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 22 Feb 2024 12:27:26 +0000
Subject: [PATCH 068/186] all keys to lowercase

---
 VERSION                            |  2 +-
 src/fdb5/database/InspectionKey.cc | 47 ------------------------------
 src/fdb5/database/Key.cc           | 26 ++++++++++++++---
 src/fdb5/database/Key.h            |  2 ++
 tests/fdb/type/test_toKey.cc       |  4 +--
 5 files changed, 27 insertions(+), 54 deletions(-)

diff --git a/VERSION b/VERSION
index 73e63502d..bbc239cb8 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.101
\ No newline at end of file
+5.11.102
diff --git a/src/fdb5/database/InspectionKey.cc b/src/fdb5/database/InspectionKey.cc
index d23698d40..a16799031 100644
--- a/src/fdb5/database/InspectionKey.cc
+++ b/src/fdb5/database/InspectionKey.cc
@@ -106,53 +106,6 @@ Key InspectionKey::canonical() const {
     return key;
 }
 
-// void InspectionKey::decode(eckit::Stream& s) {
-
-//     ASSERT(rule_ == nullptr);
-
-//     keys_.clear();
-//     names_.clear();
-
-
-//     size_t n;
-
-//     s >> n;
-//     std::string k;
-//     std::string v;
-//     for (size_t i = 0; i < n; ++i) {
-//         s >> k;
-//         s >> v;
-//         keys_[k] = v;
-//     }
-
-//     s >> n;
-//     for (size_t i = 0; i < n; ++i) {
-//         s >> k;
-//         s >> v; // this is the type (ignoring FTM)
-//         names_.push_back(k);
-//     }
-//     s >> canonical_;
-// }
-
-// void InspectionKey::encode(eckit::Stream& s) const {
-//     const TypesRegistry& registry = this->registry();
-
-//     s << keys_.size();
-//     for (eckit::StringDict::const_iterator i = keys_.begin(); i != keys_.end(); ++i) {
-//         const Type &t = registry.lookupType(i->first);
-//         s << i->first << canonicalise(i->first, i->second);
-//     }
-
-//     s << names_.size();
-//     for (eckit::StringList::const_iterator i = names_.begin(); i != names_.end(); ++i) {
-//         const Type &t = registry.lookupType(*i);
-//         s << (*i);
-//         s << t.type();
-//     }
-
-//     s << true;
-// }
-
 void InspectionKey::rule(const Rule *rule) {
     rule_ = rule;
 }
diff --git a/src/fdb5/database/Key.cc b/src/fdb5/database/Key.cc
index 956ba5ebc..822e32ab8 100644
--- a/src/fdb5/database/Key.cc
+++ b/src/fdb5/database/Key.cc
@@ -10,6 +10,7 @@
 
 #include 
 
+#include "eckit/config/Resource.h"
 #include "eckit/container/DenseSet.h"
 #include "eckit/utils/Tokenizer.h"
 
@@ -79,7 +80,7 @@ void Key::decode(eckit::Stream& s) {
     for (size_t i = 0; i < n; ++i) {
         s >> k;
         s >> v;
-        keys_[k] = v;
+        keys_[k] = lower(v);
     }
 
     s >> n;
@@ -126,9 +127,9 @@ void Key::set(const std::string &k, const std::string &v) {
     eckit::StringDict::iterator it = keys_.find(k);
     if (it == keys_.end()) {
         names_.push_back(k);
-        keys_[k] = v;
+        keys_[k] = lower(v);
     } else {
-        it->second = v;
+        it->second = lower(v);
     }
 
 }
@@ -138,7 +139,7 @@ void Key::unset(const std::string &k) {
 }
 
 void Key::push(const std::string &k, const std::string &v) {
-    keys_[k] = v;
+    keys_[k] = lower(v);
     names_.push_back(k);
 }
 
@@ -338,7 +339,24 @@ fdb5::Key::operator std::string() const {
     return toString();
 }
 
+std::string Key::lower(const std::string& value) const {
+
+    static bool toLower = eckit::Resource("fdbKeysLowercase;$FDB_KEYS_LOWERCASE", true);
+
+    if (!toLower) {
+        return value;
+    }
+
+    std::string lowerCase;
+    lowerCase.resize(value.size());
+
+    std::transform(value.begin(), value.end(), lowerCase.begin(), [](unsigned char c){ return std::tolower(c); });
+
+    return lowerCase;
+}
+
 std::string Key::canonicalise(const std::string&, const std::string& value) const {
+
     return value;
 }
 
diff --git a/src/fdb5/database/Key.h b/src/fdb5/database/Key.h
index f17eecf3e..10dcb1d4a 100644
--- a/src/fdb5/database/Key.h
+++ b/src/fdb5/database/Key.h
@@ -138,6 +138,8 @@ class Key {
 
 protected: // methods 
 
+    std::string lower(const std::string& value) const;
+
     //TODO add unit test for each type
     virtual std::string canonicalise(const std::string& keyword, const std::string& value) const;
 
diff --git a/tests/fdb/type/test_toKey.cc b/tests/fdb/type/test_toKey.cc
index 64496fe95..9407fc317 100644
--- a/tests/fdb/type/test_toKey.cc
+++ b/tests/fdb/type/test_toKey.cc
@@ -275,7 +275,7 @@ CASE( "Date - string ctor - expansion" ) {
     eckit::Translator t;
 
     EXPECT(key.canonicalValue("date") == t(now.yyyymmdd()));
-    EXPECT(key.valuesToString() == "od:0001:oper:ofb:"+t(now.yyyymmdd())+":0000:MHS:3001");
+    EXPECT(key.valuesToString() == "od:0001:oper:ofb:"+t(now.yyyymmdd())+":0000:mhs:3001");
 
     fdb5::Archiver archiver;
     fdb5::ArchiveVisitor visitor(archiver, key, data, 4);
@@ -284,7 +284,7 @@ CASE( "Date - string ctor - expansion" ) {
 
 //    std::cout << key.valuesToString() << std::endl;
     EXPECT(key.canonicalValue("date") == t(now.yyyymmdd()));
-    EXPECT(key.valuesToString() == "od:0001:oper:ofb:"+t(now.yyyymmdd())+":0000:MHS:3001");
+    EXPECT(key.valuesToString() == "od:0001:oper:ofb:"+t(now.yyyymmdd())+":0000:mhs:3001");
 
 }
 

From d0c0cdb957b3df4f1906c50b786678515a3496a6 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 22 Feb 2024 15:26:48 +0000
Subject: [PATCH 069/186] scan_dbs accepting symlinks

---
 src/fdb5/toc/TocEngine.cc | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/fdb5/toc/TocEngine.cc b/src/fdb5/toc/TocEngine.cc
index 9e0de7557..71932012a 100644
--- a/src/fdb5/toc/TocEngine.cc
+++ b/src/fdb5/toc/TocEngine.cc
@@ -45,6 +45,8 @@ void TocEngine::scan_dbs(const std::string& path, std::list& dbs) c
         return;
     }
 
+    char linkname[PATH_MAX];
+
     eckit::StdDir d(path.c_str());
     if (d == nullptr) {
         // If fdb-wipe is running in parallel, it is perfectly legit for a (non-matching)
@@ -91,6 +93,13 @@ void TocEngine::scan_dbs(const std::string& path, std::list& dbs) c
         do_stat = false;
         if (e->d_type == DT_DIR) {
             scan_dbs(full.c_str(), dbs);
+        } else if (e->d_type == DT_LNK) {
+            ssize_t r = readlink(full.c_str(), linkname, PATH_MAX);
+            if (r < 0) {
+                do_stat = true;
+            } else {
+                scan_dbs(linkname, dbs);
+            }
         } else if (e->d_type == DT_UNKNOWN) {
             do_stat = true;
         }
@@ -101,6 +110,11 @@ void TocEngine::scan_dbs(const std::string& path, std::list& dbs) c
             {
                 if(S_ISDIR(info.st_mode)) {
                     scan_dbs(full.c_str(), dbs);
+                } else if (S_ISLNK(info.st_mode)) {
+                    ssize_t r = readlink(full.c_str(), linkname, PATH_MAX);
+                    if (r >= 0) {
+                       scan_dbs(linkname, dbs);
+                    }
                 }
             }
             else Log::error() << "Cannot stat " << full << Log::syserr << std::endl;

From 5a3bc9a73a58069cd1737f5ff575501761449d7e Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 22 Feb 2024 15:27:20 +0000
Subject: [PATCH 070/186] version bump

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index bbc239cb8..ae4a38e71 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.102
+5.11.103

From 87ccf553d31e84dc1e7ac4b3095f45965a5e0dc8 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 23 Feb 2024 09:09:35 +0200
Subject: [PATCH 071/186] archival on existing (case sensitive) db

---
 src/fdb5/database/Key.cc    | 25 ++++------------
 src/fdb5/database/Key.h     |  2 --
 src/fdb5/toc/FileSpace.cc   | 59 ++++++++++++++++++++++++++++++-------
 src/fdb5/toc/FileSpace.h    |  2 +-
 src/fdb5/toc/RootManager.cc |  6 ++--
 src/fdb5/toc/TocEngine.cc   | 22 +++++---------
 6 files changed, 65 insertions(+), 51 deletions(-)

diff --git a/src/fdb5/database/Key.cc b/src/fdb5/database/Key.cc
index 822e32ab8..55481767e 100644
--- a/src/fdb5/database/Key.cc
+++ b/src/fdb5/database/Key.cc
@@ -13,6 +13,7 @@
 #include "eckit/config/Resource.h"
 #include "eckit/container/DenseSet.h"
 #include "eckit/utils/Tokenizer.h"
+#include "eckit/utils/StringTools.h"
 
 #include "metkit/mars/MarsRequest.h"
 
@@ -80,7 +81,7 @@ void Key::decode(eckit::Stream& s) {
     for (size_t i = 0; i < n; ++i) {
         s >> k;
         s >> v;
-        keys_[k] = lower(v);
+        keys_[k] = eckit::StringTools::lower(v);
     }
 
     s >> n;
@@ -127,9 +128,9 @@ void Key::set(const std::string &k, const std::string &v) {
     eckit::StringDict::iterator it = keys_.find(k);
     if (it == keys_.end()) {
         names_.push_back(k);
-        keys_[k] = lower(v);
+        keys_[k] = eckit::StringTools::lower(v);
     } else {
-        it->second = lower(v);
+        it->second = eckit::StringTools::lower(v);
     }
 
 }
@@ -139,7 +140,7 @@ void Key::unset(const std::string &k) {
 }
 
 void Key::push(const std::string &k, const std::string &v) {
-    keys_[k] = lower(v);
+    keys_[k] = eckit::StringTools::lower(v);
     names_.push_back(k);
 }
 
@@ -339,22 +340,6 @@ fdb5::Key::operator std::string() const {
     return toString();
 }
 
-std::string Key::lower(const std::string& value) const {
-
-    static bool toLower = eckit::Resource("fdbKeysLowercase;$FDB_KEYS_LOWERCASE", true);
-
-    if (!toLower) {
-        return value;
-    }
-
-    std::string lowerCase;
-    lowerCase.resize(value.size());
-
-    std::transform(value.begin(), value.end(), lowerCase.begin(), [](unsigned char c){ return std::tolower(c); });
-
-    return lowerCase;
-}
-
 std::string Key::canonicalise(const std::string&, const std::string& value) const {
 
     return value;
diff --git a/src/fdb5/database/Key.h b/src/fdb5/database/Key.h
index 10dcb1d4a..f17eecf3e 100644
--- a/src/fdb5/database/Key.h
+++ b/src/fdb5/database/Key.h
@@ -138,8 +138,6 @@ class Key {
 
 protected: // methods 
 
-    std::string lower(const std::string& value) const;
-
     //TODO add unit test for each type
     virtual std::string canonicalise(const std::string& keyword, const std::string& value) const;
 
diff --git a/src/fdb5/toc/FileSpace.cc b/src/fdb5/toc/FileSpace.cc
index fadc51f91..ab3267f92 100644
--- a/src/fdb5/toc/FileSpace.cc
+++ b/src/fdb5/toc/FileSpace.cc
@@ -8,7 +8,10 @@
  * does it submit to any jurisdiction.
  */
 
+#include "eckit/config/Resource.h"
 #include "fdb5/toc/FileSpace.h"
+#include "eckit/filesystem/StdDir.h"
+#include "eckit/utils/StringTools.h"
 
 #include "eckit/os/BackTrace.h"
 #include "eckit/exception/Exceptions.h"
@@ -40,16 +43,16 @@ TocPath FileSpace::filesystem(const Key& key, const eckit::PathName& db) const {
     // check that the database isn't present already
     // if it is, then return that path
 
-    TocPath root;
-    if (existsDB(key, db, root)) {
-        Log::debug() << "Found FDB root for key " << key << " -> " << root.directory_ << std::endl;
-        return root;
+    TocPath existingDB;
+    if (existsDB(key, db, existingDB)) {
+        Log::debug() << "Found existing DB for key " << key << " -> " << existingDB.directory_ << std::endl;
+        return existingDB;
     }
 
     Log::debug() << "FDB for key " << key << " not found, selecting a root" << std::endl;
     // Log::debug() << eckit::BackTrace::dump() << std::endl;
 
-    return TocPath{FileSpaceHandler::lookup(handler_).selectFileSystem(key, *this), ControlIdentifiers{}};
+    return TocPath{FileSpaceHandler::lookup(handler_).selectFileSystem(key, *this) / db, ControlIdentifiers{}};
 }
 
 std::vector FileSpace::enabled(const ControlIdentifier& controlIdentifier) const {
@@ -82,7 +85,43 @@ bool FileSpace::match(const std::string& s) const {
     return re_.match(s);
 }
 
-bool FileSpace::existsDB(const Key& key, const eckit::PathName& db, TocPath& root) const {
+eckit::PathName getFullDB(const eckit::PathName& path, const std::string& db) {
+
+    static bool searchCaseSensitiveDB = eckit::Resource("fdbSearchCaseSensitiveDB;$FDB_SEARCH_CASESENSITIVE_DB", true);
+
+    if (searchCaseSensitiveDB) {
+
+        eckit::StdDir d(path.path().c_str());
+        if (d == nullptr) {
+            Log::error() << "opendir(" << path << ")" << Log::syserr << std::endl;
+            throw eckit::FailedSystemCall("opendir");
+        }
+
+        // Once readdir_r finally gets deprecated and removed, we may need to 
+        // protecting readdir() as not yet guarranteed thread-safe by POSIX
+        // technically it should only be needed on a per-directory basis
+        // this should be a resursive mutex
+        // AutoLock lock(mutex_); 
+        std::string ldb = eckit::StringTools::lower(db);
+
+        for(;;)
+        {
+            struct dirent* e = d.dirent();
+            if (e == nullptr) {
+                break;
+            }
+
+            if (ldb == eckit::StringTools::lower(e->d_name)) {
+                return path / e->d_name;
+            }
+        }
+    }
+
+    return path / db;
+}
+
+
+bool FileSpace::existsDB(const Key& key, const eckit::PathName& db, TocPath& existsDB) const {
     unsigned count = 0;
     bool found = false;
 
@@ -90,15 +129,15 @@ bool FileSpace::existsDB(const Key& key, const eckit::PathName& db, TocPath& roo
     std::string matchList;
     for (RootVec::const_iterator i = roots_.begin(); i != roots_.end(); ++i) {
         if (i->enabled(ControlIdentifier::List) && i->exists()) {
-            eckit::PathName fullDB = i->path() / db;
-            eckit::PathName dbToc = i->path() / db / "toc";
+            eckit::PathName fullDB = getFullDB(i->path(), db);
+            eckit::PathName dbToc = fullDB / "toc";
             if (fullDB.exists() && dbToc.exists()) {
                 matchList += (count == 0 ? "" : ", ") + fullDB;
 
                 bool allowMultipleDbs = (fullDB / (controlfile_lookup.find(ControlIdentifier::UniqueRoot)->second)).exists();
                 if (!count || allowMultipleDbs) { // take last
-                    root.directory_ = i->path();
-                    root.controlIdentifiers_ = i->controlIdentifiers();
+                    existsDB.directory_ = fullDB;
+                    existsDB.controlIdentifiers_ = i->controlIdentifiers();
                     found = true;
                 }
                 if (!allowMultipleDbs)
diff --git a/src/fdb5/toc/FileSpace.h b/src/fdb5/toc/FileSpace.h
index 0872d0e81..b26fb6ba9 100644
--- a/src/fdb5/toc/FileSpace.h
+++ b/src/fdb5/toc/FileSpace.h
@@ -69,7 +69,7 @@ class FileSpace {
 
 private: // methods
 
-    bool existsDB(const Key& key, const eckit::PathName& db, TocPath& root) const;
+    bool existsDB(const Key& key, const eckit::PathName& db, TocPath& existsDB) const;
 
     void print( std::ostream &out ) const;
 
diff --git a/src/fdb5/toc/RootManager.cc b/src/fdb5/toc/RootManager.cc
index ca925d47f..e0624283e 100644
--- a/src/fdb5/toc/RootManager.cc
+++ b/src/fdb5/toc/RootManager.cc
@@ -631,9 +631,9 @@ TocPath RootManager::directory(const Key& key) {
 
     for (FileSpaceTable::const_iterator i = spacesTable_.begin(); i != spacesTable_.end() ; ++i) {
         if(i->match(keystr)) {
-            TocPath root = i->filesystem(key, dbpath);
-            eckit::Log::debug() << "Directory root " << root.directory_ << " dbpath " << dbpath <<  std::endl;
-            return TocPath{root.directory_ / dbpath, root.controlIdentifiers_};
+            TocPath db = i->filesystem(key, dbpath);
+            eckit::Log::debug() << "Database directory " << db.directory_ << std::endl;
+            return db;
         }
     }
 
diff --git a/src/fdb5/toc/TocEngine.cc b/src/fdb5/toc/TocEngine.cc
index 71932012a..2cf29babb 100644
--- a/src/fdb5/toc/TocEngine.cc
+++ b/src/fdb5/toc/TocEngine.cc
@@ -17,6 +17,7 @@
 
 #include "eckit/eckit.h"
 
+#include "eckit/config/Resource.h"
 #include "eckit/filesystem/LocalFileManager.h"
 #include "eckit/filesystem/LocalPathName.h"
 #include "eckit/filesystem/StdDir.h"
@@ -24,6 +25,7 @@
 #include "eckit/os/BackTrace.h"
 #include "eckit/os/Stat.h"
 #include "eckit/utils/Regex.h"
+#include "eckit/utils/StringTools.h"
 
 #include "fdb5/LibFdb5.h"
 #include "fdb5/rules/Schema.h"
@@ -45,8 +47,6 @@ void TocEngine::scan_dbs(const std::string& path, std::list& dbs) c
         return;
     }
 
-    char linkname[PATH_MAX];
-
     eckit::StdDir d(path.c_str());
     if (d == nullptr) {
         // If fdb-wipe is running in parallel, it is perfectly legit for a (non-matching)
@@ -93,13 +93,6 @@ void TocEngine::scan_dbs(const std::string& path, std::list& dbs) c
         do_stat = false;
         if (e->d_type == DT_DIR) {
             scan_dbs(full.c_str(), dbs);
-        } else if (e->d_type == DT_LNK) {
-            ssize_t r = readlink(full.c_str(), linkname, PATH_MAX);
-            if (r < 0) {
-                do_stat = true;
-            } else {
-                scan_dbs(linkname, dbs);
-            }
         } else if (e->d_type == DT_UNKNOWN) {
             do_stat = true;
         }
@@ -110,11 +103,6 @@ void TocEngine::scan_dbs(const std::string& path, std::list& dbs) c
             {
                 if(S_ISDIR(info.st_mode)) {
                     scan_dbs(full.c_str(), dbs);
-                } else if (S_ISLNK(info.st_mode)) {
-                    ssize_t r = readlink(full.c_str(), linkname, PATH_MAX);
-                    if (r >= 0) {
-                       scan_dbs(linkname, dbs);
-                    }
                 }
             }
             else Log::error() << "Cannot stat " << full << Log::syserr << std::endl;
@@ -154,10 +142,14 @@ static void matchRequestToDB(const metkit::mars::MarsRequest& rq, std::set TocEngine::databases(const std::set& keys,
                                                const std::vector& roots,
                                                const Config& config) const {
 
+    static bool searchCaseSensitiveDB = eckit::Resource("fdbSearchCaseSensitiveDB;$FDB_SEARCH_CASESENSITIVE_DB", true);
+
     std::set result;
 
     for (std::vector::const_iterator j = roots.begin(); j != roots.end(); ++j) {
@@ -187,7 +179,7 @@ std::set TocEngine::databases(const std::set& ke
                         continue;
                     }
 
-                    if (re.match(*k)) {
+                    if (re.match(searchCaseSensitiveDB ? eckit::StringTools::lower(*k) : *k)) {
                         result.insert(*k);
                     }
                 }

From 46d4166aa8c0c2f88365099c725b71f8d6b11e1a Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 23 Feb 2024 08:45:23 +0000
Subject: [PATCH 072/186] version bump

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index ae4a38e71..7dae53c49 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.103
+5.11.104

From d3afd34239799c0014ad874aff3f9b2f55f82a1b Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 1 Mar 2024 22:47:54 +0000
Subject: [PATCH 073/186] fix remote termination

---
 src/fdb5/api/RemoteFDB.cc                  |   1 +
 src/fdb5/database/ArchiveVisitor.cc        |  15 +-
 src/fdb5/remote/RemoteFieldLocation.cc     |   2 +-
 src/fdb5/remote/RemoteStore.cc             |  12 +-
 src/fdb5/remote/client/Client.cc           |  18 +-
 src/fdb5/remote/client/Client.h            |   5 +-
 src/fdb5/remote/client/ClientConnection.cc |   5 +-
 src/fdb5/remote/client/ClientConnection.h  |  19 +-
 src/fdb5/remote/server/CatalogueHandler.cc |  64 ++++--
 src/fdb5/remote/server/CatalogueHandler.h  |  13 +-
 src/fdb5/remote/server/ServerConnection.cc | 162 ++++++--------
 src/fdb5/remote/server/ServerConnection.h  |   7 +-
 src/fdb5/remote/server/StoreHandler.cc     | 243 +++------------------
 src/fdb5/remote/server/StoreHandler.h      |  19 +-
 src/fdb5/toc/TocEngine.cc                  |   3 +-
 src/fdb5/tools/fdb-write.cc                |  34 ++-
 16 files changed, 228 insertions(+), 394 deletions(-)

diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index 973fbe39e..61bceb8fd 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -44,6 +44,7 @@ struct ListHelper : BaseAPIHelper());
         eckit::Log::debug() << std::endl;
 
+        // TODO move the endpoint replacement to the server side ()
         if (elem.location().uri().scheme() == "fdb") {
             eckit::net::Endpoint fieldLocationEndpoint{elem.location().uri().host(), elem.location().uri().port()};
 
diff --git a/src/fdb5/database/ArchiveVisitor.cc b/src/fdb5/database/ArchiveVisitor.cc
index 4f71be421..77d4ba5ef 100644
--- a/src/fdb5/database/ArchiveVisitor.cc
+++ b/src/fdb5/database/ArchiveVisitor.cc
@@ -15,11 +15,11 @@
 #include "fdb5/database/Catalogue.h"
 #include "fdb5/database/Store.h"
 
-namespace {
-void CatalogueCallback(fdb5::CatalogueWriter* catalogue, const fdb5::InspectionKey &key, std::unique_ptr fieldLocation) {
-    catalogue->archive(key, std::move(fieldLocation));
-}
-}
+// namespace {
+// void CatalogueCallback(fdb5::CatalogueWriter* catalogue, const fdb5::InspectionKey &key, std::unique_ptr fieldLocation) {
+//     catalogue->archive(key, std::move(fieldLocation));
+// }
+// }
 namespace fdb5 {
 
 ArchiveVisitor::ArchiveVisitor(Archiver &owner, const Key &dataKey, const void *data, size_t size) :
@@ -35,8 +35,9 @@ bool ArchiveVisitor::selectDatum(const InspectionKey &key, const Key &full) {
     checkMissingKeys(full);
     const Key idxKey = current()->currentIndexKey();
 
-    store()->archive(idxKey, data_, size_, std::bind(&CatalogueCallback, current(), key, std::placeholders::_1));
-    
+    // store()->archive(idxKey, data_, size_, std::bind(&CatalogueCallback, current(), key, std::placeholders::_1));
+    store()->archive(idxKey, data_, size_, std::bind(&CatalogueWriter::archive, current(), key, std::placeholders::_1));
+
     return true;
 }
 
diff --git a/src/fdb5/remote/RemoteFieldLocation.cc b/src/fdb5/remote/RemoteFieldLocation.cc
index 749b2d65b..806bea6ee 100644
--- a/src/fdb5/remote/RemoteFieldLocation.cc
+++ b/src/fdb5/remote/RemoteFieldLocation.cc
@@ -87,7 +87,6 @@ eckit::DataHandle* RemoteFieldLocation::dataHandle() const {
     const std::string scheme = uri_.query("internalScheme");
     const std::string hostport = uri_.query("internalHost");
     eckit::URI remote;
-    remote.query("internalScheme", "");
     if (hostport.empty()) {
         remote = eckit::URI(scheme, uri_, "",  -1);
     }
@@ -96,6 +95,7 @@ eckit::DataHandle* RemoteFieldLocation::dataHandle() const {
         remote = eckit::URI(scheme, uri_, endpoint.host(),  endpoint.port());
         remote.query("internalHost", "");
     }
+    remote.query("internalScheme", "");
     FieldLocation* loc = FieldLocationFactory::instance().build(scheme, remote, offset_, length_, remapKey_);
 
     return store.dataHandle(*loc);
diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index e7c66e2d1..c5e4634d9 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -195,14 +195,10 @@ RemoteStore::~RemoteStore() {
     // If we have launched a thread with an async and we manage to get here, this is
     // an error. n.b. if we don't do something, we will block in the destructor
     // of std::future.
-    // if (archiver_ && archiver_->valid()) {
-    //     Log::error() << "Attempting to destruct DecoupledFDB with active archive thread" << std::endl;
-    //     eckit::Main::instance().terminate();
-    // }
-
-    ASSERT(!archivalCompleted_.valid());
-    // ASSERT(!archiver_ || !archiver_->dirty());
-//    disconnect();
+    if (archivalCompleted_.valid() || !locations_.empty()) {
+        Log::error() << "Attempting to destruct RemoteStore with active archival" << std::endl;
+        eckit::Main::instance().terminate();
+    }
 }
 
 eckit::URI RemoteStore::uri() const {
diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index 50345425a..54867b89e 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -19,7 +19,7 @@ namespace fdb5::remote {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-uint32_t Client::clientId_=0;
+// uint32_t Client::clientId_=0;
 
 // Client::Client(const eckit::net::Endpoint& endpoint, std::string domain) :
 //     endpoint_(endpoint),
@@ -31,12 +31,21 @@ uint32_t Client::clientId_=0;
 //     id_ = ++clientId_;
 // }
 
+void Client::setClientID() {
+    static std::mutex idMutex_;
+    static uint32_t clientId_ = 0;
+
+    std::lock_guard lock(idMutex_);
+    id_ = ++clientId_;
+}
+
+
+
 Client::Client(const eckit::net::Endpoint& endpoint, const std::string& defaultEndpoint) :
     connection_(ClientConnectionRouter::instance().connection(endpoint, defaultEndpoint)),
     blockingRequestId_(0) {
     
-    std::lock_guard lock(idMutex_);
-    id_ = ++clientId_;
+    setClientID();
 
     connection_.add(*this);
 }
@@ -46,8 +55,7 @@ Client::Client(const std::vector>&
     connection_(ClientConnectionRouter::instance().connection(endpoints)),
     blockingRequestId_(0) {
 
-    std::lock_guard lock(idMutex_);
-    id_ = ++clientId_;
+    setClientID();
 
     connection_.add(*this);
 }
diff --git a/src/fdb5/remote/client/Client.h b/src/fdb5/remote/client/Client.h
index 2f5b353a8..7f79b0c44 100644
--- a/src/fdb5/remote/client/Client.h
+++ b/src/fdb5/remote/client/Client.h
@@ -65,8 +65,9 @@ class Client : eckit::NonCopyable {
     ClientConnection& connection_;
 
 private:
-    inline static std::mutex idMutex_;
-    static uint32_t clientId_;
+    void setClientID();
+
+private:
     uint32_t id_;
 
     std::mutex blockingRequestMutex_;
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index e4077f9be..07e002e7c 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -102,8 +102,9 @@ ClientConnection::~ClientConnection() {
 }
 
 uint32_t ClientConnection::generateRequestID() {
-
     std::lock_guard lock(idMutex_);
+    // we do not want to re-use previous request IDs
+    ASSERT(id_ < UINT32_MAX-2);
     return ++id_;
 }
 
@@ -133,7 +134,7 @@ bool ClientConnection::connect(bool singleAttempt) {
 
         // And the connections are set up. Let everything start up!
         listeningThread_ = std::thread([this] { listeningThreadLoop(); });
-        //listeningThread_.detach();
+
         connected_ = true;
         return true;
     } catch(eckit::TooManyRetries& e) {
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index 15ea9c564..ae44a0f0e 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -40,14 +40,12 @@ class DataWriteRequest;
 
 class ClientConnection : protected Connection {
 
-public: // types
-
 public: // methods
 
     virtual ~ClientConnection();
 
-    void controlWrite(Client& client, Message msg, uint32_t requestID, bool dataListener, std::vector> data={});
-    void dataWrite(Client& client, remote::Message msg, uint32_t requestID, std::vector> data={});
+    void controlWrite(Client& client, Message msg, uint32_t requestID, bool startDataListener, std::vector> data={});
+    void dataWrite(Client& client, Message msg, uint32_t requestID, std::vector> data={});
 
     void add(Client& client);
     bool remove(uint32_t clientID);
@@ -61,26 +59,15 @@ class ClientConnection : protected Connection {
 
 private: // methods
 
-//    bool remove(uint32_t clientID);
-
-    void dataWrite(DataWriteRequest& dataWriteRequest);
-
     friend class ClientConnectionRouter;
 
     ClientConnection(const eckit::net::Endpoint& controlEndpoint, const std::string& defaultEndpoint);
 
-    // const eckit::net::Endpoint& dataEndpoint() const;
+    void dataWrite(DataWriteRequest& dataWriteRequest);
 
     // construct dictionary for protocol negotiation - to be defined in the client class
     eckit::LocalConfiguration availableFunctionality() const;
 
-    // void controlWrite(Message msg, uint32_t clientID, uint32_t requestID = 0, std::vector> data = {});
-    // void controlWrite(const void* data, size_t length);
-    // void controlRead (      void* data, size_t length);
-    // void dataWrite   (Message msg, uint32_t clientID, uint32_t requestID = 0, std::vector> data = {});
-    // void dataWrite   (const void* data, size_t length);
-    // void dataRead    (      void* data, size_t length);
- 
     void writeControlStartupMessage();
     void writeDataStartupMessage(const eckit::SessionID& serverSession);
 
diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc
index 178fa19a3..3aad56dae 100644
--- a/src/fdb5/remote/server/CatalogueHandler.cc
+++ b/src/fdb5/remote/server/CatalogueHandler.cc
@@ -30,7 +30,7 @@ namespace fdb5::remote {
 // ***************************************************************************************
 
 CatalogueHandler::CatalogueHandler(eckit::net::TCPSocket& socket, const Config& config):
-    ServerConnection(socket, config) {}
+    ServerConnection(socket, config), fdbId_(0), fdbControlConnection_(false), fdbDataConnection_(false) {}
 
 CatalogueHandler::~CatalogueHandler() {}
 
@@ -39,6 +39,17 @@ Handled CatalogueHandler::handleControl(Message message, uint32_t clientID, uint
     try {
         switch (message) {
             case Message::Schema: // request top-level schema
+                {
+                    std::lock_guard lock(handlerMutex_);
+                    if (fdbId_ == 0) {
+                        fdbId_ = clientID;
+                        fdbControlConnection_ = true;
+                        fdbDataConnection_ = !single_;
+                        numControlConnection_++;
+                        if (fdbDataConnection_)
+                            numDataConnection_++;
+                    }
+                }
                 schema(clientID, requestID, eckit::Buffer(0));
                 return Handled::Replied;
 
@@ -380,7 +391,7 @@ void CatalogueHandler::archiveBlob(const uint32_t clientID, const uint32_t reque
 
     std::map::iterator it;
     {
-        std::lock_guard lock(cataloguesMutex_);
+        std::lock_guard lock(handlerMutex_);
         it = catalogues_.find(clientID);
         if (it == catalogues_.end()) {
             std::string what("Requested unknown catalogue id: " + std::to_string(clientID));
@@ -397,29 +408,44 @@ void CatalogueHandler::archiveBlob(const uint32_t clientID, const uint32_t reque
     }
 }
 
-bool CatalogueHandler::remove(uint32_t clientID) {
+bool CatalogueHandler::remove(bool control, uint32_t clientID) {
     
-    std::stringstream ss;
-    ss << "remove " << clientID << " from " << catalogues_.size() << " clients - ids:";
-    for (auto it = catalogues_.begin(); it != catalogues_.end(); it++)
-        ss << "  " << it->first;
-    ss << std::endl;
-    eckit::Log::info() << ss.str();
-
-    std::lock_guard lock(cataloguesMutex_);
+    std::lock_guard lock(handlerMutex_);
+    if (clientID == fdbId_) {
+        if (control) {
+            fdbControlConnection_ = false;
+            numControlConnection_--;
+        } else {
+            fdbDataConnection_ = false;
+            numDataConnection_--;
+        }
+    } else {
 
-    auto it = catalogues_.find(clientID);
-    if (it != catalogues_.end()) {
-        catalogues_.erase(it);
+        auto it = catalogues_.find(clientID);
+        if (it != catalogues_.end()) {
+            if (control) {
+                it->second.controlConnection = false;
+                numControlConnection_--;
+            } else {
+                it->second.dataConnection = false;
+                numDataConnection_--;
+            }
+            if (!it->second.controlConnection && !it->second.dataConnection) {
+                catalogues_.erase(it);
+            }
+        }
     }
-
-    return catalogues_.empty();
+    return ((control ? numControlConnection_ : numDataConnection_) == 0);
 }
 
-
 CatalogueWriter& CatalogueHandler::catalogue(uint32_t id, const Key& dbKey) {
-    std::lock_guard lock(cataloguesMutex_);
-    return *((catalogues_.emplace(id, CatalogueArchiver(dbKey, config_)).first)->second.catalogue);
+    std::lock_guard lock(handlerMutex_);
+    numControlConnection_++;
+    if (!single_) {
+        numDataConnection_++;
+    }
+    return *((catalogues_.emplace(id, CatalogueArchiver(!single_, dbKey, config_)).first)->second.catalogue);
+    // return *((catalogues_[id] = std::make_pair(single_ ? 1 : 2, CatalogueArchiver(dbKey, config_))).second.catalogue);
 }
 
 }  // namespace fdb5::remote
diff --git a/src/fdb5/remote/server/CatalogueHandler.h b/src/fdb5/remote/server/CatalogueHandler.h
index 1af1f6298..90826144a 100644
--- a/src/fdb5/remote/server/CatalogueHandler.h
+++ b/src/fdb5/remote/server/CatalogueHandler.h
@@ -18,11 +18,15 @@ namespace fdb5::remote {
 //----------------------------------------------------------------------------------------------------------------------
 
 struct CatalogueArchiver {
-    CatalogueArchiver(const Key& dbKey, const Config& config) :
+    CatalogueArchiver(bool dataConnection, const Key& dbKey, const Config& config) :
+        controlConnection(true), dataConnection(dataConnection),
         catalogue(CatalogueWriterFactory::instance().build(dbKey, config)), locationsExpected(-1), locationsArchived(0) {
         archivalCompleted = fieldLocationsReceived.get_future();
     }
 
+    bool controlConnection;
+    bool dataConnection;
+    
     std::unique_ptr catalogue;
     int32_t locationsExpected;
     int32_t locationsArchived;
@@ -57,18 +61,21 @@ class CatalogueHandler : public ServerConnection {
 
     void archiveBlob(const uint32_t clientID, const uint32_t requestID, const void* data, size_t length) override;
 
-    bool remove(uint32_t clientID) override;
+    bool remove(bool control, uint32_t clientID) override;
+    // bool handlers() override;
 
     // CatalogueWriter& catalogue(uint32_t catalogueID);
     CatalogueWriter& catalogue(uint32_t catalogueID, const Key& dbKey);
 
 private:  // member
 
-    std::mutex cataloguesMutex_;
     // clientID --> 
     std::map catalogues_;
 
     FDB fdb_;
+    uint32_t fdbId_;
+    bool fdbControlConnection_;
+    bool fdbDataConnection_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index 01fbeafb7..000c595a4 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -70,7 +70,8 @@ ServerConnection::ServerConnection(eckit::net::TCPSocket& socket, const Config&
         dataListenHostname_(config.getString("dataListenHostname", "")),
         readLocationQueue_(eckit::Resource("fdbRetrieveQueueSize", 10000)), 
         archiveQueue_(eckit::Resource("fdbServerMaxQueueSize", 32)),
-        controlSocket_(socket), dataSocket_(nullptr), dataListener_(0) {
+        controlSocket_(socket), numControlConnection_(0), numDataConnection_(0),
+        dataSocket_(nullptr), dataListener_(0) {
 
     eckit::Log::debug() << "ServerConnection::ServerConnection initialized" << std::endl;
 }
@@ -79,6 +80,8 @@ ServerConnection::~ServerConnection() {
     // We don't want to die before the worker threads are cleaned up
     waitForWorkers();
 
+    std::this_thread::sleep_for(std::chrono::milliseconds(500));
+
     // And notify the client that we are done.
 //     eckit::Log::info() << "Sending exit message to client" << std::endl;
 // //    write(Message::Exit, true, 0, 0);
@@ -139,25 +142,6 @@ Handled ServerConnection::handleData(Message message, uint32_t clientID, uint32_
     return Handled::No;
 }
 
-// ServerConnection::ServerConnection(eckit::net::TCPSocket& socket, const Config& config) :
-//         config_(config),
-//         controlSocket_(socket),
-//         dataSocket_(selectDataPort()),
-//         dataListenHostname_(config.getString("dataListenHostname", "")),
-//         readLocationQueue_(eckit::Resource("fdbRetrieveQueueSize", 10000)) {
-//             eckit::Log::debug() << "ServerConnection::ServerConnection initialized" << std::endl;
-//     }
-
-// ServerConnection::~ServerConnection() {
-//     // We don't want to die before the worker threads are cleaned up
-//     waitForWorkers();
-
-//     // And notify the client that we are done.
-//     eckit::Log::info() << "Sending exit message to client" << std::endl;
-//     dataWrite(Message::Exit, 0, 0);
-//     eckit::Log::info() << "Done" << std::endl;
-// }
-
 eckit::LocalConfiguration ServerConnection::availableFunctionality() const {
     eckit::LocalConfiguration conf;
 //    Add to the configuration all the components that require to be versioned, as in the following example, with a vector of supported version numbers
@@ -168,7 +152,6 @@ eckit::LocalConfiguration ServerConnection::availableFunctionality() const {
     return conf;
 }
 
-
 void ServerConnection::initialiseConnections() {
     // Read the startup message from the client. Check that it all checks out.
 
@@ -392,77 +375,55 @@ void ServerConnection::listeningThreadLoopData() {
 
         // The archive loop is the only thing that can listen on the data socket,
         // so we don't need to to anything clever here.
-
         // n.b. we also don't need to lock on read. We are the only thing that reads.
         while (true) {
             eckit::Buffer payload = readData(hdr); // READ DATA
-            // eckit::Log::debug() << "ServerConnection::listeningThreadLoopData - got [message=" << hdr.message << ",clientID="<< hdr.clientID() << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
 
-        if (hdr.message == Message::Exit) {
-            if (remove(hdr.clientID())) {
-                eckit::Log::status() << "Exiting" << std::endl;
-                eckit::Log::info() << "Exiting" << std::endl;
+            if (hdr.message == Message::Exit) {
+                if (remove(false, hdr.clientID())) {
 
-                return;
-            }
-        } else {
-            
-            Handled handled;
-            if (payload.size() == 0) {
-                handled = handleData(hdr.message, hdr.clientID(), hdr.requestID);
-            } else {
-                handled = handleData(hdr.message, hdr.clientID(), hdr.requestID, std::move(payload));
-            }
+                    eckit::Log::status() << "Terminating DATA listener" << std::endl;
+                    eckit::Log::info() << "Terminating DATA listener" << std::endl;
 
-
-        switch (handled)
-        {
-            case Handled::Replied: // nothing to do
-                break;
-            case Handled::YesRemoveArchiveListener:
-                dataListener_--;
-                if (dataListener_ == 0) {
-                    return;
-                    // listeningThreadData.join();
+                    break;
                 }
-                break;
-            case Handled::YesRemoveReadListener:
-                dataListener_--;
-                if (dataListener_ == 0) {
-                    return;
-                    // listeningThreadData.join();
+            } else {
+                
+                Handled handled;
+                if (payload.size() == 0) {
+                    handled = handleData(hdr.message, hdr.clientID(), hdr.requestID);
+                } else {
+                    handled = handleData(hdr.message, hdr.clientID(), hdr.requestID, std::move(payload));
                 }
-                break;
-            case Handled::Yes:
-//                write(Message::Received, false, hdr.clientID(), hdr.requestID);
-                break;
-            case Handled::No:
-            default:
-                std::stringstream ss;
-                ss << "Unable to handle message " << hdr.message << " received on the data connection";
-                error(ss.str(), hdr.clientID(), hdr.requestID);
-        }
-        }
 
-            // std::cout << "listeningThreadLoopData " << hdr.message << " " << hdr.clientID() << " " << hdr.requestID << " " << ((int) handled) << std::endl;
-            // switch (handled)
-            // {
-            //     case Handled::YesRemoveArchiveListener:
-            //         dataListener_--;
-            //         if (dataListener_ == 0) {
-            //             // queue.close();
-            //             return;
-            //         }
-            //         break;
-            //     case Handled::Yes:
-            //         break;
-            //     case Handled::Replied:
-            //     case Handled::No:
-            //     default:
-            //         std::stringstream ss;
-            //         ss << "Unable to handle message " << hdr.message << " from data connection";
-            //         error(ss.str(), hdr.clientID(), hdr.requestID);
-            // }
+
+                switch (handled)
+                {
+                    case Handled::Replied: // nothing to do
+                        break;
+                    case Handled::YesRemoveArchiveListener:
+                        dataListener_--;
+                        if (dataListener_ == 0) {
+                            // return;
+                            // listeningThreadData.join();
+                        }
+                        break;
+                    case Handled::YesRemoveReadListener:
+                        dataListener_--;
+                        if (dataListener_ == 0) {
+                            // return;
+                        }
+                        break;
+                    case Handled::Yes:
+        //                write(Message::Received, false, hdr.clientID(), hdr.requestID);
+                        break;
+                    case Handled::No:
+                    default:
+                        std::stringstream ss;
+                        ss << "Unable to handle message " << hdr.message << " received on the data connection";
+                        error(ss.str(), hdr.clientID(), hdr.requestID);
+                }
+            }
         }
 
         // // Trigger cleanup of the workers
@@ -513,15 +474,14 @@ void ServerConnection::handle() {
         eckit::Log::debug() << "ServerConnection::handle - got [message=" << hdr.message << ",clientID="<< hdr.clientID() << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
 
         if (hdr.message == Message::Exit) {
-            // write(Message::Exit, true, hdr.clientID(), 0);
-            eckit::Log::info() << "Exit " << controlSocket().localPort() << "  client " <<  hdr.clientID() << std::endl;
-            write(Message::Exit, false, hdr.clientID(), 0);
+            if (remove(true, hdr.clientID())) {
 
-            if (remove(hdr.clientID())) {
-                eckit::Log::status() << "Exiting" << std::endl;
-                eckit::Log::info() << "Exiting" << std::endl;
+                write(Message::Exit, false, hdr.clientID(), 0);
 
-                return;
+                eckit::Log::status() << "Terminating CONTROL listener" << std::endl;
+                eckit::Log::info() << "Terminating CONTROL listener" << std::endl;
+
+                break;
             }
         } else {
 
@@ -555,9 +515,12 @@ void ServerConnection::handle() {
                 //     }
                 //     break;
                 case Handled::YesAddArchiveListener:
-                    dataListener_++;
-                    if (dataListener_ == 1) {
-                        listeningThreadData = std::thread([this] { listeningThreadLoopData(); });
+                    {
+                        std::lock_guard lock(handlerMutex_);
+                        dataListener_++;
+                        if (dataListener_ == 1) {
+                            listeningThreadData = std::thread([this] { listeningThreadLoopData(); });
+                        }
                     }
                     write(Message::Received, false, hdr.clientID(), hdr.requestID);
                     break;
@@ -569,9 +532,12 @@ void ServerConnection::handle() {
                 //     }
                 //     break;
                 case Handled::YesAddReadListener:
-                    dataListener_++;
-                    if (dataListener_ == 1) {
-                        listeningThreadData = std::thread([this] { listeningThreadLoopData(); });
+                    {
+                        std::lock_guard lock(handlerMutex_);
+                        dataListener_++;
+                        if (dataListener_ == 1) {
+                            listeningThreadData = std::thread([this] { listeningThreadLoopData(); });
+                        }
                     }
                     write(Message::Received, false, hdr.clientID(), hdr.requestID);
                     break;
@@ -586,6 +552,10 @@ void ServerConnection::handle() {
             }
         }
     }
+
+    if (listeningThreadData.joinable()) {
+        listeningThreadData.join();
+    }
 }
 
 void ServerConnection::handleException(std::exception_ptr e) {
diff --git a/src/fdb5/remote/server/ServerConnection.h b/src/fdb5/remote/server/ServerConnection.h
index 5338f9c47..0e89a538f 100644
--- a/src/fdb5/remote/server/ServerConnection.h
+++ b/src/fdb5/remote/server/ServerConnection.h
@@ -144,7 +144,8 @@ class ServerConnection : public Connection, public Handler {
 
 protected:
 
-    virtual bool remove(uint32_t clientID) = 0;
+    virtual bool remove(bool control, uint32_t clientID) = 0;
+    // virtual bool handlers() = 0;
 
     // std::map handlers_;
     Config config_;
@@ -164,6 +165,10 @@ class ServerConnection : public Connection, public Handler {
 
     eckit::net::TCPSocket controlSocket_;
 
+    std::mutex handlerMutex_;
+    size_t numControlConnection_;
+    size_t numDataConnection_;
+
 private:
 
     // data connection
diff --git a/src/fdb5/remote/server/StoreHandler.cc b/src/fdb5/remote/server/StoreHandler.cc
index cd7c41b8c..09f838726 100644
--- a/src/fdb5/remote/server/StoreHandler.cc
+++ b/src/fdb5/remote/server/StoreHandler.cc
@@ -88,58 +88,6 @@ Handled StoreHandler::handleControl(Message message, uint32_t clientID, uint32_t
     return Handled::No;
 }
 
-// Handled StoreHandler::handleData(Message message, uint32_t clientID, uint32_t requestID) {
-//     try {
-//         switch (message) {
-//             case Message::Flush: // notification that the client has sent all data for archival
-//                 return Handled::YesRemoveArchiveListener; // ????
-
-//             default: {
-//                 std::stringstream ss;
-//                 ss << "ERROR: Unexpected message recieved (" << message << "). ABORTING";
-//                 Log::status() << ss.str() << std::endl;
-//                 Log::error() << ss.str() << std::endl;
-//                 throw SeriousBug(ss.str(), Here());
-//             }
-//         }
-//     }
-//     catch (std::exception& e) {
-//         // n.b. more general than eckit::Exception
-//         error(e.what(), clientID, requestID);
-//     }
-//     catch (...) {
-//         error("Caught unexpected and unknown error", clientID, requestID);
-//     }
-//     return Handled::No;
-// }
-
-// Handled StoreHandler::handleData(Message message, uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) {
-//     try {
-//         switch (message) {
-//             case Message::Blob:
-//             case Message::MultiBlob:
-//                 queue(message, clientID, requestID, std::move(payload));
-//                 return Handled::Yes;
-
-//             default: {
-//                 std::stringstream ss;
-//                 ss << "ERROR: Unexpected message recieved (" << message << "). ABORTING";
-//                 Log::status() << ss.str() << std::endl;
-//                 Log::error() << ss.str() << std::endl;
-//                 throw SeriousBug(ss.str(), Here());
-//             }
-//         }
-//     }
-//     catch (std::exception& e) {
-//         // n.b. more general than eckit::Exception
-//         error(e.what(), clientID, requestID);
-//     }
-//     catch (...) {
-//         error("Caught unexpected and unknown error", clientID, requestID);
-//     }
-//     return Handled::No;
-// }
-
 void StoreHandler::read(uint32_t clientID, uint32_t requestID, const eckit::Buffer& payload) {
 
     if (!readLocationWorker_.joinable()) {
@@ -234,148 +182,6 @@ void StoreHandler::archiveBlob(const uint32_t clientID, const uint32_t requestID
     Connection::write(Message::Store, false, clientID, requestID, buffer, stream.position());
 }
 
-// struct archiveElem {
-//     uint32_t clientID;
-//     uint32_t requestID;
-//     eckit::Buffer payload;
-//     bool multiblob;
-
-//     archiveElem() : clientID(0), requestID(0), payload(0), multiblob(false) {}
-
-//     archiveElem(uint32_t clientID, uint32_t requestID, eckit::Buffer payload, bool multiblob) :
-//         clientID(clientID), requestID(requestID), payload(std::move(payload)), multiblob(multiblob) {}
-// };
-
-// size_t StoreHandler::archiveThreadLoop() {
-//     size_t totalArchived = 0;
-
-//     // Create a worker that will do the actual archiving
-
-//     static size_t queueSize(eckit::Resource("fdbServerMaxQueueSize", 32));
-//     eckit::Queue queue(queueSize);
-
-//     std::future worker = std::async(std::launch::async, [this, &queue] {
-//         size_t totalArchived = 0;
-
-//         archiveElem elem;
-
-//         try {
-//             long queuelen;
-//             while ((queuelen = queue.pop(elem)) != -1) {
-//                 if (elem.multiblob) {
-//                     // Handle MultiBlob
-
-//                     const char* firstData = static_cast(elem.payload.data());  // For pointer arithmetic
-//                     const char* charData = firstData;
-//                     while (size_t(charData - firstData) < elem.payload.size()) {
-//                         const MessageHeader* hdr = static_cast(static_cast(charData));
-//                         ASSERT(hdr->marker == StartMarker);
-//                         ASSERT(hdr->version == CurrentVersion);
-//                         ASSERT(hdr->message == Message::Blob);
-//                         ASSERT(hdr->clientID() == elem.clientID);
-//                         ASSERT(hdr->requestID == elem.requestID);
-//                         charData += sizeof(MessageHeader);
-
-//                         const void* payloadData = charData;
-//                         charData += hdr->payloadSize;
-
-//                         const decltype(EndMarker)* e = static_cast(static_cast(charData));
-//                         ASSERT(*e == EndMarker);
-//                         charData += sizeof(EndMarker);
-
-//                         archiveBlobPayload(elem.clientID, elem.requestID, payloadData, hdr->payloadSize);
-//                         totalArchived += 1;
-//                     }
-//                 }
-//                 else {
-//                     // Handle single blob
-//                     archiveBlobPayload(elem.clientID, elem.requestID, elem.payload.data(), elem.payload.size());
-//                     totalArchived += 1;
-//                 }
-//             }
-            
-//             // dataWrite(Message::Complete, 0);
-//         }
-//         catch (...) {
-//             // Ensure exception propagates across the queue back to the parent thread.
-//             queue.interrupt(std::current_exception());
-//             throw;
-//         }
-
-//         return totalArchived;
-//     });
-
-//     try {
-//         // The archive loop is the only thing that can listen on the data socket,
-//         // so we don't need to to anything clever here.
-
-//         // n.b. we also don't need to lock on read. We are the only thing that reads.
-
-//         while (true) {
-//             MessageHeader hdr;
-//             socketRead(&hdr, sizeof(hdr), dataSocket_);
-
-//             ASSERT(hdr.marker == StartMarker);
-//             ASSERT(hdr.version == CurrentVersion);
-//             //ASSERT(hdr.requestID == id);
-
-//             // Have we been told that we are done yet?
-//             if (hdr.message == Message::Flush)
-//                 break;
-
-//             ASSERT(hdr.message == Message::Blob || hdr.message == Message::MultiBlob);
-
-//             Buffer payload(receivePayload(hdr, dataSocket_));
-//             MemoryStream s(payload);
-
-//             eckit::FixedString<4> tail;
-//             socketRead(&tail, sizeof(tail), dataSocket_);
-//             ASSERT(tail == EndMarker);
-
-//             // Queueing payload
-
-//             size_t sz = payload.size();
-//             Log::debug() << "Queueing data: " << sz << std::endl;
-//             size_t queuelen = queue.emplace(archiveElem(hdr.clientID(), hdr.requestID, std::move(payload), hdr.message == Message::MultiBlob));
-//             Log::status() << "Queued data (" << queuelen << ", size=" << sz << ")" << std::endl;
-//             ;
-//             Log::debug() << "Queued data (" << queuelen << ", size=" << sz << ")"
-//                                   << std::endl;
-//             ;
-//         }
-
-//         // Trigger cleanup of the workers
-//         queue.close();
-
-//         // Complete reading the Flush instruction
-
-//         eckit::FixedString<4> tail;
-//         socketRead(&tail, sizeof(tail), dataSocket_);
-//         ASSERT(tail == EndMarker);
-
-//         // Ensure worker is done
-
-//         ASSERT(worker.valid());
-//         totalArchived = worker.get();  // n.b. use of async, get() propagates any exceptions.
-//     }
-//     catch (std::exception& e) {
-//         // n.b. more general than eckit::Exception
-//         std::string what(e.what());
-//         dataWrite(Message::Error, 0, 0, what.c_str(), what.length());
-//         queue.interrupt(std::current_exception());
-//         throw;
-//     }
-//     catch (...) {
-//         std::string what("Caught unexpected, unknown exception in retrieve worker");
-//         dataWrite(Message::Error, 0, 0, what.c_str(), what.length());
-//         queue.interrupt(std::current_exception());
-//         throw;
-//     }
-
-//     return totalArchived;
-// }
-
-
 void StoreHandler::flush(uint32_t clientID, uint32_t requestID, const eckit::Buffer& payload) {
 
     size_t numArchived = 0;
@@ -389,7 +195,9 @@ void StoreHandler::flush(uint32_t clientID, uint32_t requestID, const eckit::Buf
 
     ASSERT(numArchived == 0 || archiveFuture_.valid());
 
-    stores_[clientID]->flush();
+    auto it = stores_.find(clientID);
+    ASSERT(it != stores_.end());
+    it->second.store->flush();
 
     // if (archiveFuture_.valid()) {
     //     // Ensure that the expected number of fields have been written, and that the
@@ -409,27 +217,34 @@ void StoreHandler::flush(uint32_t clientID, uint32_t requestID, const eckit::Buf
     Log::status() << "Flush complete" << std::endl;
 }
 
-bool StoreHandler::remove(uint32_t clientID) {
-
-    std::lock_guard lock(storesMutex_);
-    std::stringstream ss;
-    ss << "remove " << clientID << " from " << stores_.size() << " clients - ids:";
-    for (auto it = stores_.begin(); it != stores_.end(); it++)
-        ss << "  " << it->first;
-    ss << std::endl;
-    eckit::Log::info() << ss.str();
+bool StoreHandler::remove(bool control, uint32_t clientID) {
+    
+    std::lock_guard lock(handlerMutex_);
 
     auto it = stores_.find(clientID);
     if (it != stores_.end()) {
-        stores_.erase(it);
+        if (control) {
+            it->second.controlConnection = false;
+            numControlConnection_--;
+        } else {
+            it->second.dataConnection = false;
+            numDataConnection_--;
+        }
+        if (!it->second.controlConnection && !it->second.dataConnection) {
+            stores_.erase(it);
+        }
     }
-
-    return stores_.empty();
+    return ((control ? numControlConnection_ : numDataConnection_) == 0);
 }
 
+// bool StoreHandler::handlers() {
+//     std::lock_guard lock(handlerMutex_);
+//     return stores_.empty();
+// }
+
 Store& StoreHandler::store(uint32_t clientID) {
 
-    std::lock_guard lock(storesMutex_);
+    std::lock_guard lock(handlerMutex_);
     auto it = stores_.find(clientID);
     if (it == stores_.end()) {
         std::string what("Requested Store has not been loaded id: " + std::to_string(clientID));
@@ -437,18 +252,22 @@ Store& StoreHandler::store(uint32_t clientID) {
         throw;
     }
 
-    return *(it->second);
+    return *(it->second.store);
 }
 
 Store& StoreHandler::store(uint32_t clientID, const Key& dbKey) {
 
-    std::lock_guard lock(storesMutex_);
+    std::lock_guard lock(handlerMutex_);
     auto it = stores_.find(clientID);
     if (it != stores_.end()) {
-        return *(it->second);
+        return *(it->second.store);
     }
 
-    return *(stores_[clientID] = StoreFactory::instance().build(dbKey, config_));
+    numControlConnection_++;
+    if (!single_) {
+        numDataConnection_++;
+    }
+    return *((stores_.emplace(clientID, StoreHelper(!single_, dbKey, config_)).first)->second.store);
 }
 
 }  // namespace fdb5::remote
diff --git a/src/fdb5/remote/server/StoreHandler.h b/src/fdb5/remote/server/StoreHandler.h
index fff40cf5d..0319e6d4b 100644
--- a/src/fdb5/remote/server/StoreHandler.h
+++ b/src/fdb5/remote/server/StoreHandler.h
@@ -15,6 +15,19 @@
 
 namespace fdb5::remote {
     
+//----------------------------------------------------------------------------------------------------------------------
+
+struct StoreHelper {
+    StoreHelper(bool dataConnection, const Key& dbKey, const Config& config) :
+        controlConnection(true), dataConnection(dataConnection),
+        store(StoreFactory::instance().build(dbKey, config)) {}
+
+    bool controlConnection;
+    bool dataConnection;
+    
+    std::unique_ptr store;
+};
+
 //----------------------------------------------------------------------------------------------------------------------
 class StoreHandler : public ServerConnection {
 public:  // methods
@@ -35,7 +48,8 @@ class StoreHandler : public ServerConnection {
     void readLocationThreadLoop();
     void writeToParent(const uint32_t clientID, const uint32_t requestID, std::unique_ptr dh);
 
-    bool remove(uint32_t clientID) override;
+    bool remove(bool control, uint32_t clientID) override;
+    // bool handlers() override;
 
     Store& store(uint32_t clientID);
     Store& store(uint32_t clientID, const Key& dbKey);
@@ -43,8 +57,7 @@ class StoreHandler : public ServerConnection {
 private:  // members
 
     // clientID --> Store
-    std::mutex storesMutex_;
-    std::map> stores_;
+    std::map stores_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/toc/TocEngine.cc b/src/fdb5/toc/TocEngine.cc
index 2cf29babb..1c253eb80 100644
--- a/src/fdb5/toc/TocEngine.cc
+++ b/src/fdb5/toc/TocEngine.cc
@@ -165,7 +165,8 @@ std::set TocEngine::databases(const std::set& ke
 
             for(std::vector::const_iterator dbpath = dbpaths.begin(); dbpath != dbpaths.end(); ++dbpath) {
 
-                Regex re("^" + *j + "/" + *dbpath + "$");
+                std::string regex = "^" + *j + "/" + *dbpath + "$";
+                Regex re(searchCaseSensitiveDB ? eckit::StringTools::lower(regex) : regex);
 
                 Log::debug() << " -> key i " << *i
                                      << " dbpath " << *dbpath
diff --git a/src/fdb5/tools/fdb-write.cc b/src/fdb5/tools/fdb-write.cc
index e038ee021..0c3d3f120 100644
--- a/src/fdb5/tools/fdb-write.cc
+++ b/src/fdb5/tools/fdb-write.cc
@@ -35,7 +35,7 @@ class FDBWrite : public fdb5::FDBTool {
 
     FDBWrite(int argc, char **argv) :
         fdb5::FDBTool(argc, argv),
-        verbose_(false) {
+        archivers_(1), verbose_(false) {
 
         options_.push_back(
                     new eckit::option::SimpleOption("include-filter",
@@ -50,6 +50,8 @@ class FDBWrite : public fdb5::FDBTool {
                                                          "List of comma separated key-values of modifiers to each message "
                                                          "int input data, e.g --modifiers=packingType=grib_ccsds,expver=0042"));
 
+        options_.push_back(new eckit::option::SimpleOption("archivers", "Number of archivers (default is 1). Input files are distributed among the user-specified archivers. For testing purposes only!"));
+
         options_.push_back(new eckit::option::SimpleOption("statistics", "Report timing statistics"));
 
         options_.push_back(new eckit::option::SimpleOption("verbose", "Print verbose output"));
@@ -58,6 +60,7 @@ class FDBWrite : public fdb5::FDBTool {
     std::string filterInclude_;
     std::string filterExclude_;
     std::string modifiers_;
+    long archivers_;
     bool verbose_;
 };
 
@@ -72,20 +75,19 @@ void FDBWrite::init(const eckit::option::CmdArgs& args)
     args.get("include-filter", filterInclude_);
     args.get("exclude-filter", filterExclude_);
     args.get("modifiers", modifiers_);
+    archivers_ = args.getLong("archivers", 1);
     verbose_ = args.getBool("verbose", false);
 }
 
 void FDBWrite::execute(const eckit::option::CmdArgs &args) {
 
-    fdb5::MessageArchiver archiver1(fdb5::Key(), false, verbose_, config(args));
-
-    archiver1.filters(filterInclude_, filterExclude_);
-    archiver1.modifiers(modifiers_);
-
-    fdb5::MessageArchiver archiver2(fdb5::Key(), false, verbose_, config(args));
-
-    archiver2.filters(filterInclude_, filterExclude_);
-    archiver2.modifiers(modifiers_);
+    std::vector> archivers;
+    for (int i=0; i a(new fdb5::MessageArchiver(fdb5::Key(), false, verbose_, config(args)));
+        a->filters(filterInclude_, filterExclude_);
+        a->modifiers(modifiers_);
+        archivers.push_back(std::move(a));
+    }
 
     for (size_t i = 0; i < args.count(); i++) {
 
@@ -95,16 +97,12 @@ void FDBWrite::execute(const eckit::option::CmdArgs &args) {
 
         std::unique_ptr dh ( path.fileHandle() );
 
-        if (i%2==0) {
-            archiver1.archive( *dh );
-        } else {
-            std::cout << "adding to archiver 2\n";
-            archiver2.archive( *dh );
-        }
+        archivers.at(i%archivers_)->archive( *dh );
     }
 
-    archiver1.flush();
-    archiver2.flush();
+    for (auto& a : archivers) {
+        a->flush();
+    }
 }
 
 //----------------------------------------------------------------------------------------------------------------------

From ec5e5573c145e1d219a49d06d801a1cc10b2d57c Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Sat, 2 Mar 2024 11:59:55 +0000
Subject: [PATCH 074/186] version bump

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 7dae53c49..7495c41d8 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.104
+5.11.105

From b5c2a775d753fbae1cf81ac4d513f16c1e9d740d Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 8 Mar 2024 12:46:20 +0000
Subject: [PATCH 075/186] fdb read returns a seekable handle

---
 src/fdb5/CMakeLists.txt    |   2 +
 src/fdb5/api/FDB.cc        |  56 +-------
 src/fdb5/io/FieldHandle.cc | 268 +++++++++++++++++++++++++++++++++++++
 src/fdb5/io/FieldHandle.h  |  94 +++++++++++++
 4 files changed, 366 insertions(+), 54 deletions(-)
 create mode 100644 src/fdb5/io/FieldHandle.cc
 create mode 100644 src/fdb5/io/FieldHandle.h

diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt
index 4bfae8bdd..498a01522 100644
--- a/src/fdb5/CMakeLists.txt
+++ b/src/fdb5/CMakeLists.txt
@@ -146,6 +146,8 @@ list( APPEND fdb5_srcs
     io/LustreFileHandle.h
     io/HandleGatherer.cc
     io/HandleGatherer.h
+    io/FieldHandle.cc
+    io/FieldHandle.h
     rules/MatchAlways.cc
     rules/MatchAlways.h
     rules/MatchAny.cc
diff --git a/src/fdb5/api/FDB.cc b/src/fdb5/api/FDB.cc
index f04547e36..ad58c4fa4 100644
--- a/src/fdb5/api/FDB.cc
+++ b/src/fdb5/api/FDB.cc
@@ -28,6 +28,7 @@
 #include "fdb5/api/helpers/FDBToolRequest.h"
 #include "fdb5/database/Key.h"
 #include "fdb5/io/HandleGatherer.h"
+#include "fdb5/io/FieldHandle.h"
 #include "fdb5/message/MessageDecoder.h"
 
 namespace fdb5 {
@@ -140,13 +141,6 @@ bool FDB::sorted(const metkit::mars::MarsRequest &request) {
     return sorted;
 }
 
-class ListElementDeduplicator : public metkit::hypercube::Deduplicator {
-public:
-    bool toReplace(const ListElement& existing, const ListElement& replacement) const override {
-        return existing.timestamp() < replacement.timestamp();
-    }
-};
-
 eckit::DataHandle* FDB::read(const eckit::URI& uri) {
     FieldLocation* loc = FieldLocationFactory::instance().build(uri.scheme(), uri);
     return loc->dataHandle();
@@ -165,54 +159,8 @@ eckit::DataHandle* FDB::read(const std::vector& uris, bool sorted) {
     
 
 eckit::DataHandle* FDB::read(ListIterator& it, bool sorted) {
-    eckit::Timer timer;
-    timer.start();
-
-    HandleGatherer result(sorted);
-    ListElement el;
-
-    static bool dedup = eckit::Resource("fdbDeduplicate;$FDB_DEDUPLICATE_FIELDS", false);
-    if (dedup) {
-        if (it.next(el)) {
-            // build the request representing the tensor-product of all retrieved fields
-            metkit::mars::MarsRequest cubeRequest = el.combinedKey().request();
-            std::vector elements{el};
-
-            while (it.next(el)) {
-                cubeRequest.merge(el.combinedKey().request());
-                elements.push_back(el);
-            }
 
-            // checking all retrieved fields against the hypercube, to remove duplicates
-            ListElementDeduplicator dedup;
-            metkit::hypercube::HyperCubePayloaded cube(cubeRequest, dedup);
-            for(auto el: elements) {
-                cube.add(el.combinedKey().request(), el);
-            }
-
-            if (cube.countVacant() > 0) {
-                std::stringstream ss;
-                ss << "No matching data for requests:" << std::endl;
-                for (auto req: cube.vacantRequests()) {
-                    ss << "    " << req << std::endl;
-                }
-                eckit::Log::warning() << ss.str() << std::endl;
-            }
-
-            for (size_t i=0; i< cube.size(); i++) {
-                ListElement element;
-                if (cube.find(i, element)) {
-                    result.add(element.location().dataHandle());
-                }
-            }
-        }
-    }
-    else {
-        while (it.next(el)) {
-            result.add(el.location().dataHandle());
-        }
-    }
-    return result.dataHandle();
+    return new FieldHandle(it);
 }
 
 eckit::DataHandle* FDB::retrieve(const metkit::mars::MarsRequest& request) {
diff --git a/src/fdb5/io/FieldHandle.cc b/src/fdb5/io/FieldHandle.cc
new file mode 100644
index 000000000..96d30938d
--- /dev/null
+++ b/src/fdb5/io/FieldHandle.cc
@@ -0,0 +1,268 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+#include 
+
+#include "eckit/io/MemoryHandle.h"
+#include "eckit/config/Resource.h"
+#include "eckit/log/Timer.h"
+#include "eckit/runtime/Metrics.h"
+#include "eckit/types/Types.h"
+
+#include "metkit/mars/MarsRequest.h"
+#include "metkit/hypercube/HyperCubePayloaded.h"
+
+#include "fdb5/io/FieldHandle.h"
+
+namespace fdb5 {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+class ListElementDeduplicator : public metkit::hypercube::Deduplicator {
+public:
+    bool toReplace(const ListElement& existing, const ListElement& replacement) const override {
+        return existing.timestamp() < replacement.timestamp();
+    }
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
+FieldHandle::FieldHandle(ListIterator& it) :
+    fields_({}), totalSize_(0), currentIdx_(0), current_(nullptr), currentMemoryHandle_(false), sorted_(false), seekable_(true) {
+    ListElement el;
+    eckit::Length largest = 0;
+
+    static bool dedup = eckit::Resource("fdbDeduplicate;$FDB_DEDUPLICATE_FIELDS", false);
+    if (dedup) {
+        if (it.next(el)) {
+            // build the request representing the tensor-product of all retrieved fields
+            metkit::mars::MarsRequest cubeRequest = el.combinedKey().request();
+            std::vector elements{el};
+
+            while (it.next(el)) {
+                cubeRequest.merge(el.combinedKey().request());
+                elements.push_back(el);
+            }
+
+            // checking all retrieved fields against the hypercube, to remove duplicates
+            ListElementDeduplicator dedup;
+            metkit::hypercube::HyperCubePayloaded cube(cubeRequest, dedup);
+            for(auto el: elements) {
+                cube.add(el.combinedKey().request(), el);
+            }
+
+            if (cube.countVacant() > 0) {
+                std::stringstream ss;
+                ss << "No matching data for requests:" << std::endl;
+                for (auto req: cube.vacantRequests()) {
+                    ss << "    " << req << std::endl;
+                }
+                eckit::Log::warning() << ss.str() << std::endl;
+            }
+
+            for (size_t i=0; i< cube.size(); i++) {
+                ListElement element;
+                if (cube.find(i, element)) {
+                    fields_.push_back(std::make_pair(el.location().length(), el.location().dataHandle()));
+                    eckit::Length len = el.location().length();
+                    totalSize_ += len;
+                    bool canSeek = el.location().dataHandle()->canSeek();
+                    if (!canSeek) {
+                        largest = std::max(largest, len);
+                        seekable_ = false;
+                    }
+                }
+            }
+        }
+    }
+    else {
+        while (it.next(el)) {
+            fields_.push_back(std::make_pair(el.location().length(), el.location().dataHandle()));
+
+            eckit::Length len = el.location().length();
+            totalSize_ += len;
+            bool canSeek = el.location().dataHandle()->canSeek();
+            if (!canSeek) {
+                largest = std::max(largest, len);
+                seekable_ = false;
+            }
+        }
+    }
+
+    if (!seekable_) {
+        // allocate a buffer that can fit the largest not seekable field (to avoid re-allocations)
+        buffer_ = eckit::Buffer(largest);
+    }
+}
+
+FieldHandle::~FieldHandle() {}
+
+void FieldHandle::openCurrent() {
+
+    if (current_ && currentMemoryHandle_) {
+        delete current_;
+        currentMemoryHandle_ = false;
+    }
+
+    if (currentIdx_ < fields_.size()) {
+
+        eckit::Length currentSize = fields_[currentIdx_].first;
+        current_ = fields_[currentIdx_].second;
+        current_->openForRead();
+
+        if (!current_->canSeek()) {
+            current_->read(buffer_.data(), currentSize);
+            current_ = new eckit::MemoryHandle(buffer_.data(), currentSize);
+            current_->openForRead();
+            currentMemoryHandle_ = true;
+        }
+    }
+}
+
+eckit::Length FieldHandle::openForRead() {
+
+    current_=0;
+    openCurrent();
+
+    return totalSize_;
+}
+
+long FieldHandle::read1(char* buffer, long length) {
+    if (currentIdx_ >= fields_.size()) {
+        return 0;
+    }
+
+    long n = current_->read(buffer, length);
+    if (n <= 0) {
+        current_->close();
+        currentIdx_++;
+        openCurrent();
+        return read1(buffer, length);
+    }
+    return n;
+}
+
+long FieldHandle::read(void* buffer, long length) {
+    char* p    = static_cast(buffer);
+    long n     = 0;
+    long total = 0;
+
+    while (length > 0 && (n = read1(p, length)) > 0) {
+        length -= n;
+        total += n;
+        p += n;
+    }
+
+    eckit::Log::debug() << "FieldHandle::read " << (total > 0 ? total : n) << std::endl;
+
+    return total > 0 ? total : n;
+}
+
+void FieldHandle::close() {
+    if (currentIdx_ != fields_.size()) {
+        current_->close();
+        if (currentMemoryHandle_) {
+            delete current_;
+        }
+    }
+    currentIdx_ = fields_.size();
+}
+
+void FieldHandle::rewind() {
+    if (currentIdx_ == 0 || seekable_) {
+        if (current_ && currentIdx_ < fields_.size()) {
+            current_->close();
+        }
+        currentIdx_ = 0;
+        openCurrent();
+    } else {
+        throw eckit::ReadError("rewind not supported");
+    }
+}
+
+void FieldHandle::print(std::ostream& s) const {
+    if (eckit::format(s) == eckit::Log::compactFormat) {
+        s << "FieldHandle";
+    }
+    else {
+        s << "FieldHandle[";
+        for (size_t i = 0; i < fields_.size(); i++) {
+            if (i != 0) {
+                s << ",(";
+            }
+            fields_[i].second->print(s);
+            s << ")";
+        }
+        s << ']';
+    }
+}
+
+eckit::Length FieldHandle::size() {
+    return totalSize_;
+}
+
+eckit::Length FieldHandle::estimate() {
+    return totalSize_;
+}
+
+// only partial seek: within the current message and forward
+bool FieldHandle::canSeek() const {
+    return true;
+}
+
+eckit::Offset FieldHandle::position() {
+    long long accumulated = 0;
+    for (size_t idx = 0; idx < currentIdx_; idx++) {
+        accumulated += fields_[idx].first;
+    }
+    return accumulated + (currentIdx_ >= fields_.size() ? eckit::Offset(0) : current_->position());
+}
+
+eckit::Offset FieldHandle::seek(const eckit::Offset& offset) {
+    if (current_ && currentIdx_ < fields_.size()) {
+        current_->close();
+    }
+
+    const long long seekto = offset;
+    long long accumulated  = 0;
+
+    if (!seekable_) { // check that the offset is within the current message of later
+        for (size_t idx = 0; idx < currentIdx_; idx++) {
+            accumulated += fields_[idx].first;
+        }
+
+        if (seekto < accumulated) {
+            throw eckit::UserError("Cannot seek backward to previous data fields");
+        }
+    }
+
+    accumulated  = 0;
+    for (currentIdx_ = 0; currentIdx_ < fields_.size(); ++currentIdx_) {
+        long long size = fields_[currentIdx_].first;
+        if (accumulated <= seekto && seekto < accumulated + size) {
+            openCurrent();
+            current_->seek(seekto - accumulated);
+            return offset;
+        }
+        accumulated += size;
+    }
+    // check if we went beyond EOF which is POSIX compliant, but we ASSERT so we find possible bugs
+    eckit::Offset beyond = seekto - accumulated;
+    ASSERT(not beyond);
+    return offset;
+}
+
+bool FieldHandle::compress(bool) {
+    return false;
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+
+}  // namespace eckit
diff --git a/src/fdb5/io/FieldHandle.h b/src/fdb5/io/FieldHandle.h
new file mode 100644
index 000000000..12ec5614d
--- /dev/null
+++ b/src/fdb5/io/FieldHandle.h
@@ -0,0 +1,94 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/// @author Emanuele Danovaro
+/// @date March 2024
+
+#pragma once
+
+#include "eckit/io/DataHandle.h"
+#include "eckit/io/Buffer.h"
+
+#include "fdb5/api/helpers/ListIterator.h"
+
+namespace fdb5 {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+class FieldHandle : public eckit::DataHandle {
+public:
+
+    // -- Contructors
+
+    FieldHandle(ListIterator& it);
+
+    // -- Destructor
+
+    ~FieldHandle() override;
+
+    // -- Operators
+
+    // -- Overridden methods
+
+    // From DataHandle
+
+    eckit::Length openForRead() override;
+
+    long read(void*, long) override;
+    void close() override;
+    void rewind() override;
+    void print(std::ostream&) const override;
+
+    eckit::Offset position() override;
+    eckit::Offset seek(const eckit::Offset&) override;
+    bool canSeek() const override;
+
+    bool compress(bool = false) override;
+
+    eckit::Length size() override;
+    eckit::Length estimate() override;
+
+    // std::string title() const override;
+
+    // // From Streamable
+
+    // void encode(Stream&) const override;
+    // const ReanimatorBase& reanimator() const override { return reanimator_; }
+
+    // // -- Class methods
+
+    // static const ClassSpec& classSpec() { return classSpec_; }
+
+private:
+    // -- Methods
+
+    void openCurrent();
+    void open();
+    long read1(char*, long);
+
+private:
+    // -- Members
+
+    std::vector> fields_;
+    eckit::Length totalSize_;
+    
+    size_t currentIdx_;
+    DataHandle* current_;
+    bool currentMemoryHandle_;
+
+    eckit::Buffer buffer_;
+    
+    bool sorted_;
+    bool seekable_;
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
+}  // namespace eckit

From 9ac8b942fafde3b4a932de14b0522ccf8f3206da Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 8 Mar 2024 12:47:24 +0000
Subject: [PATCH 076/186] version bump

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 7495c41d8..a07008d2f 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.105
+5.11.106

From d08408cf97ec70c0446330b9606d4127f1e1b7e3 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Sun, 10 Mar 2024 07:47:27 +0000
Subject: [PATCH 077/186] wip

---
 src/fdb5/io/FieldHandle.cc | 58 +++++++++++++++++++++++---------------
 src/fdb5/io/FieldHandle.h  |  4 +--
 2 files changed, 37 insertions(+), 25 deletions(-)

diff --git a/src/fdb5/io/FieldHandle.cc b/src/fdb5/io/FieldHandle.cc
index 96d30938d..bbd42d91e 100644
--- a/src/fdb5/io/FieldHandle.cc
+++ b/src/fdb5/io/FieldHandle.cc
@@ -35,7 +35,7 @@ class ListElementDeduplicator : public metkit::hypercube::DeduplicatorcanSeek();
@@ -84,7 +84,7 @@ FieldHandle::FieldHandle(ListIterator& it) :
     }
     else {
         while (it.next(el)) {
-            fields_.push_back(std::make_pair(el.location().length(), el.location().dataHandle()));
+            datahandles_.push_back(std::make_pair(el.location().length(), el.location().dataHandle()));
 
             eckit::Length len = el.location().length();
             totalSize_ += len;
@@ -98,11 +98,21 @@ FieldHandle::FieldHandle(ListIterator& it) :
 
     if (!seekable_) {
         // allocate a buffer that can fit the largest not seekable field (to avoid re-allocations)
-        buffer_ = eckit::Buffer(largest);
+        buffer_ = new char[largest];
     }
 }
 
-FieldHandle::~FieldHandle() {}
+FieldHandle::~FieldHandle() {
+    for (size_t i = 0; i < datahandles_.size(); i++) {
+        delete datahandles_[i].second;
+    }
+    if (current_ && currentMemoryHandle_) {
+        delete current_;
+    }
+    if (buffer_) {
+        delete[] buffer_;
+    }
+}
 
 void FieldHandle::openCurrent() {
 
@@ -111,15 +121,15 @@ void FieldHandle::openCurrent() {
         currentMemoryHandle_ = false;
     }
 
-    if (currentIdx_ < fields_.size()) {
+    if (currentIdx_ < datahandles_.size()) {
 
-        eckit::Length currentSize = fields_[currentIdx_].first;
-        current_ = fields_[currentIdx_].second;
+        eckit::Length currentSize = datahandles_[currentIdx_].first;
+        current_ = datahandles_[currentIdx_].second;
         current_->openForRead();
 
         if (!current_->canSeek()) {
-            current_->read(buffer_.data(), currentSize);
-            current_ = new eckit::MemoryHandle(buffer_.data(), currentSize);
+            current_->read(buffer_, currentSize);
+            current_ = new eckit::MemoryHandle(buffer_, currentSize);
             current_->openForRead();
             currentMemoryHandle_ = true;
         }
@@ -127,15 +137,16 @@ void FieldHandle::openCurrent() {
 }
 
 eckit::Length FieldHandle::openForRead() {
+    ASSERT(!current_);
 
-    current_=0;
+    currentIdx_=0;
     openCurrent();
 
     return totalSize_;
 }
 
 long FieldHandle::read1(char* buffer, long length) {
-    if (currentIdx_ >= fields_.size()) {
+    if (currentIdx_ >= datahandles_.size()) {
         return 0;
     }
 
@@ -166,18 +177,19 @@ long FieldHandle::read(void* buffer, long length) {
 }
 
 void FieldHandle::close() {
-    if (currentIdx_ != fields_.size()) {
+    if (currentIdx_ < datahandles_.size()) {
         current_->close();
         if (currentMemoryHandle_) {
             delete current_;
         }
+        current_=nullptr;
     }
-    currentIdx_ = fields_.size();
+    currentIdx_ = datahandles_.size();
 }
 
 void FieldHandle::rewind() {
     if (currentIdx_ == 0 || seekable_) {
-        if (current_ && currentIdx_ < fields_.size()) {
+        if (current_ && currentIdx_ < datahandles_.size()) {
             current_->close();
         }
         currentIdx_ = 0;
@@ -193,11 +205,11 @@ void FieldHandle::print(std::ostream& s) const {
     }
     else {
         s << "FieldHandle[";
-        for (size_t i = 0; i < fields_.size(); i++) {
+        for (size_t i = 0; i < datahandles_.size(); i++) {
             if (i != 0) {
                 s << ",(";
             }
-            fields_[i].second->print(s);
+            datahandles_[i].second->print(s);
             s << ")";
         }
         s << ']';
@@ -220,13 +232,13 @@ bool FieldHandle::canSeek() const {
 eckit::Offset FieldHandle::position() {
     long long accumulated = 0;
     for (size_t idx = 0; idx < currentIdx_; idx++) {
-        accumulated += fields_[idx].first;
+        accumulated += datahandles_[idx].first;
     }
-    return accumulated + (currentIdx_ >= fields_.size() ? eckit::Offset(0) : current_->position());
+    return accumulated + (currentIdx_ >= datahandles_.size() ? eckit::Offset(0) : current_->position());
 }
 
 eckit::Offset FieldHandle::seek(const eckit::Offset& offset) {
-    if (current_ && currentIdx_ < fields_.size()) {
+    if (current_ && currentIdx_ < datahandles_.size()) {
         current_->close();
     }
 
@@ -235,7 +247,7 @@ eckit::Offset FieldHandle::seek(const eckit::Offset& offset) {
 
     if (!seekable_) { // check that the offset is within the current message of later
         for (size_t idx = 0; idx < currentIdx_; idx++) {
-            accumulated += fields_[idx].first;
+            accumulated += datahandles_[idx].first;
         }
 
         if (seekto < accumulated) {
@@ -244,8 +256,8 @@ eckit::Offset FieldHandle::seek(const eckit::Offset& offset) {
     }
 
     accumulated  = 0;
-    for (currentIdx_ = 0; currentIdx_ < fields_.size(); ++currentIdx_) {
-        long long size = fields_[currentIdx_].first;
+    for (currentIdx_ = 0; currentIdx_ < datahandles_.size(); ++currentIdx_) {
+        long long size = datahandles_[currentIdx_].first;
         if (accumulated <= seekto && seekto < accumulated + size) {
             openCurrent();
             current_->seek(seekto - accumulated);
diff --git a/src/fdb5/io/FieldHandle.h b/src/fdb5/io/FieldHandle.h
index 12ec5614d..eb64ff47a 100644
--- a/src/fdb5/io/FieldHandle.h
+++ b/src/fdb5/io/FieldHandle.h
@@ -76,14 +76,14 @@ class FieldHandle : public eckit::DataHandle {
 private:
     // -- Members
 
-    std::vector> fields_;
+    std::vector> datahandles_;
     eckit::Length totalSize_;
     
     size_t currentIdx_;
     DataHandle* current_;
     bool currentMemoryHandle_;
 
-    eckit::Buffer buffer_;
+    char* buffer_{nullptr};
     
     bool sorted_;
     bool seekable_;

From c89ada3b52b59015901f7fb1f92b76c3617e94aa Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 11 Mar 2024 13:59:53 +0000
Subject: [PATCH 078/186] fixed fdb handler termination

---
 VERSION                      | 2 +-
 src/fdb5/remote/FdbServer.cc | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 7495c41d8..2869de7f6 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.105
+5.11.107
diff --git a/src/fdb5/remote/FdbServer.cc b/src/fdb5/remote/FdbServer.cc
index 739dfcdcf..51f2d5e0d 100644
--- a/src/fdb5/remote/FdbServer.cc
+++ b/src/fdb5/remote/FdbServer.cc
@@ -57,11 +57,13 @@ void FDBForker::run() {
         eckit::Log::info() << "FDB using Catalogue Handler" << std::endl;
         CatalogueHandler handler(socket_, config_);
         handler.handle();
+        stop();
     } 
     else if (config_.getString("type", "local") == "store" || (::getenv("FDB_IS_STORE") && ::getenv("FDB_IS_STORE")[0] == '1')) {
         eckit::Log::info() << "FDB using Store Handler" << std::endl;
         StoreHandler handler(socket_, config_);
         handler.handle();
+        stop();
     }
     // else {
     //     eckit::Log::info() << "FDB using Remote Handler" << std::endl;

From 4fb2629ae42602d967973488abf7ef2e3f956357 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 12 Mar 2024 10:53:10 +0000
Subject: [PATCH 079/186] fixed handler termination (closed archivalQueue)

---
 src/fdb5/remote/FdbServer.cc               | 19 ------------
 src/fdb5/remote/server/ServerConnection.cc | 34 ++--------------------
 2 files changed, 2 insertions(+), 51 deletions(-)

diff --git a/src/fdb5/remote/FdbServer.cc b/src/fdb5/remote/FdbServer.cc
index 51f2d5e0d..d15ddc559 100644
--- a/src/fdb5/remote/FdbServer.cc
+++ b/src/fdb5/remote/FdbServer.cc
@@ -57,19 +57,12 @@ void FDBForker::run() {
         eckit::Log::info() << "FDB using Catalogue Handler" << std::endl;
         CatalogueHandler handler(socket_, config_);
         handler.handle();
-        stop();
     } 
     else if (config_.getString("type", "local") == "store" || (::getenv("FDB_IS_STORE") && ::getenv("FDB_IS_STORE")[0] == '1')) {
         eckit::Log::info() << "FDB using Store Handler" << std::endl;
         StoreHandler handler(socket_, config_);
         handler.handle();
-        stop();
     }
-    // else {
-    //     eckit::Log::info() << "FDB using Remote Handler" << std::endl;
-    //     RemoteHandler handler(socket_, config_);
-    //     handler.handle();
-    // }
 }
 
 //----------------------------------------------------------------------------------------------------------------------
@@ -98,12 +91,8 @@ FDBServerThread::FDBServerThread(net::TCPSocket& socket, const Config& config) :
     config_(config) {}
 
 void FDBServerThread::run() {
-    std::cout << "FDBServerThread::run()" << std::endl;
     eckit::Log::info() << "FDB started handler thread" << std::endl;
 
-    // ServerConnection handler(socket_, config_);
-    // handler.handle();
-
     if (config_.getString("type", "local") == "catalogue" || (::getenv("FDB_IS_CAT") && ::getenv("FDB_IS_CAT")[0] == '1')) {
         eckit::Log::info() << "FDB using Catalogue Handler" << std::endl;
         CatalogueHandler handler(socket_, config_);
@@ -114,14 +103,6 @@ void FDBServerThread::run() {
         StoreHandler handler(socket_, config_);
         handler.handle();
     }
-    // else {
-    //     eckit::Log::info() << "FDB using Remote Handler" << std::endl;
-    //     RemoteHandler handler(socket_, config_);
-    //     handler.handle();
-    // }
-
-    // // RemoteHandler handler(socket_, config_);
-    // // handler.handle();
 }
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index 000c595a4..1a662f6d6 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -80,12 +80,6 @@ ServerConnection::~ServerConnection() {
     // We don't want to die before the worker threads are cleaned up
     waitForWorkers();
 
-    std::this_thread::sleep_for(std::chrono::milliseconds(500));
-
-    // And notify the client that we are done.
-//     eckit::Log::info() << "Sending exit message to client" << std::endl;
-// //    write(Message::Exit, true, 0, 0);
-//     write(Message::Exit, false, 0, 0);
     eckit::Log::info() << "Done" << std::endl;
 }
 
@@ -426,32 +420,14 @@ void ServerConnection::listeningThreadLoopData() {
             }
         }
 
-        // // Trigger cleanup of the workers
-        // auto q = archiveQueues_.find(archiverID);
-        // ASSERT(q != archiveQueues_.end());
-        // q->second.close();
-
-        // auto w = archiveFuture_.find(archiverID);
-        // ASSERT(w != archiveFuture_.end());
-        // // Ensure worker is done
-        // ASSERT(w->second.valid());
-        // totalArchived = worker.get();  // n.b. use of async, get() propagates any exceptions.
     }
     catch (std::exception& e) {
         // n.b. more general than eckit::Exception
         error(e.what(), hdr.clientID(), hdr.requestID);
-        // auto q = archiveQueues_.find(archiverID);
-        // if(q != archiveQueues_.end()) {
-        //     q->second.interrupt(std::current_exception());
-        // }
         throw;
     }
     catch (...) {
         error("Caught unexpected, unknown exception in retrieve worker", hdr.clientID(), hdr.requestID);
-        // auto q = archiveQueues_.find(archiverID);
-        // if(q != archiveQueues_.end()) {
-        //     q->second.interrupt(std::current_exception());
-        // }
         throw;
     }
 }
@@ -460,9 +436,6 @@ void ServerConnection::handle() {
     initialiseConnections();
  
     std::thread listeningThreadData;
-    // if (!single_) {
-    //     listeningThreadData = std::thread([this] { listeningThreadLoopData(); });
-    // }
 
     MessageHeader hdr;
 
@@ -556,6 +529,8 @@ void ServerConnection::handle() {
     if (listeningThreadData.joinable()) {
         listeningThreadData.join();
     }
+    ASSERT(archiveQueue_.empty());
+    archiveQueue_.close();
 }
 
 void ServerConnection::handleException(std::exception_ptr e) {
@@ -600,11 +575,6 @@ void ServerConnection::archiver() {
         // Start archive worker thread
         archiveFuture_ = std::async(std::launch::async, [this] { return archiveThreadLoop(); });
     }
-
-    // // Start data reader thread if double connection and we aren't already running it
-    // if (!single_ &&  !dataReader_.valid()) {
-    //     dataReader_ = std::async(std::launch::async, [this] { return listeningThreadLoopData(); });
-    // }
 }
 
 void ServerConnection::waitForWorkers() {

From 04c55018ef82e30d3f7f2b5c213a34387ac2c9e3 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 13 Mar 2024 06:43:49 +0000
Subject: [PATCH 080/186] fix seekable handle

---
 src/fdb5/database/Archiver.cc  |  8 --------
 src/fdb5/io/FieldHandle.cc     | 18 +++++++++++-------
 src/fdb5/remote/RemoteStore.cc |  9 +++++++--
 3 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/src/fdb5/database/Archiver.cc b/src/fdb5/database/Archiver.cc
index 725d4d901..98e988c81 100644
--- a/src/fdb5/database/Archiver.cc
+++ b/src/fdb5/database/Archiver.cc
@@ -31,14 +31,6 @@ Archiver::Archiver(const Config& dbConfig) :
 
 Archiver::~Archiver() {
     flush(); // certify that all sessions are flushed before closing them
-
-    // for (auto it = databases_.begin(); it != databases_.end(); it++) {
-    //     databases_.erase(it);
-    // }
-
-//    std::cout << databases_.size();
-
-    // databases_.clear(); //< explicitly delete the DBs before schemas are destroyed
 }
 
 void Archiver::archive(const Key &key, const void* data, size_t len) {
diff --git a/src/fdb5/io/FieldHandle.cc b/src/fdb5/io/FieldHandle.cc
index bbd42d91e..b0f79c193 100644
--- a/src/fdb5/io/FieldHandle.cc
+++ b/src/fdb5/io/FieldHandle.cc
@@ -70,10 +70,12 @@ FieldHandle::FieldHandle(ListIterator& it) :
             for (size_t i=0; i< cube.size(); i++) {
                 ListElement element;
                 if (cube.find(i, element)) {
-                    datahandles_.push_back(std::make_pair(el.location().length(), el.location().dataHandle()));
-                    eckit::Length len = el.location().length();
+                    eckit::Length len = element.location().length();
+                    eckit::DataHandle* dh = element.location().dataHandle();
+                    datahandles_.push_back(std::make_pair(len, dh));
+
                     totalSize_ += len;
-                    bool canSeek = el.location().dataHandle()->canSeek();
+                    bool canSeek = dh->canSeek();
                     if (!canSeek) {
                         largest = std::max(largest, len);
                         seekable_ = false;
@@ -84,11 +86,12 @@ FieldHandle::FieldHandle(ListIterator& it) :
     }
     else {
         while (it.next(el)) {
-            datahandles_.push_back(std::make_pair(el.location().length(), el.location().dataHandle()));
-
             eckit::Length len = el.location().length();
+            eckit::DataHandle* dh = el.location().dataHandle();
+            datahandles_.push_back(std::make_pair(len, dh));
+
             totalSize_ += len;
-            bool canSeek = el.location().dataHandle()->canSeek();
+            bool canSeek = dh->canSeek();
             if (!canSeek) {
                 largest = std::max(largest, len);
                 seekable_ = false;
@@ -128,7 +131,8 @@ void FieldHandle::openCurrent() {
         current_->openForRead();
 
         if (!current_->canSeek()) {
-            current_->read(buffer_, currentSize);
+            auto len = current_->read(buffer_, currentSize);
+            ASSERT(len == currentSize);
             current_ = new eckit::MemoryHandle(buffer_, currentSize);
             current_->openForRead();
             currentMemoryHandle_ = true;
diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/RemoteStore.cc
index c5e4634d9..1b67313b1 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/RemoteStore.cc
@@ -99,8 +99,12 @@ class FDBRemoteDataHandle : public DataHandle {
 
         // Are we now complete?
         if (msg.first == Message::Complete) {
-            complete_ = 0;
-            return 0;
+            if (overallPosition_ == eckit::Offset(0)) {
+                ASSERT(queue_.pop(msg) != -1);
+            } else {
+                complete_ = true;
+                return 0;
+            }
         }
 
         ASSERT(msg.first == Message::Blob);
@@ -116,6 +120,7 @@ class FDBRemoteDataHandle : public DataHandle {
 
     long bufferRead(void* pos, long sz) {
 
+
         ASSERT(currentBuffer_.size() != 0);
         ASSERT(pos_ < currentBuffer_.size());
 

From 1054385371902e034c45e5538da90d560108d6b4 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 13 Mar 2024 13:56:13 +0000
Subject: [PATCH 081/186] process termination

---
 VERSION                                    |   2 +-
 src/fdb5/api/helpers/ListIterator.cc       |   3 -
 src/fdb5/remote/Messages.h                 |   1 +
 src/fdb5/remote/client/ClientConnection.cc |  93 +++++++++++--
 src/fdb5/remote/client/ClientConnection.h  |   6 +-
 src/fdb5/remote/server/CatalogueHandler.cc |   4 +-
 src/fdb5/remote/server/ServerConnection.cc | 144 +++++++++++----------
 src/fdb5/remote/server/StoreHandler.cc     |   4 +-
 8 files changed, 165 insertions(+), 92 deletions(-)

diff --git a/VERSION b/VERSION
index 69d75dec6..a9bdc9c94 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.107
\ No newline at end of file
+5.11.109
\ No newline at end of file
diff --git a/src/fdb5/api/helpers/ListIterator.cc b/src/fdb5/api/helpers/ListIterator.cc
index 8e14b3efd..0bff75679 100644
--- a/src/fdb5/api/helpers/ListIterator.cc
+++ b/src/fdb5/api/helpers/ListIterator.cc
@@ -43,9 +43,6 @@ Key ListElement::combinedKey() const {
 }
 
 void ListElement::print(std::ostream &out, bool withLocation, bool withLength) const {
-    if (!withLocation && location_ && !location_->host().empty()) {
-        out << "host=" << location_->host() << ",";
-    }
     for (const auto& bit : keyParts_) {
         out << bit;
     }
diff --git a/src/fdb5/remote/Messages.h b/src/fdb5/remote/Messages.h
index 90cc02e99..00d14520d 100644
--- a/src/fdb5/remote/Messages.h
+++ b/src/fdb5/remote/Messages.h
@@ -46,6 +46,7 @@ enum class Message : uint16_t {
     Error,
     Stores,
     Schema,
+    Stop,
 
     // API calls to forward
     Flush = 100,
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index 07e002e7c..b1c1cea46 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -78,15 +78,19 @@ bool ClientConnection::remove(uint32_t clientID) {
         auto it = clients_.find(clientID);
 
         if (it != clients_.end()) {
-            Connection::write(Message::Exit, true, clientID, 0);
-            // TODO make the data connection dying automatically, when there are no more async writes
-            Connection::write(Message::Exit, false, clientID, 0);
+            Connection::write(Message::Stop, true, clientID, 0);
 
             clients_.erase(it);
         }
     }
 
     if (clients_.empty()) {
+        Connection::write(Message::Exit, true, 0, 0);
+        if (!single_) {
+            // TODO make the data connection dying automatically, when there are no more async writes
+            Connection::write(Message::Exit, false, 0, 0);
+        }
+
         ClientConnectionRouter::instance().deregister(*this);
     }
 
@@ -133,7 +137,8 @@ bool ClientConnection::connect(bool singleAttempt) {
         writeDataStartupMessage(serverSession);
 
         // And the connections are set up. Let everything start up!
-        listeningThread_ = std::thread([this] { listeningThreadLoop(); });
+        listeningControlThread_ = std::thread([this] { listeningControlThreadLoop(); });
+        listeningDataThread_ = std::thread([this] { listeningDataThreadLoop(); });
 
         connected_ = true;
         return true;
@@ -150,7 +155,12 @@ void ClientConnection::disconnect() {
     ASSERT(clients_.empty());
     if (connected_) {
 
-        listeningThread_.join();
+        if (listeningControlThread_.joinable()) {
+            listeningControlThread_.join();
+        }
+        if (listeningDataThread_.joinable()) {
+            listeningDataThread_.join();
+        }
 
         // Close both the control and data connections
         controlClient_.close();
@@ -328,7 +338,7 @@ eckit::SessionID ClientConnection::verifyServerStartupResponse() {
     return serverSession;
 }
 
-void ClientConnection::listeningThreadLoop() {
+void ClientConnection::listeningControlThreadLoop() {
 
     try {
 
@@ -337,15 +347,12 @@ void ClientConnection::listeningThreadLoop() {
 
         while (true) {
 
-            eckit::Buffer payload = Connection::readData(hdr);
+            eckit::Buffer payload = Connection::readControl(hdr);
 
-            eckit::Log::debug() << "ClientConnection::listeningThreadLoop - got [message=" << hdr.message << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
+            eckit::Log::debug() << "ClientConnection::listeningControlThreadLoop - got [message=" << hdr.message << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
 
             if (hdr.message == Message::Exit) {
-
-                if (clients_.empty()) {
-                    return;
-                }
+                return;
             } else {
                 if (hdr.clientID()) {
                     bool handled = false;
@@ -357,10 +364,10 @@ void ClientConnection::listeningThreadLoop() {
                         eckit::Log::status() << ss.str() << std::endl;
                         eckit::Log::error() << "Retrieving... " << ss.str() << std::endl;
                         throw eckit::SeriousBug(ss.str(), Here());
-
-                        ASSERT(false); // todo report the error
                     }
 
+                    ASSERT(hdr.control() || single_);
+
                     if (hdr.payloadSize == 0) {
                         if (it->second->blockingRequestId() == hdr.requestID) {
                             ASSERT(hdr.message == Message::Received);
@@ -397,4 +404,62 @@ void ClientConnection::listeningThreadLoop() {
     // ClientConnectionRouter::instance().deregister(*this);
 }
 
+void ClientConnection::listeningDataThreadLoop() {
+
+    try {
+
+        MessageHeader hdr;
+        eckit::FixedString<4> tail;
+
+        while (true) {
+
+            eckit::Buffer payload = Connection::readData(hdr);
+
+            eckit::Log::debug() << "ClientConnection::listeningDataThreadLoop - got [message=" << hdr.message << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
+
+            if (hdr.message == Message::Exit) {
+                return;
+            } else {
+                if (hdr.clientID()) {
+                    bool handled = false;
+                    auto it = clients_.find(hdr.clientID());
+                    if (it == clients_.end()) {
+                        std::stringstream ss;
+                        ss << "ERROR: Received [clientID="<< hdr.clientID() << ",requestID="<< hdr.requestID << ",message=" << hdr.message << ",payload=" << hdr.payloadSize << "]" << std::endl;
+                        ss << "Unexpected answer for clientID recieved (" << hdr.clientID() << "). ABORTING";
+                        eckit::Log::status() << ss.str() << std::endl;
+                        eckit::Log::error() << "Retrieving... " << ss.str() << std::endl;
+                        throw eckit::SeriousBug(ss.str(), Here());
+
+                        ASSERT(false); // todo report the error
+                    }
+
+                    ASSERT(!hdr.control());
+                    if (hdr.payloadSize == 0) {
+                        handled = it->second->handle(hdr.message, hdr.control(), hdr.requestID);
+                    }
+                    else {
+                        handled = it->second->handle(hdr.message, hdr.control(), hdr.requestID, std::move(payload));
+                    }
+
+                    if (!handled) {
+                        std::stringstream ss;
+                        ss << "ERROR: Unexpected message recieved (" << hdr.message << "). ABORTING";
+                        eckit::Log::status() << ss.str() << std::endl;
+                        eckit::Log::error() << "Client Retrieving... " << ss.str() << std::endl;
+                        throw eckit::SeriousBug(ss.str(), Here());
+                    }
+                }
+            }
+        }
+
+    // We don't want to let exceptions escape inside a worker thread.
+    } catch (const std::exception& e) {
+//        ClientConnectionRouter::instance().handleException(std::make_exception_ptr(e));
+    } catch (...) {
+//        ClientConnectionRouter::instance().handleException(std::current_exception());
+    }
+    // ClientConnectionRouter::instance().deregister(*this);
+}
+
 }  // namespace fdb5::remote
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index ae44a0f0e..51846fd60 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -75,7 +75,8 @@ class ClientConnection : protected Connection {
 
     void handleError(const MessageHeader& hdr, eckit::Buffer buffer);
 
-    void listeningThreadLoop();
+    void listeningControlThreadLoop();
+    void listeningDataThreadLoop();
     void dataWriteThreadLoop();
 
     eckit::net::TCPSocket& controlSocket() override { return controlClient_; }
@@ -96,7 +97,8 @@ class ClientConnection : protected Connection {
     std::mutex clientsMutex_;
     std::map clients_;
 
-    std::thread listeningThread_;
+    std::thread listeningControlThread_;
+    std::thread listeningDataThread_;
     
     std::mutex requestMutex_;
 
diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc
index 3aad56dae..75bda0b63 100644
--- a/src/fdb5/remote/server/CatalogueHandler.cc
+++ b/src/fdb5/remote/server/CatalogueHandler.cc
@@ -279,7 +279,7 @@ void CatalogueHandler::schema(uint32_t clientID, uint32_t requestID, eckit::Buff
         stream << schema;
     }
 
-    write(Message::Received, false, clientID, requestID, schemaBuffer.data(), stream.position());
+    write(Message::Received, true, clientID, requestID, schemaBuffer.data(), stream.position());
 }
 
 void CatalogueHandler::stores(uint32_t clientID, uint32_t requestID) {
@@ -342,7 +342,7 @@ void CatalogueHandler::stores(uint32_t clientID, uint32_t requestID) {
                 s << ee;
             }
         }
-        write(Message::Received, false, clientID, requestID, startupBuffer.data(), s.position());
+        write(Message::Received, true, clientID, requestID, startupBuffer.data(), s.position());
     }
 }
 
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index 1a662f6d6..6abc5f093 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -374,13 +374,12 @@ void ServerConnection::listeningThreadLoopData() {
             eckit::Buffer payload = readData(hdr); // READ DATA
 
             if (hdr.message == Message::Exit) {
-                if (remove(false, hdr.clientID())) {
+                ASSERT(hdr.clientID() == 0);
 
-                    eckit::Log::status() << "Terminating DATA listener" << std::endl;
-                    eckit::Log::info() << "Terminating DATA listener" << std::endl;
+                eckit::Log::status() << "Terminating DATA listener" << std::endl;
+                eckit::Log::info() << "Terminating DATA listener" << std::endl;
 
-                    break;
-                }
+                break;
             } else {
                 
                 Handled handled;
@@ -446,82 +445,91 @@ void ServerConnection::handle() {
         eckit::Buffer payload = readControl(hdr); // READ CONTROL
         eckit::Log::debug() << "ServerConnection::handle - got [message=" << hdr.message << ",clientID="<< hdr.clientID() << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
 
-        if (hdr.message == Message::Exit) {
-            if (remove(true, hdr.clientID())) {
+        if (hdr.message == Message::Stop) {
+            ASSERT(hdr.clientID());
+            remove(true, hdr.clientID());
 
-                write(Message::Exit, false, hdr.clientID(), 0);
+        } else {
+            if (hdr.message == Message::Exit) {
+                ASSERT(hdr.clientID() == 0);
+
+                write(Message::Exit, true, 0, 0);
+                if (!single_) {
+                    write(Message::Exit, false, 0, 0);
+                }
 
                 eckit::Log::status() << "Terminating CONTROL listener" << std::endl;
                 eckit::Log::info() << "Terminating CONTROL listener" << std::endl;
 
                 break;
             }
-        } else {
+            else {
 
-            Handled handled = Handled::No;
-            ASSERT(single_ || hdr.control());
+                Handled handled = Handled::No;
+                ASSERT(single_ || hdr.control());
 
-            if (payload.size()) {
-                if (hdr.control()) {
-                    handled = handleControl(hdr.message, hdr.clientID(), hdr.requestID, std::move(payload));
-                } else {
-                    handled = handleData(hdr.message, hdr.clientID(), hdr.requestID, std::move(payload));
-                }
-            } else {
-                if (hdr.control()) {
-                    handled = handleControl(hdr.message, hdr.clientID(), hdr.requestID);
+                if (payload.size()) {
+                    if (hdr.control()) {
+                        handled = handleControl(hdr.message, hdr.clientID(), hdr.requestID, std::move(payload));
+                    } else {
+                        handled = handleData(hdr.message, hdr.clientID(), hdr.requestID, std::move(payload));
+                    }
                 } else {
-                    handled = handleData(hdr.message, hdr.clientID(), hdr.requestID);
+                    if (hdr.control()) {
+                        handled = handleControl(hdr.message, hdr.clientID(), hdr.requestID);
+                    } else {
+                        handled = handleData(hdr.message, hdr.clientID(), hdr.requestID);
+                    }
                 }
-            }
-            
-
-            switch (handled)
-            {
-                case Handled::Replied: // nothing to do
-                    break;
-                // case Handled::YesRemoveArchiveListener:
-                //     dataListener_--;
-                //     if (dataListener_ == 0) {
-                //         //return;
-                //         // listeningThreadData.join();
-                //     }
-                //     break;
-                case Handled::YesAddArchiveListener:
-                    {
-                        std::lock_guard lock(handlerMutex_);
-                        dataListener_++;
-                        if (dataListener_ == 1) {
-                            listeningThreadData = std::thread([this] { listeningThreadLoopData(); });
+                
+
+                switch (handled)
+                {
+                    case Handled::Replied: // nothing to do
+                        break;
+                    // case Handled::YesRemoveArchiveListener:
+                    //     dataListener_--;
+                    //     if (dataListener_ == 0) {
+                    //         //return;
+                    //         // listeningThreadData.join();
+                    //     }
+                    //     break;
+                    case Handled::YesAddArchiveListener:
+                        {
+                            std::lock_guard lock(handlerMutex_);
+                            dataListener_++;
+                            if (dataListener_ == 1) {
+                                listeningThreadData = std::thread([this] { listeningThreadLoopData(); });
+                            }
                         }
-                    }
-                    write(Message::Received, false, hdr.clientID(), hdr.requestID);
-                    break;
-                // case Handled::YesRemoveReadListener:
-                //     dataListener_--;
-                //     if (dataListener_ == 0) {
-                //         //return;
-                //         // listeningThreadData.join();
-                //     }
-                //     break;
-                case Handled::YesAddReadListener:
-                    {
-                        std::lock_guard lock(handlerMutex_);
-                        dataListener_++;
-                        if (dataListener_ == 1) {
-                            listeningThreadData = std::thread([this] { listeningThreadLoopData(); });
+                        write(Message::Received, true, hdr.clientID(), hdr.requestID);
+                        break;
+                    // case Handled::YesRemoveReadListener:
+                    //     dataListener_--;
+                    //     if (dataListener_ == 0) {
+                    //         //return;
+                    //         // listeningThreadData.join();
+                    //     }
+                    //     break;
+                    case Handled::YesAddReadListener:
+                        {
+                            std::lock_guard lock(handlerMutex_);
+                            dataListener_++;
+                            if (dataListener_ == 1) {
+                                listeningThreadData = std::thread([this] { listeningThreadLoopData(); });
+                            }
                         }
-                    }
-                    write(Message::Received, false, hdr.clientID(), hdr.requestID);
-                    break;
-                case Handled::Yes:
-                    write(Message::Received, false, hdr.clientID(), hdr.requestID);
-                    break;
-                case Handled::No:
-                default:
-                    std::stringstream ss;
-                    ss << "Unable to handle message " << hdr.message;
-                    error(ss.str(), hdr.clientID(), hdr.requestID);
+                        write(Message::Received, true, hdr.clientID(), hdr.requestID);
+                        break;
+                    case Handled::Yes:
+                        write(Message::Received, true, hdr.clientID(), hdr.requestID);
+                        break;
+                    case Handled::No:
+                    default:
+                        std::stringstream ss;
+                        ss << "Unable to handle message " << hdr.message;
+                        error(ss.str(), hdr.clientID(), hdr.requestID);
+                }
             }
         }
     }
diff --git a/src/fdb5/remote/server/StoreHandler.cc b/src/fdb5/remote/server/StoreHandler.cc
index 09f838726..82eaf6738 100644
--- a/src/fdb5/remote/server/StoreHandler.cc
+++ b/src/fdb5/remote/server/StoreHandler.cc
@@ -179,7 +179,7 @@ void StoreHandler::archiveBlob(const uint32_t clientID, const uint32_t requestID
     MemoryStream stream(buffer);
 //    stream << archiverID;
     stream << (*loc);
-    Connection::write(Message::Store, false, clientID, requestID, buffer, stream.position());
+    Connection::write(Message::Store, true, clientID, requestID, buffer, stream.position());
 }
 
 void StoreHandler::flush(uint32_t clientID, uint32_t requestID, const eckit::Buffer& payload) {
@@ -248,7 +248,7 @@ Store& StoreHandler::store(uint32_t clientID) {
     auto it = stores_.find(clientID);
     if (it == stores_.end()) {
         std::string what("Requested Store has not been loaded id: " + std::to_string(clientID));
-        write(Message::Error, false, 0, 0, what.c_str(), what.length());
+        write(Message::Error, true, 0, 0, what.c_str(), what.length());
         throw;
     }
 

From b8085b2efff471855ccb8defa412a2375a2b5d08 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 13 Mar 2024 14:31:33 +0000
Subject: [PATCH 082/186] wait for future before dtor

---
 VERSION                                    | 2 +-
 src/fdb5/remote/client/ClientConnection.cc | 4 ++++
 src/fdb5/remote/server/ServerConnection.cc | 5 ++++-
 src/fdb5/remote/server/StoreHandler.cc     | 7 ++-----
 4 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/VERSION b/VERSION
index a9bdc9c94..66e3b5814 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.109
\ No newline at end of file
+5.11.110
\ No newline at end of file
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index b1c1cea46..68064dd49 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -155,6 +155,10 @@ void ClientConnection::disconnect() {
     ASSERT(clients_.empty());
     if (connected_) {
 
+        if (dataWriteFuture_.valid()) {
+            dataWriteFuture_.wait();
+        }
+
         if (listeningControlThread_.joinable()) {
             listeningControlThread_.join();
         }
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index 6abc5f093..db4225be5 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -80,6 +80,10 @@ ServerConnection::~ServerConnection() {
     // We don't want to die before the worker threads are cleaned up
     waitForWorkers();
 
+    if (archiveFuture_.valid()) {
+        archiveFuture_.wait();
+    }
+    
     eckit::Log::info() << "Done" << std::endl;
 }
 
@@ -363,7 +367,6 @@ size_t ServerConnection::archiveThreadLoop() {
 void ServerConnection::listeningThreadLoopData() {
 
     MessageHeader hdr;
-    uint32_t archiverID;
 
     try {
 
diff --git a/src/fdb5/remote/server/StoreHandler.cc b/src/fdb5/remote/server/StoreHandler.cc
index 82eaf6738..12462ff6f 100644
--- a/src/fdb5/remote/server/StoreHandler.cc
+++ b/src/fdb5/remote/server/StoreHandler.cc
@@ -170,15 +170,12 @@ void StoreHandler::archiveBlob(const uint32_t clientID, const uint32_t requestID
     
     Store& ss = store(clientID, dbKey);
 
-    auto futureLocation = ss.archive(idxKey, charData + s.position(), length - s.position());
+    std::unique_ptr location = ss.archive(idxKey, charData + s.position(), length - s.position());
     Log::status() << "Archiving done: " << ss_key.str() << std::endl;
     
-    auto loc = futureLocation.get();
-
     eckit::Buffer buffer(16 * 1024);
     MemoryStream stream(buffer);
-//    stream << archiverID;
-    stream << (*loc);
+    stream << (*location);
     Connection::write(Message::Store, true, clientID, requestID, buffer, stream.position());
 }
 

From 2a3fac10237f982792d992904e93791350b6eaad Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 15 Mar 2024 13:57:10 +0100
Subject: [PATCH 083/186] cleanup FDB remote blocking requests

---
 src/fdb5/remote/client/Client.cc           | 73 ++++------------------
 src/fdb5/remote/client/Client.h            |  7 ---
 src/fdb5/remote/client/ClientConnection.cc | 29 +++++----
 src/fdb5/remote/client/ClientConnection.h  | 10 +--
 src/fdb5/remote/server/ServerConnection.cc | 56 +++--------------
 src/fdb5/remote/server/ServerConnection.h  | 18 +-----
 src/fdb5/remote/server/StoreHandler.cc     | 23 -------
 7 files changed, 40 insertions(+), 176 deletions(-)

diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index 54867b89e..b681700c4 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -19,18 +19,6 @@ namespace fdb5::remote {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-// uint32_t Client::clientId_=0;
-
-// Client::Client(const eckit::net::Endpoint& endpoint, std::string domain) :
-//     endpoint_(endpoint),
-//     fullyQualifiedEndpoint_(endpoint.host()+domain, endpoint.port()),
-//     connection_(*(ClientConnectionRouter::instance().connection(*this))),
-//     blockingRequestId_(0) {
-
-//     std::lock_guard lock(idMutex_);
-//     id_ = ++clientId_;
-// }
-
 void Client::setClientID() {
     static std::mutex idMutex_;
     static uint32_t clientId_ = 0;
@@ -39,24 +27,17 @@ void Client::setClientID() {
     id_ = ++clientId_;
 }
 
-
-
 Client::Client(const eckit::net::Endpoint& endpoint, const std::string& defaultEndpoint) :
-    connection_(ClientConnectionRouter::instance().connection(endpoint, defaultEndpoint)),
-    blockingRequestId_(0) {
-    
-    setClientID();
+    connection_(ClientConnectionRouter::instance().connection(endpoint, defaultEndpoint)) {
 
+    setClientID();
     connection_.add(*this);
 }
 
-
 Client::Client(const std::vector>& endpoints) :
-    connection_(ClientConnectionRouter::instance().connection(endpoints)),
-    blockingRequestId_(0) {
+    connection_(ClientConnectionRouter::instance().connection(endpoints)) {
 
     setClientID();
-
     connection_.add(*this);
 }
 
@@ -64,43 +45,20 @@ Client::~Client() {
     connection_.remove(id_);
 }
 
-bool Client::response(uint32_t requestID) {
-    ASSERT(requestID == blockingRequestId_);
-
-    eckit::Log::debug() << " response to blocking request " << requestID << std::endl;
-
-    promise_.set_value(true);
-    return true;
-}
-
-bool Client::response(uint32_t requestID, eckit::Buffer&& payload) {
-    ASSERT(requestID == blockingRequestId_);
-
-    eckit::Log::debug() << " response to blocking request " << requestID << " - payload size: " << payload.size() << std::endl;
-
-    payloadPromise_.set_value(std::move(payload));
-    return true;
-}
-
 void Client::controlWriteCheckResponse(Message msg, uint32_t requestID, bool dataListener, const void* payload, uint32_t payloadLength) {
 
     ASSERT(requestID);
     ASSERT(!(!payloadLength ^ !payload));
     std::lock_guard lock(blockingRequestMutex_);
-    
-    promise_ = {};
-    std::future f = promise_.get_future();
-
-    blockingRequestId_=requestID;
 
+    std::vector> data;
     if (payloadLength) {
-        connection_.controlWrite(*this, msg, blockingRequestId_, dataListener, std::vector>{{payload, payloadLength}});
-    } else {
-        connection_.controlWrite(*this, msg, blockingRequestId_, dataListener);
+        data.push_back(std::make_pair(payload, payloadLength));
     }
+    std::future f = connection_.controlWrite(*this, msg, requestID, dataListener, data);
 
-    f.get();
-    blockingRequestId_=0;
+    eckit::Buffer buf = f.get();
+    ASSERT(buf.size() == 0);
 }
 
 eckit::Buffer Client::controlWriteReadResponse(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
@@ -108,21 +66,14 @@ eckit::Buffer Client::controlWriteReadResponse(Message msg, uint32_t requestID,
     ASSERT(requestID);
     ASSERT(!(!payloadLength ^ !payload));
     std::lock_guard lock(blockingRequestMutex_);
-
-    payloadPromise_ = {};
-    std::future f = payloadPromise_.get_future();
-
-    blockingRequestId_=requestID;
-
+    
+    std::vector> data{};
     if (payloadLength) {
-        connection_.controlWrite(*this, msg, blockingRequestId_, false, std::vector>{{payload, payloadLength}});
-    } else {
-        connection_.controlWrite(*this, msg, blockingRequestId_, false);
+        data.push_back(std::make_pair(payload, payloadLength));
     }
+    std::future f = connection_.controlWrite(*this, msg, requestID, false, data);
 
     eckit::Buffer buf = f.get();
-    blockingRequestId_=0;
-
     return buf;
 }
 
diff --git a/src/fdb5/remote/client/Client.h b/src/fdb5/remote/client/Client.h
index 7f79b0c44..680d504a2 100644
--- a/src/fdb5/remote/client/Client.h
+++ b/src/fdb5/remote/client/Client.h
@@ -56,10 +56,6 @@ class Client : eckit::NonCopyable {
     virtual bool handle(Message message, bool control, uint32_t requestID, eckit::Buffer&& payload) = 0;
     virtual void handleException(std::exception_ptr e) = 0;
 
-    bool response(uint32_t requestID);
-    bool response(uint32_t requestID, eckit::Buffer&& payload);
-    uint32_t blockingRequestId() { return blockingRequestId_; }
-
 protected:
     
     ClientConnection& connection_;
@@ -71,9 +67,6 @@ class Client : eckit::NonCopyable {
     uint32_t id_;
 
     std::mutex blockingRequestMutex_;
-    uint32_t blockingRequestId_;
-    std::promise promise_;
-    std::promise payloadPromise_;
 };
 
 }
\ No newline at end of file
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index 68064dd49..779e6f8df 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -186,11 +186,14 @@ eckit::LocalConfiguration ClientConnection::availableFunctionality() const {
 
 // -----------------------------------------------------------------------------------------------------
 
-void ClientConnection::controlWrite(Client& client, Message msg, uint32_t requestID, bool dataListener, std::vector> data) {
+std::future ClientConnection::controlWrite(Client& client, Message msg, uint32_t requestID, bool dataListener, std::vector> data) {
     auto it = clients_.find(client.clientId());
     ASSERT(it != clients_.end());
 
+    auto pp = promises_.emplace(requestID, std::promise{});
     Connection::write(msg, true, client.clientId(), requestID, data);
+
+    return pp.first->second.get_future();
 }
 
 void ClientConnection::dataWrite(DataWriteRequest& r) {
@@ -199,7 +202,7 @@ void ClientConnection::dataWrite(DataWriteRequest& r) {
 
 void ClientConnection::dataWrite(Client& client, remote::Message msg, uint32_t requestID, std::vector> data) {
 
-    static size_t maxQueueLength = eckit::Resource("fdbDataWriteQueueLength;$FDB_DATA_WRITE_QUEUE_LENGTH", 200);
+    static size_t maxQueueLength = eckit::Resource("fdbDataWriteQueueLength;$FDB_DATA_WRITE_QUEUE_LENGTH", 2);
     auto it = clients_.find(client.clientId());
     ASSERT(it != clients_.end());
 
@@ -230,7 +233,6 @@ void ClientConnection::dataWrite(Client& client, remote::Message msg, uint32_t r
     }
 
     dataWriteQueue_->emplace(&client, msg, requestID, std::move(buffer));
-//    Connection::write(msg, false, client.clientId(), requestID, data);
 }
 
 
@@ -286,7 +288,6 @@ void ClientConnection::writeControlStartupMessage() {
     //       essentially JSON) over the wire for flexibility.
     s << availableFunctionality().get();
 
-    // controlWrite(Message::Startup, 0, 0, std::vector>{{payload, s.position()}});
     Connection::write(Message::Startup, true, 0, 0, payload, s.position());
 }
 
@@ -298,7 +299,6 @@ void ClientConnection::writeDataStartupMessage(const eckit::SessionID& serverSes
     s << sessionID_;
     s << serverSession;
 
-    // dataWrite(Message::Startup, 0, 0, std::vector>{{payload, s.position()}});
     Connection::write(Message::Startup, false, 0, 0, payload, s.position());
 }
 
@@ -372,18 +372,21 @@ void ClientConnection::listeningControlThreadLoop() {
 
                     ASSERT(hdr.control() || single_);
 
-                    if (hdr.payloadSize == 0) {
-                        if (it->second->blockingRequestId() == hdr.requestID) {
+                    auto pp = promises_.find(hdr.requestID);
+                    if (pp != promises_.end()) {
+                        if (hdr.payloadSize == 0) {
                             ASSERT(hdr.message == Message::Received);
-                            handled = it->second->response(hdr.requestID);
+                            pp->second.set_value(eckit::Buffer(0));
                         } else {
+                            pp->second.set_value(std::move(payload));
+                        }
+                        promises_.erase(pp);
+                        handled = true;
+                    } else {
+                        if (hdr.payloadSize == 0) {
                             handled = it->second->handle(hdr.message, hdr.control(), hdr.requestID);
                         }
-                    }
-                    else {
-                        if (it->second->blockingRequestId() == hdr.requestID) {
-                            handled = it->second->response(hdr.requestID, std::move(payload));
-                        } else {
+                        else {
                             handled = it->second->handle(hdr.message, hdr.control(), hdr.requestID, std::move(payload));
                         }
                     }
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index 51846fd60..6252ff01f 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -24,12 +24,6 @@
 #include "fdb5/remote/Messages.h"
 #include "fdb5/remote/Connection.h"
 
-namespace eckit {
-
-class Buffer;
-
-}
-
 namespace fdb5::remote {
 
 class Client;
@@ -44,7 +38,7 @@ class ClientConnection : protected Connection {
 
     virtual ~ClientConnection();
 
-    void controlWrite(Client& client, Message msg, uint32_t requestID, bool startDataListener, std::vector> data={});
+    std::future controlWrite(Client& client, Message msg, uint32_t requestID, bool startDataListener, std::vector> data={});
     void dataWrite(Client& client, Message msg, uint32_t requestID, std::vector> data={});
 
     void add(Client& client);
@@ -108,6 +102,8 @@ class ClientConnection : protected Connection {
 
     bool connected_; 
 
+    std::map> promises_;
+
     std::mutex dataWriteQueueMutex_;
     std::unique_ptr> dataWriteQueue_;
     std::future dataWriteFuture_;
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index db4225be5..c42fc1efd 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -62,9 +62,6 @@ std::vector intersection(const eckit::LocalConfiguration& c1, const eckit::
 
 } // namespace
 
-
-//size_t ServerConnection::queueSize_ = eckit::Resource("fdbServerMaxQueueSize", 32);
-
 ServerConnection::ServerConnection(eckit::net::TCPSocket& socket, const Config& config) :
         Connection(), config_(config),
         dataListenHostname_(config.getString("dataListenHostname", "")),
@@ -250,7 +247,6 @@ void ServerConnection::initialiseConnections() {
         s << sessionID_;
         s << dataEndpoint;
         s << agreedConf_.get();
-        // s << storeEndpoint; // xxx single-store case only: we cant do this with multiple stores // For now, dont send the store endpoint to the client 
 
         eckit::Log::debug() << "Protocol negotiation - configuration: " << agreedConf_ < lock(handlerMutex_);
+                            dataListener_--;
+                            if (dataListener_ == 0) {
+                                return;
+                            }
+                            break;
                         }
-                        break;
+                    case Handled::Replied: // nothing to do
                     case Handled::Yes:
         //                write(Message::Received, false, hdr.clientID(), hdr.requestID);
                         break;
@@ -490,30 +475,7 @@ void ServerConnection::handle() {
                 {
                     case Handled::Replied: // nothing to do
                         break;
-                    // case Handled::YesRemoveArchiveListener:
-                    //     dataListener_--;
-                    //     if (dataListener_ == 0) {
-                    //         //return;
-                    //         // listeningThreadData.join();
-                    //     }
-                    //     break;
                     case Handled::YesAddArchiveListener:
-                        {
-                            std::lock_guard lock(handlerMutex_);
-                            dataListener_++;
-                            if (dataListener_ == 1) {
-                                listeningThreadData = std::thread([this] { listeningThreadLoopData(); });
-                            }
-                        }
-                        write(Message::Received, true, hdr.clientID(), hdr.requestID);
-                        break;
-                    // case Handled::YesRemoveReadListener:
-                    //     dataListener_--;
-                    //     if (dataListener_ == 0) {
-                    //         //return;
-                    //         // listeningThreadData.join();
-                    //     }
-                    //     break;
                     case Handled::YesAddReadListener:
                         {
                             std::lock_guard lock(handlerMutex_);
@@ -522,8 +484,6 @@ void ServerConnection::handle() {
                                 listeningThreadData = std::thread([this] { listeningThreadLoopData(); });
                             }
                         }
-                        write(Message::Received, true, hdr.clientID(), hdr.requestID);
-                        break;
                     case Handled::Yes:
                         write(Message::Received, true, hdr.clientID(), hdr.requestID);
                         break;
diff --git a/src/fdb5/remote/server/ServerConnection.h b/src/fdb5/remote/server/ServerConnection.h
index 0e89a538f..b9d48b0ca 100644
--- a/src/fdb5/remote/server/ServerConnection.h
+++ b/src/fdb5/remote/server/ServerConnection.h
@@ -102,15 +102,11 @@ class ServerConnection : public Connection, public Handler {
 
 protected:
 
-    // Handler& handler(uint32_t id);
-    // virtual Handler& handler(uint32_t id, Buffer& payload) = 0;
-
     // socket methods
     int selectDataPort();
-//    virtual void initialiseConnections();
     eckit::LocalConfiguration availableFunctionality() const;
     
-        // Worker functionality
+    // Worker functionality
     void tidyWorkers();
     void waitForWorkers();
 
@@ -120,20 +116,12 @@ class ServerConnection : public Connection, public Handler {
 
     // archival helper methods
     void archiver();
-    // emplace new ArchiveElem to archival queue
     void queue(Message message, uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload);
-    // // retrieve archival queue
-    // eckit::Queue& queue();
 
     void handleException(std::exception_ptr e) override;
 
-    // void archive(const MessageHeader& hdr);
-    // virtual size_t archiveThreadLoop() = 0;
-
 private:
 
-    // void controlWrite(Message msg, uint32_t clientID, uint32_t requestID, const void* payload = nullptr, uint32_t payloadLength = 0);
-    // void controlWrite(const void* data, size_t length);
     void listeningThreadLoopData();
 
     eckit::net::TCPSocket& controlSocket() override { return controlSocket_; }
@@ -145,9 +133,7 @@ class ServerConnection : public Connection, public Handler {
 protected:
 
     virtual bool remove(bool control, uint32_t clientID) = 0;
-    // virtual bool handlers() = 0;
 
-    // std::map handlers_;
     Config config_;
     std::string dataListenHostname_;
 
@@ -155,8 +141,6 @@ class ServerConnection : public Connection, public Handler {
 
     eckit::SessionID sessionID_;
     eckit::LocalConfiguration agreedConf_;
-    // std::mutex controlWriteMutex_;
-    // std::mutex dataWriteMutex_;
     std::thread readLocationWorker_;
     
     std::map> workerThreads_;
diff --git a/src/fdb5/remote/server/StoreHandler.cc b/src/fdb5/remote/server/StoreHandler.cc
index 12462ff6f..cc1d5e428 100644
--- a/src/fdb5/remote/server/StoreHandler.cc
+++ b/src/fdb5/remote/server/StoreHandler.cc
@@ -33,10 +33,6 @@ Handled StoreHandler::handleControl(Message message, uint32_t clientID, uint32_t
                 archiver();
                 return Handled::YesAddArchiveListener;
             
-            // case Message::Flush: // notification that the client has sent all data for archival
-            //     flush(clientID, requestID, eckit::Buffer{0});
-            //     return Handled::YesRemoveArchiveListener; // ????
-
             default: {
                 std::stringstream ss;
                 ss << "ERROR: Unexpected message recieved (" << message << "). ABORTING";
@@ -196,20 +192,6 @@ void StoreHandler::flush(uint32_t clientID, uint32_t requestID, const eckit::Buf
     ASSERT(it != stores_.end());
     it->second.store->flush();
 
-    // if (archiveFuture_.valid()) {
-    //     // Ensure that the expected number of fields have been written, and that the
-    //     // archive worker processes have been cleanly wound up.
-    //     size_t n = archiveFuture_.get();
-    //     ASSERT(numArchived == n);
-
-    //     // Do the actual flush!
-    //     Log::info() << "Flushing" << std::endl;
-    //     Log::status() << "Flushing" << std::endl;
-    //     // for (auto it = stores_.begin(); it != stores_.end(); it++) {
-    //     //     it->second->flush();
-    //     // }
-    // }
-
     Log::info() << "Flush complete" << std::endl;
     Log::status() << "Flush complete" << std::endl;
 }
@@ -234,11 +216,6 @@ bool StoreHandler::remove(bool control, uint32_t clientID) {
     return ((control ? numControlConnection_ : numDataConnection_) == 0);
 }
 
-// bool StoreHandler::handlers() {
-//     std::lock_guard lock(handlerMutex_);
-//     return stores_.empty();
-// }
-
 Store& StoreHandler::store(uint32_t clientID) {
 
     std::lock_guard lock(handlerMutex_);

From 17a9bcb8e180c16c10695b3327c0bb49a54c7805 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 15 Mar 2024 15:52:56 +0100
Subject: [PATCH 084/186] fdb remote src file cleanup

---
 src/fdb5/CMakeLists.txt                       | 36 ++++----
 src/fdb5/remote/FdbServer.cc                  |  7 +-
 src/fdb5/remote/FdbServer.h                   |  2 -
 src/fdb5/remote/RemoteConfiguration.cc        |  0
 src/fdb5/remote/RemoteConfiguration.h         | 66 --------------
 src/fdb5/remote/RemoteEngine.cc               | 89 -------------------
 src/fdb5/remote/RemoteEngine.h                | 50 -----------
 src/fdb5/remote/RemoteFieldLocation.cc        |  2 +-
 .../remote/{ => client}/RemoteCatalogue.cc    |  8 +-
 .../remote/{ => client}/RemoteCatalogue.h     |  0
 src/fdb5/remote/{ => client}/RemoteStore.cc   |  2 +-
 src/fdb5/remote/{ => client}/RemoteStore.h    |  0
 .../remote/{ => server}/AvailablePortList.cc  |  2 +-
 .../remote/{ => server}/AvailablePortList.h   |  0
 src/fdb5/remote/server/ServerConnection.cc    |  2 +-
 15 files changed, 26 insertions(+), 240 deletions(-)
 delete mode 100644 src/fdb5/remote/RemoteConfiguration.cc
 delete mode 100644 src/fdb5/remote/RemoteConfiguration.h
 delete mode 100644 src/fdb5/remote/RemoteEngine.cc
 delete mode 100644 src/fdb5/remote/RemoteEngine.h
 rename src/fdb5/remote/{ => client}/RemoteCatalogue.cc (96%)
 rename src/fdb5/remote/{ => client}/RemoteCatalogue.h (100%)
 rename src/fdb5/remote/{ => client}/RemoteStore.cc (99%)
 rename src/fdb5/remote/{ => client}/RemoteStore.h (100%)
 rename src/fdb5/remote/{ => server}/AvailablePortList.cc (99%)
 rename src/fdb5/remote/{ => server}/AvailablePortList.h (100%)

diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt
index 498a01522..7c3be791d 100644
--- a/src/fdb5/CMakeLists.txt
+++ b/src/fdb5/CMakeLists.txt
@@ -119,7 +119,6 @@ list( APPEND fdb5_srcs
     database/Key.h
     database/InspectionKey.cc
     database/InspectionKey.h
-    # database/ReadVisitor.cc
     database/ReadVisitor.h
     database/Report.cc
     database/Report.h
@@ -212,18 +211,17 @@ list( APPEND fdb5_srcs
 
 if(HAVE_FDB_REMOTE)
     list( APPEND fdb5_srcs 
-        # api/ClientFDB.cc
-        # api/ClientFDB.h
         api/RemoteFDB.cc
         api/RemoteFDB.h
-        remote/RemoteStore.cc
-        remote/RemoteStore.h
-        remote/RemoteCatalogue.h
-        remote/RemoteCatalogue.cc
-        # remote/RemoteEngine.h
-        # remote/RemoteEngine.cc
+
         remote/Connection.h
         remote/Connection.cc
+        remote/RemoteFieldLocation.h
+        remote/RemoteFieldLocation.cc
+        remote/Messages.h
+        remote/Messages.cc
+        remote/FdbServer.h
+        remote/FdbServer.cc
 
         remote/client/Client.h
         remote/client/Client.cc
@@ -231,22 +229,22 @@ if(HAVE_FDB_REMOTE)
         remote/client/ClientConnection.cc
         remote/client/ClientConnectionRouter.h
         remote/client/ClientConnectionRouter.cc
+        remote/client/RemoteStore.cc
+        remote/client/RemoteStore.h
+        remote/client/RemoteCatalogue.h
+        remote/client/RemoteCatalogue.cc
+
+        remote/server/AvailablePortList.cc
+        remote/server/AvailablePortList.h
         remote/server/CatalogueHandler.h
         remote/server/CatalogueHandler.cc
         remote/server/StoreHandler.h
         remote/server/StoreHandler.cc
         remote/server/ServerConnection.h
         remote/server/ServerConnection.cc
-        remote/RemoteConfiguration.h
-        remote/RemoteConfiguration.cc
-        remote/RemoteFieldLocation.h
-        remote/RemoteFieldLocation.cc
-        remote/Messages.h
-        remote/Messages.cc
-        remote/AvailablePortList.cc
-        remote/AvailablePortList.h
-        remote/FdbServer.h
-        remote/FdbServer.cc
+
+        # remote/RemoteConfiguration.h
+        # remote/RemoteConfiguration.cc
     )
 endif()
 
diff --git a/src/fdb5/remote/FdbServer.cc b/src/fdb5/remote/FdbServer.cc
index d15ddc559..c7f053634 100644
--- a/src/fdb5/remote/FdbServer.cc
+++ b/src/fdb5/remote/FdbServer.cc
@@ -18,12 +18,9 @@
 
 #include "fdb5/remote/FdbServer.h"
 
-#include "fdb5/remote/AvailablePortList.h"
-// #include "fdb5/remote/Handler.h"
-//#include "fdb5/remote/CatalogueHandler.h"
-#include "fdb5/remote/server/StoreHandler.h"
+#include "fdb5/remote/server/AvailablePortList.h"
 #include "fdb5/remote/server/CatalogueHandler.h"
-#include "fdb5/remote/server/ServerConnection.h"
+#include "fdb5/remote/server/StoreHandler.h"
 #include "eckit/config/Resource.h"
 
 using namespace eckit;
diff --git a/src/fdb5/remote/FdbServer.h b/src/fdb5/remote/FdbServer.h
index bbc6535eb..bce69f6b9 100644
--- a/src/fdb5/remote/FdbServer.h
+++ b/src/fdb5/remote/FdbServer.h
@@ -29,8 +29,6 @@
 #include "fdb5/config/Config.h"
 #include "fdb5/LibFdb5.h"
 
-#include "fdb5/remote/AvailablePortList.h"
-
 namespace fdb5::remote {
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/RemoteConfiguration.cc b/src/fdb5/remote/RemoteConfiguration.cc
deleted file mode 100644
index e69de29bb..000000000
diff --git a/src/fdb5/remote/RemoteConfiguration.h b/src/fdb5/remote/RemoteConfiguration.h
deleted file mode 100644
index 92ac41dea..000000000
--- a/src/fdb5/remote/RemoteConfiguration.h
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/// @author Simon Smart
-/// @date   Mar 2018
-
-#ifndef fdb5_remote_RemoteConfiguration_H
-#define fdb5_remote_RemoteConfiguration_H
-
-namespace eckit { class Stream; }
-
-namespace fdb5 {
-
-class FDB;
-
-//----------------------------------------------------------------------------------------------------------------------
-
-// This class handles negotiation between client/server for which functionality will be used
-// over the wire
-//
-// n.b. This includes cases where there is version mismatches
-//      --> The over-the-wire negotiation needs to take this into account.
-
-#if 0
-
-// TODO
-
-class RemoteConfiguration {
-
-public: // methods
-
-    RemoteConfiguration();
-    RemoteConfiguration(eckit::Stream& s);
-
-
-
-
-private: // methods
-
-    void encode(eckit::Stream& s) const override;
-
-private: // members
-
-    friend eckit::Stream& operator<< (eckit::Stream& s, const RemoteConfiguration& rc) {
-        rc.encode(s);
-        return s;
-    }
-
-
-};
-
-#endif
-
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace fdb5
-
-#endif // fdb5_remote_RemoteFDB_H
diff --git a/src/fdb5/remote/RemoteEngine.cc b/src/fdb5/remote/RemoteEngine.cc
deleted file mode 100644
index 0ee0b54da..000000000
--- a/src/fdb5/remote/RemoteEngine.cc
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-#include "fdb5/remote/RemoteEngine.h"
-
-// #include "eckit/io/Buffer.h"
-// #include "eckit/serialisation/MemoryStream.h"
-
-#include "fdb5/config/Config.h"
-// #include "fdb5/remote/client/Client.h"
-
-
-namespace fdb5::remote {
-
-// class RemoteEngineClient : public Client {
-
-// public:
-
-//     RemoteEngineClient(const Config& config) :
-//         Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port"))) {
-//     }
-
-// private: // methods
-
-//     // Client
-//     bool handle(remote::Message message, uint32_t requestID) override { return false; }
-//     bool handle(remote::Message message, uint32_t requestID, eckit::net::Endpoint endpoint, eckit::Buffer&& payload) override { return false; }
-//     void handleException(std::exception_ptr e) override { NOTIMP; }
-
-//     const Key& key() const override { NOTIMP; }
-
-// private: // members
-
-// };
-
-//----------------------------------------------------------------------------------------------------------------------
-
-std::string RemoteEngine::name() const {
-    return RemoteEngine::typeName();
-}
-
-std::string RemoteEngine::dbType() const {
-    return RemoteEngine::typeName();
-}
-
-bool RemoteEngine::canHandle(const eckit::URI& uri) const
-{
-    return uri.scheme() == "fdb";
-}
-
-std::vector RemoteEngine::visitableLocations(const Config& config) const
-{
-    // if (!client_) {
-    //     client_.reset(new RemoteEngineClient(config));
-    // }
-    // ASSERT(client_);
-
-    return std::vector {}; //databases(Key(), CatalogueRootManager(config).visitableRoots(InspectionKey()), config);
-}
-
-std::vector RemoteEngine::visitableLocations(const metkit::mars::MarsRequest& request, const Config& config) const
-{
-    // if (!client_) {
-    //     client_.reset(new RemoteEngineClient(config));
-    // }
-    // ASSERT(client_);
-
-    return std::vector {}; //databases(request, CatalogueRootManager(config).visitableRoots(request), config);
-}
-
-void RemoteEngine::print(std::ostream& out) const
-{
-    out << "RemoteEngine()";
-}
-
-static EngineBuilder remote_builder;
-
-//static eckit::LocalFileManager manager_toc("toc");
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace fdb5::remote
diff --git a/src/fdb5/remote/RemoteEngine.h b/src/fdb5/remote/RemoteEngine.h
deleted file mode 100644
index 295de620b..000000000
--- a/src/fdb5/remote/RemoteEngine.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/// @author Emanuele Danovaro
-/// @date   October 2023
-
-#pragma once
-
-#include "fdb5/database/Engine.h"
-
-namespace fdb5::remote {
-
-class RemoteEngineClient;
-
-//----------------------------------------------------------------------------------------------------------------------
-
-class RemoteEngine : public fdb5::Engine {
-
-public: // methods
-
-    static const char* typeName() { return "remote"; }
-
-protected: // methods
-
-    virtual std::string name() const override;
-
-    virtual std::string dbType() const override;
-
-    virtual bool canHandle(const eckit::URI& path) const override;
-
-    virtual std::vector visitableLocations(const Config& config) const override;
-    virtual std::vector visitableLocations(const metkit::mars::MarsRequest& rq, const Config& config) const override;
-
-    virtual void print( std::ostream &out ) const override;
-
-private:
-    // mutable std::unique_ptr client_;
-
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace fdb5::remote
diff --git a/src/fdb5/remote/RemoteFieldLocation.cc b/src/fdb5/remote/RemoteFieldLocation.cc
index 806bea6ee..ee5f404e5 100644
--- a/src/fdb5/remote/RemoteFieldLocation.cc
+++ b/src/fdb5/remote/RemoteFieldLocation.cc
@@ -14,7 +14,7 @@
  */
 
 #include "fdb5/remote/RemoteFieldLocation.h"
-#include "fdb5/remote/RemoteStore.h"
+#include "fdb5/remote/client/RemoteStore.h"
 #include "fdb5/remote/client/ClientConnectionRouter.h"
 #include "fdb5/LibFdb5.h"
 
diff --git a/src/fdb5/remote/RemoteCatalogue.cc b/src/fdb5/remote/client/RemoteCatalogue.cc
similarity index 96%
rename from src/fdb5/remote/RemoteCatalogue.cc
rename to src/fdb5/remote/client/RemoteCatalogue.cc
index 31b90f374..b36a01f77 100644
--- a/src/fdb5/remote/RemoteCatalogue.cc
+++ b/src/fdb5/remote/client/RemoteCatalogue.cc
@@ -13,8 +13,7 @@
 #include "eckit/serialisation/MemoryStream.h"
 
 #include "fdb5/LibFdb5.h"
-#include "fdb5/remote/RemoteCatalogue.h"
-#include "fdb5/remote/RemoteEngine.h"
+#include "fdb5/remote/client/RemoteCatalogue.h"
 
 #include 
 
@@ -148,8 +147,7 @@ bool RemoteCatalogue::exists() const {NOTIMP;}
 void RemoteCatalogue::checkUID() const {}
 
 eckit::URI RemoteCatalogue::uri() const {
-    return eckit::URI(/*RemoteEngine::typeName()*/ "fdb", controlEndpoint().host(), controlEndpoint().port());
-    // return eckit::URI(TocEngine::typeName(), basePath());
+    return eckit::URI("fdb", controlEndpoint().host(), controlEndpoint().port());
 }
 
 void RemoteCatalogue::loadSchema() {
@@ -220,7 +218,7 @@ void RemoteCatalogue::allMasked(std::set>&
 void RemoteCatalogue::print( std::ostream &out ) const {NOTIMP;}
 
 std::string RemoteCatalogue::type() const {
-    return "remote"; // RemoteEngine::typeName();
+    return "remote";
 }
 bool RemoteCatalogue::open() {
     return true;
diff --git a/src/fdb5/remote/RemoteCatalogue.h b/src/fdb5/remote/client/RemoteCatalogue.h
similarity index 100%
rename from src/fdb5/remote/RemoteCatalogue.h
rename to src/fdb5/remote/client/RemoteCatalogue.h
diff --git a/src/fdb5/remote/RemoteStore.cc b/src/fdb5/remote/client/RemoteStore.cc
similarity index 99%
rename from src/fdb5/remote/RemoteStore.cc
rename to src/fdb5/remote/client/RemoteStore.cc
index 1b67313b1..f78d1b8cd 100644
--- a/src/fdb5/remote/RemoteStore.cc
+++ b/src/fdb5/remote/client/RemoteStore.cc
@@ -22,7 +22,7 @@
 #include "fdb5/LibFdb5.h"
 #include "fdb5/rules/Rule.h"
 #include "fdb5/database/FieldLocation.h"
-#include "fdb5/remote/RemoteStore.h"
+#include "fdb5/remote/client/RemoteStore.h"
 #include "fdb5/remote/RemoteFieldLocation.h"
 #include "fdb5/io/FDBFileHandle.h"
 
diff --git a/src/fdb5/remote/RemoteStore.h b/src/fdb5/remote/client/RemoteStore.h
similarity index 100%
rename from src/fdb5/remote/RemoteStore.h
rename to src/fdb5/remote/client/RemoteStore.h
diff --git a/src/fdb5/remote/AvailablePortList.cc b/src/fdb5/remote/server/AvailablePortList.cc
similarity index 99%
rename from src/fdb5/remote/AvailablePortList.cc
rename to src/fdb5/remote/server/AvailablePortList.cc
index 03bd0c67f..fd2828002 100644
--- a/src/fdb5/remote/AvailablePortList.cc
+++ b/src/fdb5/remote/server/AvailablePortList.cc
@@ -13,7 +13,7 @@
  * (Project ID: 671951) www.nextgenio.eu
  */
 
-#include "fdb5/remote/AvailablePortList.h"
+#include "fdb5/remote/server/AvailablePortList.h"
 
 #include 
 #include 
diff --git a/src/fdb5/remote/AvailablePortList.h b/src/fdb5/remote/server/AvailablePortList.h
similarity index 100%
rename from src/fdb5/remote/AvailablePortList.h
rename to src/fdb5/remote/server/AvailablePortList.h
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index c42fc1efd..1ecdad95a 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -27,7 +27,7 @@
 #include "fdb5/fdb5_version.h"
 #include "fdb5/api/helpers/FDBToolRequest.h"
 #include "fdb5/database/Key.h"
-#include "fdb5/remote/AvailablePortList.h"
+#include "fdb5/remote/server/AvailablePortList.h"
 #include "fdb5/remote/Messages.h"
 #include "fdb5/remote/RemoteFieldLocation.h"
 #include "fdb5/api/FDB.h"

From a284d068b8a984761c6d36cb46ab598d8dc9f80b Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 15 Mar 2024 15:53:59 +0100
Subject: [PATCH 085/186] tag

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 66e3b5814..cdeed24e0 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.110
\ No newline at end of file
+5.11.111

From cc3f2fad598e9696e0c4b542bf969419a2b54bd8 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Sun, 17 Mar 2024 08:13:28 +0000
Subject: [PATCH 086/186] fix message buffering (read in chunks)

---
 src/fdb5/io/FieldHandle.cc | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/fdb5/io/FieldHandle.cc b/src/fdb5/io/FieldHandle.cc
index b0f79c193..182b2373b 100644
--- a/src/fdb5/io/FieldHandle.cc
+++ b/src/fdb5/io/FieldHandle.cc
@@ -131,8 +131,17 @@ void FieldHandle::openCurrent() {
         current_->openForRead();
 
         if (!current_->canSeek()) {
-            auto len = current_->read(buffer_, currentSize);
-            ASSERT(len == currentSize);
+            long len = 0;
+            long toRead = currentSize;
+            long read = 0;
+            char *buf = buffer_;
+            while (toRead>0 && (len = current_->read(buf, toRead))>0) {
+                toRead -= len;
+                buf += len;
+                read += len;
+            }
+
+            ASSERT(read == currentSize);
             current_ = new eckit::MemoryHandle(buffer_, currentSize);
             current_->openForRead();
             currentMemoryHandle_ = true;

From 75e7cdc75a72d6ec3ede2899d0eba58a2ca7e66e Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 20 Mar 2024 16:10:36 +0000
Subject: [PATCH 087/186] PR comments

---
 src/fdb5/database/EntryVisitMechanism.cc     |   2 +-
 src/fdb5/remote/client/RemoteCatalogue.cc    |   1 -
 src/fdb5/remote/server/DataStoreStrategies.h | 409 -------------------
 src/fdb5/remote/server/ServerConnection.cc   |   2 +-
 src/fdb5/remote/server/ServerConnection.h    |   2 +-
 src/fdb5/remote/server/StoreHandler.cc       |   2 +-
 6 files changed, 4 insertions(+), 414 deletions(-)
 delete mode 100644 src/fdb5/remote/server/DataStoreStrategies.h

diff --git a/src/fdb5/database/EntryVisitMechanism.cc b/src/fdb5/database/EntryVisitMechanism.cc
index f8f70fb58..da8ed9ef6 100644
--- a/src/fdb5/database/EntryVisitMechanism.cc
+++ b/src/fdb5/database/EntryVisitMechanism.cc
@@ -59,7 +59,7 @@ void EntryVisitor::catalogueComplete(const Catalogue& catalogue) {
         ASSERT(currentCatalogue_ == &catalogue);
     }
     currentCatalogue_ = nullptr;
-    // currentStore_ = nullptr;
+    currentStore_.reset();
     currentIndex_ = nullptr;
 }
 
diff --git a/src/fdb5/remote/client/RemoteCatalogue.cc b/src/fdb5/remote/client/RemoteCatalogue.cc
index b36a01f77..b886cad84 100644
--- a/src/fdb5/remote/client/RemoteCatalogue.cc
+++ b/src/fdb5/remote/client/RemoteCatalogue.cc
@@ -44,7 +44,6 @@ void RemoteCatalogue::sendArchiveData(uint32_t id, const Key& key, std::unique_p
 
     Buffer keyBuffer(4096);
     MemoryStream keyStream(keyBuffer);
-//    keyStream << dbKey_;
     keyStream << currentIndexKey_;
     keyStream << key;
 
diff --git a/src/fdb5/remote/server/DataStoreStrategies.h b/src/fdb5/remote/server/DataStoreStrategies.h
deleted file mode 100644
index 8e6d9c66e..000000000
--- a/src/fdb5/remote/server/DataStoreStrategies.h
+++ /dev/null
@@ -1,409 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/// @file   DataStoreStrategies.h
-/// @date   Mar 1998
-/// @author Baudouin Raoult
-/// @author Tiago Quintino
-
-#pragma once
-
-#include "eckit/memory/NonCopyable.h"
-
-namespace fdb5 {
-
-//----------------------------------------------------------------------------------------------------------------------
-
-template< class T >
-class DataStoreStrategies : private NonCopyable {
-public:
-    static const T& selectFileSystem(const std::vector& fileSystems, const std::string& s);
-
-    static const T& leastUsed(const std::vector& fileSystems);
-    static const T& leastUsedPercent(const std::vector& fileSystems);
-    static const T& roundRobin(const std::vector& fileSystems);
-    static const T& pureRandom(const std::vector& fileSystems);
-    static const T& weightedRandom(const std::vector& fileSystems);
-    static const T& weightedRandomPercent(const std::vector& fileSystems);
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-struct DataStoreSize {
-    unsigned long long available;
-    unsigned long long total;
-    DataStoreSize() :
-        available(0), total(0) {}
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-template< class T >
-struct Candidate {
-
-    const T* datastore_;
-
-    DataStoreSize size_;
-
-    double probability_;
-
-    Candidate(const T* datastore) :
-        datastore_(datastore) {}
-
-    void print(std::ostream& s) const {
-        s << "Candidate(datastore=" << datastore_->asString() << ",total=" << total() << ",available=" << available()
-          << ",percent=" << percent() << ",probability=" << probability_ << ")";
-    }
-
-    friend std::ostream& operator<<(std::ostream& s, const Candidate& v) {
-        v.print(s);
-        return s;
-    }
-
-    const T& datastore() const { return *datastore_; }
-
-    double probability() const { return probability_; }
-    void probability(double p) { probability_ = p; }
-
-    long percent() const { return long(100. * (double(size_.available) / size_.total)); }
-
-    unsigned long long total() const { return size_.total; }
-
-    unsigned long long available() const { return size_.available; }
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-template< class T >
-const T& DataStoreStrategies::selectFileSystem(const std::vector& fileSystems, const std::string& s) {
-    Log::info() << "DataStoreStrategies::selectFileSystem is " << s << std::endl;
-
-    if (s == "roundRobin") {
-        return DataStoreStrategies::roundRobin(fileSystems);
-    }
-
-    if (s == "weightedRandom") {
-        return DataStoreStrategies::weightedRandom(fileSystems);
-    }
-
-    if (s == "pureRandom") {
-        return DataStoreStrategies::pureRandom(fileSystems);
-    }
-
-    if (s == "weightedRandomPercent") {
-        return DataStoreStrategies::weightedRandomPercent(fileSystems);
-    }
-
-    if (s == "leastUsedPercent") {
-        return DataStoreStrategies::leastUsedPercent(fileSystems);
-    }
-
-    return DataStoreStrategies::leastUsed(fileSystems);
-}
-
-template< class T >
-const T& DataStoreStrategies::leastUsed(const std::vector& fileSystems) {
-    unsigned long long free = 0;
-    Ordinal best            = 0;
-    Ordinal checked         = 0;
-
-    ASSERT(fileSystems.size() != 0);
-
-    for (Ordinal i = 0; i < fileSystems.size(); i++) {
-        // Log::info() << "leastUsed: " << fileSystems[i] << " " << fileSystems[i].available() << std::endl;
-        if (fileSystems[i].available()) {
-            DataStoreSize fs;
-
-            try {
-                fileSystems[i].fileSystemSize(fs);
-            }
-            catch (std::exception& e) {
-                Log::error() << "** " << e.what() << " Caught in " << Here() << std::endl;
-                Log::error() << "** Exception is ignored" << std::endl;
-                Log::error() << "Cannot stat " << fileSystems[i] << Log::syserr << std::endl;
-                continue;
-            }
-
-            if (fs.available >= free || checked == 0) {
-                free = fs.available;
-                best = i;
-                checked++;
-            }
-        }
-    }
-
-    if (!checked) {
-        throw Retry(std::string("No available filesystem (") + fileSystems[0] + ")");
-    }
-
-    Log::info() << "Filespace strategy leastUsed selected " << fileSystems[best] << " " << Bytes(free) << " available"
-                << std::endl;
-
-    return fileSystems[best];
-}
-
-template< class T >
-const T& DataStoreStrategies::leastUsedPercent(const std::vector& fileSystems) {
-    long percent = 0;
-    size_t best  = 0;
-
-    ASSERT(fileSystems.size() != 0);
-
-    for (size_t i = 0; i < fileSystems.size(); ++i) {
-        Candidate candidate(&fileSystems[i]);
-
-        Log::info() << "leastUsedPercent: " << fileSystems[i] << " " << fileSystems[i].available() << std::endl;
-        if (fileSystems[i].available()) {
-            DataStoreSize fs;
-
-            try {
-                fileSystems[i].fileSystemSize(candidate.size_);
-            }
-            catch (std::exception& e) {
-                Log::error() << "** " << e.what() << " Caught in " << Here() << std::endl;
-                Log::error() << "** Exception is ignored" << std::endl;
-                Log::error() << "Cannot stat " << fileSystems[i] << Log::syserr << std::endl;
-                continue;
-            }
-
-            if (candidate.percent() >= percent) {
-                percent = candidate.percent();
-                best    = i;
-            }
-        }
-    }
-
-    Log::info() << "Filespace strategy leastUsedPercent selected " << fileSystems[best] << " " << percent
-                << "% available" << std::endl;
-
-    return fileSystems[best];
-}
-
-typedef void (*compute_probability_t)(Candidate&);
-
-static void computePercent(Candidate& c) {
-    c.probability_ = double(c.percent());
-}
-
-static void computeAvailable(Candidate& c) {
-    c.probability_ = double(c.available());
-}
-
-static void computeIdentity(Candidate& c) {
-    c.probability_ = 1;
-}
-
-static void computeNull(Candidate& c) {
-    c.probability_ = 0;
-}
-
-static std::vector findCandidates(const std::vector& fileSystems,
-                                             compute_probability_t probability) {
-
-    ASSERT(fileSystems.size() != 0);
-
-    static Resource candidateFileSystemPercent("candidateFileSystem", 99);
-
-    std::vector result;
-
-    for (size_t i = 0; i < fileSystems.size(); ++i) {
-
-        Candidate candidate(&fileSystems[i]);
-
-        if (fileSystems[i].available()) {
-
-            try {
-                fileSystems[i].fileSystemSize(candidate.size_);
-            }
-            catch (std::exception& e) {
-                Log::error() << "** " << e.what() << " Caught in " << Here() << std::endl;
-                Log::error() << "** Exception is ignored" << std::endl;
-                Log::error() << "Cannot stat " << fileSystems[i] << Log::syserr << std::endl;
-                continue;
-            }
-
-            if (candidate.total() == 0) {
-                Log::warning() << "Cannot get total size of " << fileSystems[i] << std::endl;
-                return std::vector();
-            }
-
-            if (candidate.percent() <= candidateFileSystemPercent) {
-
-                probability(candidate);
-
-                //                Log::info() << candidate << std::endl;
-
-                result.push_back(candidate);
-            }
-        }
-    }
-
-    return result;
-}
-
-template< class T >
-const T& DataStoreStrategies::roundRobin(const std::vector& fileSystems) {
-    std::vector candidates = findCandidates(fileSystems, &computeNull);
-
-    if (candidates.empty()) {
-        return leastUsed(fileSystems);
-    }
-
-    static long value = -1;
-
-    if (value < 0) {
-        value = ::getpid();
-    }
-
-    value++;
-    value %= candidates.size();
-
-    Log::info() << "Filespace strategy roundRobin selected " << candidates[value].datastore() << " " << value << " out of "
-                << candidates.size() << std::endl;
-
-    return candidates[value].datastore();
-}
-
-template< class T >
-static void attenuateProbabilities(std::vector& candidates) {
-
-    ASSERT(!candidates.empty());
-
-    static double attenuation = Resource("attenuateFileSpacePeakProbability", 0.);
-
-    ASSERT(attenuation >= 0.);
-    ASSERT(attenuation <= 1.);
-
-    if (attenuation == 0.) {
-        return;
-    }
-
-    // compute mean
-
-    double mean = 0.;
-    for (std::vector::const_iterator i = candidates.begin(); i != candidates.end(); ++i) {
-        mean += i->probability();
-    }
-
-    mean /= candidates.size();
-
-    //    // compute variance
-
-    //    double variance = 0.;
-    //    for(std::vector::const_iterator i = candidates.begin(); i != candidates.end(); ++i) {
-    //        double diff = (i->probability() - mean);
-    //        variance += diff*diff;
-    //    }
-
-    //    variance /= candidates.size();
-
-    //    // compute stddev
-
-    //    double stddev = std::sqrt(variance);
-
-    //    // attenuate the peaks that exceed the stddev to the stddev value
-    //    double max = mean + attenuation * stddev;
-    //    for(std::vector::iterator i = candidates.begin(); i != candidates.end(); ++i) {
-    //        if(i->probability() > max) {
-    //            i->probability(max);
-    //        }
-    //    }
-
-
-    for (std::vector::iterator i = candidates.begin(); i != candidates.end(); ++i) {
-        double p    = i->probability();
-        double newp = attenuation * mean + (1. - attenuation) * p;
-        i->probability(newp);
-    }
-}
-
-
-template< class T >
-static const T& chooseByProbabylity(const char* strategy, const std::vector& candidates) {
-
-    double total = 0;
-    for (std::vector::const_iterator i = candidates.begin(); i != candidates.end(); ++i) {
-        //        Log::info() << "probability " << i->probability() << std::endl;
-        total += i->probability();
-    }
-
-    double choice = (double(random()) / double(RAND_MAX));
-
-    //    Log::info() << "choice " << choice << std::endl;
-
-    choice *= total;
-
-    std::vector::const_iterator select = candidates.begin();
-
-    double lower = 0;
-    double upper = 0;
-    for (std::vector::const_iterator i = candidates.begin(); i != candidates.end(); ++i) {
-
-        upper += i->probability();
-
-        //        Log::info() << "Choice " << choice << " total = " << total << " lower = " << lower << " upper = " <<
-        //        upper << std::endl;
-
-        if (choice >= lower && choice < upper) {
-            select = i;
-            break;
-        }
-
-        lower = upper;
-    }
-
-    Log::info() << "Filespace strategy " << strategy << " selected " << select->datastore() << " "
-                << Bytes(select->available()) << " available" << std::endl;
-
-    return select->datastore();
-}
-
-template< class T >
-const T& DataStoreStrategies::pureRandom(const std::vector& fileSystems) {
-    std::vector candidates = findCandidates(fileSystems, &computeIdentity);
-
-    if (candidates.empty()) {
-        return leastUsed(fileSystems);
-    }
-
-    attenuateProbabilities(candidates); /* has no effect */
-
-    return chooseByProbabylity("pureRandom", candidates);
-}
-
-template< class T >
-const T& DataStoreStrategies::weightedRandom(const std::vector& fileSystems) {
-    std::vector candidates = findCandidates(fileSystems, &computeAvailable);
-
-    if (candidates.empty()) {
-        return leastUsed(fileSystems);
-    }
-
-    attenuateProbabilities(candidates);
-
-    return chooseByProbabylity("weightedRandom", candidates);
-}
-
-template< class T >
-const T& DataStoreStrategies::weightedRandomPercent(const std::vector& fileSystems) {
-    std::vector candidates = findCandidates(fileSystems, &computePercent);
-
-    if (candidates.empty()) {
-        return leastUsed(fileSystems);
-    }
-
-    attenuateProbabilities(candidates);
-
-    return chooseByProbabylity("weightedRandomPercent", candidates);
-}
-
-//----------------------------------------------------------------------------------------------------------------------
-
-}  // namespace eckit
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index 1ecdad95a..4a736d19b 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -137,7 +137,7 @@ Handled ServerConnection::handleData(Message message, uint32_t clientID, uint32_
     return Handled::No;
 }
 
-eckit::LocalConfiguration ServerConnection::availableFunctionality() const {
+constexpr eckit::LocalConfiguration ServerConnection::availableFunctionality() {
     eckit::LocalConfiguration conf;
 //    Add to the configuration all the components that require to be versioned, as in the following example, with a vector of supported version numbers
     std::vector remoteFieldLocationVersions = {1};
diff --git a/src/fdb5/remote/server/ServerConnection.h b/src/fdb5/remote/server/ServerConnection.h
index b9d48b0ca..2c05f6e59 100644
--- a/src/fdb5/remote/server/ServerConnection.h
+++ b/src/fdb5/remote/server/ServerConnection.h
@@ -104,7 +104,7 @@ class ServerConnection : public Connection, public Handler {
 
     // socket methods
     int selectDataPort();
-    eckit::LocalConfiguration availableFunctionality() const;
+    constexpr eckit::LocalConfiguration availableFunctionality();
     
     // Worker functionality
     void tidyWorkers();
diff --git a/src/fdb5/remote/server/StoreHandler.cc b/src/fdb5/remote/server/StoreHandler.cc
index cc1d5e428..b0f7530a4 100644
--- a/src/fdb5/remote/server/StoreHandler.cc
+++ b/src/fdb5/remote/server/StoreHandler.cc
@@ -119,7 +119,7 @@ void StoreHandler::writeToParent(const uint32_t clientID, const uint32_t request
         Log::status() << "Reading: " << requestID << std::endl;
         // Write the data to the parent, in chunks if necessary.
 
-        Buffer writeBuffer(10 * 1024 * 1024);
+        Buffer writeBuffer(4 * 1024 * 1024 - 2048); // slightly smaller than 4MiB to nicely fit in a TCP window with scale factor 6
         long dataRead;
 
         dh->openForRead();

From 9f29eaf53ddf461da489f2f7736dd4ec94a068a9 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Sun, 24 Mar 2024 07:38:56 +0000
Subject: [PATCH 088/186] debug macro

---
 src/fdb5/LibFdb5.cc                         |  2 +-
 src/fdb5/api/DistFDB.cc                     | 36 ++++++++--------
 src/fdb5/api/FDB.cc                         |  6 +--
 src/fdb5/api/FDBFactory.cc                  |  6 +--
 src/fdb5/api/FDBStats.cc                    |  6 +--
 src/fdb5/api/LocalFDB.cc                    | 20 ++++-----
 src/fdb5/api/RandomFDB.cc                   |  2 +-
 src/fdb5/api/RemoteFDB.cc                   |  4 +-
 src/fdb5/api/SelectFDB.cc                   | 14 +++---
 src/fdb5/api/helpers/FDBToolRequest.cc      |  4 +-
 src/fdb5/config/Config.cc                   |  4 +-
 src/fdb5/database/AxisRegistry.cc           |  2 +-
 src/fdb5/database/BaseArchiveVisitor.cc     |  2 +-
 src/fdb5/database/Catalogue.cc              |  8 ++--
 src/fdb5/database/EntryVisitMechanism.cc    |  4 +-
 src/fdb5/database/FieldLocation.cc          |  4 +-
 src/fdb5/database/Index.cc                  |  2 +-
 src/fdb5/database/Inspector.cc              |  4 +-
 src/fdb5/database/Manager.cc                | 16 +++----
 src/fdb5/database/MultiRetrieveVisitor.cc   | 12 +++---
 src/fdb5/database/Report.cc                 |  8 ++--
 src/fdb5/database/RetrieveVisitor.cc        |  2 +-
 src/fdb5/database/Store.cc                  |  4 +-
 src/fdb5/io/FieldHandle.cc                  |  3 +-
 src/fdb5/io/LustreFileHandle.h              |  2 +-
 src/fdb5/message/MessageArchiver.cc         |  6 +--
 src/fdb5/rados/RadosStore.cc                |  2 +-
 src/fdb5/remote/Connection.cc               |  2 +-
 src/fdb5/remote/Messages.cc                 | 47 ++-------------------
 src/fdb5/remote/client/ClientConnection.cc  | 10 ++---
 src/fdb5/remote/client/RemoteCatalogue.cc   | 10 ++---
 src/fdb5/remote/client/RemoteStore.cc       |  2 +-
 src/fdb5/remote/server/AvailablePortList.cc |  4 +-
 src/fdb5/remote/server/CatalogueHandler.cc  |  4 +-
 src/fdb5/remote/server/ServerConnection.cc  | 12 +++---
 src/fdb5/remote/server/ServerConnection.h   |  2 +-
 src/fdb5/remote/server/StoreHandler.cc      |  8 ++--
 src/fdb5/rules/Predicate.cc                 |  4 +-
 src/fdb5/rules/Schema.cc                    |  2 +-
 src/fdb5/toc/EnvVarFileSpaceHandler.cc      |  2 +-
 src/fdb5/toc/ExpverFileSpaceHandler.cc      | 14 +++---
 src/fdb5/toc/FileSpace.cc                   |  6 +--
 src/fdb5/toc/Root.cc                        |  2 +-
 src/fdb5/toc/RootManager.cc                 | 32 +++++++-------
 src/fdb5/toc/TocCatalogueReader.cc          |  8 ++--
 src/fdb5/toc/TocCatalogueWriter.cc          |  4 +-
 src/fdb5/toc/TocEngine.cc                   | 18 ++++----
 src/fdb5/toc/TocHandler.cc                  | 32 +++++++-------
 src/fdb5/toc/TocIndex.cc                    |  4 +-
 src/fdb5/toc/TocSerialisationVersion.cc     |  6 ++-
 src/fdb5/toc/TocStore.cc                    |  8 ++--
 src/fdb5/tools/FDBInspect.cc                | 12 +++---
 src/fdb5/tools/fdb-move.cc                  | 10 ++---
 tests/regressions/FDB-310/fdb-url.cc        |  5 ++-
 54 files changed, 209 insertions(+), 246 deletions(-)

diff --git a/src/fdb5/LibFdb5.cc b/src/fdb5/LibFdb5.cc
index 450530b56..6bbe46d3e 100644
--- a/src/fdb5/LibFdb5.cc
+++ b/src/fdb5/LibFdb5.cc
@@ -81,7 +81,7 @@ static unsigned getUserEnvRemoteProtocol() {
     static unsigned fdbRemoteProtocolVersion =
         eckit::Resource("fdbRemoteProtocolVersion;$FDB5_REMOTE_PROTOCOL_VERSION", 0);
     if (fdbRemoteProtocolVersion) {
-        eckit::Log::debug() << "fdbRemoteProtocolVersion overidde to version: " << fdbRemoteProtocolVersion
+        LOG_DEBUG_LIB(LibFdb5) << "fdbRemoteProtocolVersion overidde to version: " << fdbRemoteProtocolVersion
                             << std::endl;
     }
     return 0;  // no version override
diff --git a/src/fdb5/api/DistFDB.cc b/src/fdb5/api/DistFDB.cc
index c5a423cfd..fbbe524b5 100644
--- a/src/fdb5/api/DistFDB.cc
+++ b/src/fdb5/api/DistFDB.cc
@@ -74,21 +74,21 @@ void DistFDB::archive(const Key& key, const void* data, size_t length) {
 
     std::vector laneIndices;
 
-    //Log::debug() << "Number of lanes: " << lanes_.size() << std::endl;
-    //Log::debug() << "Lane indices: ";
-    //for (const auto& i : laneIndices) Log::debug() << i << ", ";
-    //Log::debug() << std::endl;
+    //LOG_DEBUG_LIB(LibFdb5) << "Number of lanes: " << lanes_.size() << std::endl;
+    //LOG_DEBUG_LIB(LibFdb5) << "Lane indices: ";
+    //for (const auto& i : laneIndices) LOG_DEBUG_LIB(LibFdb5) << i << ", ";
+    //LOG_DEBUG_LIB(LibFdb5) << std::endl;
 
     hash_.hashOrder(key.keyDict(), laneIndices);
 
-    //Log::debug() << "Number of lanes: " << lanes_.size() << std::endl;
-    //Log::debug() << "Lane indices: ";
-    //for (const auto& i : laneIndices) Log::debug() << i << ", ";
-    //Log::debug() << std::endl;
+    //LOG_DEBUG_LIB(LibFdb5) << "Number of lanes: " << lanes_.size() << std::endl;
+    //LOG_DEBUG_LIB(LibFdb5) << "Lane indices: ";
+    //for (const auto& i : laneIndices) LOG_DEBUG_LIB(LibFdb5) << i << ", ";
+    //LOG_DEBUG_LIB(LibFdb5) << std::endl;
 
     // Given an order supplied by the Rendezvous hash, try the FDB in order until
     // one works. n.b. Errors are unacceptable once the FDB is dirty.
-    Log::debug() << "Attempting dist FDB archive" << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Attempting dist FDB archive" << std::endl;
 
     decltype(laneIndices)::const_iterator it = laneIndices.begin();
     decltype(laneIndices)::const_iterator end = laneIndices.end();
@@ -180,7 +180,7 @@ auto DistFDB::queryInternal(const FDBToolRequest& request, const QueryFN& fn) ->
 
 
 ListIterator DistFDB::list(const FDBToolRequest& request) {
-    Log::debug() << "DistFDB::list() : " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "DistFDB::list() : " << request << std::endl;
     return queryInternal(request,
                          [](FDB& fdb, const FDBToolRequest& request) {
                             return fdb.list(request);
@@ -188,7 +188,7 @@ ListIterator DistFDB::list(const FDBToolRequest& request) {
 }
 
 ListIterator DistFDB::inspect(const metkit::mars::MarsRequest& request) {
-    Log::debug() << "DistFDB::inspect() : " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "DistFDB::inspect() : " << request << std::endl;
     return queryInternal(request,
                          [](FDB& fdb, const FDBToolRequest& request) {
                             return fdb.inspect(request.request());
@@ -196,7 +196,7 @@ ListIterator DistFDB::inspect(const metkit::mars::MarsRequest& request) {
 }
 
 DumpIterator DistFDB::dump(const FDBToolRequest& request, bool simple) {
-    Log::debug() << "DistFDB::dump() : " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "DistFDB::dump() : " << request << std::endl;
     return queryInternal(request,
                          [simple](FDB& fdb, const FDBToolRequest& request) {
                             return fdb.dump(request, simple);
@@ -204,7 +204,7 @@ DumpIterator DistFDB::dump(const FDBToolRequest& request, bool simple) {
 }
 
 StatusIterator DistFDB::status(const FDBToolRequest& request) {
-    Log::debug() << "DistFDB::status() : " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "DistFDB::status() : " << request << std::endl;
     return queryInternal(request,
                          [](FDB& fdb, const FDBToolRequest& request) {
                             return fdb.status(request);
@@ -212,7 +212,7 @@ StatusIterator DistFDB::status(const FDBToolRequest& request) {
 }
 
 WipeIterator DistFDB::wipe(const FDBToolRequest& request, bool doit, bool porcelain, bool unsafeWipeAll) {
-    Log::debug() << "DistFDB::wipe() : " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "DistFDB::wipe() : " << request << std::endl;
     return queryInternal(request,
                          [doit, porcelain, unsafeWipeAll](FDB& fdb, const FDBToolRequest& request) {
                             return fdb.wipe(request, doit, porcelain, unsafeWipeAll);
@@ -220,7 +220,7 @@ WipeIterator DistFDB::wipe(const FDBToolRequest& request, bool doit, bool porcel
 }
 
 PurgeIterator DistFDB::purge(const FDBToolRequest& request, bool doit, bool porcelain) {
-    Log::debug() << "DistFDB::purge() : " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "DistFDB::purge() : " << request << std::endl;
     return queryInternal(request,
                          [doit, porcelain](FDB& fdb, const FDBToolRequest& request) {
                             return fdb.purge(request, doit, porcelain);
@@ -228,7 +228,7 @@ PurgeIterator DistFDB::purge(const FDBToolRequest& request, bool doit, bool porc
 }
 
 StatsIterator DistFDB::stats(const FDBToolRequest &request) {
-    Log::debug() << "DistFDB::stats() : " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "DistFDB::stats() : " << request << std::endl;
     return queryInternal(request,
                          [](FDB& fdb, const FDBToolRequest& request) {
                             return fdb.stats(request);
@@ -238,7 +238,7 @@ StatsIterator DistFDB::stats(const FDBToolRequest &request) {
 ControlIterator DistFDB::control(const FDBToolRequest& request,
                                  ControlAction action,
                                  ControlIdentifiers identifiers) {
-    Log::debug() << "DistFDB::control() : " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "DistFDB::control() : " << request << std::endl;
     return queryInternal(request,
                          [action, identifiers](FDB& fdb, const FDBToolRequest& request) {
                             return fdb.control(request, action, identifiers);
@@ -247,7 +247,7 @@ ControlIterator DistFDB::control(const FDBToolRequest& request,
 
 
 MoveIterator DistFDB::move(const FDBToolRequest& request, const eckit::URI& dest) {
-    Log::debug() << "DistFDB::move() : " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "DistFDB::move() : " << request << std::endl;
     return queryInternal(request,
                          [dest](FDB& fdb, const FDBToolRequest& request) {
                             return fdb.move(request, dest);
diff --git a/src/fdb5/api/FDB.cc b/src/fdb5/api/FDB.cc
index ad58c4fa4..99f709ba2 100644
--- a/src/fdb5/api/FDB.cc
+++ b/src/fdb5/api/FDB.cc
@@ -79,7 +79,7 @@ void FDB::archive(const metkit::mars::MarsRequest& request, eckit::DataHandle& h
             ss << "FDB archive - found unexpected message" << std::endl;
             ss << "  user request:"  << std::endl << "    " << request << std::endl;
             ss << "  unexpected message:" << std::endl << "    " << key << std::endl;
-            eckit::Log::debug() << ss.str();
+            LOG_DEBUG_LIB(LibFdb5) << ss.str();
             throw eckit::UserError(ss.str(), Here());
         }
         archive(key, msg.data(), msg.length());
@@ -92,7 +92,7 @@ void FDB::archive(const metkit::mars::MarsRequest& request, eckit::DataHandle& h
         for (auto vacantRequest : cube.vacantRequests()) {
             ss << "    " << vacantRequest << std::endl;
         }
-        eckit::Log::debug() << ss.str();
+        LOG_DEBUG_LIB(LibFdb5) << ss.str();
         throw eckit::UserError(ss.str(), Here());
     }
 }
@@ -136,7 +136,7 @@ bool FDB::sorted(const metkit::mars::MarsRequest &request) {
         eckit::Log::userInfo() << "Using optimise" << std::endl;
     }
 
-    eckit::Log::debug() << "fdb5::FDB::retrieve() Sorted? " << sorted << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "fdb5::FDB::retrieve() Sorted? " << sorted << std::endl;
 
     return sorted;
 }
diff --git a/src/fdb5/api/FDBFactory.cc b/src/fdb5/api/FDBFactory.cc
index 4091d4302..6c48026ad 100644
--- a/src/fdb5/api/FDBFactory.cc
+++ b/src/fdb5/api/FDBFactory.cc
@@ -50,7 +50,7 @@ FDBBase::FDBBase(const Config& config, const std::string& name) :
         controlIdentifiers_ |= ControlIdentifier::Wipe;
     }
 
-    eckit::Log::debug() << "FDBBase: " << config << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "FDBBase: " << config << std::endl;
 }
 
 
@@ -114,7 +114,7 @@ std::unique_ptr FDBFactory::build(const Config& config) {
 
     std::string key = actualConfig.getString("type", "local");
 
-    eckit::Log::debug() << "Selecting FDB implementation: " << key << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Selecting FDB implementation: " << key << std::endl;
 
     eckit::AutoLock lock(mutex_);
 
@@ -127,7 +127,7 @@ std::unique_ptr FDBFactory::build(const Config& config) {
     }
 
     std::unique_ptr ret = it->second->make(actualConfig);
-    eckit::Log::debug() << "Constructed FDB implementation: " << *ret << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Constructed FDB implementation: " << *ret << std::endl;
     return ret;
 }
 
diff --git a/src/fdb5/api/FDBStats.cc b/src/fdb5/api/FDBStats.cc
index 8376eeb68..b1cafc670 100644
--- a/src/fdb5/api/FDBStats.cc
+++ b/src/fdb5/api/FDBStats.cc
@@ -76,7 +76,7 @@ void FDBStats::addArchive(size_t length, eckit::Timer& timer, size_t nfields) {
     elapsedArchive_ += elapsed;
     sumArchiveTimingSquared_ += elapsed * elapsed;
 
-    Log::debug() << "Archive count: " << numArchive_
+    LOG_DEBUG_LIB(LibFdb5) << "Archive count: " << numArchive_
                          << ", size: " << Bytes(length)
                          << ", total: " << Bytes(bytesArchive_)
                          << ", time: " << Seconds(elapsed)
@@ -97,7 +97,7 @@ void FDBStats::addRetrieve(size_t length, eckit::Timer& timer) {
     elapsedRetrieve_ += elapsed;
     sumRetrieveTimingSquared_ += elapsed * elapsed;
 
-    Log::debug() << "Retrieve count: " << numRetrieve_
+    LOG_DEBUG_LIB(LibFdb5) << "Retrieve count: " << numRetrieve_
                          << ", size: " << Bytes(length)
                          << ", total: " << Bytes(bytesRetrieve_)
                          << ", time: " << Seconds(elapsed)
@@ -113,7 +113,7 @@ void FDBStats::addFlush(eckit::Timer& timer) {
     elapsedFlush_ += elapsed;
     sumFlushTimingSquared_ += elapsed * elapsed;
 
-    Log::debug() << "Flush count: " << numFlush_
+    LOG_DEBUG_LIB(LibFdb5) << "Flush count: " << numFlush_
                          << ", time: " << elapsed << "s"
                          << ", total: " << elapsedFlush_ << "s" << std::endl;
 
diff --git a/src/fdb5/api/LocalFDB.cc b/src/fdb5/api/LocalFDB.cc
index b8644ac33..5627f973e 100644
--- a/src/fdb5/api/LocalFDB.cc
+++ b/src/fdb5/api/LocalFDB.cc
@@ -48,7 +48,7 @@ namespace fdb5 {
 void LocalFDB::archive(const Key& key, const void* data, size_t length) {
 
     if (!archiver_) {
-        Log::debug() << *this << ": Constructing new archiver" << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << *this << ": Constructing new archiver" << std::endl;
         archiver_.reset(new Archiver(config_));
     }
 
@@ -58,7 +58,7 @@ void LocalFDB::archive(const Key& key, const void* data, size_t length) {
 ListIterator LocalFDB::inspect(const metkit::mars::MarsRequest &request) {
 
     if (!inspector_) {
-        Log::debug() << *this << ": Constructing new retriever" << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << *this << ": Constructing new retriever" << std::endl;
         inspector_.reset(new Inspector(config_));
     }
 
@@ -82,44 +82,44 @@ APIIterator LocalFDB::queryInternal(const FDBTo
 }
 
 ListIterator LocalFDB::list(const FDBToolRequest& request) {
-    Log::debug() << "LocalFDB::list() : " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::list() : " << request << std::endl;
     return queryInternal(request);
 }
 
 DumpIterator LocalFDB::dump(const FDBToolRequest &request, bool simple) {
-    Log::debug() << "LocalFDB::dump() : " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::dump() : " << request << std::endl;
     return queryInternal(request, simple);
 }
 
 StatusIterator LocalFDB::status(const FDBToolRequest &request) {
-    Log::debug() << "LocalFDB::status() : " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::status() : " << request << std::endl;
     return queryInternal(request);
 }
 
 WipeIterator LocalFDB::wipe(const FDBToolRequest &request, bool doit, bool porcelain, bool unsafeWipeAll) {
-    Log::debug() << "LocalFDB::wipe() : " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::wipe() : " << request << std::endl;
     return queryInternal(request, doit, porcelain, unsafeWipeAll);
 }
 
 MoveIterator LocalFDB::move(const FDBToolRequest& request, const eckit::URI& dest) {
-    Log::debug() << "LocalFDB::move() : " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::move() : " << request << std::endl;
     return queryInternal(request, dest);
 }
 
 PurgeIterator LocalFDB::purge(const FDBToolRequest& request, bool doit, bool porcelain) {
-    Log::debug() << "LocalFDB::purge() : " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::purge() : " << request << std::endl;
     return queryInternal(request, doit, porcelain);
 }
 
 StatsIterator LocalFDB::stats(const FDBToolRequest& request) {
-    Log::debug() << "LocalFDB::stats() : " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::stats() : " << request << std::endl;
     return queryInternal(request);
 }
 
 ControlIterator LocalFDB::control(const FDBToolRequest& request,
                                   ControlAction action,
                                   ControlIdentifiers identifiers) {
-    Log::debug() << "LocalFDB::control() : " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::control() : " << request << std::endl;
     return queryInternal(request, action, identifiers);
 }
 
diff --git a/src/fdb5/api/RandomFDB.cc b/src/fdb5/api/RandomFDB.cc
index a9221d764..69c6946cc 100644
--- a/src/fdb5/api/RandomFDB.cc
+++ b/src/fdb5/api/RandomFDB.cc
@@ -52,7 +52,7 @@ class RandomFDBBuilder : public FDBBuilderBase {
         std::random_device rd;
         int choice = std::uniform_int_distribution(0, fdbConfigs.size()-1)(rd);
 
-        Log::debug() << "Constructing random API instance: " << choice+1
+        LOG_DEBUG_LIB(LibFdb5) << "Constructing random API instance: " << choice+1
                               << " / " << fdbConfigs.size() << std::endl;
 
         ASSERT(choice >= 0);
diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index 61bceb8fd..fc595d596 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -135,7 +135,7 @@ RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name
         if (numAliases == 0) {
             eckit::net::Endpoint storeEndpoint{store};
             storesReadMapping_[storeEndpoint] = storeEndpoint;
-            Log::debug() << "store endpoint: " << storeEndpoint << " default data location endpoint: " << storeEndpoint << std::endl;
+            LOG_DEBUG_LIB(LibFdb5) << "store endpoint: " << storeEndpoint << " default data location endpoint: " << storeEndpoint << std::endl;
         } else {
             for (size_t j=0; j() << "store endpoint: " << alias << " default data location endpoint: " << store << std::endl;
+                LOG_DEBUG_LIB(LibFdb5) << "store endpoint: " << alias << " default data location endpoint: " << store << std::endl;
             }
         }
     }
diff --git a/src/fdb5/api/SelectFDB.cc b/src/fdb5/api/SelectFDB.cc
index db81d4a3a..586ee5b19 100644
--- a/src/fdb5/api/SelectFDB.cc
+++ b/src/fdb5/api/SelectFDB.cc
@@ -138,7 +138,7 @@ auto SelectFDB::queryInternal(const FDBToolRequest& request, const QueryFN& fn)
 }
 
 ListIterator SelectFDB::list(const FDBToolRequest& request) {
-    Log::debug() << "SelectFDB::list() >> " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::list() >> " << request << std::endl;
     return queryInternal(request,
                          [](FDB& fdb, const FDBToolRequest& request) {
                             return fdb.list(request);
@@ -146,7 +146,7 @@ ListIterator SelectFDB::list(const FDBToolRequest& request) {
 }
 
 DumpIterator SelectFDB::dump(const FDBToolRequest& request, bool simple) {
-    Log::debug() << "SelectFDB::dump() >> " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::dump() >> " << request << std::endl;
     return queryInternal(request,
                          [simple](FDB& fdb, const FDBToolRequest& request) {
                             return fdb.dump(request, simple);
@@ -154,7 +154,7 @@ DumpIterator SelectFDB::dump(const FDBToolRequest& request, bool simple) {
 }
 
 StatusIterator SelectFDB::status(const FDBToolRequest& request) {
-    Log::debug() << "SelectFDB::status() >> " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::status() >> " << request << std::endl;
     return queryInternal(request,
                          [](FDB& fdb, const FDBToolRequest& request) {
                             return fdb.status(request);
@@ -162,7 +162,7 @@ StatusIterator SelectFDB::status(const FDBToolRequest& request) {
 }
 
 WipeIterator SelectFDB::wipe(const FDBToolRequest& request, bool doit, bool porcelain, bool unsafeWipeAll) {
-    Log::debug() << "SelectFDB::wipe() >> " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::wipe() >> " << request << std::endl;
     return queryInternal(request,
                          [doit, porcelain, unsafeWipeAll](FDB& fdb, const FDBToolRequest& request) {
                             return fdb.wipe(request, doit, porcelain, unsafeWipeAll);
@@ -170,7 +170,7 @@ WipeIterator SelectFDB::wipe(const FDBToolRequest& request, bool doit, bool porc
 }
 
 PurgeIterator SelectFDB::purge(const FDBToolRequest& request, bool doit, bool porcelain) {
-    Log::debug() << "SelectFDB::purge() >> " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::purge() >> " << request << std::endl;
     return queryInternal(request,
                          [doit, porcelain](FDB& fdb, const FDBToolRequest& request) {
                             return fdb.purge(request, doit, porcelain);
@@ -178,7 +178,7 @@ PurgeIterator SelectFDB::purge(const FDBToolRequest& request, bool doit, bool po
 }
 
 StatsIterator SelectFDB::stats(const FDBToolRequest &request) {
-    Log::debug() << "SelectFDB::stats() >> " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::stats() >> " << request << std::endl;
     return queryInternal(request,
                          [](FDB& fdb, const FDBToolRequest& request) {
                             return fdb.stats(request);
@@ -188,7 +188,7 @@ StatsIterator SelectFDB::stats(const FDBToolRequest &request) {
 ControlIterator SelectFDB::control(const FDBToolRequest& request,
                                    ControlAction action,
                                    ControlIdentifiers identifiers) {
-    Log::debug() << "SelectFDB::control >> " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::control >> " << request << std::endl;
     return queryInternal(request,
                          [action, identifiers](FDB& fdb, const FDBToolRequest& request) {
                             return fdb.control(request, action, identifiers);
diff --git a/src/fdb5/api/helpers/FDBToolRequest.cc b/src/fdb5/api/helpers/FDBToolRequest.cc
index 330178fbb..45762478c 100644
--- a/src/fdb5/api/helpers/FDBToolRequest.cc
+++ b/src/fdb5/api/helpers/FDBToolRequest.cc
@@ -44,7 +44,7 @@ std::vector FDBToolRequest::requestsFromString(const std::string
     ASSERT(parsedRequests.size() == 1);
 
     for (const auto& r : parsedRequests) {
-        eckit::Log::debug() << "Parsed request: " << static_cast(r) << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "Parsed request: " << static_cast(r) << std::endl;
         checkMinimumKeys(r, minimumKeys);
     }
 
@@ -75,7 +75,7 @@ std::vector FDBToolRequest::requestsFromString(const std::string
                     request.unsetValues(param);
                 }
             }*/
-            eckit::Log::debug() << "Expanded request: " << request << std::endl;
+            LOG_DEBUG_LIB(LibFdb5) << "Expanded request: " << request << std::endl;
             requests.emplace_back(FDBToolRequest(request, false, minimumKeys));
         }
     }
diff --git a/src/fdb5/config/Config.cc b/src/fdb5/config/Config.cc
index 67b05dec5..7db4124ef 100644
--- a/src/fdb5/config/Config.cc
+++ b/src/fdb5/config/Config.cc
@@ -68,7 +68,7 @@ Config::Config() : schemaPath_(""), schemaPathInitialised_(false) {
 }
 
 Config Config::make(const eckit::PathName& path, const eckit::Configuration& userConfig) {
-    eckit::Log::debug() << "Using FDB configuration file: " << path << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Using FDB configuration file: " << path << std::endl;
     Config cfg{YAMLConfiguration(path)};
     cfg.set("configSource", path);
     cfg.userConfig_ = std::make_shared(userConfig);
@@ -213,7 +213,7 @@ void Config::initializeSchemaPath() const {
     }
 
     schemaPathInitialised_ = true;
-    eckit::Log::debug() << "Using FDB schema: " << schemaPath_ << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Using FDB schema: " << schemaPath_ << std::endl;
 }
 
 PathName Config::configPath() const {
diff --git a/src/fdb5/database/AxisRegistry.cc b/src/fdb5/database/AxisRegistry.cc
index a79007e5f..c25e965a1 100644
--- a/src/fdb5/database/AxisRegistry.cc
+++ b/src/fdb5/database/AxisRegistry.cc
@@ -56,7 +56,7 @@ void AxisRegistry::deduplicate(const keyword_t& keyword, std::shared_ptr
     }
     else {
 //        dedups++;
-//        eckit::Log::debug() << dedups << " deduped axis [" << *ptr << "]" << std::endl;
+//        LOG_DEBUG_LIB(LibFdb5) << dedups << " deduped axis [" << *ptr << "]" << std::endl;
         ptr = *it;
     }
 }
diff --git a/src/fdb5/database/BaseArchiveVisitor.cc b/src/fdb5/database/BaseArchiveVisitor.cc
index 0ef41086f..9338d35c1 100644
--- a/src/fdb5/database/BaseArchiveVisitor.cc
+++ b/src/fdb5/database/BaseArchiveVisitor.cc
@@ -26,7 +26,7 @@ BaseArchiveVisitor::BaseArchiveVisitor(Archiver &owner, const Key &dataKey) :
 }
 
 bool BaseArchiveVisitor::selectDatabase(const Key &dbKey, const Key&) {
-    eckit::Log::debug() << "selectDatabase " << dbKey << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "selectDatabase " << dbKey << std::endl;
     owner_.selectDatabase(dbKey);
     ASSERT(owner_.catalogue_);
     owner_.catalogue_->deselectIndex();
diff --git a/src/fdb5/database/Catalogue.cc b/src/fdb5/database/Catalogue.cc
index 6b8e25de5..845c46074 100644
--- a/src/fdb5/database/Catalogue.cc
+++ b/src/fdb5/database/Catalogue.cc
@@ -91,7 +91,7 @@ std::unique_ptr CatalogueReaderFactory::build(const Key& dbKey,
     eckit::AutoLock lock(mutex_);
     auto j = builders_.find(nameLowercase);
 
-    eckit::Log::debug() << "Looking for CatalogueReaderBuilder [" << nameLowercase << "]" << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Looking for CatalogueReaderBuilder [" << nameLowercase << "]" << std::endl;
 
     if (j == builders_.end()) {
         eckit::Log::error() << "No CatalogueReaderBuilder for [" << nameLowercase << "]" << std::endl;
@@ -111,7 +111,7 @@ std::unique_ptr CatalogueReaderFactory::build(const eckit::URI&
     eckit::AutoLock lock(mutex_);
     auto j = builders_.find(nameLowercase);
 
-    eckit::Log::debug() << "Looking for CatalogueReaderBuilder [" << nameLowercase << "]" << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Looking for CatalogueReaderBuilder [" << nameLowercase << "]" << std::endl;
 
     if (j == builders_.end()) {
         eckit::Log::error() << "No CatalogueReaderBuilder for [" << nameLowercase << "]" << std::endl;
@@ -185,7 +185,7 @@ std::unique_ptr CatalogueWriterFactory::build(const Key& dbKey,
     eckit::AutoLock lock(mutex_);
     auto j = builders_.find(nameLowercase);
 
-    eckit::Log::debug() << "Looking for CatalogueWriterBuilder [" << nameLowercase << "]" << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Looking for CatalogueWriterBuilder [" << nameLowercase << "]" << std::endl;
 
     if (j == builders_.end()) {
         eckit::Log::error() << "No CatalogueWriterBuilder for [" << nameLowercase << "]" << std::endl;
@@ -205,7 +205,7 @@ std::unique_ptr CatalogueWriterFactory::build(const eckit::URI&
     eckit::AutoLock lock(mutex_);
     auto j = builders_.find(nameLowercase);
 
-    eckit::Log::debug() << "Looking for CatalogueWriterBuilder [" << nameLowercase << "]" << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Looking for CatalogueWriterBuilder [" << nameLowercase << "]" << std::endl;
 
     if (j == builders_.end()) {
         eckit::Log::error() << "No CatalogueWriterBuilder for [" << nameLowercase << "]" << std::endl;
diff --git a/src/fdb5/database/EntryVisitMechanism.cc b/src/fdb5/database/EntryVisitMechanism.cc
index da8ed9ef6..3f02d6c40 100644
--- a/src/fdb5/database/EntryVisitMechanism.cc
+++ b/src/fdb5/database/EntryVisitMechanism.cc
@@ -99,7 +99,7 @@ void EntryVisitMechanism::visit(const FDBToolRequest& request, EntryVisitor& vis
 
     // TODO: Put minimim keys check into FDBToolRequest.
 
-    Log::debug() << "REQUEST ====> " << request.request() << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "REQUEST ====> " << request.request() << std::endl;
 
     try {
 
@@ -110,7 +110,7 @@ void EntryVisitMechanism::visit(const FDBToolRequest& request, EntryVisitor& vis
         // And do the visitation
         for (const URI& uri : uris) {
 
-            Log::debug() << "FDB processing URI " << uri << std::endl;
+            LOG_DEBUG_LIB(LibFdb5) << "FDB processing URI " << uri << std::endl;
 
             std::unique_ptr catalogue = CatalogueReaderFactory::instance().build(uri, dbConfig_);
             ASSERT(catalogue->open());
diff --git a/src/fdb5/database/FieldLocation.cc b/src/fdb5/database/FieldLocation.cc
index c626abd15..437fdaac6 100644
--- a/src/fdb5/database/FieldLocation.cc
+++ b/src/fdb5/database/FieldLocation.cc
@@ -62,7 +62,7 @@ FieldLocation* FieldLocationFactory::build(const std::string& name, const eckit:
 
     auto j = builders_.find(name);
 
-    eckit::Log::debug() << "Looking for FieldLocationBuilder [" << name << "]" << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Looking for FieldLocationBuilder [" << name << "]" << std::endl;
 
     if (j == builders_.end()) {
         eckit::Log::error() << "No FieldLocationBuilder for [" << name << "]" << std::endl;
@@ -81,7 +81,7 @@ FieldLocation* FieldLocationFactory::build(const std::string& name, const eckit:
 
     auto j = builders_.find(name);
 
-    eckit::Log::debug() << "Looking for FieldLocationBuilder [" << name << "]" << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Looking for FieldLocationBuilder [" << name << "]" << std::endl;
 
     if (j == builders_.end()) {
         eckit::Log::error() << "No FieldLocationBuilder for [" << name << "]" << std::endl;
diff --git a/src/fdb5/database/Index.cc b/src/fdb5/database/Index.cc
index 488b0332d..fa458d402 100755
--- a/src/fdb5/database/Index.cc
+++ b/src/fdb5/database/Index.cc
@@ -128,7 +128,7 @@ void IndexBase::encodeLegacy(eckit::Stream& s, const int version) const {
 
 void IndexBase::put(const InspectionKey &key, const Field &field) {
 
-    eckit::Log::debug() << "FDB Index " << indexer_ << " " << key << " -> " << field << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "FDB Index " << indexer_ << " " << key << " -> " << field << std::endl;
 
     axes_.insert(key);
     add(key, field);
diff --git a/src/fdb5/database/Inspector.cc b/src/fdb5/database/Inspector.cc
index 73a5bb5ce..b25f36cab 100644
--- a/src/fdb5/database/Inspector.cc
+++ b/src/fdb5/database/Inspector.cc
@@ -49,7 +49,7 @@ bool InspectIterator::next(ListElement& elem) {
 //----------------------------------------------------------------------------------------------------------------------
 
 static void purgeCatalogue(Key& key, CatalogueReader*& db) {
-    Log::debug() << "Purging DB with key " << key << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Purging DB with key " << key << std::endl;
     delete db;
 }
 
@@ -67,7 +67,7 @@ ListIterator Inspector::inspect(const metkit::mars::MarsRequest& request,
     InspectIterator* iterator = new InspectIterator();
     MultiRetrieveVisitor visitor(notifyee, *iterator, databases_, dbConfig_);
 
-    Log::debug() << "Using schema: " << schema << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Using schema: " << schema << std::endl;
 
     schema.expand(request, visitor);
 
diff --git a/src/fdb5/database/Manager.cc b/src/fdb5/database/Manager.cc
index 0998c856c..0a516f76c 100644
--- a/src/fdb5/database/Manager.cc
+++ b/src/fdb5/database/Manager.cc
@@ -74,7 +74,7 @@ static const EngineTable& readEngineTypes(const eckit::PathName enginesFile) {
     // Sensible defaults if not configured
 
     if(!enginesFile.exists()) {
-        eckit::Log::debug() << "FDB Engines file not found: assuming Engine 'toc' Regex '.*'" << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "FDB Engines file not found: assuming Engine 'toc' Regex '.*'" << std::endl;
         table.push_back(EngineType("toc", ".*"));
         return table;
     }
@@ -83,7 +83,7 @@ static const EngineTable& readEngineTypes(const eckit::PathName enginesFile) {
 
     std::ifstream in(enginesFile.localPath());
 
-    eckit::Log::debug() << "Loading FDB engines from " << enginesFile << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Loading FDB engines from " << enginesFile << std::endl;
 
     if (!in) {
         eckit::Log::error() << enginesFile << eckit::Log::syserr << std::endl;
@@ -280,12 +280,12 @@ std::string Manager::engine(const URI& uri)
 // {
 //     std::set engines = Manager::engines(key);
 
-//     Log::debug() << "Matching engines for key " << key << " -> " << engines << std::endl;
+//     LOG_DEBUG_LIB(LibFdb5) << "Matching engines for key " << key << " -> " << engines << std::endl;
 
 //     std::vector r; // union of all locations
 
 //     for(std::set::const_iterator i = engines.begin(); i != engines.end(); ++i) {
-//         Log::debug() << "Selected FDB engine " << *i << std::endl;
+//         LOG_DEBUG_LIB(LibFdb5) << "Selected FDB engine " << *i << std::endl;
 //         std::vector p = Engine::backend(*i).allLocations(key, config_);
 //         r.insert(r.end(), p.begin(), p.end());
 //     }
@@ -298,12 +298,12 @@ std::vector Manager::visitableLocations(const metkit::mars::MarsRequ
 
     std::set engines = Manager::engines(rq, all);
 
-    Log::debug() << "Matching engines for request " << rq << " -> " << engines << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Matching engines for request " << rq << " -> " << engines << std::endl;
 
     std::vector r; // union of all locations
 
     for(std::set::const_iterator i = engines.begin(); i != engines.end(); ++i) {
-        Log::debug() << "Selected FDB engine " << *i << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "Selected FDB engine " << *i << std::endl;
         std::vector p;
         if (all) {
             p = Engine::backend(*i).visitableLocations(config_);
@@ -321,12 +321,12 @@ std::vector Manager::visitableLocations(const metkit::mars::MarsRequ
 
 //     std::set engines = Manager::engines(key);
 
-//     Log::debug() << "Matching engines for key " << key << " -> " << engines << std::endl;
+//     LOG_DEBUG_LIB(LibFdb5) << "Matching engines for key " << key << " -> " << engines << std::endl;
 
 //     std::vector r; // union of all locations
 
 //     for(std::set::const_iterator i = engines.begin(); i != engines.end(); ++i) {
-//         Log::debug() << "Selected FDB engine " << *i << std::endl;
+//         LOG_DEBUG_LIB(LibFdb5) << "Selected FDB engine " << *i << std::endl;
 //         std::vector p = Engine::backend(*i).writableLocations(key, config_);
 //         r.insert(r.end(), p.begin(), p.end());
 //     }
diff --git a/src/fdb5/database/MultiRetrieveVisitor.cc b/src/fdb5/database/MultiRetrieveVisitor.cc
index 3dcc88db5..37a299cd4 100644
--- a/src/fdb5/database/MultiRetrieveVisitor.cc
+++ b/src/fdb5/database/MultiRetrieveVisitor.cc
@@ -44,7 +44,7 @@ MultiRetrieveVisitor::~MultiRetrieveVisitor() {
 
 bool MultiRetrieveVisitor::selectDatabase(const Key& key, const Key&) {
 
-	eckit::Log::debug() << "FDB5 selectDatabase " << key  << std::endl;
+	LOG_DEBUG_LIB(LibFdb5) << "FDB5 selectDatabase " << key  << std::endl;
 
     /* is it the current DB ? */
 
@@ -58,7 +58,7 @@ bool MultiRetrieveVisitor::selectDatabase(const Key& key, const Key&) {
     /* is the DB already open ? */
 
     if(databases_.exists(key)) {
-        eckit::Log::debug() << "FDB5 Reusing database " << key << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "FDB5 Reusing database " << key << std::endl;
         catalogue_ = databases_.access(key);
         return true;
     }
@@ -75,10 +75,10 @@ bool MultiRetrieveVisitor::selectDatabase(const Key& key, const Key&) {
         return false;
     }
 
-    eckit::Log::debug() << "selectDatabase opening database " << key << " (type=" << newCatalogue->type() << ")" << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "selectDatabase opening database " << key << " (type=" << newCatalogue->type() << ")" << std::endl;
 
     if (!newCatalogue->open()) {
-        eckit::Log::debug() << "Database does not exist " << key << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "Database does not exist " << key << std::endl;
         return false;
     } else {
         catalogue_ = newCatalogue.release();
@@ -89,13 +89,13 @@ bool MultiRetrieveVisitor::selectDatabase(const Key& key, const Key&) {
 
 bool MultiRetrieveVisitor::selectIndex(const Key& key, const Key&) {
     ASSERT(catalogue_);
-    eckit::Log::debug() << "selectIndex " << key << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "selectIndex " << key << std::endl;
     return catalogue_->selectIndex(key);
 }
 
 bool MultiRetrieveVisitor::selectDatum(const InspectionKey& key, const Key& full) {
     ASSERT(catalogue_);
-    eckit::Log::debug() << "selectDatum " << key << ", " << full << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "selectDatum " << key << ", " << full << std::endl;
 
     Field field;
     if (catalogue_->retrieve(key, field)) {
diff --git a/src/fdb5/database/Report.cc b/src/fdb5/database/Report.cc
index cf3333159..ade021912 100644
--- a/src/fdb5/database/Report.cc
+++ b/src/fdb5/database/Report.cc
@@ -42,7 +42,7 @@ void Report::append(const dbtype_t& dbtype, DbStats stats)
 
 Report& Report::operator+=(const Report& rhs) {
 
-    Log::debug() << "Collating reports" << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Collating reports" << std::endl;
 
     // union of dbtypes
 
@@ -58,7 +58,7 @@ Report& Report::operator+=(const Report& rhs) {
     // collate DB stats
 
     for(std::map::const_iterator i = rhs.dbStats_.begin(); i != rhs.dbStats_.end(); ++i) {
-        Log::debug() << "dbtype " << i->first << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "dbtype " << i->first << std::endl;
         std::map::iterator j = dbStats_.find(i->first);
         if(j != dbStats_.end()) {
             j->second.add(i->second);
@@ -71,7 +71,7 @@ Report& Report::operator+=(const Report& rhs) {
     // collate Index stats
 
     for(std::map::const_iterator i = rhs.indexStats_.begin(); i != rhs.indexStats_.end(); ++i) {
-        Log::debug() << "dbtype " << i->first << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "dbtype " << i->first << std::endl;
         std::map::iterator j = indexStats_.find(i->first);
         if(j != indexStats_.end()) {
             j->second.add(i->second);
@@ -84,7 +84,7 @@ Report& Report::operator+=(const Report& rhs) {
     // collate Data stats
 
     for(std::map::const_iterator i = rhs.dataStats_.begin(); i != rhs.dataStats_.end(); ++i) {
-        Log::debug() << "dbtype " << i->first << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "dbtype " << i->first << std::endl;
         std::map::iterator j = dataStats_.find(i->first);
         if(j != dataStats_.end()) {
             j->second.add(i->second);
diff --git a/src/fdb5/database/RetrieveVisitor.cc b/src/fdb5/database/RetrieveVisitor.cc
index 350fd587f..d5b35c316 100644
--- a/src/fdb5/database/RetrieveVisitor.cc
+++ b/src/fdb5/database/RetrieveVisitor.cc
@@ -42,7 +42,7 @@ bool RetrieveVisitor::selectDatabase(const Key& key, const Key&) {
         }
     }
 
-    eckit::Log::debug() << "selectDatabase " << key << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "selectDatabase " << key << std::endl;
     catalogue_ = CatalogueReaderFactory::instance().build(key, fdb5::Config()).get();
 
     // If this database is locked for retrieval then it "does not exist"
diff --git a/src/fdb5/database/Store.cc b/src/fdb5/database/Store.cc
index 243e57f7b..48d2c0157 100644
--- a/src/fdb5/database/Store.cc
+++ b/src/fdb5/database/Store.cc
@@ -87,7 +87,7 @@ std::unique_ptr StoreFactory::build(const Key& key, const Config& config)
     eckit::AutoLock lock(mutex_);
     auto j = builders_.find(nameLowercase);
 
-    eckit::Log::debug() << "Looking for StoreBuilder [" << nameLowercase << "]" << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Looking for StoreBuilder [" << nameLowercase << "]" << std::endl;
 
     if (j == builders_.end()) {
         eckit::Log::error() << "No StoreBuilder for [" << nameLowercase << "]" << std::endl;
@@ -107,7 +107,7 @@ std::unique_ptr StoreFactory::build(const eckit::URI& uri, const Config&
     eckit::AutoLock lock(mutex_);
     auto j = builders_.find(nameLowercase);
 
-    eckit::Log::debug() << "Looking for StoreBuilder [" << nameLowercase << "]" << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Looking for StoreBuilder [" << nameLowercase << "]" << std::endl;
 
     if (j == builders_.end()) {
         eckit::Log::error() << "No StoreBuilder for [" << nameLowercase << "]" << std::endl;
diff --git a/src/fdb5/io/FieldHandle.cc b/src/fdb5/io/FieldHandle.cc
index 182b2373b..af1c71ab6 100644
--- a/src/fdb5/io/FieldHandle.cc
+++ b/src/fdb5/io/FieldHandle.cc
@@ -19,6 +19,7 @@
 #include "metkit/mars/MarsRequest.h"
 #include "metkit/hypercube/HyperCubePayloaded.h"
 
+#include "fdb5/LibFdb5.h"
 #include "fdb5/io/FieldHandle.h"
 
 namespace fdb5 {
@@ -184,7 +185,7 @@ long FieldHandle::read(void* buffer, long length) {
         p += n;
     }
 
-    eckit::Log::debug() << "FieldHandle::read " << (total > 0 ? total : n) << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "FieldHandle::read " << (total > 0 ? total : n) << std::endl;
 
     return total > 0 ? total : n;
 }
diff --git a/src/fdb5/io/LustreFileHandle.h b/src/fdb5/io/LustreFileHandle.h
index 942122ee5..da1fe77ec 100644
--- a/src/fdb5/io/LustreFileHandle.h
+++ b/src/fdb5/io/LustreFileHandle.h
@@ -62,7 +62,7 @@ class LustreFileHandle : public HANDLE {
 
         /* From the docs: llapi_file_create closes the file descriptor. You must re-open the file afterwards */
 
-        eckit::Log::debug() << "Creating Lustre file " << path
+        LOG_DEBUG_LIB(LibFdb5) << "Creating Lustre file " << path
                                     << " with " << stripe_.count_ << " stripes "
                                     << "of " << eckit::Bytes(stripe_.size_)
                                     << std::endl;
diff --git a/src/fdb5/message/MessageArchiver.cc b/src/fdb5/message/MessageArchiver.cc
index 2b46120e8..d6cdfbafd 100644
--- a/src/fdb5/message/MessageArchiver.cc
+++ b/src/fdb5/message/MessageArchiver.cc
@@ -54,14 +54,14 @@ static std::vector str_to_requests(const std::string&
 
     std::string rs = std::string("retrieve,")  + str;
 
-    Log::debug() << "Parsing request string : " << rs << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Parsing request string : " << rs << std::endl;
 
     std::istringstream in(rs);
     metkit::mars::MarsParser parser(in);
 
     std::vector p = parser.parse();
 
-    Log::debug() << "Parsed requests:" << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Parsed requests:" << std::endl;
     for (auto j = p.begin(); j != p.end(); ++j) {
         j->dump(Log::debug());
     }
@@ -73,7 +73,7 @@ static std::vector str_to_requests(const std::string&
 
     std::vector v = expand.expand(p);
 
-    Log::debug() << "Expanded requests:" << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Expanded requests:" << std::endl;
     for (auto j = v.begin(); j != v.end(); ++j) {
         j->dump(Log::debug());
     }
diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc
index ef44535cc..8f9cbadcb 100644
--- a/src/fdb5/rados/RadosStore.cc
+++ b/src/fdb5/rados/RadosStore.cc
@@ -122,7 +122,7 @@ eckit::DataHandle *RadosStore::createFileHandle(const eckit::PathName &path) {
 
 //    static size_t sizeBuffer = eckit::Resource("fdbBufferSize", 64 * 1024 * 1024);
 
-    eckit::Log::debug() << "Creating RadosWriteHandle to " << path
+    LOG_DEBUG_LIB(LibFdb5) << "Creating RadosWriteHandle to " << path
 //                                 << " with buffer of " << eckit::Bytes(sizeBuffer)
                                  << std::endl;
 
diff --git a/src/fdb5/remote/Connection.cc b/src/fdb5/remote/Connection.cc
index e8669f6d3..b15ef945a 100644
--- a/src/fdb5/remote/Connection.cc
+++ b/src/fdb5/remote/Connection.cc
@@ -108,7 +108,7 @@ void Connection::write(remote::Message msg, bool control, uint32_t clientID, uin
 
     // std::cout << "WRITE [" << "endpoint=" << ((control || single_) ? controlSocket() : dataSocket()).remotePort() << ",message=" << message.message << ",clientID=" << message.clientID() << ",requestID=" << message.requestID << ",payload=" << message.payloadSize << "]" << std::endl;
 
-    eckit::Log::debug() << "Connection::write [message=" << msg << ",clientID=" << message.clientID() << ",requestID=" << requestID << ",data=" << data.size() << ",payload=" << payloadLength << "]" << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Connection::write [message=" << msg << ",clientID=" << message.clientID() << ",requestID=" << requestID << ",data=" << data.size() << ",payload=" << payloadLength << "]" << std::endl;
 
     std::lock_guard lock((control || single_) ? controlMutex_ : dataMutex_);
     writeUnsafe(control, &message, sizeof(message));
diff --git a/src/fdb5/remote/Messages.cc b/src/fdb5/remote/Messages.cc
index e30fc1ced..6f9df650c 100644
--- a/src/fdb5/remote/Messages.cc
+++ b/src/fdb5/remote/Messages.cc
@@ -15,24 +15,19 @@
 
 #include "fdb5/remote/Messages.h"
 
-
-//#include "eckit/serialisation/Stream.h"
-
 using namespace eckit;
 
-
-namespace fdb5 {
-namespace remote {
+namespace fdb5::remote {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-
 std::ostream& operator<<(std::ostream& s, const Message& m) {
     switch (m) {
         case Message::None: s << "None"; break;
         case Message::Exit: s << "Exit"; break;
         case Message::Startup: s << "Startup"; break;
         case Message::Error: s << "Error"; break;
+        case Message::Stop: s << "Stop"; break;
         case Message::Stores: s << "Stores"; break;
         case Message::Schema: s << "Schema"; break;
 
@@ -64,15 +59,6 @@ std::ostream& operator<<(std::ostream& s, const Message& m) {
     return s;
 }
 
-// MessageHeader::MessageHeader(Message message, bool control, const Handler& clientID, uint32_t requestID, uint32_t payloadSize) :
-//     marker(StartMarker),
-//     version(CurrentVersion),
-//     message(message),
-//     clientID_((clientID.clientId()<<1) + (control ? 1 : 0)),
-//     requestID(requestID),
-//     payloadSize(payloadSize) {}
-
-
 MessageHeader::MessageHeader(Message message, bool control, uint32_t clientID, uint32_t requestID, uint32_t payloadSize) :
     marker(StartMarker),
     version(CurrentVersion),
@@ -81,33 +67,6 @@ MessageHeader::MessageHeader(Message message, bool control, uint32_t clientID, u
     requestID(requestID),
     payloadSize(payloadSize) {}
 
-//ClassSpec MessageHeader::classSpec_ = {&MessageHeader::classSpec(), "MessageHeader",};
-//Reanimator MessageHeader::reanimator_;
-//
-//
-//MessageHeader::MessageHeader(Message message, uint16_t payloadSize=0) :
-//    marker(StartMarker),
-//    version(CurrentVersion),
-//    message(message),
-//    payloadSize(payloadSize) {}
-//
-//MessageHeader::MessageHeader(Stream &s) :
-//    marker(s) {
-//    s >> version;
-//    uint16_t tmp;
-//    s >> tmp;
-//    message = static_cast(tmp);
-//    s >> payloadSize;
-//}
-//
-//void MessageHeader::encode(Stream &s) const {
-//    s << marker;
-//    s << version;
-//    s << static_cast(message);
-//    s << payloadSize;
-//}
-
 //----------------------------------------------------------------------------------------------------------------------
 
-} // namespace remote
-} // namespace fdb5
+} // namespace fdb5::remote
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index 779e6f8df..cff6f105b 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -60,7 +60,7 @@ class DataWriteRequest {
 ClientConnection::ClientConnection(const eckit::net::Endpoint& controlEndpoint, const std::string& defaultEndpoint):
     controlEndpoint_(controlEndpoint), defaultEndpoint_(defaultEndpoint), id_(1), connected_(false), dataWriteQueue_(nullptr) {
 
-    eckit::Log::debug() << "ClientConnection::ClientConnection() controlEndpoint: " << controlEndpoint << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "ClientConnection::ClientConnection() controlEndpoint: " << controlEndpoint << std::endl;
 }
 
 void ClientConnection::add(Client& client) {
@@ -125,14 +125,14 @@ bool ClientConnection::connect(bool singleAttempt) {
 
     try {
         // Connect to server, and check that the server is happy on the response 
-        eckit::Log::debug() << "Connecting to host: " << controlEndpoint_ << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "Connecting to host: " << controlEndpoint_ << std::endl;
         controlClient_.connect(controlEndpoint_, fdbMaxConnectRetries, fdbConnectTimeout);
 
         writeControlStartupMessage();
         eckit::SessionID serverSession = verifyServerStartupResponse();
 
         // Connect to the specified data port
-        eckit::Log::debug() << "Received data endpoint from host: " << dataEndpoint_ << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "Received data endpoint from host: " << dataEndpoint_ << std::endl;
         dataClient_.connect(dataEndpoint_, fdbMaxConnectRetries, fdbConnectTimeout);
         writeDataStartupMessage(serverSession);
 
@@ -353,7 +353,7 @@ void ClientConnection::listeningControlThreadLoop() {
 
             eckit::Buffer payload = Connection::readControl(hdr);
 
-            eckit::Log::debug() << "ClientConnection::listeningControlThreadLoop - got [message=" << hdr.message << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
+            LOG_DEBUG_LIB(LibFdb5) << "ClientConnection::listeningControlThreadLoop - got [message=" << hdr.message << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
 
             if (hdr.message == Message::Exit) {
                 return;
@@ -422,7 +422,7 @@ void ClientConnection::listeningDataThreadLoop() {
 
             eckit::Buffer payload = Connection::readData(hdr);
 
-            eckit::Log::debug() << "ClientConnection::listeningDataThreadLoop - got [message=" << hdr.message << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
+            LOG_DEBUG_LIB(LibFdb5) << "ClientConnection::listeningDataThreadLoop - got [message=" << hdr.message << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
 
             if (hdr.message == Message::Exit) {
                 return;
diff --git a/src/fdb5/remote/client/RemoteCatalogue.cc b/src/fdb5/remote/client/RemoteCatalogue.cc
index b886cad84..984c8861c 100644
--- a/src/fdb5/remote/client/RemoteCatalogue.cc
+++ b/src/fdb5/remote/client/RemoteCatalogue.cc
@@ -124,7 +124,7 @@ void RemoteCatalogue::flush() {
         MemoryStream s(sendBuf);
         s << numLocations_;
 
-        eckit::Log::debug() << " RemoteCatalogue::flush - flushing " << numLocations_ << " fields" << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << " RemoteCatalogue::flush - flushing " << numLocations_ << " fields" << std::endl;
 
         // The flush call is blocking
         uint32_t id = generateRequestID();
@@ -154,7 +154,7 @@ void RemoteCatalogue::loadSchema() {
     // (outside of the catalogue) 
 
     if (!schema_) {
-        eckit::Log::debug() << "RemoteCatalogue::loadSchema()" << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "RemoteCatalogue::loadSchema()" << std::endl;
 
         // send dbkey to remote.
         eckit::Buffer keyBuffer(4096);
@@ -170,14 +170,14 @@ void RemoteCatalogue::loadSchema() {
 }
 
 bool RemoteCatalogue::handle(Message message, bool control, uint32_t requestID) {
-    eckit::Log::debug() << *this << " - Received [message=" << ((uint) message) << ",requestID=" << requestID << "]" << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << *this << " - Received [message=" << ((uint) message) << ",requestID=" << requestID << "]" << std::endl;
     NOTIMP;
     return false;
 }
 bool RemoteCatalogue::handle(Message message, bool control, uint32_t requestID, eckit::Buffer&& payload) {
-    eckit::Log::debug() << *this << " - Received [message=" << ((uint) message) << ",requestID=" << requestID << ",payloadSize=" << payload.size() << "]" << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << *this << " - Received [message=" << ((uint) message) << ",requestID=" << requestID << ",payloadSize=" << payload.size() << "]" << std::endl;
     // if (message == Message::Schema) {
-    //     eckit::Log::debug() << "RemoteCatalogue::handle received payload size: " << payload.size() << std::endl;
+    //     LOG_DEBUG_LIB(LibFdb5) << "RemoteCatalogue::handle received payload size: " << payload.size() << std::endl;
     //     MemoryStream s(payload);
     //     schema_ = std::unique_ptr(eckit::Reanimator::reanimate(s));
     //     return true;
diff --git a/src/fdb5/remote/client/RemoteStore.cc b/src/fdb5/remote/client/RemoteStore.cc
index f78d1b8cd..14e28045c 100644
--- a/src/fdb5/remote/client/RemoteStore.cc
+++ b/src/fdb5/remote/client/RemoteStore.cc
@@ -296,7 +296,7 @@ void RemoteStore::flush() {
             MemoryStream s(sendBuf);
             s << stats.numArchive();
 
-            eckit::Log::debug() << " RemoteStore::flush - flushing " << stats.numArchive() << " fields" << std::endl;
+            LOG_DEBUG_LIB(LibFdb5) << " RemoteStore::flush - flushing " << stats.numArchive() << " fields" << std::endl;
             // The flush call is blocking
             uint32_t id = generateRequestID();
             controlWriteCheckResponse(Message::Flush, id, false, sendBuf, s.position());
diff --git a/src/fdb5/remote/server/AvailablePortList.cc b/src/fdb5/remote/server/AvailablePortList.cc
index fd2828002..112c66f55 100644
--- a/src/fdb5/remote/server/AvailablePortList.cc
+++ b/src/fdb5/remote/server/AvailablePortList.cc
@@ -46,7 +46,7 @@ std::set readServices() {
 
     std::ifstream in(servicesFile.localPath());
 
-    Log::debug() << "Scanning for ports to avoid in services file " << servicesFile << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Scanning for ports to avoid in services file " << servicesFile << std::endl;
 
     if (!in) {
         Log::error() << servicesFile << Log::syserr << std::endl;
@@ -84,7 +84,7 @@ std::set readServices() {
                 const std::string& port     = s[1];
                 const std::string& protocol = s[2];
 
-                Log::debug() << "Skipping port " << port << " service " << sname << " protocol " << protocol << std::endl;
+                LOG_DEBUG_LIB(LibFdb5) << "Skipping port " << port << " service " << sname << " protocol " << protocol << std::endl;
 
                 int p = toInt(port);
                 portsToSkip.insert(p);
diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc
index 75bda0b63..51d00b9c8 100644
--- a/src/fdb5/remote/server/CatalogueHandler.cc
+++ b/src/fdb5/remote/server/CatalogueHandler.cc
@@ -299,7 +299,7 @@ void CatalogueHandler::stores(uint32_t clientID, uint32_t requestID) {
         }
     }
 
-    Log::debug() << "Client " << clientIPaddress << " from network '" << clientNetwork << "'" << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Client " << clientIPaddress << " from network '" << clientNetwork << "'" << std::endl;
 
     ASSERT(config_.has("stores"));
     std::map> stores;
@@ -387,7 +387,7 @@ void CatalogueHandler::archiveBlob(const uint32_t clientID, const uint32_t reque
 
     std::unique_ptr location(eckit::Reanimator::reanimate(s));
 
-    Log::debug() << "CatalogueHandler::archiveBlob key: " << idxKey << key << "  location: " << location->uri() << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "CatalogueHandler::archiveBlob key: " << idxKey << key << "  location: " << location->uri() << std::endl;
 
     std::map::iterator it;
     {
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index 4a736d19b..0d9f5b4e9 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -70,7 +70,7 @@ ServerConnection::ServerConnection(eckit::net::TCPSocket& socket, const Config&
         controlSocket_(socket), numControlConnection_(0), numDataConnection_(0),
         dataSocket_(nullptr), dataListener_(0) {
 
-    eckit::Log::debug() << "ServerConnection::ServerConnection initialized" << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "ServerConnection::ServerConnection initialized" << std::endl;
 }
 
 ServerConnection::~ServerConnection() {
@@ -137,7 +137,7 @@ Handled ServerConnection::handleData(Message message, uint32_t clientID, uint32_
     return Handled::No;
 }
 
-constexpr eckit::LocalConfiguration ServerConnection::availableFunctionality() {
+eckit::LocalConfiguration ServerConnection::availableFunctionality() const {
     eckit::LocalConfiguration conf;
 //    Add to the configuration all the components that require to be versioned, as in the following example, with a vector of supported version numbers
     std::vector remoteFieldLocationVersions = {1};
@@ -185,7 +185,7 @@ void ServerConnection::initialiseConnections() {
         
         std::vector rflCommon = intersection(clientAvailableFunctionality, serverConf, "RemoteFieldLocation");
         if (rflCommon.size() > 0) {
-            eckit::Log::debug() << "Protocol negotiation - RemoteFieldLocation version " << rflCommon.back() << std::endl;
+            LOG_DEBUG_LIB(LibFdb5) << "Protocol negotiation - RemoteFieldLocation version " << rflCommon.back() << std::endl;
             agreedConf_.set("RemoteFieldLocation", rflCommon.back());
         } else {
             compatibleProtocol = false;
@@ -211,7 +211,7 @@ void ServerConnection::initialiseConnections() {
                 }
             }
 
-            eckit::Log::debug() << "Protocol negotiation - NumberOfConnections " << ncSelected << std::endl;
+            LOG_DEBUG_LIB(LibFdb5) << "Protocol negotiation - NumberOfConnections " << ncSelected << std::endl;
             agreedConf_.set("NumberOfConnections", ncSelected);
             single_ = (ncSelected == 1);
         }
@@ -248,7 +248,7 @@ void ServerConnection::initialiseConnections() {
         s << dataEndpoint;
         s << agreedConf_.get();
 
-        eckit::Log::debug() << "Protocol negotiation - configuration: " << agreedConf_ <>{{startupBuffer.data(), s.position()}});
     }
@@ -431,7 +431,7 @@ void ServerConnection::handle() {
         //tidyWorkers();
 
         eckit::Buffer payload = readControl(hdr); // READ CONTROL
-        eckit::Log::debug() << "ServerConnection::handle - got [message=" << hdr.message << ",clientID="<< hdr.clientID() << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "ServerConnection::handle - got [message=" << hdr.message << ",clientID="<< hdr.clientID() << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
 
         if (hdr.message == Message::Stop) {
             ASSERT(hdr.clientID());
diff --git a/src/fdb5/remote/server/ServerConnection.h b/src/fdb5/remote/server/ServerConnection.h
index 2c05f6e59..b9d48b0ca 100644
--- a/src/fdb5/remote/server/ServerConnection.h
+++ b/src/fdb5/remote/server/ServerConnection.h
@@ -104,7 +104,7 @@ class ServerConnection : public Connection, public Handler {
 
     // socket methods
     int selectDataPort();
-    constexpr eckit::LocalConfiguration availableFunctionality();
+    eckit::LocalConfiguration availableFunctionality() const;
     
     // Worker functionality
     void tidyWorkers();
diff --git a/src/fdb5/remote/server/StoreHandler.cc b/src/fdb5/remote/server/StoreHandler.cc
index b0f7530a4..859c87139 100644
--- a/src/fdb5/remote/server/StoreHandler.cc
+++ b/src/fdb5/remote/server/StoreHandler.cc
@@ -94,7 +94,7 @@ void StoreHandler::read(uint32_t clientID, uint32_t requestID, const eckit::Buff
 
     std::unique_ptr location(eckit::Reanimator::reanimate(s));
 
-    Log::debug() << "Queuing for read: " << requestID << " " << *location << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Queuing for read: " << requestID << " " << *location << std::endl;
 
     std::unique_ptr dh;
     dh.reset(location->dataHandle());
@@ -123,7 +123,7 @@ void StoreHandler::writeToParent(const uint32_t clientID, const uint32_t request
         long dataRead;
 
         dh->openForRead();
-        Log::debug() << "Reading: " << requestID << " dh size: " << dh->size()
+        LOG_DEBUG_LIB(LibFdb5) << "Reading: " << requestID << " dh size: " << dh->size()
                               << std::endl;
 
         while ((dataRead = dh->read(writeBuffer, writeBuffer.size())) != 0) {
@@ -132,13 +132,13 @@ void StoreHandler::writeToParent(const uint32_t clientID, const uint32_t request
 
         // And when we are done, add a complete message.
 
-        Log::debug() << "Writing retrieve complete message: " << requestID
+        LOG_DEBUG_LIB(LibFdb5) << "Writing retrieve complete message: " << requestID
                               << std::endl;
 
         write(Message::Complete, false, clientID, requestID);
 
         Log::status() << "Done retrieve: " << requestID << std::endl;
-        Log::debug() << "Done retrieve: " << requestID << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "Done retrieve: " << requestID << std::endl;
     }
     catch (std::exception& e) {
         // n.b. more general than eckit::Exception
diff --git a/src/fdb5/rules/Predicate.cc b/src/fdb5/rules/Predicate.cc
index 5f932d880..1f485eefa 100644
--- a/src/fdb5/rules/Predicate.cc
+++ b/src/fdb5/rules/Predicate.cc
@@ -24,8 +24,8 @@ eckit::Reanimator Predicate::reanimator_;
 Predicate::Predicate(const std::string &keyword, Matcher *matcher) :
     matcher_(matcher),
     keyword_(keyword) {
-    //    dump(eckit::Log::debug());
-    //    eckit::Log::debug() << std::endl;
+    //    dump(Log::debug());
+    //    LOG_DEBUG_LIB(LibFdb5) << std::endl;
 }
 
 Predicate::Predicate(eckit::Stream& s) {
diff --git a/src/fdb5/rules/Schema.cc b/src/fdb5/rules/Schema.cc
index 7d006ee7f..366f9362c 100644
--- a/src/fdb5/rules/Schema.cc
+++ b/src/fdb5/rules/Schema.cc
@@ -168,7 +168,7 @@ void Schema::load(const eckit::PathName &path, bool replace) {
 
     path_ = path;
 
-    eckit::Log::debug() << "Loading FDB rules from " << path << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Loading FDB rules from " << path << std::endl;
 
     std::ifstream in(path.localPath());
     if (!in) {
diff --git a/src/fdb5/toc/EnvVarFileSpaceHandler.cc b/src/fdb5/toc/EnvVarFileSpaceHandler.cc
index 64cace0b7..0a04013c9 100644
--- a/src/fdb5/toc/EnvVarFileSpaceHandler.cc
+++ b/src/fdb5/toc/EnvVarFileSpaceHandler.cc
@@ -46,7 +46,7 @@ eckit::PathName EnvVarFileSpaceHandler::selectFileSystem(const Key& key, const F
 
     AutoLock lock(mutex_);
 
-    Log::debug() << "Selecting a file system based on environment variable " << fdbFileSpaceSHandlerEnvVarName_ << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Selecting a file system based on environment variable " << fdbFileSpaceSHandlerEnvVarName_ << std::endl;
 
     const char* value  = ::getenv(fdbFileSpaceSHandlerEnvVarName_.c_str());
     if(value) {
diff --git a/src/fdb5/toc/ExpverFileSpaceHandler.cc b/src/fdb5/toc/ExpverFileSpaceHandler.cc
index 2331e4e88..c99f62c7d 100644
--- a/src/fdb5/toc/ExpverFileSpaceHandler.cc
+++ b/src/fdb5/toc/ExpverFileSpaceHandler.cc
@@ -42,7 +42,7 @@ ExpverFileSpaceHandler::~ExpverFileSpaceHandler() {
 
 void ExpverFileSpaceHandler::load() const {
 
-    Log::debug() << "Loading " << fdbExpverFileSystems_ << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Loading " << fdbExpverFileSystems_ << std::endl;
 
     std::ifstream in(fdbExpverFileSystems_.localPath());
 
@@ -138,7 +138,7 @@ eckit::PathName ExpverFileSpaceHandler::append(const std::string& expver, const
         }
 
         if(s[0] == expver) {
-            Log::debug() << "Found expver " << expver << " " << path << " in " << fdbExpverFileSystems_ << std::endl;
+            LOG_DEBUG_LIB(LibFdb5) << "Found expver " << expver << " " << path << " in " << fdbExpverFileSystems_ << std::endl;
             return PathName(s[1]);
         }
     }
@@ -156,7 +156,7 @@ eckit::PathName ExpverFileSpaceHandler::append(const std::string& expver, const
 
     // append to the file
 
-    Log::debug() << "Appending expver " << expver << " " << path << " to " << fdbExpverFileSystems_ << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Appending expver " << expver << " " << path << " to " << fdbExpverFileSystems_ << std::endl;
 
     of << expver << " " << path << std::endl;
 
@@ -171,7 +171,7 @@ PathName ExpverFileSpaceHandler::select(const Key& key, const FileSpace& fs) con
 }
 
 static bool expver_is_valid(const std::string& str) {
-    Log::debug() << "Validating expver string [" << str << "]" << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Validating expver string [" << str << "]" << std::endl;
     return (str.size() == 4) and std::find_if_not(str.begin(), str.end(), isalnum) == str.end();
 }
 
@@ -194,11 +194,11 @@ eckit::PathName ExpverFileSpaceHandler::selectFileSystem(const Key& key, const F
     if(not expver_is_valid(expver))
         throw eckit::BadValue("Invalid expver value " + expver, Here());
 
-    Log::debug() << "Selecting file system for expver [" << expver << "]" << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Selecting file system for expver [" << expver << "]" << std::endl;
 
     PathTable::const_iterator itr = table_.find(expver);
     if(itr != table_.end()) {
-        Log::debug() << "Found expver " << expver << " " << itr->second << " in " << fdbExpverFileSystems_ << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "Found expver " << expver << " " << itr->second << " in " << fdbExpverFileSystems_ << std::endl;
 
         if (!fdbRootDirectory.empty() && itr->second != fdbRootDirectory) {
             Log::warning() << "Existing root directory " << itr->second << " does not match FDB5_ROOT. Using existing" << std::endl;
@@ -222,7 +222,7 @@ eckit::PathName ExpverFileSpaceHandler::selectFileSystem(const Key& key, const F
             throw BadParameter(msg.str());
         }
 
-        Log::debug() << "Using root directory specified by FDB5_ROOT: " << fdbRootDirectory << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "Using root directory specified by FDB5_ROOT: " << fdbRootDirectory << std::endl;
         maybe = fdbRootDirectory;
     }
 
diff --git a/src/fdb5/toc/FileSpace.cc b/src/fdb5/toc/FileSpace.cc
index ab3267f92..567b25b35 100644
--- a/src/fdb5/toc/FileSpace.cc
+++ b/src/fdb5/toc/FileSpace.cc
@@ -45,12 +45,12 @@ TocPath FileSpace::filesystem(const Key& key, const eckit::PathName& db) const {
 
     TocPath existingDB;
     if (existsDB(key, db, existingDB)) {
-        Log::debug() << "Found existing DB for key " << key << " -> " << existingDB.directory_ << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "Found existing DB for key " << key << " -> " << existingDB.directory_ << std::endl;
         return existingDB;
     }
 
-    Log::debug() << "FDB for key " << key << " not found, selecting a root" << std::endl;
-    // Log::debug() << eckit::BackTrace::dump() << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "FDB for key " << key << " not found, selecting a root" << std::endl;
+    // LOG_DEBUG_LIB(LibFdb5) << eckit::BackTrace::dump() << std::endl;
 
     return TocPath{FileSpaceHandler::lookup(handler_).selectFileSystem(key, *this) / db, ControlIdentifiers{}};
 }
diff --git a/src/fdb5/toc/Root.cc b/src/fdb5/toc/Root.cc
index 2fbc6c187..794ec6f25 100644
--- a/src/fdb5/toc/Root.cc
+++ b/src/fdb5/toc/Root.cc
@@ -47,7 +47,7 @@ bool Root::exists() const {
             Log::warning() << "FDB root " << path_ << " " << Log::syserr << std::endl;
             exists_ = false;
         }
-        Log::debug() << "Root " << *this << (exists_ ? " exists" : " does NOT exists") << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "Root " << *this << (exists_ ? " exists" : " does NOT exists") << std::endl;
         checked_ = true;
     }
     
diff --git a/src/fdb5/toc/RootManager.cc b/src/fdb5/toc/RootManager.cc
index e0624283e..96ab2f864 100644
--- a/src/fdb5/toc/RootManager.cc
+++ b/src/fdb5/toc/RootManager.cc
@@ -52,7 +52,7 @@ class DbPathNamer {
     DbPathNamer(const std::string& keyregex, const std::string& format) :
         format_(format){
         crack(keyregex);
-        eckit::Log::debug() << "Building " << *this << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "Building " << *this << std::endl;
     }
 
     /// Full match of the incomming key with the key regex
@@ -236,7 +236,7 @@ static const DbPathNamerTable& readDbNamers(const Config& config) {
 
     if(filename.exists()) {
 
-        eckit::Log::debug() << "Loading FDB DBPathNames from " << fdbDbNamesFile << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "Loading FDB DBPathNames from " << fdbDbNamesFile << std::endl;
 
         std::ifstream in(filename.localPath());
 
@@ -299,7 +299,7 @@ static std::vector readRoots(const eckit::PathName& fdbRootsFile) {
 
     std::ifstream in(fdbRootsFile.localPath());
 
-    eckit::Log::debug() << "Loading FDB roots from " << fdbRootsFile << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Loading FDB roots from " << fdbRootsFile << std::endl;
 
     if (!in) {
         eckit::Log::error() << fdbRootsFile << eckit::Log::syserr << std::endl;
@@ -417,7 +417,7 @@ static FileSpaceTable parseFileSpacesFile(const eckit::PathName& fdbHome) {
     eckit::PathName fdbSpacesFile = eckit::Resource("fdbSpacesFile;$FDB_SPACES_FILE", fdbHome / "etc/fdb/spaces");
     std::ifstream in(fdbSpacesFile.localPath());
 
-    eckit::Log::debug() << "Loading FDB file spaces from " << fdbSpacesFile << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Loading FDB file spaces from " << fdbSpacesFile << std::endl;
 
     if (!in) {
         throw eckit::ReadError(fdbSpacesFile, Here());
@@ -570,14 +570,14 @@ std::string RootManager::dbPathName(const Key& key)
     for (DbPathNamerTable::const_iterator i = dbPathNamers_.begin(); i != dbPathNamers_.end() ; ++i) {
         if(i->match(key)) {
             dbpath = i->name(key);
-            eckit::Log::debug() << "DbName is " << dbpath << " for key " << key <<  std::endl;
+            LOG_DEBUG_LIB(LibFdb5) << "DbName is " << dbpath << " for key " << key <<  std::endl;
             return dbpath;
         }
     }
 
     // default naming convention for DB's
     dbpath = key.valuesToString();
-    eckit::Log::debug() << "Using default naming convention for key " << key << " -> " << dbpath <<  std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Using default naming convention for key " << key << " -> " << dbpath <<  std::endl;
     return dbpath;
 }
 
@@ -587,7 +587,7 @@ std::vector RootManager::possibleDbPathNames(const InspectionKey& k
     for (DbPathNamerTable::const_iterator i = dbPathNamers_.begin(); i != dbPathNamers_.end() ; ++i) {
         if(i->match(key, missing)) {
             std::string dbpath = i->namePartial(key, missing);
-            eckit::Log::debug() << "Matched " << *i << " with key " << key << " resulting in dbpath " << dbpath << std::endl;
+            LOG_DEBUG_LIB(LibFdb5) << "Matched " << *i << " with key " << key << " resulting in dbpath " << dbpath << std::endl;
             result.push_back(dbpath);
         }
     }
@@ -605,7 +605,7 @@ std::vector RootManager::possibleDbPathNames(const InspectionKey& k
     }
     result.push_back(oss.str());
 
-    eckit::Log::debug() << "Using default naming convention for key " << key << " -> " << result.back() <<  std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Using default naming convention for key " << key << " -> " << result.back() <<  std::endl;
 
     return result;
 }
@@ -615,7 +615,7 @@ TocPath RootManager::directory(const Key& key) {
 
     PathName dbpath = dbPathName(key);
 
-    eckit::Log::debug() << "Choosing directory for key " << key << " dbpath " << dbpath << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Choosing directory for key " << key << " dbpath " << dbpath << std::endl;
 
     // override root location for testing purposes only
 
@@ -632,7 +632,7 @@ TocPath RootManager::directory(const Key& key) {
     for (FileSpaceTable::const_iterator i = spacesTable_.begin(); i != spacesTable_.end() ; ++i) {
         if(i->match(keystr)) {
             TocPath db = i->filesystem(key, dbpath);
-            eckit::Log::debug() << "Database directory " << db.directory_ << std::endl;
+            LOG_DEBUG_LIB(LibFdb5) << "Database directory " << db.directory_ << std::endl;
             return db;
         }
     }
@@ -665,24 +665,24 @@ std::vector RootManager::visitableRoots(const std::set&
     std::transform(keys.begin(), keys.end(), std::back_inserter(keystrings),
                    [](const Key& k) { return k.valuesToString(); });
 
-    Log::debug() << "RootManager::visitableRoots() trying to match keys " << keystrings << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "RootManager::visitableRoots() trying to match keys " << keystrings << std::endl;
 
     for (const auto& space : spacesTable_) {
 
         bool matched = false;
         for (const std::string& k : keystrings) {
             if (space.match(k) || k.empty()) {
-                Log::debug() << "MATCH space " << space << std::endl;
+                LOG_DEBUG_LIB(LibFdb5) << "MATCH space " << space << std::endl;
                 space.enabled(ControlIdentifier::List,roots);
                 matched = true;
                 break;
             }
         }
 
-        if (!matched) Log::debug() << "FAIL to match space " << space << std::endl;
+        if (!matched) LOG_DEBUG_LIB(LibFdb5) << "FAIL to match space " << space << std::endl;
     }
 
-    Log::debug() << "Visitable Roots " << roots << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Visitable Roots " << roots << std::endl;
 
     return std::vector(roots.begin(), roots.end());
 }
@@ -715,7 +715,7 @@ std::vector RootManager::canArchiveRoots(const Key& key) {
         }
     }
 
-    Log::debug() << "Roots supporting archival " << roots << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Roots supporting archival " << roots << std::endl;
 
     return std::vector(roots.begin(), roots.end());
 }
@@ -733,7 +733,7 @@ std::vector RootManager::canMoveToRoots(const Key& key) {
         }
     }
 
-    Log::debug() << "Roots supporting wiping " << roots << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Roots supporting wiping " << roots << std::endl;
 
     return std::vector(roots.begin(), roots.end());
 }
diff --git a/src/fdb5/toc/TocCatalogueReader.cc b/src/fdb5/toc/TocCatalogueReader.cc
index adb5892fc..bdb3d9cc5 100644
--- a/src/fdb5/toc/TocCatalogueReader.cc
+++ b/src/fdb5/toc/TocCatalogueReader.cc
@@ -33,7 +33,7 @@ TocCatalogueReader::TocCatalogueReader(const eckit::URI& uri, const fdb5::Config
 }
 
 TocCatalogueReader::~TocCatalogueReader() {
-    eckit::Log::debug() << "Closing DB " << (*this) << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Closing DB " << (*this) << std::endl;
 }
 
 void TocCatalogueReader::loadIndexesAndRemap() {
@@ -62,7 +62,7 @@ bool TocCatalogueReader::selectIndex(const Key &idxKey) {
         }
     }
 
-    eckit::Log::debug() << "TocCatalogueReader::selectIndex " << idxKey << ", found "
+    LOG_DEBUG_LIB(LibFdb5) << "TocCatalogueReader::selectIndex " << idxKey << ", found "
                                 << matching_.size() << " matche(s)" << std::endl;
 
     return (matching_.size() != 0);
@@ -105,8 +105,8 @@ void TocCatalogueReader::close() {
 }
 
 bool TocCatalogueReader::retrieve(const InspectionKey& key, Field& field) const {
-    eckit::Log::debug() << "Trying to retrieve key " << key << std::endl;
-    eckit::Log::debug() << "Scanning indexes " << matching_.size() << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Trying to retrieve key " << key << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Scanning indexes " << matching_.size() << std::endl;
 
     for (auto m = matching_.begin(); m != matching_.end(); ++m) {
         const Index& idx((*m)->first);
diff --git a/src/fdb5/toc/TocCatalogueWriter.cc b/src/fdb5/toc/TocCatalogueWriter.cc
index 21e8bed4e..a58c091cb 100644
--- a/src/fdb5/toc/TocCatalogueWriter.cc
+++ b/src/fdb5/toc/TocCatalogueWriter.cc
@@ -108,7 +108,7 @@ bool TocCatalogueWriter::open() {
 
 void TocCatalogueWriter::clean() {
 
-    eckit::Log::debug() << "Closing path " << directory_ << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Closing path " << directory_ << std::endl;
 
     flush(); // closes the TOC entries & indexes but not data files
 
@@ -380,7 +380,7 @@ void TocCatalogueWriter::compactSubTocIndexes() {
 
     if (useSubToc() && anythingWrittenToSubToc()) {
 
-        eckit::Log::debug() << "compacting sub tocs" << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "compacting sub tocs" << std::endl;
 
         for (IndexStore::iterator j = fullIndexes_.begin(); j != fullIndexes_.end(); ++j) {
             Index& idx = j->second;
diff --git a/src/fdb5/toc/TocEngine.cc b/src/fdb5/toc/TocEngine.cc
index 1c253eb80..99be2c22d 100644
--- a/src/fdb5/toc/TocEngine.cc
+++ b/src/fdb5/toc/TocEngine.cc
@@ -154,7 +154,7 @@ std::set TocEngine::databases(const std::set& ke
 
     for (std::vector::const_iterator j = roots.begin(); j != roots.end(); ++j) {
 
-        Log::debug() << "Scanning for TOC FDBs in root " << *j << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "Scanning for TOC FDBs in root " << *j << std::endl;
 
         std::list dbs;
         scan_dbs(*j, dbs);
@@ -168,13 +168,13 @@ std::set TocEngine::databases(const std::set& ke
                 std::string regex = "^" + *j + "/" + *dbpath + "$";
                 Regex re(searchCaseSensitiveDB ? eckit::StringTools::lower(regex) : regex);
 
-                Log::debug() << " -> key i " << *i
+                LOG_DEBUG_LIB(LibFdb5) << " -> key i " << *i
                                      << " dbpath " << *dbpath
                                      << " pathregex " << re << std::endl;
 
                 for (std::list::const_iterator k = dbs.begin(); k != dbs.end(); ++k) {
 
-                    Log::debug() << "    -> db " << *k << std::endl;
+                    LOG_DEBUG_LIB(LibFdb5) << "    -> db " << *k << std::endl;
 
                     if(result.find(*k) != result.end()) {
                         continue;
@@ -188,7 +188,7 @@ std::set TocEngine::databases(const std::set& ke
         }
     }
 
-    Log::debug() << "TocEngine::databases() results " << result << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "TocEngine::databases() results " << result << std::endl;
 
     return result;
 }
@@ -201,7 +201,7 @@ std::vector TocEngine::databases(const Key& key,
 
     matchKeyToDB(key, keys, regexForMissingValues, config);
 
-    Log::debug() << "Matched DB schemas for key " << key << " -> keys " << keys << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Matched DB schemas for key " << key << " -> keys " << keys << std::endl;
 
     std::set databasesMatchRegex(databases(keys, roots, config));
 
@@ -210,7 +210,7 @@ std::vector TocEngine::databases(const Key& key,
         try {
             TocHandler toc(path, config);
             if (toc.databaseKey().match(key)) {
-                Log::debug() << " found match with " << path << std::endl;
+                LOG_DEBUG_LIB(LibFdb5) << " found match with " << path << std::endl;
                 result.push_back(eckit::URI(TocEngine::typeName(), path));
             }
         } catch (eckit::Exception& e) {
@@ -231,7 +231,7 @@ std::vector TocEngine::databases(const metkit::mars::MarsRequest& re
 //    matchRequestToDB(request, keys, regexForMissingValues, config);
     matchRequestToDB(request, keys, "", config);
 
-    Log::debug() << "Matched DB schemas for request " << request << " -> keys " << keys << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Matched DB schemas for request " << request << " -> keys " << keys << std::endl;
 
     std::set databasesMatchRegex(databases(keys, roots, config));
 
@@ -243,11 +243,11 @@ std::vector TocEngine::databases(const metkit::mars::MarsRequest& re
                     path = path.dirName();
                 path = path.realName();
 
-                Log::debug() << "FDB processing Path " << path << std::endl;
+                LOG_DEBUG_LIB(LibFdb5) << "FDB processing Path " << path << std::endl;
 
                 TocHandler toc(path, config);
                 if (toc.databaseKey().partialMatch(request)) {
-                    Log::debug() << " found match with " << path << std::endl;
+                    LOG_DEBUG_LIB(LibFdb5) << " found match with " << path << std::endl;
                     result.push_back(eckit::URI(TocEngine::typeName(), path));
                 }
             }
diff --git a/src/fdb5/toc/TocHandler.cc b/src/fdb5/toc/TocHandler.cc
index 3b84c7cdf..71d90c866 100644
--- a/src/fdb5/toc/TocHandler.cc
+++ b/src/fdb5/toc/TocHandler.cc
@@ -161,7 +161,7 @@ TocHandler::TocHandler(const eckit::PathName& path, const Key& parentKey) :
         Key key(databaseKey());
         if (!parentKey.empty() && parentKey != key) {
 
-            eckit::Log::debug() << "Opening (remapped) toc with differing parent key: "
+            LOG_DEBUG_LIB(LibFdb5) << "Opening (remapped) toc with differing parent key: "
                                          << key << " --> " << parentKey << std::endl;
 
             if (parentKey.size() != key.size()) {
@@ -181,7 +181,7 @@ TocHandler::TocHandler(const eckit::PathName& path, const Key& parentKey) :
                 }
             }
 
-            eckit::Log::debug() << "Key remapping: " << remapKey_ << std::endl;
+            LOG_DEBUG_LIB(LibFdb5) << "Key remapping: " << remapKey_ << std::endl;
         }
     }
 }
@@ -234,7 +234,7 @@ void TocHandler::openForAppend() {
 
     ASSERT(fd_ == -1);
 
-    eckit::Log::debug() << "Opening for append TOC " << tocPath_ << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Opening for append TOC " << tocPath_ << std::endl;
 
     int iomode = O_WRONLY | O_APPEND;
 #ifdef O_NOATIME
@@ -261,7 +261,7 @@ void TocHandler::openForRead() const {
 
     writeMode_ = false;
 
-    eckit::Log::debug() << "Opening for read TOC " << tocPath_ << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Opening for read TOC " << tocPath_ << std::endl;
 
     int iomode = O_RDONLY;
 #ifdef O_NOATIME
@@ -321,7 +321,7 @@ void TocHandler::append(TocRecord &r, size_t payloadSize ) {
     ASSERT(fd_ != -1);
     ASSERT(not cachedToc_);
 
-    Log::debug() << "Writing toc entry: " << (int)r.header_.tag_ << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Writing toc entry: " << (int)r.header_.tag_ << std::endl;
 
     // Obtain the rounded size, and set it in the record header.
     size_t roundedSize = roundRecord(r, payloadSize);
@@ -431,17 +431,17 @@ bool TocHandler::readNext( TocRecord &r, bool walkSubTocs, bool hideSubTocEntrie
                 std::pair key(absPath.baseName(), 0);
                 if (maskedEntries_.find(key) != maskedEntries_.end()) {
                     if (!readMasked){
-                        Log::debug() << "SubToc ignored by mask: " << path << std::endl;
+                        LOG_DEBUG_LIB(LibFdb5) << "SubToc ignored by mask: " << path << std::endl;
                         continue;
                     }
                     // This is a masked subtoc, so it is valid for it to not exist.
                     if (!absPath.exists()) {
-                        Log::debug() << "SubToc does not exist: " << path << std::endl;
+                        LOG_DEBUG_LIB(LibFdb5) << "SubToc does not exist: " << path << std::endl;
                         continue;
                     }
                 }
 
-                eckit::Log::debug() << "Opening SUB_TOC: " << absPath << " " << parentKey_ << std::endl;
+                LOG_DEBUG_LIB(LibFdb5) << "Opening SUB_TOC: " << absPath << " " << parentKey_ << std::endl;
 
                 subTocRead_.reset(new TocHandler(absPath, parentKey_));
                 subTocRead_->openForRead();
@@ -467,12 +467,12 @@ bool TocHandler::readNext( TocRecord &r, bool walkSubTocs, bool hideSubTocEntrie
                 std::pair key(absPath.baseName(), offset);
                 if (maskedEntries_.find(key) != maskedEntries_.end()) {
                     if(!readMasked){
-                        Log::debug() << "Index ignored by mask: " << path << ":" << offset << std::endl;
+                        LOG_DEBUG_LIB(LibFdb5) << "Index ignored by mask: " << path << ":" << offset << std::endl;
                         continue;
                     }
                     // This is a masked index, so it is valid for it to not exist.
                     if (!absPath.exists()) {
-                        Log::debug() << "Index does not exist: " << path << ":" << offset << std::endl;
+                        LOG_DEBUG_LIB(LibFdb5) << "Index does not exist: " << path << ":" << offset << std::endl;
                         continue;
                     }
                 }
@@ -582,7 +582,7 @@ void TocHandler::close() const {
     }
 
     if ( fd_ >= 0 ) {
-        eckit::Log::debug() << "Closing TOC " << tocPath_ << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "Closing TOC " << tocPath_ << std::endl;
         if(dirty_) {
             SYSCALL2( eckit::fdatasync(fd_), tocPath_ );
             dirty_ = false;
@@ -733,12 +733,12 @@ void TocHandler::writeInitRecord(const Key &key) {
     size_t len = readNext(*r);
     if (len == 0) {
 
-        eckit::Log::debug() << "Initializing FDB TOC in " << tocPath_ << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "Initializing FDB TOC in " << tocPath_ << std::endl;
 
         if (!isSubToc_) {
 
             /* Copy schema first */
-            eckit::Log::debug() << "Copy schema from "
+            LOG_DEBUG_LIB(LibFdb5) << "Copy schema from "
                                << dbConfig_.schemaPath()
                                << " to "
                                << schemaPath_
@@ -817,7 +817,7 @@ void TocHandler::writeSubTocRecord(const TocHandler& subToc) {
     s << off_t{0};
     append(*r, s.position());
 
-    eckit::Log::debug() << "Write TOC_SUB_TOC " << path << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Write TOC_SUB_TOC " << path << std::endl;
 }
 
 
@@ -843,7 +843,7 @@ void TocHandler::writeIndexRecord(const Index& index) {
             index_.encode(s, r->header_.serialisationVersion_);
             handler_.append(*r, s.position());
 
-            eckit::Log::debug() << "Write TOC_INDEX " << location.uri().path().baseName() << " - " << location.offset() << " " << index_.type() << std::endl;
+            LOG_DEBUG_LIB(LibFdb5) << "Write TOC_INDEX " << location.uri().path().baseName() << " - " << location.offset() << " " << index_.type() << std::endl;
         }
 
     private:
@@ -1401,7 +1401,7 @@ size_t TocHandler::buildClearRecord(TocRecord &r, const Index &index) {
             s << location.offset();
             ASSERT(sz_ == 0);
             sz_ = s.position();
-            eckit::Log::debug() << "Write TOC_CLEAR " << location.uri().path().baseName() << " - " << location.offset() << std::endl;
+            LOG_DEBUG_LIB(LibFdb5) << "Write TOC_CLEAR " << location.uri().path().baseName() << " - " << location.offset() << std::endl;
         }
 
         size_t size() const { return sz_; }
diff --git a/src/fdb5/toc/TocIndex.cc b/src/fdb5/toc/TocIndex.cc
index b5aa13bac..40a857b35 100644
--- a/src/fdb5/toc/TocIndex.cc
+++ b/src/fdb5/toc/TocIndex.cc
@@ -95,7 +95,7 @@ bool TocIndex::get(const InspectionKey &key, const Key &remapKey, Field &field)
 
 void TocIndex::open() {
     if (!btree_) {
-        eckit::Log::debug() << "Opening " << *this << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "Opening " << *this << std::endl;
         btree_.reset(BTreeIndexFactory::build(type_, location_.path_, mode_ == TocIndex::READ, location_.offset_));
         if (mode_ == TocIndex::READ && preloadBTree_) btree_->preload();
     }
@@ -119,7 +119,7 @@ void TocIndex::reopen() {
 
 void TocIndex::close() {
     if (btree_) {
-        eckit::Log::debug() << "Closing " << *this << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "Closing " << *this << std::endl;
         btree_.reset();
     }
 }
diff --git a/src/fdb5/toc/TocSerialisationVersion.cc b/src/fdb5/toc/TocSerialisationVersion.cc
index 78fe0ab36..6a759ab9e 100644
--- a/src/fdb5/toc/TocSerialisationVersion.cc
+++ b/src/fdb5/toc/TocSerialisationVersion.cc
@@ -9,6 +9,8 @@
  */
 
 #include "eckit/config/Resource.h"
+
+#include "fdb5/LibFdb5.h"
 #include "fdb5/toc/TocSerialisationVersion.h"
 
 namespace fdb5 {
@@ -19,7 +21,7 @@ static unsigned getUserEnvSerialisationVersion() {
         eckit::Resource("fdbSerialisationVersion;$FDB5_SERIALISATION_VERSION", 0);
     
     if (fdbSerialisationVersion && fdbSerialisationVersion != TocSerialisationVersion::defaulted()) {
-        eckit::Log::debug() << "fdbSerialisationVersion overidde to version: " << fdbSerialisationVersion << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "fdbSerialisationVersion overidde to version: " << fdbSerialisationVersion << std::endl;
     }
     return fdbSerialisationVersion; // default is 0 (not defined by user/service)
 }
@@ -31,7 +33,7 @@ TocSerialisationVersion::TocSerialisationVersion(const fdb5::Config& config) {
     } else {
         static int tocSerialisationVersion = config.getInt("tocSerialisationVersion", 0);
         if (tocSerialisationVersion && tocSerialisationVersion != TocSerialisationVersion::defaulted()) {
-            eckit::Log::debug() << "tocSerialisationVersion overidde to version: " << tocSerialisationVersion << std::endl;
+            LOG_DEBUG_LIB(LibFdb5) << "tocSerialisationVersion overidde to version: " << tocSerialisationVersion << std::endl;
             used_ = tocSerialisationVersion;
         } else {
             used_ = defaulted();
diff --git a/src/fdb5/toc/TocStore.cc b/src/fdb5/toc/TocStore.cc
index edd1426c3..973aab34a 100644
--- a/src/fdb5/toc/TocStore.cc
+++ b/src/fdb5/toc/TocStore.cc
@@ -123,14 +123,14 @@ eckit::DataHandle *TocStore::createFileHandle(const eckit::PathName &path) {
 
     if(stripeLustre()) {
 
-        eckit::Log::debug() << "Creating LustreFileHandle to " << path
+        LOG_DEBUG_LIB(LibFdb5) << "Creating LustreFileHandle to " << path
                                      << " buffer size " << sizeBuffer
                                      << std::endl;
 
         return new LustreFileHandle(path, sizeBuffer, stripeDataLustreSettings());
     }
 
-    eckit::Log::debug() << "Creating FDBFileHandle to " << path
+    LOG_DEBUG_LIB(LibFdb5) << "Creating FDBFileHandle to " << path
                                  << " with buffer of " << eckit::Bytes(sizeBuffer)
                                  << std::endl;
 
@@ -144,7 +144,7 @@ eckit::DataHandle *TocStore::createAsyncHandle(const eckit::PathName &path) {
 
     if(stripeLustre()) {
 
-        eckit::Log::debug() << "Creating LustreFileHandle to " << path
+        LOG_DEBUG_LIB(LibFdb5) << "Creating LustreFileHandle to " << path
                                      << " with " << nbBuffers
                                      << " buffer each with " << eckit::Bytes(sizeBuffer)
                                      << std::endl;
@@ -268,7 +268,7 @@ void TocStore::remove(const Key& key) const {
     while ((dp = ::readdir(dirp)) != NULL) {
         if (strstr( dp->d_name, ".data")) {
             eckit::PathName dataFile = src_db / dp->d_name;
-            eckit::Log::debug() << "Removing " << dataFile << std::endl;
+            LOG_DEBUG_LIB(LibFdb5) << "Removing " << dataFile << std::endl;
             dataFile.unlink(false);
         }
     }
diff --git a/src/fdb5/tools/FDBInspect.cc b/src/fdb5/tools/FDBInspect.cc
index db5d88c58..3b18a4792 100644
--- a/src/fdb5/tools/FDBInspect.cc
+++ b/src/fdb5/tools/FDBInspect.cc
@@ -62,7 +62,7 @@ void FDBInspect::init(const eckit::option::CmdArgs &args) {
 
 void FDBInspect::execute(const eckit::option::CmdArgs &args) {
 
-    eckit::Log::debug() << " FDBInspect minimum-keys " << minimumKeys_ << " @ " << Here() << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << " FDBInspect minimum-keys " << minimumKeys_ << " @ " << Here() << std::endl;
 
     bool all = args.getBool("all", false);
 
@@ -73,7 +73,7 @@ void FDBInspect::execute(const eckit::option::CmdArgs &args) {
 
     bool ignoreErrors = args.getBool("ignore-errors", true);
     if (ignoreErrors) {
-        Log::debug() << "Errors ignored where possible" << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "Errors ignored where possible" << std::endl;
         fail_ = false;
     }
 
@@ -81,10 +81,10 @@ void FDBInspect::execute(const eckit::option::CmdArgs &args) {
 
     if (all) {
         Key dbKey;
-        Log::debug() << "KEY =====> " << dbKey << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "KEY =====> " << dbKey << std::endl;
         std::vector dbs = Manager().visitableLocations(dbKey);
         for (std::vector::const_iterator j = dbs.begin(); j != dbs.end(); ++j) {
-            Log::debug() << "Visitable FDB DB location " << *j << std::endl;
+            LOG_DEBUG_LIB(LibFdb5) << "Visitable FDB DB location " << *j << std::endl;
             paths.push_back(*j);
         }
 
@@ -111,7 +111,7 @@ void FDBInspect::execute(const eckit::option::CmdArgs &args) {
 
             ToolRequest req(args(i), force ? std::vector() : minimumKeys_);
 
-            Log::debug() << "KEY =====> " << req.key() << std::endl;
+            LOG_DEBUG_LIB(LibFdb5) << "KEY =====> " << req.key() << std::endl;
             std::vector dbs = Manager().visitableLocations(req.key());
             for (std::vector::const_iterator j = dbs.begin(); j != dbs.end(); ++j) {
                 paths.push_back(*j);
@@ -149,7 +149,7 @@ void FDBInspect::execute(const eckit::option::CmdArgs &args) {
 
         }
 
-        Log::debug() << "FDBInspect processing path " << path << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "FDBInspect processing path " << path << std::endl;
         process(path , args);
 
     }
diff --git a/src/fdb5/tools/fdb-move.cc b/src/fdb5/tools/fdb-move.cc
index 62a1353cc..d44a2d612 100644
--- a/src/fdb5/tools/fdb-move.cc
+++ b/src/fdb5/tools/fdb-move.cc
@@ -120,8 +120,8 @@ class MoveProducer : public eckit::distributed::Producer {
             throw FDBToolException(ss.str());
         }
 
-        Log::debug() << "Request:     " << request << std::endl;
-        Log::debug() << "Destination: " << destination << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "Request:     " << request << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "Destination: " << destination << std::endl;
 
         moveIterator_ = new fdb_moveiterator_t(fdb_.move(request, destination));
     }
@@ -134,7 +134,7 @@ class MoveProducer : public eckit::distributed::Producer {
 
         fdb5::MoveElement elem;
         if (moveIterator_->next(elem)) {
-            Log::debug() << "MoveProducer " << elem << std::endl;
+            LOG_DEBUG_LIB(LibFdb5) << "MoveProducer " << elem << std::endl;
             if (!elem.sync()) {
                 list_.push_back(elem);
                 elem.encode(message);
@@ -191,7 +191,7 @@ class MoveWorker : public eckit::distributed::Consumer {
 
     void consume(eckit::distributed::Message& message) override {
         fdb5::FileCopy fileCopy(message);
-        Log::debug() << "MoveWorker " << fileCopy << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "MoveWorker " << fileCopy << std::endl;
 
         count_++;
         fileCopy.execute();
@@ -203,7 +203,7 @@ class MoveWorker : public eckit::distributed::Consumer {
     // virtual void getNextMessage(eckit::Message& message) const;
     // virtual void failure(eckit::Message &message);
     void shutdown(eckit::distributed::Message & message) override {
-        eckit::Log::debug() << "Shuting down MoveWorker..." << std::endl;
+        LOG_DEBUG_LIB(LibFdb5) << "Shuting down MoveWorker..." << std::endl;
         message << count_;
     }
 
diff --git a/tests/regressions/FDB-310/fdb-url.cc b/tests/regressions/FDB-310/fdb-url.cc
index e8b15a137..054de6e95 100644
--- a/tests/regressions/FDB-310/fdb-url.cc
+++ b/tests/regressions/FDB-310/fdb-url.cc
@@ -18,6 +18,7 @@
 #include "metkit/mars/MarsParser.h"
 #include "metkit/mars/MarsExpension.h"
 
+#include "fdb5/LibFdb5.h"
 #include "fdb5/api/FDB.h"
 #include "fdb5/message/MessageDecoder.h"
 #include "fdb5/io/HandleGatherer.h"
@@ -99,10 +100,10 @@ void FDBUrl::execute(const eckit::option::CmdArgs &args) {
             fdb5::ListIterator it = fdb.inspect(request);
             while (it.next(el)) {
                 eckit::URI uri = el.location().fullUri();
-                eckit::Log::debug() << uri << std::endl;
+                LOG_DEBUG_LIB(fdb5::LibFdb5) << uri << std::endl;
 
                 std::string uriStr = uri.asRawString();
-                eckit::Log::debug() << uriStr << std::endl;
+                LOG_DEBUG_LIB(fdb5::LibFdb5) << uriStr << std::endl;
 
                 eckit::URI newUri(uriStr);
                 handles.add(fdb.read(newUri));

From e97ebc833f4a1ecd2bafa5014620ad16013d7d6b Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 25 Mar 2024 10:28:34 +0000
Subject: [PATCH 089/186] version bump

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index cdeed24e0..f4371502c 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.111
+5.11.112
\ No newline at end of file

From e8baf96dfa22a5f614e4d6a9777a6063a049e4b6 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 2 Apr 2024 14:55:27 +0200
Subject: [PATCH 090/186] fix single connection + archival ack

---
 src/fdb5/io/FieldHandle.cc                 |  6 +++++-
 src/fdb5/remote/Connection.cc              |  2 +-
 src/fdb5/remote/client/ClientConnection.cc | 23 ++++++++++++----------
 src/fdb5/remote/client/RemoteStore.cc      |  8 ++++++--
 src/fdb5/remote/client/RemoteStore.h       |  1 +
 src/fdb5/remote/server/ServerConnection.cc |  6 +++---
 6 files changed, 29 insertions(+), 17 deletions(-)

diff --git a/src/fdb5/io/FieldHandle.cc b/src/fdb5/io/FieldHandle.cc
index af1c71ab6..625623128 100644
--- a/src/fdb5/io/FieldHandle.cc
+++ b/src/fdb5/io/FieldHandle.cc
@@ -142,7 +142,11 @@ void FieldHandle::openCurrent() {
                 read += len;
             }
 
-            ASSERT(read == currentSize);
+            if (read != currentSize) {
+                std::stringstream ss;
+                ss << "Error reading from " << *current_ << " - read " << read << ", expected " << currentSize;
+                throw eckit::ReadError(ss.str());
+            }
             current_ = new eckit::MemoryHandle(buffer_, currentSize);
             current_->openForRead();
             currentMemoryHandle_ = true;
diff --git a/src/fdb5/remote/Connection.cc b/src/fdb5/remote/Connection.cc
index b15ef945a..a17661861 100644
--- a/src/fdb5/remote/Connection.cc
+++ b/src/fdb5/remote/Connection.cc
@@ -63,7 +63,7 @@ eckit::Buffer Connection::read(bool control, MessageHeader& hdr) {
 
     ASSERT(hdr.marker == StartMarker);
     ASSERT(hdr.version == CurrentVersion);
-    ASSERT(hdr.control() == control);
+    ASSERT(single_ || hdr.control() == control);
 
     eckit::Buffer payload{hdr.payloadSize};
     if (hdr.payloadSize > 0) {
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index cff6f105b..44f49ce59 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -131,14 +131,16 @@ bool ClientConnection::connect(bool singleAttempt) {
         writeControlStartupMessage();
         eckit::SessionID serverSession = verifyServerStartupResponse();
 
-        // Connect to the specified data port
-        LOG_DEBUG_LIB(LibFdb5) << "Received data endpoint from host: " << dataEndpoint_ << std::endl;
-        dataClient_.connect(dataEndpoint_, fdbMaxConnectRetries, fdbConnectTimeout);
-        writeDataStartupMessage(serverSession);
+        if (!single_) {
+            // Connect to the specified data port
+            LOG_DEBUG_LIB(LibFdb5) << "Received data endpoint from host: " << dataEndpoint_ << std::endl;
+            dataClient_.connect(dataEndpoint_, fdbMaxConnectRetries, fdbConnectTimeout);
+            writeDataStartupMessage(serverSession);
+
+            listeningDataThread_ = std::thread([this] { listeningDataThreadLoop(); });
+        }
 
-        // And the connections are set up. Let everything start up!
         listeningControlThread_ = std::thread([this] { listeningControlThreadLoop(); });
-        listeningDataThread_ = std::thread([this] { listeningDataThreadLoop(); });
 
         connected_ = true;
         return true;
@@ -181,6 +183,9 @@ eckit::LocalConfiguration ClientConnection::availableFunctionality() const {
     eckit::LocalConfiguration conf;
     std::vector remoteFieldLocationVersions = {1};
     conf.set("RemoteFieldLocation", remoteFieldLocationVersions);
+    std::vector numberOfConnections = {1,2};
+    conf.set("NumberOfConnections", numberOfConnections);
+    conf.set("PreferSingleConnection", false);
     return conf;
 }
 
@@ -202,7 +207,7 @@ void ClientConnection::dataWrite(DataWriteRequest& r) {
 
 void ClientConnection::dataWrite(Client& client, remote::Message msg, uint32_t requestID, std::vector> data) {
 
-    static size_t maxQueueLength = eckit::Resource("fdbDataWriteQueueLength;$FDB_DATA_WRITE_QUEUE_LENGTH", 2);
+    static size_t maxQueueLength = eckit::Resource("fdbDataWriteQueueLength;$FDB_DATA_WRITE_QUEUE_LENGTH", 32);
     auto it = clients_.find(client.clientId());
     ASSERT(it != clients_.end());
 
@@ -347,13 +352,12 @@ void ClientConnection::listeningControlThreadLoop() {
     try {
 
         MessageHeader hdr;
-        eckit::FixedString<4> tail;
 
         while (true) {
 
             eckit::Buffer payload = Connection::readControl(hdr);
 
-            LOG_DEBUG_LIB(LibFdb5) << "ClientConnection::listeningControlThreadLoop - got [message=" << hdr.message << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
+            LOG_DEBUG_LIB(LibFdb5) << "ClientConnection::listeningControlThreadLoop - got [message=" << hdr.message << ",clientID=" << hdr.clientID() << ",control=" << hdr.control() << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
 
             if (hdr.message == Message::Exit) {
                 return;
@@ -416,7 +420,6 @@ void ClientConnection::listeningDataThreadLoop() {
     try {
 
         MessageHeader hdr;
-        eckit::FixedString<4> tail;
 
         while (true) {
 
diff --git a/src/fdb5/remote/client/RemoteStore.cc b/src/fdb5/remote/client/RemoteStore.cc
index 14e28045c..ac7cb8b03 100644
--- a/src/fdb5/remote/client/RemoteStore.cc
+++ b/src/fdb5/remote/client/RemoteStore.cc
@@ -241,7 +241,10 @@ void RemoteStore::archive(const Key& key, const void *data, eckit::Length length
         }
     }
 
-    locations_[id] = catalogue_archive;
+    {
+        std::lock_guard lock(locationMutex_);
+        locations_[id] = catalogue_archive;
+    }
 
     Buffer keyBuffer(4096);
     MemoryStream keyStream(keyBuffer);
@@ -365,8 +368,9 @@ bool RemoteStore::handle(Message message, bool control, uint32_t requestID, ecki
     switch (message) {
     
         case Message::Store: { // received a Field location from the remote store, can forward to the archiver for the indexing
+            std::lock_guard lock(locationMutex_);
+
             auto it = locations_.find(requestID);
-            ASSERT(it != locations_.end());
             if (it != locations_.end()) {
                 MemoryStream s(payload);
                 std::unique_ptr location(eckit::Reanimator::reanimate(s));
diff --git a/src/fdb5/remote/client/RemoteStore.h b/src/fdb5/remote/client/RemoteStore.h
index d8a36be71..e2be1661f 100644
--- a/src/fdb5/remote/client/RemoteStore.h
+++ b/src/fdb5/remote/client/RemoteStore.h
@@ -100,6 +100,7 @@ class RemoteStore : public Store, public Client {
     std::map> messageQueues_;
     MessageQueue retrieveMessageQueue_;
 
+    std::mutex locationMutex_;
     std::map fieldLocation)>> locations_;
     FDBStats internalStats_;
     FDBStats archivalStats_;
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index 0d9f5b4e9..3058149b9 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -116,7 +116,7 @@ Handled ServerConnection::handleData(Message message, uint32_t clientID, uint32_
             case Message::Blob:
             case Message::MultiBlob:
                 queue(message, clientID, requestID, std::move(payload));
-                return Handled::Yes;
+                return Handled::Replied;
 
             default: {
                 std::stringstream ss;
@@ -142,7 +142,7 @@ eckit::LocalConfiguration ServerConnection::availableFunctionality() const {
 //    Add to the configuration all the components that require to be versioned, as in the following example, with a vector of supported version numbers
     std::vector remoteFieldLocationVersions = {1};
     conf.set("RemoteFieldLocation", remoteFieldLocationVersions);
-    std::vector numberOfConnections = {2};
+    std::vector numberOfConnections = {1,2};
     conf.set("NumberOfConnections", numberOfConnections);
     return conf;
 }
@@ -480,7 +480,7 @@ void ServerConnection::handle() {
                         {
                             std::lock_guard lock(handlerMutex_);
                             dataListener_++;
-                            if (dataListener_ == 1) {
+                            if (dataListener_ == 1 && !single_) {
                                 listeningThreadData = std::thread([this] { listeningThreadLoopData(); });
                             }
                         }

From 6c3ff1b934c07453093391dba399ee30dfb90c7e Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 2 Apr 2024 15:12:21 +0200
Subject: [PATCH 091/186] version bump

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index f4371502c..b58b26e18 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.112
\ No newline at end of file
+5.11.113

From 26344a863e5bacf4a55544023c90379a7cd78a76 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 4 Apr 2024 17:50:59 +0200
Subject: [PATCH 092/186] fix multiple flushes + lazy creation of futures

---
 src/fdb5/api/FDB.cc                        | 60 +++++++++++--
 src/fdb5/api/FDB.h                         |  4 +-
 src/fdb5/api/fdb_c.cc                      |  2 +-
 src/fdb5/database/ArchiveVisitor.cc        | 11 +--
 src/fdb5/database/Archiver.cc              | 18 ++--
 src/fdb5/database/Archiver.h               |  3 +-
 src/fdb5/database/BaseArchiveVisitor.cc    | 19 +++--
 src/fdb5/database/BaseArchiveVisitor.h     |  2 +-
 src/fdb5/database/Catalogue.h              |  2 +-
 src/fdb5/database/Store.h                  |  4 +-
 src/fdb5/io/FieldHandle.cc                 | 13 +--
 src/fdb5/io/FieldHandle.h                  |  9 ++
 src/fdb5/rados/RadosStore.cc               | 18 ++--
 src/fdb5/rados/RadosStore.h                |  6 +-
 src/fdb5/remote/FdbServer.cc               |  3 -
 src/fdb5/remote/client/RemoteCatalogue.cc  | 32 ++-----
 src/fdb5/remote/client/RemoteCatalogue.h   |  5 +-
 src/fdb5/remote/client/RemoteStore.cc      | 98 ++++++++++------------
 src/fdb5/remote/client/RemoteStore.h       | 16 ++--
 src/fdb5/remote/server/CatalogueHandler.cc |  9 +-
 src/fdb5/remote/server/CatalogueHandler.h  | 12 ++-
 src/fdb5/toc/AdoptVisitor.cc               |  8 +-
 src/fdb5/toc/TocCatalogueReader.h          |  2 +-
 src/fdb5/toc/TocCatalogueWriter.cc         | 20 +++--
 src/fdb5/toc/TocCatalogueWriter.h          |  3 +-
 src/fdb5/toc/TocCommon.cc                  |  3 +-
 src/fdb5/toc/TocCommon.h                   |  2 -
 src/fdb5/toc/TocHandler.cc                 |  6 +-
 src/fdb5/toc/TocHandler.h                  |  2 +
 src/fdb5/toc/TocStore.cc                   | 19 +++--
 src/fdb5/toc/TocStore.h                    |  4 +-
 31 files changed, 212 insertions(+), 203 deletions(-)

diff --git a/src/fdb5/api/FDB.cc b/src/fdb5/api/FDB.cc
index 99f709ba2..3fc20f99f 100644
--- a/src/fdb5/api/FDB.cc
+++ b/src/fdb5/api/FDB.cc
@@ -156,16 +156,66 @@ eckit::DataHandle* FDB::read(const std::vector& uris, bool sorted) {
     }
     return result.dataHandle();
 }
-    
 
-eckit::DataHandle* FDB::read(ListIterator& it, bool sorted) {
+eckit::DataHandle* FDB::read(bool seekable, ListIterator& it, bool sorted) {
 
-    return new FieldHandle(it);
+    if (seekable) {
+        return new FieldHandle(it);
+    }
+
+    eckit::Timer timer;
+    timer.start();
+
+    HandleGatherer result(sorted);
+    ListElement el;
+
+    static bool dedup = eckit::Resource("fdbDeduplicate;$FDB_DEDUPLICATE_FIELDS", false);
+    if (dedup) {
+        if (it.next(el)) {
+            // build the request representing the tensor-product of all retrieved fields
+            metkit::mars::MarsRequest cubeRequest = el.combinedKey().request();
+            std::vector elements{el};
+
+            while (it.next(el)) {
+                cubeRequest.merge(el.combinedKey().request());
+                elements.push_back(el);
+            }
+
+            // checking all retrieved fields against the hypercube, to remove duplicates
+            ListElementDeduplicator dedup;
+            metkit::hypercube::HyperCubePayloaded cube(cubeRequest, dedup);
+            for(auto el: elements) {
+                cube.add(el.combinedKey().request(), el);
+            }
+
+            if (cube.countVacant() > 0) {
+                std::stringstream ss;
+                ss << "No matching data for requests:" << std::endl;
+                for (auto req: cube.vacantRequests()) {
+                    ss << "    " << req << std::endl;
+                }
+                eckit::Log::warning() << ss.str() << std::endl;
+            }
+
+            for (size_t i=0; i< cube.size(); i++) {
+                ListElement element;
+                if (cube.find(i, element)) {
+                    result.add(element.location().dataHandle());
+                }
+            }
+        }
+    }
+    else {
+        while (it.next(el)) {
+            result.add(el.location().dataHandle());
+        }
+    }
+    return result.dataHandle();
 }
 
-eckit::DataHandle* FDB::retrieve(const metkit::mars::MarsRequest& request) {
+eckit::DataHandle* FDB::retrieve(const metkit::mars::MarsRequest& request, bool seekable) {
     ListIterator it = inspect(request);
-    return read(it, sorted(request));
+    return read(seekable, it, sorted(request));
 }
 
 ListIterator FDB::inspect(const metkit::mars::MarsRequest& request) {
diff --git a/src/fdb5/api/FDB.h b/src/fdb5/api/FDB.h
index cb9879363..5d938cd46 100644
--- a/src/fdb5/api/FDB.h
+++ b/src/fdb5/api/FDB.h
@@ -85,9 +85,9 @@ class FDB {
 
     eckit::DataHandle* read(const std::vector& uris, bool sorted = false);
 
-    eckit::DataHandle* read(ListIterator& it, bool sorted = false);
+    eckit::DataHandle* read(bool seekable, ListIterator& it, bool sorted = false);
 
-    eckit::DataHandle* retrieve(const metkit::mars::MarsRequest& request);
+    eckit::DataHandle* retrieve(const metkit::mars::MarsRequest& request, bool seekable = false);
 
     ListIterator inspect(const metkit::mars::MarsRequest& request);
 
diff --git a/src/fdb5/api/fdb_c.cc b/src/fdb5/api/fdb_c.cc
index 9c2fa3363..e64f5fef8 100644
--- a/src/fdb5/api/fdb_c.cc
+++ b/src/fdb5/api/fdb_c.cc
@@ -349,7 +349,7 @@ int fdb_retrieve(fdb_handle_t* fdb, fdb_request_t* req, fdb_datareader_t* dr) {
         ASSERT(fdb);
         ASSERT(req);
         ASSERT(dr);
-        dr->set(fdb->retrieve(req->request()));
+        dr->set(fdb->retrieve(req->request(), true));
     });
 }
 int fdb_flush(fdb_handle_t* fdb) {
diff --git a/src/fdb5/database/ArchiveVisitor.cc b/src/fdb5/database/ArchiveVisitor.cc
index 77d4ba5ef..254a7453e 100644
--- a/src/fdb5/database/ArchiveVisitor.cc
+++ b/src/fdb5/database/ArchiveVisitor.cc
@@ -15,11 +15,6 @@
 #include "fdb5/database/Catalogue.h"
 #include "fdb5/database/Store.h"
 
-// namespace {
-// void CatalogueCallback(fdb5::CatalogueWriter* catalogue, const fdb5::InspectionKey &key, std::unique_ptr fieldLocation) {
-//     catalogue->archive(key, std::move(fieldLocation));
-// }
-// }
 namespace fdb5 {
 
 ArchiveVisitor::ArchiveVisitor(Archiver &owner, const Key &dataKey, const void *data, size_t size) :
@@ -31,12 +26,10 @@ ArchiveVisitor::ArchiveVisitor(Archiver &owner, const Key &dataKey, const void *
 
 bool ArchiveVisitor::selectDatum(const InspectionKey &key, const Key &full) {
 
-    // eckit::Log::info() << "selectDatum " << key << ", " << full << " " << size_ << std::endl;
     checkMissingKeys(full);
-    const Key idxKey = current()->currentIndexKey();
+    const Key idxKey = catalogue()->currentIndexKey();
 
-    // store()->archive(idxKey, data_, size_, std::bind(&CatalogueCallback, current(), key, std::placeholders::_1));
-    store()->archive(idxKey, data_, size_, std::bind(&CatalogueWriter::archive, current(), key, std::placeholders::_1));
+    store()->archive(idxKey, data_, size_, std::bind(&CatalogueWriter::archive, catalogue(), key, std::placeholders::_1));
 
     return true;
 }
diff --git a/src/fdb5/database/Archiver.cc b/src/fdb5/database/Archiver.cc
index 98e988c81..ec5a19a80 100644
--- a/src/fdb5/database/Archiver.cc
+++ b/src/fdb5/database/Archiver.cc
@@ -26,8 +26,7 @@ namespace fdb5 {
 
 Archiver::Archiver(const Config& dbConfig) :
     dbConfig_(dbConfig),
-    catalogue_(nullptr),
-    store_(nullptr) {}
+    db_(nullptr) {}
 
 Archiver::~Archiver() {
     flush(); // certify that all sessions are flushed before closing them
@@ -56,8 +55,8 @@ void Archiver::archive(const Key &key, BaseArchiveVisitor& visitor) {
 
 void Archiver::flush() {
     for (auto i = databases_.begin(); i != databases_.end(); ++i) {
-        i->second.store_->flush();      // flush the store
-        i->second.catalogue_->flush();  // flush the catalogue
+        // flush the store, pass the number of flushed fields to the catalogue
+        i->second.catalogue_->flush(i->second.store_->flush());
     }
 }
 
@@ -66,8 +65,7 @@ void Archiver::selectDatabase(const Key &dbKey) {
     auto i = databases_.find(dbKey);
 
     if (i != databases_.end() ) {
-        catalogue_ = i->second.catalogue_.get();
-        store_ = i->second.store_.get();
+        db_ = &(i->second);
         i->second.time_ = ::time(0);
         return;
     }
@@ -86,8 +84,7 @@ void Archiver::selectDatabase(const Key &dbKey) {
             }
         }
         if (found) {
-            databases_[oldK].store_->flush();
-            databases_[oldK].catalogue_->flush();
+            databases_[oldK].catalogue_->flush(databases_[oldK].store_->flush());
             
             eckit::Log::info() << "Closing database " << *databases_[oldK].catalogue_ << std::endl;
             databases_.erase(oldK);
@@ -105,10 +102,7 @@ void Archiver::selectDatabase(const Key &dbKey) {
     }
 
     std::unique_ptr str = cat->buildStore();
-    catalogue_ = cat.get();
-    store_ = str.get();
-
-    databases_[dbKey] = Database{::time(0), std::move(cat), std::move(str)};
+    db_ = &(databases_[dbKey] = Database{::time(0), std::move(cat), std::move(str)});
 }
 
 void Archiver::print(std::ostream &out) const {
diff --git a/src/fdb5/database/Archiver.h b/src/fdb5/database/Archiver.h
index 0beab4267..2bbf637a5 100644
--- a/src/fdb5/database/Archiver.h
+++ b/src/fdb5/database/Archiver.h
@@ -80,8 +80,7 @@ class Archiver : public eckit::NonCopyable {
 
     std::vector prev_;
 
-    CatalogueWriter* catalogue_;
-    Store* store_;
+    Database* db_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/database/BaseArchiveVisitor.cc b/src/fdb5/database/BaseArchiveVisitor.cc
index 9338d35c1..d5748d0ac 100644
--- a/src/fdb5/database/BaseArchiveVisitor.cc
+++ b/src/fdb5/database/BaseArchiveVisitor.cc
@@ -28,16 +28,14 @@ BaseArchiveVisitor::BaseArchiveVisitor(Archiver &owner, const Key &dataKey) :
 bool BaseArchiveVisitor::selectDatabase(const Key &dbKey, const Key&) {
     LOG_DEBUG_LIB(LibFdb5) << "selectDatabase " << dbKey << std::endl;
     owner_.selectDatabase(dbKey);
-    ASSERT(owner_.catalogue_);
-    owner_.catalogue_->deselectIndex();
+    catalogue()->deselectIndex();
 
     return true;
 }
 
 bool BaseArchiveVisitor::selectIndex(const Key &idxKey, const Key&) {
     // eckit::Log::info() << "selectIndex " << key << std::endl;
-    ASSERT(owner_.catalogue_);
-    return owner_.catalogue_->selectIndex(idxKey);
+    return catalogue()->selectIndex(idxKey);
 }
 
 void BaseArchiveVisitor::checkMissingKeys(const Key &full) {
@@ -47,16 +45,19 @@ void BaseArchiveVisitor::checkMissingKeys(const Key &full) {
 }
 
 const Schema& BaseArchiveVisitor::databaseSchema() const {
-    ASSERT(current());
-    return current()->schema();
+    return catalogue()->schema();
 }
 
-CatalogueWriter* BaseArchiveVisitor::current() const {
-    return owner_.catalogue_;
+CatalogueWriter* BaseArchiveVisitor::catalogue() const {
+    ASSERT(owner_.db_);
+    ASSERT(owner_.db_->catalogue_);
+    return owner_.db_->catalogue_.get();
 }
 
 Store* BaseArchiveVisitor::store() const {
-    return owner_.store_;
+    ASSERT(owner_.db_);
+    ASSERT(owner_.db_->store_);
+    return owner_.db_->store_.get();
 }
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/database/BaseArchiveVisitor.h b/src/fdb5/database/BaseArchiveVisitor.h
index c3d7dd071..8d8e55f1f 100644
--- a/src/fdb5/database/BaseArchiveVisitor.h
+++ b/src/fdb5/database/BaseArchiveVisitor.h
@@ -45,7 +45,7 @@ class BaseArchiveVisitor : public WriteVisitor {
 
     virtual const Schema& databaseSchema() const;
 
-    fdb5::CatalogueWriter* current() const;
+    fdb5::CatalogueWriter* catalogue() const;
     fdb5::Store* store() const;
 
 protected: // members
diff --git a/src/fdb5/database/Catalogue.h b/src/fdb5/database/Catalogue.h
index d8f050f74..17e90f8c8 100644
--- a/src/fdb5/database/Catalogue.h
+++ b/src/fdb5/database/Catalogue.h
@@ -88,7 +88,7 @@ class Catalogue {
 
     virtual std::string type() const = 0;
     virtual bool open() = 0;
-    virtual void flush() = 0;
+    virtual void flush(size_t archivedFields) = 0;
     virtual void clean() = 0;
     virtual void close() = 0;
 
diff --git a/src/fdb5/database/Store.h b/src/fdb5/database/Store.h
index ecf3e5695..f39a88018 100644
--- a/src/fdb5/database/Store.h
+++ b/src/fdb5/database/Store.h
@@ -46,7 +46,7 @@ class Store {
 
     virtual std::string type() const = 0;
     virtual bool open() = 0;
-    virtual void flush() = 0;
+    virtual size_t flush() = 0;
     virtual void close() = 0;
 
 //    virtual std::string owner() const = 0;
@@ -58,7 +58,7 @@ class Store {
     virtual void remove(const Key& key) const { NOTIMP; }
 
     virtual eckit::URI uri() const = 0;
-    
+
 };
 
 
diff --git a/src/fdb5/io/FieldHandle.cc b/src/fdb5/io/FieldHandle.cc
index 625623128..f89ddefb4 100644
--- a/src/fdb5/io/FieldHandle.cc
+++ b/src/fdb5/io/FieldHandle.cc
@@ -26,15 +26,6 @@ namespace fdb5 {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-class ListElementDeduplicator : public metkit::hypercube::Deduplicator {
-public:
-    bool toReplace(const ListElement& existing, const ListElement& replacement) const override {
-        return existing.timestamp() < replacement.timestamp();
-    }
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
 FieldHandle::FieldHandle(ListIterator& it) :
     datahandles_({}), totalSize_(0), currentIdx_(0), current_(nullptr), currentMemoryHandle_(false), buffer_(nullptr), sorted_(false), seekable_(true) {
     ListElement el;
@@ -179,6 +170,8 @@ long FieldHandle::read1(char* buffer, long length) {
 }
 
 long FieldHandle::read(void* buffer, long length) {
+    long requested = length;
+
     char* p    = static_cast(buffer);
     long n     = 0;
     long total = 0;
@@ -189,7 +182,7 @@ long FieldHandle::read(void* buffer, long length) {
         p += n;
     }
 
-    LOG_DEBUG_LIB(LibFdb5) << "FieldHandle::read " << (total > 0 ? total : n) << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "FieldHandle::read - requested: " << requested << "  read: " << (total > 0 ? total : n) << std::endl;
 
     return total > 0 ? total : n;
 }
diff --git a/src/fdb5/io/FieldHandle.h b/src/fdb5/io/FieldHandle.h
index eb64ff47a..679279283 100644
--- a/src/fdb5/io/FieldHandle.h
+++ b/src/fdb5/io/FieldHandle.h
@@ -22,6 +22,15 @@ namespace fdb5 {
 
 //----------------------------------------------------------------------------------------------------------------------
 
+class ListElementDeduplicator : public metkit::hypercube::Deduplicator {
+public:
+    bool toReplace(const ListElement& existing, const ListElement& replacement) const override {
+        return existing.timestamp() < replacement.timestamp();
+    }
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
 class FieldHandle : public eckit::DataHandle {
 public:
 
diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc
index 8f9cbadcb..cdc2ad274 100644
--- a/src/fdb5/rados/RadosStore.cc
+++ b/src/fdb5/rados/RadosStore.cc
@@ -29,15 +29,15 @@ namespace fdb5 {
 //----------------------------------------------------------------------------------------------------------------------
 
 RadosStore::RadosStore(const Key& key, const Config& config) :
-    Store(), directory_("mars:"+key.valuesToString()) {}
+    Store(), directory_("mars:"+key.valuesToString()), archivedFields_(0) {}
 
 RadosStore(const Key& key, const Config& config, const eckit::net::Endpoint& controlEndpoint) :
-    Store(), directory_("mars:"+key.valuesToString()) {
+    Store(), directory_("mars:"+key.valuesToString()), archivedFields_(0) {
     NOTIMP;
 }
 
 RadosStore::RadosStore(const eckit::URI& uri, const Config& config) :
-    Store(), directory_("mars:"+uri.path().dirName()) {}
+    Store(), directory_("mars:"+uri.path().dirName()), archivedFields_(0) {}
 
 eckit::URI RadosStore::uri() const {
     return URI("rados", directory_);
@@ -54,7 +54,7 @@ eckit::DataHandle* RadosStore::retrieve(Field& field, Key& remapKey) const {
 }
 
 std::unique_ptr RadosStore::archive(const uint32_t, const Key& key, const void *data, eckit::Length length) {
-    dirty_ = true;
+    archivedFields_++;
 
     eckit::PathName dataPath = getDataPath(key);
     eckit::URI dataUri("rados", dataPath);
@@ -70,16 +70,18 @@ std::unique_ptr RadosStore::archive(const uint32_t, const Key& ke
     return std::unique_ptr(new RadosFieldLocation(dataUri, position, length));
 }
 
-void RadosStore::flush() {
-    if (!dirty_) {
-        return;
+size_t RadosStore::flush() {
+    if (archivedFields_ == 0) {
+        return 0;
     }
 
     // ensure consistent state before writing Toc entry
 
     flushDataHandles();
 
-    dirty_ = false;
+    size_t out = archivedFields_;
+    archivedFields_ = 0;
+    return out;
 }
 
 void RadosStore::close() {
diff --git a/src/fdb5/rados/RadosStore.h b/src/fdb5/rados/RadosStore.h
index 4b03c31de..7bf8eba9e 100644
--- a/src/fdb5/rados/RadosStore.h
+++ b/src/fdb5/rados/RadosStore.h
@@ -39,7 +39,7 @@ class RadosStore : public Store {
     eckit::URI uri() const override;
 
     bool open() override { return true; }
-    void flush() override;
+    size_t flush() override;
     void close() override;
 
     void checkUID() const override { /* nothing to do */ }
@@ -78,8 +78,8 @@ class RadosStore : public Store {
 
     PathStore   dataPaths_;
     eckit::PathName directory_;
-
-    mutable bool dirty_;
+    
+    size_t archivedFields_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/FdbServer.cc b/src/fdb5/remote/FdbServer.cc
index c7f053634..e9e9d53be 100644
--- a/src/fdb5/remote/FdbServer.cc
+++ b/src/fdb5/remote/FdbServer.cc
@@ -47,9 +47,6 @@ void FDBForker::run() {
 
     eckit::Log::info() << "FDB forked pid " << ::getpid() << std::endl;
 
-    // ServerConnection handler(socket_, config_);
-    // handler.handle();
-    
     if (config_.getString("type", "local") == "catalogue" || (::getenv("FDB_IS_CAT") && ::getenv("FDB_IS_CAT")[0] == '1')) {
         eckit::Log::info() << "FDB using Catalogue Handler" << std::endl;
         CatalogueHandler handler(socket_, config_);
diff --git a/src/fdb5/remote/client/RemoteCatalogue.cc b/src/fdb5/remote/client/RemoteCatalogue.cc
index 984c8861c..8c2596a53 100644
--- a/src/fdb5/remote/client/RemoteCatalogue.cc
+++ b/src/fdb5/remote/client/RemoteCatalogue.cc
@@ -60,9 +60,6 @@ void RemoteCatalogue::sendArchiveData(uint32_t id, const Key& key, std::unique_p
 
 void RemoteCatalogue::archive(const InspectionKey& key, std::unique_ptr fieldLocation) {
 
-    // eckit::Timer timer;
-    // timer.start();
-
     ASSERT(!key.empty());
     ASSERT(fieldLocation);
 
@@ -85,9 +82,6 @@ void RemoteCatalogue::archive(const InspectionKey& key, std::unique_ptr{buffer, stream.position()});
 
     dataWrite(Message::Blob, id, payloads);
-    // timer.stop();
-
-    // archivalStats_.addArchive(0, timer, 0);
 }
 
 bool RemoteCatalogue::selectIndex(const Key& idxKey) {
@@ -110,13 +104,12 @@ const Schema& RemoteCatalogue::schema() const {
     return *schema_;
 }
 
-void RemoteCatalogue::flush() {
+void RemoteCatalogue::flush(size_t archivedFields) {
 
-    // Timer timer;
+    std::lock_guard lock(archiveMutex_);
 
-    // timer.start();
+    ASSERT(archivedFields == numLocations_);
 
-    std::lock_guard lock(archiveMutex_);
     // Flush only does anything if there is an ongoing archive();
     if (numLocations_ > 0) {
 
@@ -132,9 +125,6 @@ void RemoteCatalogue::flush() {
 
         numLocations_ = 0;
     }
-
-    // timer.stop();
-    // internalStats_.addFlush(timer);
 }
 
 void RemoteCatalogue::clean() {NOTIMP;}
@@ -189,17 +179,6 @@ void RemoteCatalogue::handleException(std::exception_ptr e) {
     NOTIMP;
 }
 
-// Catalogue Reader
-// DbStats RemoteCatalogue::stats() const {
-//     NOTIMP;
-// }
-// bool RemoteCatalogue::axis(const std::string& keyword, eckit::StringSet& s) const {
-//     NOTIMP;
-// }
-// bool RemoteCatalogue::retrieve(const InspectionKey& key, Field& field) const{
-//     NOTIMP;
-// }
-
 void RemoteCatalogue::overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) {NOTIMP;}
 void RemoteCatalogue::index(const InspectionKey& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) {NOTIMP;}
 void RemoteCatalogue::reconsolidate(){NOTIMP;}
@@ -214,7 +193,10 @@ void RemoteCatalogue::control(const ControlAction& action, const ControlIdentifi
 std::vector RemoteCatalogue::indexes(bool sorted) const {NOTIMP;}
 void RemoteCatalogue::maskIndexEntry(const Index& index) const {NOTIMP;}
 void RemoteCatalogue::allMasked(std::set>& metadata, std::set& data) const {NOTIMP;}
-void RemoteCatalogue::print( std::ostream &out ) const {NOTIMP;}
+void RemoteCatalogue::print( std::ostream &out ) const {
+    out << "RemoteCatalogue(endpoint=" << controlEndpoint() << ",clientID=" << clientId() << ")";
+}
+
 
 std::string RemoteCatalogue::type() const {
     return "remote";
diff --git a/src/fdb5/remote/client/RemoteCatalogue.h b/src/fdb5/remote/client/RemoteCatalogue.h
index d4d15c043..08c573b61 100644
--- a/src/fdb5/remote/client/RemoteCatalogue.h
+++ b/src/fdb5/remote/client/RemoteCatalogue.h
@@ -53,7 +53,7 @@ class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public C
     void print( std::ostream &out ) const override;
     std::string type() const override;
     bool open() override;
-    void flush() override;
+    void flush(size_t archivedFields) override;
     void clean() override;
     void close() override;
     bool exists() const override;
@@ -87,9 +87,6 @@ class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public C
 
     std::mutex archiveMutex_;
     size_t numLocations_;
-
-    // // not owning
-    // RemoteCatalogueArchiver* archiver_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/client/RemoteStore.cc b/src/fdb5/remote/client/RemoteStore.cc
index ac7cb8b03..6215bcfe6 100644
--- a/src/fdb5/remote/client/RemoteStore.cc
+++ b/src/fdb5/remote/client/RemoteStore.cc
@@ -87,7 +87,6 @@ class FDBRemoteDataHandle : public DataHandle {
         if (currentBuffer_.size() != 0) return bufferRead(pos, sz);
 
         // If we are in the DataHandle, then there MUST be data to read
-
         RemoteStore::StoredMessage msg = std::make_pair(remote::Message{}, eckit::Buffer{0});
         ASSERT(queue_.pop(msg) != -1);
 
@@ -183,14 +182,14 @@ RemoteStore::RemoteStore(const Key& dbKey, const Config& config) :
     Client(storeEndpoints(config)),
     dbKey_(dbKey), config_(config),
     retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
-    dirty_(false), flushRequested_(false) {}
+    fieldsArchived_(0), locationsReceived_(0) {}
 
 // this is used only in retrieval, with an URI already referring to an accessible Store
 RemoteStore::RemoteStore(const eckit::URI& uri, const Config& config) :
     Client(eckit::net::Endpoint(uri.hostport()), uri.hostport()),
     dbKey_(Key()), config_(config),
     retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
-    dirty_(false), flushRequested_(false) { 
+    fieldsArchived_(0), locationsReceived_(0) { 
 
     // no need to set the local_ flag on the read path
     ASSERT(uri.scheme() == "fdb");
@@ -201,7 +200,7 @@ RemoteStore::~RemoteStore() {
     // an error. n.b. if we don't do something, we will block in the destructor
     // of std::future.
     if (archivalCompleted_.valid() || !locations_.empty()) {
-        Log::error() << "Attempting to destruct RemoteStore with active archival" << std::endl;
+        Log::error() << "Attempting to destruct RemoteStore with active archival - location to receive: " << locations_.size() << " archivalCompleted_.valid() " << archivalCompleted_.valid() << std::endl;
         eckit::Main::instance().terminate();
     }
 }
@@ -220,9 +219,6 @@ eckit::DataHandle* RemoteStore::retrieve(Field& field) const {
 
 void RemoteStore::archive(const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive) {
 
-    eckit::Timer timer;
-    timer.start();
-
     ASSERT(!key.empty());
     ASSERT(data);
     ASSERT(length != 0);
@@ -230,16 +226,13 @@ void RemoteStore::archive(const Key& key, const void *data, eckit::Length length
     uint32_t id = connection_.generateRequestID();
     {
         std::lock_guard lock(archiveMutex_);
-        if (!dirty_) { // if this is the first archival request, notify the server
-            ASSERT(archivalStats_.numArchive() == 0);
-            ASSERT(!archivalCompleted_.valid());
+        if (fieldsArchived_ == 0) { // if this is the first archival request, notify the server
             ASSERT(locations_.size() == 0);
-            archivalCompleted_ = fieldLocationsReceived_.get_future();
 
             controlWriteCheckResponse(Message::Store, id, true);
-            dirty_=true;
         }
     }
+    fieldsArchived_++;
 
     {
         std::lock_guard lock(locationMutex_);
@@ -256,60 +249,57 @@ void RemoteStore::archive(const Key& key, const void *data, eckit::Length length
     payloads.push_back(std::pair{data, length});
 
     dataWrite(Message::Blob, id, payloads);
-
-    timer.stop();
-
-    archivalStats_.addArchive(length, timer, 1);
 }
 
 bool RemoteStore::open() {
     return true;
 }
 
-FDBStats RemoteStore::archivalCompleted() {
+size_t RemoteStore::flush() {
 
-    if (flushRequested_ && (archivalStats_.numArchive() == archivalStats_.numLocation()) && locations_.empty()) {
-        fieldLocationsReceived_.set_value(archivalStats_);
+    // Flush only does anything if there is an ongoing archive();
+    if (fieldsArchived_ == 0) {
+        return 0;
     }
 
-    FDBStats stats = archivalCompleted_.get();
-
-    ASSERT(locations_.empty());
-    archivalStats_ = FDBStats{};
-    return stats;
-}
-
-void RemoteStore::flush() {
-
-    Timer timer;
-
-    timer.start();
+    LOG_DEBUG_LIB(LibFdb5) << " RemoteStore::flush - fieldsArchived_ " << fieldsArchived_ << " locationsReceived_ " << locationsReceived_ << std::endl;
 
-    flushRequested_ = true;
-
-    // Flush only does anything if there is an ongoing archive();
-    std::lock_guard lock(archiveMutex_);
-    if (archivalCompleted_.valid()) {
+    size_t locations;
+    bool wait = true;
+    {
+        std::lock_guard lock(locationMutex_);
+        if ((fieldsArchived_ > locationsReceived_) || !locations_.empty()) {
+            promiseArchivalCompleted_ = std::promise{};
+            archivalCompleted_ = promiseArchivalCompleted_.get_future();
+        } else {
+            wait = false;
+        }
+    }
 
+    if (wait) {
         // wait for archival completion (received all fieldLocations)
-        FDBStats stats = archivalCompleted();
-
-        if (stats.numArchive() > 0) {
-            Buffer sendBuf(1024);
-            MemoryStream s(sendBuf);
-            s << stats.numArchive();
+        archivalCompleted_.wait();
+        locations = archivalCompleted_.get();
+    } else {
+        locations = locationsReceived_;
+    }
 
-            LOG_DEBUG_LIB(LibFdb5) << " RemoteStore::flush - flushing " << stats.numArchive() << " fields" << std::endl;
-            // The flush call is blocking
-            uint32_t id = generateRequestID();
-            controlWriteCheckResponse(Message::Flush, id, false, sendBuf, s.position());
-        }
-        dirty_ = false;
+    ASSERT(locations_.empty());
+    if (locations > 0) {
+        Buffer sendBuf(1024);
+        MemoryStream s(sendBuf);
+        s << locations;
+
+        LOG_DEBUG_LIB(LibFdb5) << " RemoteStore::flush - flushing " << locations << " fields" << std::endl;
+        // The flush call is blocking
+        uint32_t id = generateRequestID();
+        controlWriteCheckResponse(Message::Flush, id, false, sendBuf, s.position());
     }
 
-    timer.stop();
-    flushRequested_ = false;
-    internalStats_.addFlush(timer);
+    fieldsArchived_ = 0;
+    locationsReceived_ = 0;
+
+    return locations;
 }
 
 void RemoteStore::close() {
@@ -382,10 +372,10 @@ bool RemoteStore::handle(Message message, bool control, uint32_t requestID, ecki
                 }
 
                 locations_.erase(it);
-                archivalStats_.addLocation();
+                locationsReceived_++;
 
-                if (flushRequested_ && (archivalStats_.numArchive() == archivalStats_.numLocation()) && locations_.empty()) {
-                    fieldLocationsReceived_.set_value(archivalStats_);
+                if (archivalCompleted_.valid() && (fieldsArchived_ == locationsReceived_) && locations_.empty()) {
+                    promiseArchivalCompleted_.set_value(locationsReceived_);
                 }
                 return true;
             }
diff --git a/src/fdb5/remote/client/RemoteStore.h b/src/fdb5/remote/client/RemoteStore.h
index e2be1661f..ec6f5e4c6 100644
--- a/src/fdb5/remote/client/RemoteStore.h
+++ b/src/fdb5/remote/client/RemoteStore.h
@@ -47,7 +47,7 @@ class RemoteStore : public Store, public Client {
     eckit::URI uri() const override;
 
     bool open() override;
-    void flush() override;
+    size_t flush() override;
     void close() override;
 
     void checkUID() const override { }
@@ -77,10 +77,6 @@ class RemoteStore : public Store, public Client {
 
 private: // methods
 
-    FDBStats archivalCompleted();
-
-    void flush(FDBStats& stats);
-
     // handlers for incoming messages - to be defined in the client class
     bool handle(Message message, bool control, uint32_t requestID) override;
     bool handle(Message message, bool control, uint32_t requestID, eckit::Buffer&& payload) override;
@@ -102,13 +98,11 @@ class RemoteStore : public Store, public Client {
 
     std::mutex locationMutex_;
     std::map fieldLocation)>> locations_;
-    FDBStats internalStats_;
-    FDBStats archivalStats_;
-    std::promise fieldLocationsReceived_;
-    std::future archivalCompleted_;
+    size_t fieldsArchived_;
+    size_t locationsReceived_;
+    std::promise promiseArchivalCompleted_;
+    std::future archivalCompleted_;
     std::mutex archiveMutex_;
-    bool dirty_;
-    bool flushRequested_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc
index 51d00b9c8..b56effb2e 100644
--- a/src/fdb5/remote/server/CatalogueHandler.cc
+++ b/src/fdb5/remote/server/CatalogueHandler.cc
@@ -365,11 +365,14 @@ void CatalogueHandler::flush(uint32_t clientID, uint32_t requestID, eckit::Buffe
     ASSERT(it != catalogues_.end());
 
     it->second.locationsExpected = numArchived;
+    it->second.archivalCompleted = it->second.fieldLocationsReceived.get_future();
+
     if (it->second.locationsArchived < numArchived) {
-        it->second.archivalCompleted.get();
+        it->second.archivalCompleted.wait();
+        it->second.fieldLocationsReceived = std::promise{};
     }
 
-    it->second.catalogue->flush();
+    it->second.catalogue->flush(numArchived);
 
     Log::info() << "Flush complete" << std::endl;
     Log::status() << "Flush complete" << std::endl;
@@ -403,7 +406,7 @@ void CatalogueHandler::archiveBlob(const uint32_t clientID, const uint32_t reque
     it->second.catalogue->selectIndex(idxKey);
     it->second.catalogue->archive(key, std::move(location));
     it->second.locationsArchived++;
-    if (it->second.locationsExpected > 0 && it->second.locationsExpected == it->second.locationsArchived) {
+    if (it->second.archivalCompleted.valid() && it->second.locationsExpected == it->second.locationsArchived) {
         it->second.fieldLocationsReceived.set_value(it->second.locationsExpected);
     }
 }
diff --git a/src/fdb5/remote/server/CatalogueHandler.h b/src/fdb5/remote/server/CatalogueHandler.h
index 90826144a..095711828 100644
--- a/src/fdb5/remote/server/CatalogueHandler.h
+++ b/src/fdb5/remote/server/CatalogueHandler.h
@@ -20,19 +20,17 @@ namespace fdb5::remote {
 struct CatalogueArchiver {
     CatalogueArchiver(bool dataConnection, const Key& dbKey, const Config& config) :
         controlConnection(true), dataConnection(dataConnection),
-        catalogue(CatalogueWriterFactory::instance().build(dbKey, config)), locationsExpected(-1), locationsArchived(0) {
-        archivalCompleted = fieldLocationsReceived.get_future();
-    }
+        catalogue(CatalogueWriterFactory::instance().build(dbKey, config)), locationsExpected(0), locationsArchived(0) {}
 
     bool controlConnection;
     bool dataConnection;
     
     std::unique_ptr catalogue;
-    int32_t locationsExpected;
-    int32_t locationsArchived;
+    size_t locationsExpected;
+    size_t locationsArchived;
 
-    std::promise fieldLocationsReceived;
-    std::future archivalCompleted;
+    std::promise fieldLocationsReceived;
+    std::future archivalCompleted;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/toc/AdoptVisitor.cc b/src/fdb5/toc/AdoptVisitor.cc
index 254272ea8..3988d3b9a 100644
--- a/src/fdb5/toc/AdoptVisitor.cc
+++ b/src/fdb5/toc/AdoptVisitor.cc
@@ -35,11 +35,11 @@ bool AdoptVisitor::selectDatum(const InspectionKey &key, const Key &full) {
     // Log::info() << "selectDatum " << key << ", " << full << " " << length_ << std::endl;
     checkMissingKeys(full);
 
-    CatalogueWriter* catalogue = current();
-    ASSERT(catalogue);
+    CatalogueWriter* cat = catalogue();
+    ASSERT(cat);
 
-    if (catalogue->type() == TocEngine::typeName()) {
-        catalogue->index(key, eckit::URI("file", path_), offset_, length_);
+    if (cat->type() == TocEngine::typeName()) {
+        cat->index(key, eckit::URI("file", path_), offset_, length_);
         return true;
     }
     return false;
diff --git a/src/fdb5/toc/TocCatalogueReader.h b/src/fdb5/toc/TocCatalogueReader.h
index d0eef481b..6068a1839 100644
--- a/src/fdb5/toc/TocCatalogueReader.h
+++ b/src/fdb5/toc/TocCatalogueReader.h
@@ -43,7 +43,7 @@ class TocCatalogueReader : public TocCatalogue, public CatalogueReader {
     void deselectIndex() override;
 
     bool open() override;
-    void flush() override {}
+    void flush(size_t archivedFields) override {}
     void clean() override {}
     void close() override;
     
diff --git a/src/fdb5/toc/TocCatalogueWriter.cc b/src/fdb5/toc/TocCatalogueWriter.cc
index a58c091cb..cbc600bdc 100644
--- a/src/fdb5/toc/TocCatalogueWriter.cc
+++ b/src/fdb5/toc/TocCatalogueWriter.cc
@@ -33,7 +33,8 @@ namespace fdb5 {
 
 TocCatalogueWriter::TocCatalogueWriter(const Key &dbKey, const fdb5::Config& config) :
     TocCatalogue(dbKey, config),
-    umask_(config.umask()) {
+    umask_(config.umask()),
+    archivedLocations_(0) {
     writeInitRecord(dbKey);
     TocCatalogue::loadSchema();
     TocCatalogue::checkUID();
@@ -41,7 +42,8 @@ TocCatalogueWriter::TocCatalogueWriter(const Key &dbKey, const fdb5::Config& con
 
 TocCatalogueWriter::TocCatalogueWriter(const eckit::URI &uri, const fdb5::Config& config) :
     TocCatalogue(uri.path(), ControlIdentifiers{}, config),
-    umask_(config.umask()) {
+    umask_(config.umask()),
+    archivedLocations_(0) {
     writeInitRecord(TocCatalogue::key());
     TocCatalogue::loadSchema();
     TocCatalogue::checkUID();
@@ -110,7 +112,7 @@ void TocCatalogueWriter::clean() {
 
     LOG_DEBUG_LIB(LibFdb5) << "Closing path " << directory_ << std::endl;
 
-    flush(); // closes the TOC entries & indexes but not data files
+    flush(archivedLocations_); // closes the TOC entries & indexes but not data files
 
     compactSubTocIndexes();
 
@@ -123,7 +125,7 @@ void TocCatalogueWriter::close() {
 }
 
 void TocCatalogueWriter::index(const InspectionKey &key, const eckit::URI &uri, eckit::Offset offset, eckit::Length length) {
-    dirty_ = true;
+    archivedLocations_++;
 
     if (current_.null()) {
         ASSERT(!currentIndexKey_.empty());
@@ -301,7 +303,7 @@ bool TocCatalogueWriter::enabled(const ControlIdentifier& controlIdentifier) con
 }
 
 void TocCatalogueWriter::archive(const InspectionKey& key, std::unique_ptr fieldLocation) {
-    dirty_ = true;
+    archivedLocations_++;
 
     if (current_.null()) {
         ASSERT(!currentIndexKey_.empty());
@@ -316,14 +318,16 @@ void TocCatalogueWriter::archive(const InspectionKey& key, std::unique_ptr(-1)),
-    userUID_(::getuid()),
-    dirty_(false) {}
+    userUID_(::getuid()) {}
 
 void TocCommon::checkUID() const {
     static bool fdbOnlyCreatorCanWrite = eckit::Resource("fdbOnlyCreatorCanWrite", true);
diff --git a/src/fdb5/toc/TocCommon.h b/src/fdb5/toc/TocCommon.h
index d6caf21b4..6d967207a 100644
--- a/src/fdb5/toc/TocCommon.h
+++ b/src/fdb5/toc/TocCommon.h
@@ -50,8 +50,6 @@ class TocCommon {
 
     mutable uid_t dbUID_;
     uid_t userUID_;
-
-    mutable bool dirty_;
 };
 
 }
diff --git a/src/fdb5/toc/TocHandler.cc b/src/fdb5/toc/TocHandler.cc
index 71d90c866..b5cdd040e 100644
--- a/src/fdb5/toc/TocHandler.cc
+++ b/src/fdb5/toc/TocHandler.cc
@@ -131,7 +131,8 @@ TocHandler::TocHandler(const eckit::PathName& directory, const Config& config) :
     cachedToc_(nullptr),
     count_(0),
     enumeratedMaskedEntries_(false),
-    writeMode_(false)
+    writeMode_(false),
+    dirty_(false)
 {
     // An override to enable using sub tocs without configurations being passed in, for ease
     // of debugging
@@ -154,7 +155,8 @@ TocHandler::TocHandler(const eckit::PathName& path, const Key& parentKey) :
     cachedToc_(nullptr),
     count_(0),
     enumeratedMaskedEntries_(false),
-    writeMode_(false)
+    writeMode_(false),
+    dirty_(false)
 {
     /// Are we remapping a mounted DB?
     if (exists()) {
diff --git a/src/fdb5/toc/TocHandler.h b/src/fdb5/toc/TocHandler.h
index d55d13789..7b5c2281c 100644
--- a/src/fdb5/toc/TocHandler.h
+++ b/src/fdb5/toc/TocHandler.h
@@ -262,6 +262,8 @@ class TocHandler : public TocCommon, private eckit::NonCopyable {
 
     mutable bool enumeratedMaskedEntries_;
     mutable bool writeMode_;
+
+    mutable bool dirty_;
 };
 
 
diff --git a/src/fdb5/toc/TocStore.cc b/src/fdb5/toc/TocStore.cc
index 973aab34a..1cbcd164f 100644
--- a/src/fdb5/toc/TocStore.cc
+++ b/src/fdb5/toc/TocStore.cc
@@ -35,10 +35,10 @@ namespace fdb5 {
 //----------------------------------------------------------------------------------------------------------------------
 
 TocStore::TocStore(const Key& key, const Config& config) :
-    Store(), TocCommon(StoreRootManager(config).directory(key).directory_) {}
+    Store(), TocCommon(StoreRootManager(config).directory(key).directory_), archivedFields_(0) {}
 
 TocStore::TocStore(const eckit::URI& uri, const Config& config) :
-    Store(), TocCommon(uri.path().dirName()) {}
+    Store(), TocCommon(uri.path().dirName()), archivedFields_(0) {}
 
 eckit::URI TocStore::uri() const {
     return URI("file", directory_);
@@ -53,8 +53,7 @@ eckit::DataHandle* TocStore::retrieve(Field& field) const {
 }
 
 std::unique_ptr TocStore::archive(const Key& key, const void *data, eckit::Length length) {
-    
-    dirty_ = true;
+    archivedFields_++;
 
     eckit::PathName dataPath = getDataPath(key);
 
@@ -69,16 +68,18 @@ std::unique_ptr TocStore::archive(const Key& key, const void *dat
     return std::unique_ptr(new TocFieldLocation(dataPath, position, length, Key()));
 }
 
-void TocStore::flush() {
-    if (!dirty_) {
-        return;
+size_t TocStore::flush() {
+    if (archivedFields_ == 0) {
+        return 0;
     }
 
     // ensure consistent state before writing Toc entry
-
     flushDataHandles();
 
-    dirty_ = false;
+    size_t out = archivedFields_;
+    archivedFields_ = 0;
+
+    return out;
 }
 
 void TocStore::close() {
diff --git a/src/fdb5/toc/TocStore.h b/src/fdb5/toc/TocStore.h
index 30d7be60f..2c972a6a7 100644
--- a/src/fdb5/toc/TocStore.h
+++ b/src/fdb5/toc/TocStore.h
@@ -42,7 +42,7 @@ class TocStore : public Store, public TocCommon {
     eckit::URI uri() const override;
 
     bool open() override { return true; }
-    void flush() override;
+    size_t flush() override;
     void close() override;
 
     void checkUID() const override { TocCommon::checkUID(); }
@@ -84,7 +84,7 @@ class TocStore : public Store, public TocCommon {
     HandleStore handles_;    ///< stores the DataHandles being used by the Session
 
     mutable PathStore   dataPaths_;
-
+    size_t archivedFields_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------

From 27001248cabfbe6efe54e974af1ce36ac2433522 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 5 Apr 2024 19:08:22 +0200
Subject: [PATCH 093/186] version bump

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index b58b26e18..923cb8c33 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.113
+5.11.114
\ No newline at end of file

From e8413de529a7f529411776526b1f26d82fe7ddaf Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 10 Apr 2024 08:00:03 +0200
Subject: [PATCH 094/186] added request expansion to C api

---
 src/fdb5/api/fdb_c.cc       | 12 +++++++++
 src/fdb5/api/fdb_c.h        |  6 +++++
 tests/fdb/api/test_fdb_c.cc | 51 +++++++++++++++++++++++++++++++++++++
 3 files changed, 69 insertions(+)

diff --git a/src/fdb5/api/fdb_c.cc b/src/fdb5/api/fdb_c.cc
index e64f5fef8..ac198d1ff 100644
--- a/src/fdb5/api/fdb_c.cc
+++ b/src/fdb5/api/fdb_c.cc
@@ -14,6 +14,7 @@
 #include "eckit/runtime/Main.h"
 
 #include "metkit/mars/MarsRequest.h"
+#include "metkit/mars/MarsExpension.h"
 
 #include "fdb5/fdb5_version.h"
 #include "fdb5/api/FDB.h"
@@ -58,6 +59,12 @@ struct fdb_request_t {
         req->request_ = metkit::mars::MarsRequest::parse(str);
         return req;
     }
+    void expand() {
+        bool inherit = false;
+        bool strict = true;
+        metkit::mars::MarsExpension expand(inherit, strict);
+        request_ = expand.expand(request_);
+    }
     const metkit::mars::MarsRequest request() const { return request_; }
 private:
     metkit::mars::MarsRequest request_;
@@ -403,6 +410,11 @@ int fdb_request_add(fdb_request_t* req, const char* param, const char* values[],
         req->values(param, values, numValues);
     });
 }
+int fdb_expand_request(fdb_request_t* req) {
+    return wrapApiFunction([req]{
+        req->expand();
+    });
+}
 int fdb_delete_request(fdb_request_t* req) {
     return wrapApiFunction([req]{
         ASSERT(req);
diff --git a/src/fdb5/api/fdb_c.h b/src/fdb5/api/fdb_c.h
index dc725a757..68ebac070 100644
--- a/src/fdb5/api/fdb_c.h
+++ b/src/fdb5/api/fdb_c.h
@@ -139,6 +139,12 @@ int fdb_new_request(fdb_request_t** req);
  */
 int fdb_request_add(fdb_request_t* req, const char* param, const char* values[], int numValues);
 
+/** Expand a Request
+ * \param req Request instance
+ * \returns Return code (#FdbErrorValues)
+ */
+int fdb_expand_request(fdb_request_t* req);
+
 /** Deallocates Request object and associated resources.
  * \param req Request instance
  * \returns Return code (#FdbErrorValues)
diff --git a/tests/fdb/api/test_fdb_c.cc b/tests/fdb/api/test_fdb_c.cc
index e3c9d432f..ea3fbe235 100644
--- a/tests/fdb/api/test_fdb_c.cc
+++ b/tests/fdb/api/test_fdb_c.cc
@@ -429,6 +429,57 @@ CASE( "fdb_c - retrieve" ) {
 }
 
 
+CASE( "fdb_c - expand" ) {
+
+    fdb_handle_t* fdb;
+    fdb_new_handle(&fdb);
+    fdb_request_t* request;
+    fdb_new_request(&request);
+    fdb_request_add1(request, "domain", "g");
+    fdb_request_add1(request, "stream", "oper");
+    fdb_request_add1(request, "levtype", "pl");
+    fdb_request_add1(request, "levelist", "300");
+    const char* dates[] = {"20191110", "to", "20191111"};
+    fdb_request_add(request, "date", dates, 3);
+    fdb_request_add1(request, "time", "0000");
+    fdb_request_add1(request, "step", "0");
+    fdb_request_add1(request, "param", "138");
+    fdb_request_add1(request, "class", "rd");
+    fdb_request_add1(request, "type", "an");
+    fdb_request_add1(request, "expver", "xxxx");
+
+    char buf[1000];
+    char grib[4];
+    long read = 0;
+    long size;
+    fdb_datareader_t* dr;
+    fdb_new_datareader(&dr);
+    EXPECT(fdb_retrieve(fdb, request, dr) == FDB_ERROR_GENERAL_EXCEPTION);
+
+    EXPECT(fdb_expand_request(request) == FDB_SUCCESS);
+
+    EXPECT(fdb_retrieve(fdb, request, dr) == FDB_SUCCESS);
+    fdb_datareader_open(dr, &size);
+    EXPECT_NOT_EQUAL(0, size);
+    fdb_datareader_read(dr, grib, 4, &read);
+    EXPECT_EQUAL(4, read);
+    EXPECT_EQUAL(0, strncmp(grib, "GRIB", 4));
+    fdb_datareader_tell(dr, &read);
+    EXPECT_EQUAL(4, read);
+    fdb_datareader_seek(dr, 3);
+    fdb_datareader_tell(dr, &read);
+    EXPECT_EQUAL(3, read);
+    fdb_datareader_skip(dr, 3);
+    fdb_datareader_tell(dr, &read);
+    EXPECT_EQUAL(6, read);
+    fdb_datareader_read(dr, buf, 1000, &read);
+    EXPECT_EQUAL(1000, read);
+    fdb_datareader_tell(dr, &read);
+    EXPECT_EQUAL(1006, read);
+    fdb_delete_datareader(dr);
+}
+
+
 //----------------------------------------------------------------------------------------------------------------------
 
 }  // namespace test

From 856cdc5efed8f56f9d41613592e22232387b2b83 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 10 Apr 2024 12:03:36 +0200
Subject: [PATCH 095/186] added tokenizing of MarsRequest metadata

---
 src/fdb5/api/fdb_c.cc       | 28 ++++++++++++++++++++++++-
 src/fdb5/api/fdb_c.h        |  9 ++++++++
 tests/fdb/api/test_fdb_c.cc | 41 +++++++++++++++++++++++++++++++++++--
 3 files changed, 75 insertions(+), 3 deletions(-)

diff --git a/src/fdb5/api/fdb_c.cc b/src/fdb5/api/fdb_c.cc
index ac198d1ff..3d9e76dae 100644
--- a/src/fdb5/api/fdb_c.cc
+++ b/src/fdb5/api/fdb_c.cc
@@ -15,6 +15,7 @@
 
 #include "metkit/mars/MarsRequest.h"
 #include "metkit/mars/MarsExpension.h"
+#include "eckit/utils/Tokenizer.h"
 
 #include "fdb5/fdb5_version.h"
 #include "fdb5/api/FDB.h"
@@ -46,11 +47,27 @@ struct fdb_request_t {
     fdb_request_t(std::string str) {
         request_ = metkit::mars::MarsRequest(str);
     }
+    size_t values(const char* name, char** values[]) {
+        std::string n(name);
+        std::vector vv = request_.values(name);
+
+        *values = new char*[vv.size()];
+        for (size_t i = 0; i < vv.size(); i++) {
+            (*values)[i] = new char[vv[i].size()+1];
+            strncpy((*values)[i], vv[i].c_str(), vv[i].size());
+            (*values)[i][vv[i].size()] = '\0';
+        }
+        return vv.size();
+    }
     void values(const char* name, const char* values[], int numValues) {
         std::string n(name);
         std::vector vv;
+        Tokenizer parse("/");
+
         for (int i=0; i result;
+        	parse(values[i], result);
+            vv.insert(std::end(vv), std::begin(result), std::end(result));
         }
         request_.values(n, vv);
     }
@@ -63,7 +80,9 @@ struct fdb_request_t {
         bool inherit = false;
         bool strict = true;
         metkit::mars::MarsExpension expand(inherit, strict);
+        std::cout << "expand()  PRE   " << request_ << std::endl;
         request_ = expand.expand(request_);
+        std::cout << "expand()  POST  " << request_ << std::endl;
     }
     const metkit::mars::MarsRequest request() const { return request_; }
 private:
@@ -410,6 +429,13 @@ int fdb_request_add(fdb_request_t* req, const char* param, const char* values[],
         req->values(param, values, numValues);
     });
 }
+int fdb_request_get(fdb_request_t* req, const char* param, char** values[], size_t* numValues) {
+    return wrapApiFunction([req, param, values, numValues] {
+        ASSERT(req);
+        ASSERT(param);
+        *numValues = req->values(param, values);
+    });
+}
 int fdb_expand_request(fdb_request_t* req) {
     return wrapApiFunction([req]{
         req->expand();
diff --git a/src/fdb5/api/fdb_c.h b/src/fdb5/api/fdb_c.h
index 68ebac070..e3371c61a 100644
--- a/src/fdb5/api/fdb_c.h
+++ b/src/fdb5/api/fdb_c.h
@@ -139,6 +139,15 @@ int fdb_new_request(fdb_request_t** req);
  */
 int fdb_request_add(fdb_request_t* req, const char* param, const char* values[], int numValues);
 
+/** Get the Metadata values associated to a Request metadata
+ * \param req Request instance
+ * \param param Metadata name
+ * \param values Metadata values
+ * \param numValues number of metadata values
+ * \returns Return code (#FdbErrorValues)
+ */
+int fdb_request_get(fdb_request_t* req, const char* param, char** values[], size_t* numValues);
+
 /** Expand a Request
  * \param req Request instance
  * \returns Return code (#FdbErrorValues)
diff --git a/tests/fdb/api/test_fdb_c.cc b/tests/fdb/api/test_fdb_c.cc
index ea3fbe235..38ccff07f 100644
--- a/tests/fdb/api/test_fdb_c.cc
+++ b/tests/fdb/api/test_fdb_c.cc
@@ -454,9 +454,20 @@ CASE( "fdb_c - expand" ) {
     long size;
     fdb_datareader_t* dr;
     fdb_new_datareader(&dr);
-    EXPECT(fdb_retrieve(fdb, request, dr) == FDB_ERROR_GENERAL_EXCEPTION);
+    EXPECT_EQUAL(fdb_retrieve(fdb, request, dr), FDB_ERROR_GENERAL_EXCEPTION);
 
-    EXPECT(fdb_expand_request(request) == FDB_SUCCESS);
+    EXPECT_EQUAL(fdb_expand_request(request), FDB_SUCCESS);
+
+    size_t numValues;
+    char** values;
+    
+    fdb_request_get(request, "date", &values, &numValues);
+    EXPECT_EQUAL(numValues, 2);
+    EXPECT_EQUAL(0, strncmp(values[0], "20191110", 8));
+    EXPECT_EQUAL(0, strncmp(values[1], "20191111", 8));
+    delete values[0];
+    delete values[1];
+    delete values;
 
     EXPECT(fdb_retrieve(fdb, request, dr) == FDB_SUCCESS);
     fdb_datareader_open(dr, &size);
@@ -477,6 +488,32 @@ CASE( "fdb_c - expand" ) {
     fdb_datareader_tell(dr, &read);
     EXPECT_EQUAL(1006, read);
     fdb_delete_datareader(dr);
+
+    fdb_request_add1(request, "date", "20191110/to/20191115/by/2");
+
+    fdb_request_get(request, "date", &values, &numValues);
+    EXPECT_EQUAL(numValues, 5);
+    EXPECT_EQUAL(0, strncmp(values[0], "20191110", 8));
+    EXPECT_EQUAL(0, strncmp(values[1], "to", 2));
+    EXPECT_EQUAL(0, strncmp(values[2], "20191115", 8));
+    EXPECT_EQUAL(0, strncmp(values[3], "by", 2));
+    EXPECT_EQUAL(0, strncmp(values[4], "2", 1));
+    for (size_t i = 0; i
Date: Wed, 10 Apr 2024 12:08:45 +0200
Subject: [PATCH 096/186] added tokenizing of MarsRequest metadata

---
 src/fdb5/api/fdb_c.cc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/fdb5/api/fdb_c.cc b/src/fdb5/api/fdb_c.cc
index 3d9e76dae..3fcc0f43e 100644
--- a/src/fdb5/api/fdb_c.cc
+++ b/src/fdb5/api/fdb_c.cc
@@ -80,9 +80,7 @@ struct fdb_request_t {
         bool inherit = false;
         bool strict = true;
         metkit::mars::MarsExpension expand(inherit, strict);
-        std::cout << "expand()  PRE   " << request_ << std::endl;
         request_ = expand.expand(request_);
-        std::cout << "expand()  POST  " << request_ << std::endl;
     }
     const metkit::mars::MarsRequest request() const { return request_; }
 private:

From d52e86a2a5a427397734caeeb189656904b73d5b Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 10 Apr 2024 14:18:39 +0200
Subject: [PATCH 097/186] updated step-range canonical unit

---
 tests/fdb/type/test_toKey.cc | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/tests/fdb/type/test_toKey.cc b/tests/fdb/type/test_toKey.cc
index 9407fc317..0357e5e99 100644
--- a/tests/fdb/type/test_toKey.cc
+++ b/tests/fdb/type/test_toKey.cc
@@ -147,23 +147,18 @@ CASE( "Step & ClimateDaily - expansion" ) {
     EXPECT(key.canonicalValue("step") == "0-1");
 
     key.set("step", "30m");
-    // std::cout << key.get("step") << " " << key.canonicalValue("step") << std::endl;
     EXPECT(key.canonicalValue("step") == "30m");
 
     key.set("step", "60m");
-    // std::cout << key.get("step") << " " << key.canonicalValue("step") << std::endl;
     EXPECT(key.canonicalValue("step") == "1");
 
     key.set("step", "30m-60m");
-    // std::cout << key.get("step") << " " << key.canonicalValue("step") << std::endl;
-    EXPECT(key.canonicalValue("step") == "30m-60m");
+    EXPECT(key.canonicalValue("step") == "30m-1");
 
     key.set("step", "30m-1");
-    // std::cout << key.get("step") << " " << key.canonicalValue("step") << std::endl;
-    EXPECT(key.canonicalValue("step") == "30m-60m");
+    EXPECT(key.canonicalValue("step") == "30m-1");
 
     key.set("step", "60m-120m");
-    // std::cout << key.get("step") << " " << key.canonicalValue("step") << std::endl;
     EXPECT(key.canonicalValue("step") == "1-2");
 }
 
@@ -282,7 +277,6 @@ CASE( "Date - string ctor - expansion" ) {
     config.schema().expand(key, visitor);
     key.rule(visitor.rule());
 
-//    std::cout << key.valuesToString() << std::endl;
     EXPECT(key.canonicalValue("date") == t(now.yyyymmdd()));
     EXPECT(key.valuesToString() == "od:0001:oper:ofb:"+t(now.yyyymmdd())+":0000:mhs:3001");
 

From 6034b7201ca0b454e3a79648ec9eeba41f0223e6 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 10 Apr 2024 20:04:07 +0200
Subject: [PATCH 098/186] version bump

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 923cb8c33..7084a22fe 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.114
\ No newline at end of file
+5.11.115

From e1e2dd445f4a2f86050012322e511296255c6064 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 15 Apr 2024 14:32:47 +0100
Subject: [PATCH 099/186] fix catalogue indexing with async store archival

---
 src/fdb5/database/ArchiveVisitor.cc | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/fdb5/database/ArchiveVisitor.cc b/src/fdb5/database/ArchiveVisitor.cc
index 254a7453e..b953be644 100644
--- a/src/fdb5/database/ArchiveVisitor.cc
+++ b/src/fdb5/database/ArchiveVisitor.cc
@@ -15,6 +15,13 @@
 #include "fdb5/database/Catalogue.h"
 #include "fdb5/database/Store.h"
 
+namespace {
+void CatalogueCallback(fdb5::CatalogueWriter* catalogue, const fdb5::Key &idxKey, const fdb5::InspectionKey &key, std::unique_ptr fieldLocation) {
+    catalogue->selectIndex(idxKey);
+    catalogue->archive(key, std::move(fieldLocation));
+}
+}
+
 namespace fdb5 {
 
 ArchiveVisitor::ArchiveVisitor(Archiver &owner, const Key &dataKey, const void *data, size_t size) :
@@ -29,7 +36,7 @@ bool ArchiveVisitor::selectDatum(const InspectionKey &key, const Key &full) {
     checkMissingKeys(full);
     const Key idxKey = catalogue()->currentIndexKey();
 
-    store()->archive(idxKey, data_, size_, std::bind(&CatalogueWriter::archive, catalogue(), key, std::placeholders::_1));
+    store()->archive(idxKey, data_, size_, std::bind(&CatalogueCallback, catalogue(), idxKey, key, std::placeholders::_1));
 
     return true;
 }

From 3667c5e2658c7df399220acd5a5bd6e7aea63d9c Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 15 Apr 2024 16:07:46 +0100
Subject: [PATCH 100/186] propagated catalogue index key in Catalogue::archive

---
 src/fdb5/database/ArchiveVisitor.cc        | 9 +--------
 src/fdb5/database/Catalogue.h              | 2 +-
 src/fdb5/remote/client/RemoteCatalogue.cc  | 4 ++--
 src/fdb5/remote/client/RemoteCatalogue.h   | 2 +-
 src/fdb5/remote/server/CatalogueHandler.cc | 2 +-
 src/fdb5/toc/TocCatalogueWriter.cc         | 6 +++++-
 src/fdb5/toc/TocCatalogueWriter.h          | 2 +-
 7 files changed, 12 insertions(+), 15 deletions(-)

diff --git a/src/fdb5/database/ArchiveVisitor.cc b/src/fdb5/database/ArchiveVisitor.cc
index b953be644..7f629cdf3 100644
--- a/src/fdb5/database/ArchiveVisitor.cc
+++ b/src/fdb5/database/ArchiveVisitor.cc
@@ -15,13 +15,6 @@
 #include "fdb5/database/Catalogue.h"
 #include "fdb5/database/Store.h"
 
-namespace {
-void CatalogueCallback(fdb5::CatalogueWriter* catalogue, const fdb5::Key &idxKey, const fdb5::InspectionKey &key, std::unique_ptr fieldLocation) {
-    catalogue->selectIndex(idxKey);
-    catalogue->archive(key, std::move(fieldLocation));
-}
-}
-
 namespace fdb5 {
 
 ArchiveVisitor::ArchiveVisitor(Archiver &owner, const Key &dataKey, const void *data, size_t size) :
@@ -36,7 +29,7 @@ bool ArchiveVisitor::selectDatum(const InspectionKey &key, const Key &full) {
     checkMissingKeys(full);
     const Key idxKey = catalogue()->currentIndexKey();
 
-    store()->archive(idxKey, data_, size_, std::bind(&CatalogueCallback, catalogue(), idxKey, key, std::placeholders::_1));
+    store()->archive(idxKey, data_, size_, std::bind(&CatalogueWriter::archive, catalogue(), idxKey, key, std::placeholders::_1));
 
     return true;
 }
diff --git a/src/fdb5/database/Catalogue.h b/src/fdb5/database/Catalogue.h
index 17e90f8c8..9a661c6c8 100644
--- a/src/fdb5/database/Catalogue.h
+++ b/src/fdb5/database/Catalogue.h
@@ -164,7 +164,7 @@ class CatalogueWriter : virtual public Catalogue  {
 
     virtual const Index& currentIndex() = 0;
     virtual const Key currentIndexKey() = 0;
-    virtual void archive(const InspectionKey& key, std::unique_ptr fieldLocation) = 0;
+    virtual void archive(const Key& idxKey, const InspectionKey& key, std::unique_ptr fieldLocation) = 0;
     virtual void overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) = 0;
     virtual void index(const InspectionKey& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) = 0;
     virtual void reconsolidate() = 0;
diff --git a/src/fdb5/remote/client/RemoteCatalogue.cc b/src/fdb5/remote/client/RemoteCatalogue.cc
index 8c2596a53..b4261e85c 100644
--- a/src/fdb5/remote/client/RemoteCatalogue.cc
+++ b/src/fdb5/remote/client/RemoteCatalogue.cc
@@ -58,7 +58,7 @@ void RemoteCatalogue::sendArchiveData(uint32_t id, const Key& key, std::unique_p
     dataWrite(Message::Blob, id, payloads);
 }
 
-void RemoteCatalogue::archive(const InspectionKey& key, std::unique_ptr fieldLocation) {
+void RemoteCatalogue::archive(const Key& idxKey, const InspectionKey& key, std::unique_ptr fieldLocation) {
 
     ASSERT(!key.empty());
     ASSERT(fieldLocation);
@@ -74,7 +74,7 @@ void RemoteCatalogue::archive(const InspectionKey& key, std::unique_ptr fieldLocation) override;
+    void archive(const Key& idxKey, const InspectionKey& key, std::unique_ptr fieldLocation) override;
     void overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) override;
     void index(const InspectionKey& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) override;
     void reconsolidate() override;
diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc
index b56effb2e..4956898cb 100644
--- a/src/fdb5/remote/server/CatalogueHandler.cc
+++ b/src/fdb5/remote/server/CatalogueHandler.cc
@@ -404,7 +404,7 @@ void CatalogueHandler::archiveBlob(const uint32_t clientID, const uint32_t reque
     }
 
     it->second.catalogue->selectIndex(idxKey);
-    it->second.catalogue->archive(key, std::move(location));
+    it->second.catalogue->archive(idxKey, key, std::move(location));
     it->second.locationsArchived++;
     if (it->second.archivalCompleted.valid() && it->second.locationsExpected == it->second.locationsArchived) {
         it->second.fieldLocationsReceived.set_value(it->second.locationsExpected);
diff --git a/src/fdb5/toc/TocCatalogueWriter.cc b/src/fdb5/toc/TocCatalogueWriter.cc
index cbc600bdc..a47902523 100644
--- a/src/fdb5/toc/TocCatalogueWriter.cc
+++ b/src/fdb5/toc/TocCatalogueWriter.cc
@@ -302,12 +302,16 @@ bool TocCatalogueWriter::enabled(const ControlIdentifier& controlIdentifier) con
     return TocCatalogue::enabled(controlIdentifier);
 }
 
-void TocCatalogueWriter::archive(const InspectionKey& key, std::unique_ptr fieldLocation) {
+void TocCatalogueWriter::archive(const Key& idxKey, const InspectionKey& key, std::unique_ptr fieldLocation) {
     archivedLocations_++;
 
     if (current_.null()) {
         ASSERT(!currentIndexKey_.empty());
         selectIndex(currentIndexKey_);
+    } else {
+        if(currentIndexKey_ != idxKey) {
+            selectIndex(idxKey);
+        }
     }
 
     Field field(std::move(fieldLocation), currentIndex().timestamp());
diff --git a/src/fdb5/toc/TocCatalogueWriter.h b/src/fdb5/toc/TocCatalogueWriter.h
index 09cafb940..1ea7b4058 100644
--- a/src/fdb5/toc/TocCatalogueWriter.h
+++ b/src/fdb5/toc/TocCatalogueWriter.h
@@ -71,7 +71,7 @@ class TocCatalogueWriter : public TocCatalogue, public CatalogueWriter {
     void clean() override;
     void close() override;
 
-    void archive(const InspectionKey& key, std::unique_ptr fieldLocation) override;
+    void archive(const Key& idxKey, const InspectionKey& key, std::unique_ptr fieldLocation) override;
     void reconsolidateIndexesAndTocs();
 
     virtual void print( std::ostream &out ) const override;

From 87948dfc445b3cf756d3658a19ab6078fa040ef0 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 16 Apr 2024 07:14:30 +0100
Subject: [PATCH 101/186] added comment

---
 src/fdb5/toc/TocCatalogueWriter.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/fdb5/toc/TocCatalogueWriter.cc b/src/fdb5/toc/TocCatalogueWriter.cc
index a47902523..3af947b4a 100644
--- a/src/fdb5/toc/TocCatalogueWriter.cc
+++ b/src/fdb5/toc/TocCatalogueWriter.cc
@@ -54,6 +54,7 @@ TocCatalogueWriter::~TocCatalogueWriter() {
     close();
 }
 
+// selectIndex is called during schema traversal and in case of out-of-order fieldLocation archival
 bool TocCatalogueWriter::selectIndex(const Key& idxKey) {
     currentIndexKey_ = idxKey;
 
@@ -309,6 +310,7 @@ void TocCatalogueWriter::archive(const Key& idxKey, const InspectionKey& key, st
         ASSERT(!currentIndexKey_.empty());
         selectIndex(currentIndexKey_);
     } else {
+        // in case of async archival (out of order store/catalogue archival), currentIndexKey_ can differ from the indexKey used for store archival. Reset it
         if(currentIndexKey_ != idxKey) {
             selectIndex(idxKey);
         }

From 63dfd6d7ca8a0a3caaf2ab68c225d28491ba211d Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 16 Apr 2024 07:16:00 +0100
Subject: [PATCH 102/186] version bump

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 7084a22fe..f12437f50 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.115
+5.11.116

From 36ad61055f166666d4d931640386e4f86891d5c6 Mon Sep 17 00:00:00 2001
From: Chris Bradley 
Date: Fri, 26 Apr 2024 13:52:52 +0100
Subject: [PATCH 103/186] Add archival status logging

---
 src/fdb5/remote/client/RemoteCatalogue.cc | 2 ++
 src/fdb5/remote/client/RemoteStore.cc     | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/src/fdb5/remote/client/RemoteCatalogue.cc b/src/fdb5/remote/client/RemoteCatalogue.cc
index b4261e85c..a14b8a16e 100644
--- a/src/fdb5/remote/client/RemoteCatalogue.cc
+++ b/src/fdb5/remote/client/RemoteCatalogue.cc
@@ -82,6 +82,8 @@ void RemoteCatalogue::archive(const Key& idxKey, const InspectionKey& key, std::
     payloads.push_back(std::pair{buffer, stream.position()});
 
     dataWrite(Message::Blob, id, payloads);
+
+    eckit::Log::status() << "FieldLocation " << numLocations_ << "enqueued for catalogue archival" << std::endl;
 }
 
 bool RemoteCatalogue::selectIndex(const Key& idxKey) {
diff --git a/src/fdb5/remote/client/RemoteStore.cc b/src/fdb5/remote/client/RemoteStore.cc
index 6215bcfe6..8fbe5d844 100644
--- a/src/fdb5/remote/client/RemoteStore.cc
+++ b/src/fdb5/remote/client/RemoteStore.cc
@@ -249,6 +249,8 @@ void RemoteStore::archive(const Key& key, const void *data, eckit::Length length
     payloads.push_back(std::pair{data, length});
 
     dataWrite(Message::Blob, id, payloads);
+
+    eckit::Log::status() << "Field " << fieldsArchived_ << " enqueued for store archival" << std::endl;
 }
 
 bool RemoteStore::open() {

From 17ae0bbb93abb7b8f05394212886f74d754271be Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 30 Apr 2024 11:27:12 +0100
Subject: [PATCH 104/186] fdb remote stats

---
 src/fdb5/api/RemoteFDB.cc                  |  5 +++++
 src/fdb5/api/RemoteFDB.h                   |  2 +-
 src/fdb5/remote/RemoteFieldLocation.cc     |  4 ++++
 src/fdb5/remote/server/CatalogueHandler.cc | 14 ++++++++++++++
 src/fdb5/remote/server/CatalogueHandler.h  |  1 +
 tests/fdb/type/test_toKey.cc               |  2 +-
 6 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index fc595d596..fa9aa2cf9 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -33,6 +33,7 @@ struct BaseAPIHelper {
 };
 
 // using ListHelper = BaseAPIHelper;
+using StatsHelper = BaseAPIHelper;
 
 struct ListHelper : BaseAPIHelper {
 
@@ -236,6 +237,10 @@ ListIterator RemoteFDB::inspect(const metkit::mars::MarsRequest& request) {
     return forwardApiCall(InspectHelper(), request);
 }
 
+StatsIterator RemoteFDB::stats(const FDBToolRequest& request) {
+    return forwardApiCall(StatsHelper(), request);
+}
+
 void RemoteFDB::print(std::ostream& s) const {
     s << "RemoteFDB(...)";
 }
diff --git a/src/fdb5/api/RemoteFDB.h b/src/fdb5/api/RemoteFDB.h
index 9e32120c8..f0f8b75bd 100644
--- a/src/fdb5/api/RemoteFDB.h
+++ b/src/fdb5/api/RemoteFDB.h
@@ -56,7 +56,7 @@ class RemoteFDB : public LocalFDB, public remote::Client {
 
     PurgeIterator purge(const FDBToolRequest& request, bool doit, bool porcelain) override { NOTIMP; }
 
-    StatsIterator stats(const FDBToolRequest& request) override { NOTIMP; }
+    StatsIterator stats(const FDBToolRequest& request) override;
 
     ControlIterator control(const FDBToolRequest& request,
                             ControlAction action,
diff --git a/src/fdb5/remote/RemoteFieldLocation.cc b/src/fdb5/remote/RemoteFieldLocation.cc
index ee5f404e5..348cba9b2 100644
--- a/src/fdb5/remote/RemoteFieldLocation.cc
+++ b/src/fdb5/remote/RemoteFieldLocation.cc
@@ -143,6 +143,10 @@ class FdbURIManager : public eckit::URIManager {
         return f.path().partHandle(ol, ll);
     }
 
+    eckit::PathName path(const eckit::URI& u) const override {
+        return eckit::PathName{u.name()};
+    }
+
     virtual std::string asString(const eckit::URI& uri) const override {
         std::string q = uri.query();
         if (!q.empty())
diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc
index 4956898cb..1846a9843 100644
--- a/src/fdb5/remote/server/CatalogueHandler.cc
+++ b/src/fdb5/remote/server/CatalogueHandler.cc
@@ -102,6 +102,10 @@ Handled CatalogueHandler::handleControl(Message message, uint32_t clientID, uint
                 inspect(clientID, requestID, std::move(payload));
                 return Handled::Yes;
 
+            case Message::Stats: // inspect request. Location are sent aynchronously over the data connection
+                stats(clientID, requestID, std::move(payload));
+                return Handled::Yes;
+
             case Message::Flush: // flush catalogue
                 flush(clientID, requestID, std::move(payload));
                 return Handled::Yes;
@@ -217,6 +221,12 @@ struct InspectHelper : public BaseHelper {
     }
 };
 
+struct StatsHelper : public BaseHelper {
+    StatsIterator apiCall(FDB& fdb, const FDBToolRequest& request) const {
+        return fdb.stats(request);
+    }
+};
+
 template 
 void CatalogueHandler::forwardApiCall(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) {
     HelperClass helper;
@@ -261,6 +271,10 @@ void CatalogueHandler::inspect(uint32_t clientID, uint32_t requestID, eckit::Buf
     forwardApiCall(clientID, requestID, std::move(payload));
 }
 
+void CatalogueHandler::stats(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) {
+    forwardApiCall(clientID, requestID, std::move(payload));
+}
+
 void CatalogueHandler::schema(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) {
 
     eckit::Buffer schemaBuffer(256*1024);
diff --git a/src/fdb5/remote/server/CatalogueHandler.h b/src/fdb5/remote/server/CatalogueHandler.h
index 095711828..b61223239 100644
--- a/src/fdb5/remote/server/CatalogueHandler.h
+++ b/src/fdb5/remote/server/CatalogueHandler.h
@@ -54,6 +54,7 @@ class CatalogueHandler : public ServerConnection {
     void flush(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload);
     void list(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload);
     void inspect(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload);
+    void stats(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload);
     void schema(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload);
     void stores(uint32_t clientID, uint32_t requestID);
 
diff --git a/tests/fdb/type/test_toKey.cc b/tests/fdb/type/test_toKey.cc
index 0357e5e99..337372f64 100644
--- a/tests/fdb/type/test_toKey.cc
+++ b/tests/fdb/type/test_toKey.cc
@@ -133,7 +133,7 @@ CASE( "Step & ClimateDaily - expansion" ) {
     EXPECT(key.canonicalValue("date") == "0427");
     EXPECT(key.canonicalValue("time") == "0000");
 
-    std::cout << key.valuesToString() << std::endl;
+    // std::cout << key.valuesToString() << std::endl;
 
     EXPECT(key.valuesToString() == "0427:dacl:0000:ei:7799:g:pb:pl:2-12:99:100:50:129.128");
 

From a6dd9ec0f3e7c726db855e860ac6c64af42e24af Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 30 Apr 2024 13:13:42 +0100
Subject: [PATCH 105/186] tag

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index f12437f50..8423a1d92 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.116
+5.11.117

From a59cff64e801aa419aae77679e297810a167498f Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 30 Apr 2024 17:00:45 +0100
Subject: [PATCH 106/186] TocStats check for data file existence

---
 VERSION                  |  2 +-
 src/fdb5/toc/TocStats.cc | 22 +++++++++++-----------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/VERSION b/VERSION
index 8423a1d92..f4f8ed553 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.117
+5.11.118
\ No newline at end of file
diff --git a/src/fdb5/toc/TocStats.cc b/src/fdb5/toc/TocStats.cc
index 25a6e6061..1b585f46a 100644
--- a/src/fdb5/toc/TocStats.cc
+++ b/src/fdb5/toc/TocStats.cc
@@ -258,19 +258,19 @@ void TocStatsReportVisitor::visitDatum(const Field& field, const std::string& fi
     const eckit::PathName& indexPath = currentIndex_->location().uri().path();
 
     if (dataPath != lastDataPath_) {
-
-        if (allDataFiles_.find(dataPath) == allDataFiles_.end()) {
-
-            if (dataPath.dirName().sameAs(directory_)) {
-                dbStats->ownedFilesSize_ += dataPath.size();
-                dbStats->ownedFilesCount_++;
-            } else {
-                dbStats->adoptedFilesSize_ += dataPath.size();
-                dbStats->adoptedFilesCount_++;
+        if (dataPath.exists()) {
+            if (allDataFiles_.find(dataPath) == allDataFiles_.end()) {
+
+                if (dataPath.dirName().sameAs(directory_)) {
+                    dbStats->ownedFilesSize_ += dataPath.size();
+                    dbStats->ownedFilesCount_++;
+                } else {
+                    dbStats->adoptedFilesSize_ += dataPath.size();
+                    dbStats->adoptedFilesCount_++;
+                }
+                allDataFiles_.insert(dataPath);
             }
-            allDataFiles_.insert(dataPath);
         }
-
         lastDataPath_ = dataPath;
     }
 

From ac9d98f282dcacf148440f39bb3eb1be058df7a4 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 1 May 2024 17:00:43 +0100
Subject: [PATCH 107/186] FDB remote - fix race condition blocking requests

---
 VERSION                                    |  2 +-
 src/fdb5/remote/client/Client.cc           |  8 ++------
 src/fdb5/remote/client/ClientConnection.cc |  8 ++++++--
 src/fdb5/remote/client/ClientConnection.h  |  2 +-
 src/fdb5/toc/TocEngine.cc                  | 11 +++++------
 5 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/VERSION b/VERSION
index f4f8ed553..d936cf640 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.118
\ No newline at end of file
+5.11.119
diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index b681700c4..d7e975305 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -55,9 +55,8 @@ void Client::controlWriteCheckResponse(Message msg, uint32_t requestID, bool dat
     if (payloadLength) {
         data.push_back(std::make_pair(payload, payloadLength));
     }
-    std::future f = connection_.controlWrite(*this, msg, requestID, dataListener, data);
 
-    eckit::Buffer buf = f.get();
+    eckit::Buffer buf = connection_.controlWrite(*this, msg, requestID, dataListener, data);
     ASSERT(buf.size() == 0);
 }
 
@@ -71,10 +70,7 @@ eckit::Buffer Client::controlWriteReadResponse(Message msg, uint32_t requestID,
     if (payloadLength) {
         data.push_back(std::make_pair(payload, payloadLength));
     }
-    std::future f = connection_.controlWrite(*this, msg, requestID, false, data);
-
-    eckit::Buffer buf = f.get();
-    return buf;
+    return connection_.controlWrite(*this, msg, requestID, false, data);
 }
 
 void Client::dataWrite(remote::Message msg, uint32_t requestID, std::vector> data) {
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index 44f49ce59..20395f145 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -191,14 +191,16 @@ eckit::LocalConfiguration ClientConnection::availableFunctionality() const {
 
 // -----------------------------------------------------------------------------------------------------
 
-std::future ClientConnection::controlWrite(Client& client, Message msg, uint32_t requestID, bool dataListener, std::vector> data) {
+eckit::Buffer ClientConnection::controlWrite(Client& client, Message msg, uint32_t requestID, bool dataListener, std::vector> data) {
     auto it = clients_.find(client.clientId());
     ASSERT(it != clients_.end());
 
     auto pp = promises_.emplace(requestID, std::promise{});
+    std::future f = pp.first->second.get_future();
+
     Connection::write(msg, true, client.clientId(), requestID, data);
 
-    return pp.first->second.get_future();
+    return f.get();
 }
 
 void ClientConnection::dataWrite(DataWriteRequest& r) {
@@ -419,6 +421,8 @@ void ClientConnection::listeningDataThreadLoop() {
 
     try {
 
+        LOG_DEBUG_LIB(LibFdb5) << "ClientConnection::listeningDataThreadLoop started" << std::endl;
+
         MessageHeader hdr;
 
         while (true) {
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index 6252ff01f..37973d983 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -38,7 +38,7 @@ class ClientConnection : protected Connection {
 
     virtual ~ClientConnection();
 
-    std::future controlWrite(Client& client, Message msg, uint32_t requestID, bool startDataListener, std::vector> data={});
+    eckit::Buffer controlWrite(Client& client, Message msg, uint32_t requestID, bool startDataListener, std::vector> data={});
     void dataWrite(Client& client, Message msg, uint32_t requestID, std::vector> data={});
 
     void add(Client& client);
diff --git a/src/fdb5/toc/TocEngine.cc b/src/fdb5/toc/TocEngine.cc
index 99be2c22d..f238ec6a0 100644
--- a/src/fdb5/toc/TocEngine.cc
+++ b/src/fdb5/toc/TocEngine.cc
@@ -14,6 +14,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include "eckit/eckit.h"
 
@@ -24,7 +25,6 @@
 #include "eckit/log/Log.h"
 #include "eckit/os/BackTrace.h"
 #include "eckit/os/Stat.h"
-#include "eckit/utils/Regex.h"
 #include "eckit/utils/StringTools.h"
 
 #include "fdb5/LibFdb5.h"
@@ -148,8 +148,6 @@ std::set TocEngine::databases(const std::set& ke
                                                const std::vector& roots,
                                                const Config& config) const {
 
-    static bool searchCaseSensitiveDB = eckit::Resource("fdbSearchCaseSensitiveDB;$FDB_SEARCH_CASESENSITIVE_DB", true);
-
     std::set result;
 
     for (std::vector::const_iterator j = roots.begin(); j != roots.end(); ++j) {
@@ -166,12 +164,13 @@ std::set TocEngine::databases(const std::set& ke
             for(std::vector::const_iterator dbpath = dbpaths.begin(); dbpath != dbpaths.end(); ++dbpath) {
 
                 std::string regex = "^" + *j + "/" + *dbpath + "$";
-                Regex re(searchCaseSensitiveDB ? eckit::StringTools::lower(regex) : regex);
+                std::regex reg(regex, std::regex_constants::syntax_option_type::icase | std::regex_constants::syntax_option_type::optimize);
 
                 LOG_DEBUG_LIB(LibFdb5) << " -> key i " << *i
                                      << " dbpath " << *dbpath
-                                     << " pathregex " << re << std::endl;
+                                     << " pathregex " << regex << std::endl;
 
+                std::smatch m;
                 for (std::list::const_iterator k = dbs.begin(); k != dbs.end(); ++k) {
 
                     LOG_DEBUG_LIB(LibFdb5) << "    -> db " << *k << std::endl;
@@ -180,7 +179,7 @@ std::set TocEngine::databases(const std::set& ke
                         continue;
                     }
 
-                    if (re.match(searchCaseSensitiveDB ? eckit::StringTools::lower(*k) : *k)) {
+                    if (std::regex_match(*k, m, reg)) {
                         result.insert(*k);
                     }
                 }

From b3e6111c62ebf0d0f288f99ace74ffd9a65f8e1b Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 1 May 2024 18:33:56 +0100
Subject: [PATCH 108/186] fix regex options

---
 VERSION                   | 2 +-
 src/fdb5/toc/TocEngine.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/VERSION b/VERSION
index d936cf640..e28590c3e 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.119
+5.11.120
\ No newline at end of file
diff --git a/src/fdb5/toc/TocEngine.cc b/src/fdb5/toc/TocEngine.cc
index f238ec6a0..34cc14a22 100644
--- a/src/fdb5/toc/TocEngine.cc
+++ b/src/fdb5/toc/TocEngine.cc
@@ -164,7 +164,7 @@ std::set TocEngine::databases(const std::set& ke
             for(std::vector::const_iterator dbpath = dbpaths.begin(); dbpath != dbpaths.end(); ++dbpath) {
 
                 std::string regex = "^" + *j + "/" + *dbpath + "$";
-                std::regex reg(regex, std::regex_constants::syntax_option_type::icase | std::regex_constants::syntax_option_type::optimize);
+                std::regex reg(regex, std::regex::icase | std::regex::optimize);
 
                 LOG_DEBUG_LIB(LibFdb5) << " -> key i " << *i
                                      << " dbpath " << *dbpath

From 395596ee31649e0db439603701c5df61fb7ead1a Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 2 May 2024 14:23:40 +0100
Subject: [PATCH 109/186] sync action + connection termination

---
 .github/workflows/sync.yml                 | 26 ++++++++++++++++++++++
 src/fdb5/remote/client/Client.cc           |  1 -
 src/fdb5/remote/client/Client.h            |  2 --
 src/fdb5/remote/client/ClientConnection.cc | 16 +++++++------
 src/fdb5/remote/client/ClientConnection.h  |  4 +++-
 5 files changed, 38 insertions(+), 11 deletions(-)
 create mode 100644 .github/workflows/sync.yml

diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml
new file mode 100644
index 000000000..50b8478ee
--- /dev/null
+++ b/.github/workflows/sync.yml
@@ -0,0 +1,26 @@
+name: sync
+ 
+# Controls when the workflow will run
+on:
+ 
+  # Trigger the workflow on all pushes
+  push:
+    branches:
+    - '**'
+    tags:
+    - '**'
+ 
+  # Trigger the workflow when a branch or tag is deleted
+  delete: ~
+ 
+jobs:
+ 
+  # Calls a reusable CI workflow to sync the current with a remote repository.
+  #   It will correctly handle addition of any new and removal of existing Git objects.
+  sync:
+    name: sync
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/sync.yml@v2
+    secrets:
+      target_repository: mars/fdb5
+      target_username: ClonedDuck
+      target_token: ${{ secrets.BITBUCKET_PAT }}
diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index d7e975305..a89e26180 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -8,7 +8,6 @@
  * does it submit to any jurisdiction.
  */
 
-#include 
 
 #include "fdb5/LibFdb5.h"
 
diff --git a/src/fdb5/remote/client/Client.h b/src/fdb5/remote/client/Client.h
index 680d504a2..0937720e0 100644
--- a/src/fdb5/remote/client/Client.h
+++ b/src/fdb5/remote/client/Client.h
@@ -10,8 +10,6 @@
 
 #pragma once
 
-#include 
-
 #include "eckit/config/Configuration.h"
 #include "eckit/memory/NonCopyable.h"
 #include "eckit/net/Endpoint.h"
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index 20395f145..cc56955f0 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -58,7 +58,7 @@ class DataWriteRequest {
 
 
 ClientConnection::ClientConnection(const eckit::net::Endpoint& controlEndpoint, const std::string& defaultEndpoint):
-    controlEndpoint_(controlEndpoint), defaultEndpoint_(defaultEndpoint), id_(1), connected_(false), dataWriteQueue_(nullptr) {
+    controlEndpoint_(controlEndpoint), defaultEndpoint_(defaultEndpoint), id_(1), connected_(false), controlStopping_(false), dataStopping_(false), dataWriteQueue_(nullptr) {
 
     LOG_DEBUG_LIB(LibFdb5) << "ClientConnection::ClientConnection() controlEndpoint: " << controlEndpoint << std::endl;
 }
@@ -85,8 +85,9 @@ bool ClientConnection::remove(uint32_t clientID) {
     }
 
     if (clients_.empty()) {
-        Connection::write(Message::Exit, true, 0, 0);
-        if (!single_) {
+        if (!controlStopping_)
+            Connection::write(Message::Exit, true, 0, 0);
+        if (!single_ && !dataStopping_) {
             // TODO make the data connection dying automatically, when there are no more async writes
             Connection::write(Message::Exit, false, 0, 0);
         }
@@ -143,13 +144,12 @@ bool ClientConnection::connect(bool singleAttempt) {
         listeningControlThread_ = std::thread([this] { listeningControlThreadLoop(); });
 
         connected_ = true;
-        return true;
     } catch(eckit::TooManyRetries& e) {
         if (controlClient_.isConnected()) {
             controlClient_.close();
         }
     }
-    return false;
+    return connected_;
 }
 
 void ClientConnection::disconnect() {
@@ -191,7 +191,7 @@ eckit::LocalConfiguration ClientConnection::availableFunctionality() const {
 
 // -----------------------------------------------------------------------------------------------------
 
-eckit::Buffer ClientConnection::controlWrite(Client& client, Message msg, uint32_t requestID, bool dataListener, std::vector> data) {
+eckit::Buffer&& ClientConnection::controlWrite(Client& client, Message msg, uint32_t requestID, bool dataListener, std::vector> data) {
     auto it = clients_.find(client.clientId());
     ASSERT(it != clients_.end());
 
@@ -200,7 +200,7 @@ eckit::Buffer ClientConnection::controlWrite(Client& client, Message msg, uint32
 
     Connection::write(msg, true, client.clientId(), requestID, data);
 
-    return f.get();
+    return std::move(f.get());
 }
 
 void ClientConnection::dataWrite(DataWriteRequest& r) {
@@ -362,6 +362,7 @@ void ClientConnection::listeningControlThreadLoop() {
             LOG_DEBUG_LIB(LibFdb5) << "ClientConnection::listeningControlThreadLoop - got [message=" << hdr.message << ",clientID=" << hdr.clientID() << ",control=" << hdr.control() << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
 
             if (hdr.message == Message::Exit) {
+                controlStopping_ = true;
                 return;
             } else {
                 if (hdr.clientID()) {
@@ -432,6 +433,7 @@ void ClientConnection::listeningDataThreadLoop() {
             LOG_DEBUG_LIB(LibFdb5) << "ClientConnection::listeningDataThreadLoop - got [message=" << hdr.message << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
 
             if (hdr.message == Message::Exit) {
+                dataStopping_ = true;
                 return;
             } else {
                 if (hdr.clientID()) {
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index 37973d983..86650a9d6 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -38,7 +38,7 @@ class ClientConnection : protected Connection {
 
     virtual ~ClientConnection();
 
-    eckit::Buffer controlWrite(Client& client, Message msg, uint32_t requestID, bool startDataListener, std::vector> data={});
+    eckit::Buffer&& controlWrite(Client& client, Message msg, uint32_t requestID, bool startDataListener, std::vector> data={});
     void dataWrite(Client& client, Message msg, uint32_t requestID, std::vector> data={});
 
     void add(Client& client);
@@ -101,6 +101,8 @@ class ClientConnection : protected Connection {
     uint32_t id_;
 
     bool connected_; 
+    bool controlStopping_; 
+    bool dataStopping_; 
 
     std::map> promises_;
 

From 4b6fe5c017e64fdf3c0813f9a2466598bd29d68a Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 4 Jun 2024 14:20:19 +0200
Subject: [PATCH 110/186] fix FDBRemoteStore read to cope with optimised
 MultiHandle read

---
 src/fdb5/api/RemoteFDB.cc                  |  6 +--
 src/fdb5/remote/client/Client.cc           |  4 +-
 src/fdb5/remote/client/Client.h            |  2 +-
 src/fdb5/remote/client/ClientConnection.cc | 12 +++--
 src/fdb5/remote/client/ClientConnection.h  |  2 +-
 src/fdb5/remote/client/RemoteCatalogue.cc  |  2 +-
 src/fdb5/remote/client/RemoteStore.cc      | 58 ++++++++++++++--------
 7 files changed, 52 insertions(+), 34 deletions(-)

diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index fa9aa2cf9..17fb36620 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -116,7 +116,7 @@ RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name
     Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port")), "") {
 
     uint32_t id = generateRequestID();
-    eckit::Buffer buf = controlWriteReadResponse(remote::Message::Stores, id);
+    eckit::Buffer buf = controlWriteReadResponse(remote::Message::Stores, id).get();
     eckit::MemoryStream s(buf);
     size_t numStores;
     s >> numStores;
@@ -164,8 +164,8 @@ RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name
     }
 
     id = generateRequestID();
-    buf = controlWriteReadResponse(remote::Message::Schema, id);
-    eckit::MemoryStream s2(buf);
+    eckit::Buffer buf2 = controlWriteReadResponse(remote::Message::Schema, id).get();
+    eckit::MemoryStream s2(buf2);
 
     fdb5::Schema* schema = eckit::Reanimator::reanimate(s2);
 
diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index a89e26180..31582f047 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -55,11 +55,11 @@ void Client::controlWriteCheckResponse(Message msg, uint32_t requestID, bool dat
         data.push_back(std::make_pair(payload, payloadLength));
     }
 
-    eckit::Buffer buf = connection_.controlWrite(*this, msg, requestID, dataListener, data);
+    eckit::Buffer buf = connection_.controlWrite(*this, msg, requestID, dataListener, data).get();
     ASSERT(buf.size() == 0);
 }
 
-eckit::Buffer Client::controlWriteReadResponse(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
+std::future Client::controlWriteReadResponse(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
 
     ASSERT(requestID);
     ASSERT(!(!payloadLength ^ !payload));
diff --git a/src/fdb5/remote/client/Client.h b/src/fdb5/remote/client/Client.h
index 0937720e0..a41791050 100644
--- a/src/fdb5/remote/client/Client.h
+++ b/src/fdb5/remote/client/Client.h
@@ -45,7 +45,7 @@ class Client : eckit::NonCopyable {
 
     // blocking requests
     void controlWriteCheckResponse(Message msg, uint32_t requestID, bool dataListener, const void* payload=nullptr, uint32_t payloadLength=0);
-    eckit::Buffer controlWriteReadResponse (Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
+    std::future controlWriteReadResponse (Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
 
     void dataWrite(remote::Message msg, uint32_t requestID, std::vector> data={});
     
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index cc56955f0..fe1aa9c59 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -85,9 +85,11 @@ bool ClientConnection::remove(uint32_t clientID) {
     }
 
     if (clients_.empty()) {
-        if (!controlStopping_)
-            Connection::write(Message::Exit, true, 0, 0);
-        if (!single_ && !dataStopping_) {
+        Connection::write(Message::Exit, true, 0, 0);
+        if (!single_) {
+        // if (!controlStopping_)
+        //     Connection::write(Message::Exit, true, 0, 0);
+        // if (!single_ && !dataStopping_) {
             // TODO make the data connection dying automatically, when there are no more async writes
             Connection::write(Message::Exit, false, 0, 0);
         }
@@ -191,7 +193,7 @@ eckit::LocalConfiguration ClientConnection::availableFunctionality() const {
 
 // -----------------------------------------------------------------------------------------------------
 
-eckit::Buffer&& ClientConnection::controlWrite(Client& client, Message msg, uint32_t requestID, bool dataListener, std::vector> data) {
+std::future ClientConnection::controlWrite(Client& client, Message msg, uint32_t requestID, bool dataListener, std::vector> data) {
     auto it = clients_.find(client.clientId());
     ASSERT(it != clients_.end());
 
@@ -200,7 +202,7 @@ eckit::Buffer&& ClientConnection::controlWrite(Client& client, Message msg, uint
 
     Connection::write(msg, true, client.clientId(), requestID, data);
 
-    return std::move(f.get());
+    return f;
 }
 
 void ClientConnection::dataWrite(DataWriteRequest& r) {
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index 86650a9d6..884edadbb 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -38,7 +38,7 @@ class ClientConnection : protected Connection {
 
     virtual ~ClientConnection();
 
-    eckit::Buffer&& controlWrite(Client& client, Message msg, uint32_t requestID, bool startDataListener, std::vector> data={});
+    std::future controlWrite(Client& client, Message msg, uint32_t requestID, bool startDataListener, std::vector> data={});
     void dataWrite(Client& client, Message msg, uint32_t requestID, std::vector> data={});
 
     void add(Client& client);
diff --git a/src/fdb5/remote/client/RemoteCatalogue.cc b/src/fdb5/remote/client/RemoteCatalogue.cc
index a14b8a16e..3a528f982 100644
--- a/src/fdb5/remote/client/RemoteCatalogue.cc
+++ b/src/fdb5/remote/client/RemoteCatalogue.cc
@@ -154,7 +154,7 @@ void RemoteCatalogue::loadSchema() {
         keyStream << dbKey_;
         
         uint32_t id = generateRequestID();
-        eckit::Buffer buf = controlWriteReadResponse(Message::Schema, id, keyBuffer, keyStream.position());
+        eckit::Buffer buf = controlWriteReadResponse(Message::Schema, id, keyBuffer, keyStream.position()).get();
 
         eckit::MemoryStream s(buf);
         schema_.reset(eckit::Reanimator::reanimate(s));
diff --git a/src/fdb5/remote/client/RemoteStore.cc b/src/fdb5/remote/client/RemoteStore.cc
index 8fbe5d844..421d69821 100644
--- a/src/fdb5/remote/client/RemoteStore.cc
+++ b/src/fdb5/remote/client/RemoteStore.cc
@@ -84,34 +84,50 @@ class FDBRemoteDataHandle : public DataHandle {
 
         if (complete_) return 0;
 
-        if (currentBuffer_.size() != 0) return bufferRead(pos, sz);
+        long total = 0;
+        long n;
+        char* p    = static_cast(pos);
+        if (currentBuffer_.size() != 0) {
+            n = bufferRead(pos, sz);
+            sz -= n;
+            total += n;
+            p+=n;
+        }
 
-        // If we are in the DataHandle, then there MUST be data to read
-        RemoteStore::StoredMessage msg = std::make_pair(remote::Message{}, eckit::Buffer{0});
-        ASSERT(queue_.pop(msg) != -1);
+        while (sz > 0 && !complete_) {
 
-        // Handle any remote errors communicated from the server
-        if (msg.first == Message::Error) {
-            std::string errmsg(static_cast(msg.second.data()), msg.second.size());
-            throw RemoteFDBException(errmsg, remoteEndpoint_);
-        }
+            // If we are in the DataHandle, then there MUST be data to read
+            RemoteStore::StoredMessage msg = std::make_pair(remote::Message{}, eckit::Buffer{0});
+            ASSERT(queue_.pop(msg) != -1);
 
-        // Are we now complete?
-        if (msg.first == Message::Complete) {
-            if (overallPosition_ == eckit::Offset(0)) {
-                ASSERT(queue_.pop(msg) != -1);
-            } else {
-                complete_ = true;
-                return 0;
+            // Handle any remote errors communicated from the server
+            if (msg.first == Message::Error) {
+                std::string errmsg(static_cast(msg.second.data()), msg.second.size());
+                throw RemoteFDBException(errmsg, remoteEndpoint_);
             }
-        }
 
-        ASSERT(msg.first == Message::Blob);
+            // Are we now complete?
+            if (msg.first == Message::Complete) {
+                if (overallPosition_ == eckit::Offset(0)) {
+                    ASSERT(queue_.pop(msg) != -1);
+                } else {
+                    complete_ = true;
+                    return total;
+                }
+            }
 
-        // Otherwise return the data!
-        std::swap(currentBuffer_, msg.second);
+            ASSERT(msg.first == Message::Blob);
 
-        return bufferRead(pos, sz);
+            // Otherwise return the data!
+            std::swap(currentBuffer_, msg.second);
+            
+            n = bufferRead(p, sz);
+            sz -= n;
+            total += n;
+            p+=n;
+
+        }
+        return total;
     }
 
     // A helper function that returns some, or all, of a buffer that has

From f1bdbecee607668812214d7bfead7e1119ea5a6e Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 4 Jun 2024 14:42:33 +0200
Subject: [PATCH 111/186] version bump

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index e28590c3e..8d5602eca 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.120
\ No newline at end of file
+5.11.121

From 12ff64a82440adecc8711fdbaead19b48b3fb412 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 1 Jul 2024 05:53:28 +0100
Subject: [PATCH 112/186] fixed dataWriteQueue initialisation

---
 src/fdb5/api/RemoteFDB.cc                  |  8 +++-----
 src/fdb5/remote/client/Client.cc           |  4 ++--
 src/fdb5/remote/client/Client.h            |  2 +-
 src/fdb5/remote/client/ClientConnection.cc | 22 +++++++++++++---------
 src/fdb5/remote/client/RemoteCatalogue.cc  | 10 ++++------
 src/fdb5/remote/client/RemoteStore.cc      | 15 +++++++--------
 6 files changed, 30 insertions(+), 31 deletions(-)

diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index 17fb36620..5e34f7314 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -115,8 +115,7 @@ RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name
     LocalFDB(config, name),
     Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port")), "") {
 
-    uint32_t id = generateRequestID();
-    eckit::Buffer buf = controlWriteReadResponse(remote::Message::Stores, id).get();
+    eckit::Buffer buf = controlWriteReadResponse(remote::Message::Stores, generateRequestID());
     eckit::MemoryStream s(buf);
     size_t numStores;
     s >> numStores;
@@ -163,8 +162,7 @@ RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name
         fieldLocationEndpoints.push_back("");
     }
 
-    id = generateRequestID();
-    eckit::Buffer buf2 = controlWriteReadResponse(remote::Message::Schema, id).get();
+    eckit::Buffer buf2 = controlWriteReadResponse(remote::Message::Schema, generateRequestID());
     eckit::MemoryStream s2(buf2);
 
     fdb5::Schema* schema = eckit::Reanimator::reanimate(s2);
@@ -193,7 +191,7 @@ auto RemoteFDB::forwardApiCall(const HelperClass& helper, const FDBToolRequest&
     // Ensure we have an entry in the message queue before we trigger anything that
     // will result in return messages
 
-    uint32_t id = connection_.generateRequestID();
+    uint32_t id = generateRequestID();
     auto entry = messageQueues_.emplace(id, std::make_shared(HelperClass::queueSize()));
     ASSERT(entry.second);
     std::shared_ptr messageQueue(entry.first->second);
diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index 31582f047..44386a750 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -59,7 +59,7 @@ void Client::controlWriteCheckResponse(Message msg, uint32_t requestID, bool dat
     ASSERT(buf.size() == 0);
 }
 
-std::future Client::controlWriteReadResponse(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
+eckit::Buffer Client::controlWriteReadResponse(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
 
     ASSERT(requestID);
     ASSERT(!(!payloadLength ^ !payload));
@@ -69,7 +69,7 @@ std::future Client::controlWriteReadResponse(Message msg, uint32_
     if (payloadLength) {
         data.push_back(std::make_pair(payload, payloadLength));
     }
-    return connection_.controlWrite(*this, msg, requestID, false, data);
+    return eckit::Buffer{std::move(connection_.controlWrite(*this, msg, requestID, false, data).get())};
 }
 
 void Client::dataWrite(remote::Message msg, uint32_t requestID, std::vector> data) {
diff --git a/src/fdb5/remote/client/Client.h b/src/fdb5/remote/client/Client.h
index a41791050..0937720e0 100644
--- a/src/fdb5/remote/client/Client.h
+++ b/src/fdb5/remote/client/Client.h
@@ -45,7 +45,7 @@ class Client : eckit::NonCopyable {
 
     // blocking requests
     void controlWriteCheckResponse(Message msg, uint32_t requestID, bool dataListener, const void* payload=nullptr, uint32_t payloadLength=0);
-    std::future controlWriteReadResponse (Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
+    eckit::Buffer controlWriteReadResponse (Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0);
 
     void dataWrite(remote::Message msg, uint32_t requestID, std::vector> data={});
     
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index fe1aa9c59..cf78abdb3 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -215,19 +215,23 @@ void ClientConnection::dataWrite(Client& client, remote::Message msg, uint32_t r
     auto it = clients_.find(client.clientId());
     ASSERT(it != clients_.end());
 
-    if (!dataWriteFuture_.valid()) {
+    {
+        std::lock_guard lock(dataWriteQueueMutex_);
 
-        {
-            // Reset the queue after previous done/errors
-            std::lock_guard lock(dataWriteQueueMutex_);
-            ASSERT(!dataWriteQueue_);
+        if (!dataWriteFuture_.valid()) {
 
-            dataWriteQueue_.reset(new eckit::Queue{maxQueueLength});
-        }
+            {
+                // Reset the queue after previous done/errors
+                // std::lock_guard lock(dataWriteQueueMutex_);
+                // TODO
+                ASSERT(!dataWriteQueue_);
 
-        dataWriteFuture_ = std::async(std::launch::async, [this] { return dataWriteThreadLoop(); });
-    }
+                dataWriteQueue_.reset(new eckit::Queue{maxQueueLength});
+            }
 
+            dataWriteFuture_ = std::async(std::launch::async, [this] { return dataWriteThreadLoop(); });
+        }
+    }
     uint32_t payloadLength = 0;
     for (auto d: data) {
         ASSERT(d.first);
diff --git a/src/fdb5/remote/client/RemoteCatalogue.cc b/src/fdb5/remote/client/RemoteCatalogue.cc
index 3a528f982..65fb525e2 100644
--- a/src/fdb5/remote/client/RemoteCatalogue.cc
+++ b/src/fdb5/remote/client/RemoteCatalogue.cc
@@ -63,7 +63,7 @@ void RemoteCatalogue::archive(const Key& idxKey, const InspectionKey& key, std::
     ASSERT(!key.empty());
     ASSERT(fieldLocation);
 
-    uint32_t id = connection_.generateRequestID();
+    uint32_t id = generateRequestID();
     {
         std::lock_guard lock(archiveMutex_);
         if (numLocations_ == 0) { // if this is the first archival request, notify the server
@@ -122,8 +122,7 @@ void RemoteCatalogue::flush(size_t archivedFields) {
         LOG_DEBUG_LIB(LibFdb5) << " RemoteCatalogue::flush - flushing " << numLocations_ << " fields" << std::endl;
 
         // The flush call is blocking
-        uint32_t id = generateRequestID();
-        controlWriteCheckResponse(Message::Flush, id, false, sendBuf, s.position());
+        controlWriteCheckResponse(Message::Flush, generateRequestID(), false, sendBuf, s.position());
 
         numLocations_ = 0;
     }
@@ -153,8 +152,7 @@ void RemoteCatalogue::loadSchema() {
         eckit::MemoryStream keyStream(keyBuffer);
         keyStream << dbKey_;
         
-        uint32_t id = generateRequestID();
-        eckit::Buffer buf = controlWriteReadResponse(Message::Schema, id, keyBuffer, keyStream.position()).get();
+        eckit::Buffer buf = controlWriteReadResponse(Message::Schema, generateRequestID(), keyBuffer, keyStream.position());
 
         eckit::MemoryStream s(buf);
         schema_.reset(eckit::Reanimator::reanimate(s));
@@ -162,7 +160,7 @@ void RemoteCatalogue::loadSchema() {
 }
 
 bool RemoteCatalogue::handle(Message message, bool control, uint32_t requestID) {
-    LOG_DEBUG_LIB(LibFdb5) << *this << " - Received [message=" << ((uint) message) << ",requestID=" << requestID << "]" << std::endl;
+    Log::warning() << *this << " - Received [message=" << ((uint) message) << ",requestID=" << requestID << "]" << std::endl;
     NOTIMP;
     return false;
 }
diff --git a/src/fdb5/remote/client/RemoteStore.cc b/src/fdb5/remote/client/RemoteStore.cc
index 421d69821..2c27e6e62 100644
--- a/src/fdb5/remote/client/RemoteStore.cc
+++ b/src/fdb5/remote/client/RemoteStore.cc
@@ -239,8 +239,8 @@ void RemoteStore::archive(const Key& key, const void *data, eckit::Length length
     ASSERT(data);
     ASSERT(length != 0);
 
-    uint32_t id = connection_.generateRequestID();
-    {
+    uint32_t id = generateRequestID();
+    {   // send the archival request
         std::lock_guard lock(archiveMutex_);
         if (fieldsArchived_ == 0) { // if this is the first archival request, notify the server
             ASSERT(locations_.size() == 0);
@@ -248,12 +248,12 @@ void RemoteStore::archive(const Key& key, const void *data, eckit::Length length
             controlWriteCheckResponse(Message::Store, id, true);
         }
     }
-    fieldsArchived_++;
-
-    {
+    {   // store the callback, associated with the request id - to be done BEFORE sending the data
         std::lock_guard lock(locationMutex_);
         locations_[id] = catalogue_archive;
     }
+    fieldsArchived_++;
+
 
     Buffer keyBuffer(4096);
     MemoryStream keyStream(keyBuffer);
@@ -310,8 +310,7 @@ size_t RemoteStore::flush() {
 
         LOG_DEBUG_LIB(LibFdb5) << " RemoteStore::flush - flushing " << locations << " fields" << std::endl;
         // The flush call is blocking
-        uint32_t id = generateRequestID();
-        controlWriteCheckResponse(Message::Flush, id, false, sendBuf, s.position());
+        controlWriteCheckResponse(Message::Flush, generateRequestID(), false, sendBuf, s.position());
     }
 
     fieldsArchived_ = 0;
@@ -450,7 +449,7 @@ eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation, c
     s << fieldLocation;
     s << remapKey;
 
-    uint32_t id = connection_.generateRequestID();
+    uint32_t id = generateRequestID();
     controlWriteCheckResponse(fdb5::remote::Message::Read, id, true, encodeBuffer, s.position());
 
     return new FDBRemoteDataHandle(id, fieldLocation.length(), retrieveMessageQueue_, controlEndpoint());

From cf314bb2df5b25239f091ab58beaa1d3b1afbd43 Mon Sep 17 00:00:00 2001
From: Chris Bradley 
Date: Mon, 1 Jul 2024 11:01:15 +0100
Subject: [PATCH 113/186] Allow RemoteFieldLocation construction from uri

---
 src/fdb5/remote/RemoteFieldLocation.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/fdb5/remote/RemoteFieldLocation.cc b/src/fdb5/remote/RemoteFieldLocation.cc
index 348cba9b2..a8f4690a6 100644
--- a/src/fdb5/remote/RemoteFieldLocation.cc
+++ b/src/fdb5/remote/RemoteFieldLocation.cc
@@ -54,8 +54,9 @@ RemoteFieldLocation::RemoteFieldLocation(const eckit::net::Endpoint& endpoint, c
         remoteLocation.remapKey()) {}
 
 RemoteFieldLocation::RemoteFieldLocation(const eckit::URI& uri) :
-    FieldLocation(eckit::URI("fdb", uri)) {
-    NOTIMP;
+    FieldLocation(uri) {
+
+    ASSERT(uri.scheme() == "fdb");
 }
 
 RemoteFieldLocation::RemoteFieldLocation(const eckit::URI& uri, const eckit::Offset& offset, const eckit::Length& length, const Key& remapKey) :

From ecf4d3393b80201b4145ead2fa51b760b3b76fa3 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 1 Jul 2024 11:02:24 +0100
Subject: [PATCH 114/186] protected clients_ and promises_ with mutexes

---
 src/fdb5/remote/client/ClientConnection.cc | 75 ++++++++++++++--------
 src/fdb5/remote/client/ClientConnection.h  |  1 +
 2 files changed, 49 insertions(+), 27 deletions(-)

diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index cf78abdb3..63ce0e753 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -72,9 +72,11 @@ void ClientConnection::add(Client& client) {
 }
 
 bool ClientConnection::remove(uint32_t clientID) {
+
+    std::lock_guard lock(clientsMutex_);
+
     if (clientID > 0) {
 
-        //std::lock_guard lock(clientsMutex_);
         auto it = clients_.find(clientID);
 
         if (it != clients_.end()) {
@@ -194,12 +196,18 @@ eckit::LocalConfiguration ClientConnection::availableFunctionality() const {
 // -----------------------------------------------------------------------------------------------------
 
 std::future ClientConnection::controlWrite(Client& client, Message msg, uint32_t requestID, bool dataListener, std::vector> data) {
+
+    std::lock_guard lock(clientsMutex_);
+
     auto it = clients_.find(client.clientId());
     ASSERT(it != clients_.end());
 
-    auto pp = promises_.emplace(requestID, std::promise{});
-    std::future f = pp.first->second.get_future();
-
+    std::future f;
+    {
+        std::lock_guard lock(promisesMutex_);
+        auto pp = promises_.emplace(requestID, std::promise{}).first;
+        f = pp->second.get_future();
+    }
     Connection::write(msg, true, client.clientId(), requestID, data);
 
     return f;
@@ -373,20 +381,28 @@ void ClientConnection::listeningControlThreadLoop() {
             } else {
                 if (hdr.clientID()) {
                     bool handled = false;
-                    auto it = clients_.find(hdr.clientID());
-                    if (it == clients_.end()) {
-                        std::stringstream ss;
-                        ss << "ERROR: Received [clientID="<< hdr.clientID() << ",requestID="<< hdr.requestID << ",message=" << hdr.message << ",payload=" << hdr.payloadSize << "]" << std::endl;
-                        ss << "Unexpected answer for clientID recieved (" << hdr.clientID() << "). ABORTING";
-                        eckit::Log::status() << ss.str() << std::endl;
-                        eckit::Log::error() << "Retrieving... " << ss.str() << std::endl;
-                        throw eckit::SeriousBug(ss.str(), Here());
+                    Client* client = nullptr;
+                    {
+                        std::lock_guard lock(clientsMutex_);
+
+                        auto it = clients_.find(hdr.clientID());
+                        if (it == clients_.end()) {
+                            std::stringstream ss;
+                            ss << "ERROR: Received [clientID="<< hdr.clientID() << ",requestID="<< hdr.requestID << ",message=" << hdr.message << ",payload=" << hdr.payloadSize << "]" << std::endl;
+                            ss << "Unexpected answer for clientID recieved (" << hdr.clientID() << "). ABORTING";
+                            eckit::Log::status() << ss.str() << std::endl;
+                            eckit::Log::error() << "Retrieving... " << ss.str() << std::endl;
+                            throw eckit::SeriousBug(ss.str(), Here());
+                        }
+                        client = it->second;
                     }
 
+                    ASSERT(client);
                     ASSERT(hdr.control() || single_);
 
                     auto pp = promises_.find(hdr.requestID);
                     if (pp != promises_.end()) {
+                        std::lock_guard lock(promisesMutex_);
                         if (hdr.payloadSize == 0) {
                             ASSERT(hdr.message == Message::Received);
                             pp->second.set_value(eckit::Buffer(0));
@@ -397,16 +413,16 @@ void ClientConnection::listeningControlThreadLoop() {
                         handled = true;
                     } else {
                         if (hdr.payloadSize == 0) {
-                            handled = it->second->handle(hdr.message, hdr.control(), hdr.requestID);
+                            handled = client->handle(hdr.message, hdr.control(), hdr.requestID);
                         }
                         else {
-                            handled = it->second->handle(hdr.message, hdr.control(), hdr.requestID, std::move(payload));
+                            handled = client->handle(hdr.message, hdr.control(), hdr.requestID, std::move(payload));
                         }
                     }
 
                     if (!handled) {
                         std::stringstream ss;
-                        ss << "ERROR: Unexpected message recieved (" << hdr.message << "). ABORTING";
+                        ss << "ERROR: Unexpected message recieved [message=" << hdr.message << ",clientID=" << hdr.clientID() << ",requestID=" << hdr.requestID << "]. ABORTING";
                         eckit::Log::status() << ss.str() << std::endl;
                         eckit::Log::error() << "Client Retrieving... " << ss.str() << std::endl;
                         throw eckit::SeriousBug(ss.str(), Here());
@@ -444,24 +460,29 @@ void ClientConnection::listeningDataThreadLoop() {
             } else {
                 if (hdr.clientID()) {
                     bool handled = false;
-                    auto it = clients_.find(hdr.clientID());
-                    if (it == clients_.end()) {
-                        std::stringstream ss;
-                        ss << "ERROR: Received [clientID="<< hdr.clientID() << ",requestID="<< hdr.requestID << ",message=" << hdr.message << ",payload=" << hdr.payloadSize << "]" << std::endl;
-                        ss << "Unexpected answer for clientID recieved (" << hdr.clientID() << "). ABORTING";
-                        eckit::Log::status() << ss.str() << std::endl;
-                        eckit::Log::error() << "Retrieving... " << ss.str() << std::endl;
-                        throw eckit::SeriousBug(ss.str(), Here());
-
-                        ASSERT(false); // todo report the error
+                    Client* client = nullptr;
+                    {
+                        std::lock_guard lock(clientsMutex_);
+
+                        auto it = clients_.find(hdr.clientID());
+                        if (it == clients_.end()) {
+                            std::stringstream ss;
+                            ss << "ERROR: Received [clientID="<< hdr.clientID() << ",requestID="<< hdr.requestID << ",message=" << hdr.message << ",payload=" << hdr.payloadSize << "]" << std::endl;
+                            ss << "Unexpected answer for clientID recieved (" << hdr.clientID() << "). ABORTING";
+                            eckit::Log::status() << ss.str() << std::endl;
+                            eckit::Log::error() << "Retrieving... " << ss.str() << std::endl;
+                            throw eckit::SeriousBug(ss.str(), Here());
+                        }
+                        client = it->second;
                     }
 
+                    ASSERT(client);
                     ASSERT(!hdr.control());
                     if (hdr.payloadSize == 0) {
-                        handled = it->second->handle(hdr.message, hdr.control(), hdr.requestID);
+                        handled = client->handle(hdr.message, hdr.control(), hdr.requestID);
                     }
                     else {
-                        handled = it->second->handle(hdr.message, hdr.control(), hdr.requestID, std::move(payload));
+                        handled = client->handle(hdr.message, hdr.control(), hdr.requestID, std::move(payload));
                     }
 
                     if (!handled) {
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index 884edadbb..b8a772c64 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -104,6 +104,7 @@ class ClientConnection : protected Connection {
     bool controlStopping_; 
     bool dataStopping_; 
 
+    std::mutex promisesMutex_;
     std::map> promises_;
 
     std::mutex dataWriteQueueMutex_;

From 78622a18da4daee539c83011de5a8b69c4e9be93 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 1 Jul 2024 12:58:10 +0100
Subject: [PATCH 115/186] version bump

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 8d5602eca..81a17fbe4 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.121
+5.11.122
\ No newline at end of file

From f1c81150eb8c9903f63c8b87e11bd310a553cbb7 Mon Sep 17 00:00:00 2001
From: Chris Bradley 
Date: Tue, 2 Jul 2024 15:35:55 +0100
Subject: [PATCH 116/186] Remove early return

---
 src/fdb5/api/RemoteFDB.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index 5e34f7314..bd7c3e4eb 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -39,7 +39,6 @@ struct ListHelper : BaseAPIHelper() << "ListHelper::valueFromStream - original location: ";
         elem.location().dump(eckit::Log::debug());

From e2f48347e621066f1cd952d5ed7c6b2e175c592f Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 2 Jul 2024 17:07:51 +0100
Subject: [PATCH 117/186] pool of FDBs in CatalogueHandler

---
 src/fdb5/remote/server/CatalogueHandler.cc | 12 ++++++++++--
 src/fdb5/remote/server/CatalogueHandler.h  |  4 +++-
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc
index 1846a9843..95382a2ef 100644
--- a/src/fdb5/remote/server/CatalogueHandler.cc
+++ b/src/fdb5/remote/server/CatalogueHandler.cc
@@ -237,14 +237,22 @@ void CatalogueHandler::forwardApiCall(uint32_t clientID, uint32_t requestID, eck
     helper.extraDecode(s);
 
     // Construct worker thread to feed responses back to client
-
     ASSERT(workerThreads_.find(requestID) == workerThreads_.end());
 
+    {
+        std::lock_guard lock(fdbMutex_);
+        auto it = fdbs_.find(clientID);
+        if (it == fdbs_.end()) {
+            fdbs_[clientID];
+        }
+    }
+
     workerThreads_.emplace(
         requestID, std::async(std::launch::async, [request, clientID, requestID, helper, this]() {
 
             try {
-                auto iterator = helper.apiCall(fdb_, request);
+                auto it = fdbs_.find(clientID);
+                auto iterator = helper.apiCall(it->second, request);
                 typename decltype(iterator)::value_type elem;
                 while (iterator.next(elem)) {
                     auto encoded(helper.encode(elem, *this));
diff --git a/src/fdb5/remote/server/CatalogueHandler.h b/src/fdb5/remote/server/CatalogueHandler.h
index b61223239..151752ce2 100644
--- a/src/fdb5/remote/server/CatalogueHandler.h
+++ b/src/fdb5/remote/server/CatalogueHandler.h
@@ -70,8 +70,10 @@ class CatalogueHandler : public ServerConnection {
 
     // clientID --> 
     std::map catalogues_;
+    std::map fdbs_;
+
+    std::mutex fdbMutex_;
 
-    FDB fdb_;
     uint32_t fdbId_;
     bool fdbControlConnection_;
     bool fdbDataConnection_;

From f1360aed9fbbc2ef5a21e76743f2c54c6d3973ac Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 2 Jul 2024 17:10:40 +0100
Subject: [PATCH 118/186] version bump

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 81a17fbe4..76d55bb22 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.122
\ No newline at end of file
+5.11.123
\ No newline at end of file

From 049e11bd59d0418f6a8b5541796920f2eba13d23 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 3 Jul 2024 10:40:32 +0100
Subject: [PATCH 119/186] remoteFDB concurrecy

---
 VERSION                                    |  2 +-
 src/fdb5/remote/Connection.cc              |  1 +
 src/fdb5/remote/client/ClientConnection.cc | 44 ++++++-------
 src/fdb5/remote/server/CatalogueHandler.cc | 76 ++++++++++++++++------
 src/fdb5/remote/server/CatalogueHandler.h  |  3 +-
 src/fdb5/remote/server/ServerConnection.cc |  1 +
 src/fdb5/remote/server/ServerConnection.h  |  1 +
 src/fdb5/remote/server/StoreHandler.cc     | 10 ++-
 src/fdb5/remote/server/StoreHandler.h      |  1 -
 9 files changed, 90 insertions(+), 49 deletions(-)

diff --git a/VERSION b/VERSION
index 76d55bb22..fccd8bbd0 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.123
\ No newline at end of file
+5.11.124
\ No newline at end of file
diff --git a/src/fdb5/remote/Connection.cc b/src/fdb5/remote/Connection.cc
index a17661861..f3df2dffc 100644
--- a/src/fdb5/remote/Connection.cc
+++ b/src/fdb5/remote/Connection.cc
@@ -131,6 +131,7 @@ void Connection::write(remote::Message msg, bool control, uint32_t clientID, uin
 //     write(msg, false, clientID, requestID, data);
 // }
 void Connection::error(const std::string& msg, uint32_t clientID, uint32_t requestID) {
+    eckit::Log::error() << "[clientID=" << clientID << ",requestID=" << requestID << "]  " << msg << std::endl;
     write(Message::Error, false, clientID, requestID, std::vector>{{msg.c_str(), msg.length()}});
 }
 // void Connection::error(const std::string& msg, const Handler& clientID, uint32_t requestID) {
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index 63ce0e753..b5244ca40 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -65,8 +65,8 @@ ClientConnection::ClientConnection(const eckit::net::Endpoint& controlEndpoint,
 
 void ClientConnection::add(Client& client) {
     std::lock_guard lock(clientsMutex_);
-    auto it = clients_.find(client.id());
-    ASSERT(it == clients_.end());
+    // auto it = clients_.find(client.id());
+    // ASSERT(it == clients_.end());
 
     clients_[client.id()] = &client;
 }
@@ -197,10 +197,9 @@ eckit::LocalConfiguration ClientConnection::availableFunctionality() const {
 
 std::future ClientConnection::controlWrite(Client& client, Message msg, uint32_t requestID, bool dataListener, std::vector> data) {
 
-    std::lock_guard lock(clientsMutex_);
-
-    auto it = clients_.find(client.clientId());
-    ASSERT(it != clients_.end());
+    // std::lock_guard lock(clientsMutex_);
+    // auto it = clients_.find(client.clientId());
+    // ASSERT(it != clients_.end());
 
     std::future f;
     {
@@ -381,23 +380,7 @@ void ClientConnection::listeningControlThreadLoop() {
             } else {
                 if (hdr.clientID()) {
                     bool handled = false;
-                    Client* client = nullptr;
-                    {
-                        std::lock_guard lock(clientsMutex_);
 
-                        auto it = clients_.find(hdr.clientID());
-                        if (it == clients_.end()) {
-                            std::stringstream ss;
-                            ss << "ERROR: Received [clientID="<< hdr.clientID() << ",requestID="<< hdr.requestID << ",message=" << hdr.message << ",payload=" << hdr.payloadSize << "]" << std::endl;
-                            ss << "Unexpected answer for clientID recieved (" << hdr.clientID() << "). ABORTING";
-                            eckit::Log::status() << ss.str() << std::endl;
-                            eckit::Log::error() << "Retrieving... " << ss.str() << std::endl;
-                            throw eckit::SeriousBug(ss.str(), Here());
-                        }
-                        client = it->second;
-                    }
-
-                    ASSERT(client);
                     ASSERT(hdr.control() || single_);
 
                     auto pp = promises_.find(hdr.requestID);
@@ -412,6 +395,23 @@ void ClientConnection::listeningControlThreadLoop() {
                         promises_.erase(pp);
                         handled = true;
                     } else {
+
+                        Client* client = nullptr;
+                        {
+                            std::lock_guard lock(clientsMutex_);
+
+                            auto it = clients_.find(hdr.clientID());
+                            if (it == clients_.end()) {
+                                std::stringstream ss;
+                                ss << "ERROR: Received [clientID="<< hdr.clientID() << ",requestID="<< hdr.requestID << ",message=" << hdr.message << ",payload=" << hdr.payloadSize << "]" << std::endl;
+                                ss << "Unexpected answer for clientID recieved (" << hdr.clientID() << "). ABORTING";
+                                eckit::Log::status() << ss.str() << std::endl;
+                                eckit::Log::error() << "Retrieving... " << ss.str() << std::endl;
+                                throw eckit::SeriousBug(ss.str(), Here());
+                            }
+                            client = it->second;
+                        }
+
                         if (hdr.payloadSize == 0) {
                             handled = client->handle(hdr.message, hdr.control(), hdr.requestID);
                         }
diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc
index 95382a2ef..464702150 100644
--- a/src/fdb5/remote/server/CatalogueHandler.cc
+++ b/src/fdb5/remote/server/CatalogueHandler.cc
@@ -30,7 +30,7 @@ namespace fdb5::remote {
 // ***************************************************************************************
 
 CatalogueHandler::CatalogueHandler(eckit::net::TCPSocket& socket, const Config& config):
-    ServerConnection(socket, config), fdbId_(0), fdbControlConnection_(false), fdbDataConnection_(false) {}
+    ServerConnection(socket, config), fdbControlConnection_(false), fdbDataConnection_(false) {}
 
 CatalogueHandler::~CatalogueHandler() {}
 
@@ -41,8 +41,9 @@ Handled CatalogueHandler::handleControl(Message message, uint32_t clientID, uint
             case Message::Schema: // request top-level schema
                 {
                     std::lock_guard lock(handlerMutex_);
-                    if (fdbId_ == 0) {
-                        fdbId_ = clientID;
+                    auto it = fdbs_.find(clientID);
+                    if (it == fdbs_.end()) {
+                        fdbs_[clientID];
                         fdbControlConnection_ = true;
                         fdbDataConnection_ = !single_;
                         numControlConnection_++;
@@ -62,9 +63,9 @@ Handled CatalogueHandler::handleControl(Message message, uint32_t clientID, uint
                 archiver();
                 return Handled::YesAddArchiveListener;
 
-            case Message::Flush: // notification that the client has sent all data locations for archival
-                flush(clientID, requestID, eckit::Buffer{0});
-                return Handled::Yes;
+            // case Message::Flush: // notification that the client has sent all data locations for archival
+            //     flush(clientID, requestID, eckit::Buffer{0});
+            //     return Handled::Yes;
 
             default: {
                 std::stringstream ss;
@@ -251,8 +252,15 @@ void CatalogueHandler::forwardApiCall(uint32_t clientID, uint32_t requestID, eck
         requestID, std::async(std::launch::async, [request, clientID, requestID, helper, this]() {
 
             try {
-                auto it = fdbs_.find(clientID);
-                auto iterator = helper.apiCall(it->second, request);
+                FDB* fdb = nullptr;
+                {
+                    std::lock_guard lock(handlerMutex_);
+                    auto it = fdbs_.find(clientID);
+                    ASSERT(it != fdbs_.end());
+                    fdb = &it->second;
+                    ASSERT(fdb);
+                }
+                auto iterator = helper.apiCall(*fdb, request);
                 typename decltype(iterator)::value_type elem;
                 while (iterator.next(elem)) {
                     auto encoded(helper.encode(elem, *this));
@@ -375,24 +383,41 @@ void CatalogueHandler::stores(uint32_t clientID, uint32_t requestID) {
 
 
 void CatalogueHandler::flush(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) {
-    
+        
+    ASSERT(payload.size() > 0);
+
     size_t numArchived = 0;
-    
-    if (payload.size() > 0) {
-        MemoryStream s(payload);
-        s >> numArchived;
+    MemoryStream s(payload);
+    s >> numArchived;
+
+    if (numArchived == 0) {
+        return;
     }
 
     auto it = catalogues_.find(clientID);
     ASSERT(it != catalogues_.end());
 
-    it->second.locationsExpected = numArchived;
-    it->second.archivalCompleted = it->second.fieldLocationsReceived.get_future();
+    // std::cout << "flush " << clientID << " archived " << it->second.locationsArchived << " expected " << it->second.locationsExpected << std::endl;
+
+    {
+        std::lock_guard lock(fieldLocationsMutex_);
+        it->second.locationsExpected = numArchived;     // setting locationsExpected also means that a flush has been requested
+        it->second.archivalCompleted = it->second.fieldLocationsReceived.get_future();
+        // std::cout << "flush post lock " << clientID << " archived " << it->second.locationsArchived << " expected " << it->second.locationsExpected << std::endl;
+        if (it->second.locationsArchived == numArchived) {
+            it->second.fieldLocationsReceived.set_value(numArchived);
+        }
+    }
 
-    if (it->second.locationsArchived < numArchived) {
-        it->second.archivalCompleted.wait();
+    // std::cout << "flush wait " << clientID << " archived " << it->second.locationsArchived << " expected " << it->second.locationsExpected << std::endl;
+    it->second.archivalCompleted.wait();
+    {
+        std::lock_guard lock(fieldLocationsMutex_);
         it->second.fieldLocationsReceived = std::promise{};
+        it->second.locationsExpected = 0;
+        it->second.locationsArchived = 0;
     }
+    // std::cout << "flush post wait " << clientID << " archived " << it->second.locationsArchived << " expected " << it->second.locationsExpected << std::endl;
 
     it->second.catalogue->flush(numArchived);
 
@@ -427,16 +452,25 @@ void CatalogueHandler::archiveBlob(const uint32_t clientID, const uint32_t reque
 
     it->second.catalogue->selectIndex(idxKey);
     it->second.catalogue->archive(idxKey, key, std::move(location));
-    it->second.locationsArchived++;
-    if (it->second.archivalCompleted.valid() && it->second.locationsExpected == it->second.locationsArchived) {
-        it->second.fieldLocationsReceived.set_value(it->second.locationsExpected);
+    {
+        // std::cout << "archiveBlob " << clientID << " archived " << it->second.locationsArchived << " expected " << it->second.locationsExpected << std::endl;
+        std::lock_guard lock(fieldLocationsMutex_);
+        // std::cout << "archiveBlob post mutex " << std::endl;
+        it->second.locationsArchived++;
+        if (it->second.locationsExpected != 0 && it->second.archivalCompleted.valid() && it->second.locationsExpected == it->second.locationsArchived) {
+            // std::cout << "archiveBlob set_value " << std::endl;
+            it->second.fieldLocationsReceived.set_value(it->second.locationsExpected);
+        }
     }
 }
 
 bool CatalogueHandler::remove(bool control, uint32_t clientID) {
     
     std::lock_guard lock(handlerMutex_);
-    if (clientID == fdbId_) {
+
+    // is the client an FDB
+    auto it = fdbs_.find(clientID);
+    if (it != fdbs_.end()) {
         if (control) {
             fdbControlConnection_ = false;
             numControlConnection_--;
diff --git a/src/fdb5/remote/server/CatalogueHandler.h b/src/fdb5/remote/server/CatalogueHandler.h
index 151752ce2..f60e151b2 100644
--- a/src/fdb5/remote/server/CatalogueHandler.h
+++ b/src/fdb5/remote/server/CatalogueHandler.h
@@ -73,8 +73,9 @@ class CatalogueHandler : public ServerConnection {
     std::map fdbs_;
 
     std::mutex fdbMutex_;
+    std::mutex fieldLocationsMutex_;
 
-    uint32_t fdbId_;
+    // uint32_t fdbId_;
     bool fdbControlConnection_;
     bool fdbDataConnection_;
 };
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index 3058149b9..a72d92d8b 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -560,6 +560,7 @@ void ServerConnection::waitForWorkers() {
         eckit::Log::error() << "Thread complete" << std::endl;
     }
 
+    std::lock_guard lock(readLocationMutex_);
     if (readLocationWorker_.joinable()) {
         readLocationWorker_.join();
     }
diff --git a/src/fdb5/remote/server/ServerConnection.h b/src/fdb5/remote/server/ServerConnection.h
index b9d48b0ca..16f10b038 100644
--- a/src/fdb5/remote/server/ServerConnection.h
+++ b/src/fdb5/remote/server/ServerConnection.h
@@ -141,6 +141,7 @@ class ServerConnection : public Connection, public Handler {
 
     eckit::SessionID sessionID_;
     eckit::LocalConfiguration agreedConf_;
+    std::mutex readLocationMutex_;
     std::thread readLocationWorker_;
     
     std::map> workerThreads_;
diff --git a/src/fdb5/remote/server/StoreHandler.cc b/src/fdb5/remote/server/StoreHandler.cc
index 859c87139..c19f7678b 100644
--- a/src/fdb5/remote/server/StoreHandler.cc
+++ b/src/fdb5/remote/server/StoreHandler.cc
@@ -86,8 +86,11 @@ Handled StoreHandler::handleControl(Message message, uint32_t clientID, uint32_t
 
 void StoreHandler::read(uint32_t clientID, uint32_t requestID, const eckit::Buffer& payload) {
 
-    if (!readLocationWorker_.joinable()) {
-        readLocationWorker_ = std::thread([this] { readLocationThreadLoop(); });
+    {
+        std::lock_guard lock(readLocationMutex_);
+        if (!readLocationWorker_.joinable()) {
+            readLocationWorker_ = std::thread([this] { readLocationThreadLoop(); });
+        }
     }
 
     MemoryStream s(payload);
@@ -188,6 +191,7 @@ void StoreHandler::flush(uint32_t clientID, uint32_t requestID, const eckit::Buf
 
     ASSERT(numArchived == 0 || archiveFuture_.valid());
 
+    std::lock_guard lock(handlerMutex_);
     auto it = stores_.find(clientID);
     ASSERT(it != stores_.end());
     it->second.store->flush();
@@ -199,7 +203,6 @@ void StoreHandler::flush(uint32_t clientID, uint32_t requestID, const eckit::Buf
 bool StoreHandler::remove(bool control, uint32_t clientID) {
     
     std::lock_guard lock(handlerMutex_);
-
     auto it = stores_.find(clientID);
     if (it != stores_.end()) {
         if (control) {
@@ -222,6 +225,7 @@ Store& StoreHandler::store(uint32_t clientID) {
     auto it = stores_.find(clientID);
     if (it == stores_.end()) {
         std::string what("Requested Store has not been loaded id: " + std::to_string(clientID));
+        Log::error() << what << std::endl;
         write(Message::Error, true, 0, 0, what.c_str(), what.length());
         throw;
     }
diff --git a/src/fdb5/remote/server/StoreHandler.h b/src/fdb5/remote/server/StoreHandler.h
index 0319e6d4b..74b2adec1 100644
--- a/src/fdb5/remote/server/StoreHandler.h
+++ b/src/fdb5/remote/server/StoreHandler.h
@@ -49,7 +49,6 @@ class StoreHandler : public ServerConnection {
     void writeToParent(const uint32_t clientID, const uint32_t requestID, std::unique_ptr dh);
 
     bool remove(bool control, uint32_t clientID) override;
-    // bool handlers() override;
 
     Store& store(uint32_t clientID);
     Store& store(uint32_t clientID, const Key& dbKey);

From 7fdc98b7a6573ac3abacc1f15b41ae79ef1a8587 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 3 Jul 2024 12:53:47 +0100
Subject: [PATCH 120/186] remoteFDB connection teardown

---
 VERSION                                    |  2 +-
 src/fdb5/remote/client/ClientConnection.cc | 17 +++++++---
 src/fdb5/remote/server/ServerConnection.cc | 38 +++++++++++++++++++---
 3 files changed, 47 insertions(+), 10 deletions(-)

diff --git a/VERSION b/VERSION
index fccd8bbd0..97954dceb 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.124
\ No newline at end of file
+5.11.125
\ No newline at end of file
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index b5244ca40..dd82f63e4 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -87,13 +87,20 @@ bool ClientConnection::remove(uint32_t clientID) {
     }
 
     if (clients_.empty()) {
-        Connection::write(Message::Exit, true, 0, 0);
         if (!single_) {
-        // if (!controlStopping_)
-        //     Connection::write(Message::Exit, true, 0, 0);
-        // if (!single_ && !dataStopping_) {
             // TODO make the data connection dying automatically, when there are no more async writes
-            Connection::write(Message::Exit, false, 0, 0);
+            try {
+                // all done - disconnecting
+                Connection::write(Message::Exit, false, 0, 0);
+            } catch(...) {
+                // if connection is already down, no need to escalate 
+            }
+        }
+        try {
+            // all done - disconnecting
+            Connection::write(Message::Exit, true, 0, 0);
+        } catch(...) {
+            // if connection is already down, no need to escalate 
         }
 
         ClientConnectionRouter::instance().deregister(*this);
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index a72d92d8b..fb872b0a1 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -441,10 +441,20 @@ void ServerConnection::handle() {
             if (hdr.message == Message::Exit) {
                 ASSERT(hdr.clientID() == 0);
 
-                write(Message::Exit, true, 0, 0);
-                if (!single_) {
-                    write(Message::Exit, false, 0, 0);
-                }
+                // if (!single_) {
+                //     try {
+                //         // all done - disconnecting
+                //         Connection::write(Message::Exit, false, 0, 0);
+                //     } catch(...) {
+                //         // if connection is already down, no need to escalate 
+                //     }
+                // }
+                // try {
+                //     // all done - disconnecting
+                //     Connection::write(Message::Exit, true, 0, 0);
+                // } catch(...) {
+                //     // if connection is already down, no need to escalate 
+                // }
 
                 eckit::Log::status() << "Terminating CONTROL listener" << std::endl;
                 eckit::Log::info() << "Terminating CONTROL listener" << std::endl;
@@ -502,6 +512,26 @@ void ServerConnection::handle() {
     }
     ASSERT(archiveQueue_.empty());
     archiveQueue_.close();
+
+    if (!single_) {
+        try {
+            // all done - disconnecting
+            Connection::write(Message::Exit, false, 0, 0);
+        } catch(...) {
+            // if connection is already down, no need to escalate 
+        }
+    }
+    try {
+        // all done - disconnecting
+        Connection::write(Message::Exit, true, 0, 0);
+    } catch(...) {
+        // if connection is already down, no need to escalate 
+    }
+
+    // write(Message::Exit, true, 0, 0);
+    // if (!single_) {
+    //     write(Message::Exit, false, 0, 0);
+    // }
 }
 
 void ServerConnection::handleException(std::exception_ptr e) {

From 36e7bc9f4f788a2be1a2001d204f1bc9500b589a Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 3 Jul 2024 15:24:42 +0100
Subject: [PATCH 121/186] fixed remoteStore builder

---
 src/fdb5/remote/client/RemoteStore.cc      | 2 ++
 src/fdb5/remote/server/ServerConnection.cc | 5 -----
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/src/fdb5/remote/client/RemoteStore.cc b/src/fdb5/remote/client/RemoteStore.cc
index 2c27e6e62..3430aa4ed 100644
--- a/src/fdb5/remote/client/RemoteStore.cc
+++ b/src/fdb5/remote/client/RemoteStore.cc
@@ -456,9 +456,11 @@ eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation, c
 }
 
 RemoteStore& RemoteStore::get(const eckit::URI& uri) {
+    static std::mutex storeMutex_;
     // we memoise one read store for each endpoint. Do not need to have one for each key
     static std::map> readStores_;
 
+    std::lock_guard lock(storeMutex_);
     const std::string& endpoint = uri.hostport();
     auto it = readStores_.find(endpoint);
     if (it != readStores_.end()) {
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index fb872b0a1..3fe55f107 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -527,11 +527,6 @@ void ServerConnection::handle() {
     } catch(...) {
         // if connection is already down, no need to escalate 
     }
-
-    // write(Message::Exit, true, 0, 0);
-    // if (!single_) {
-    //     write(Message::Exit, false, 0, 0);
-    // }
 }
 
 void ServerConnection::handleException(std::exception_ptr e) {

From 5df53a6ca3a4b6d6fd49015c667c2ef1d39c4877 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 3 Jul 2024 17:44:14 +0100
Subject: [PATCH 122/186] concurrent remote readers

---
 src/fdb5/remote/client/RemoteStore.cc | 36 +++++++++++++++++++--------
 src/fdb5/remote/client/RemoteStore.h  |  3 ++-
 2 files changed, 28 insertions(+), 11 deletions(-)

diff --git a/src/fdb5/remote/client/RemoteStore.cc b/src/fdb5/remote/client/RemoteStore.cc
index 3430aa4ed..92009439a 100644
--- a/src/fdb5/remote/client/RemoteStore.cc
+++ b/src/fdb5/remote/client/RemoteStore.cc
@@ -54,7 +54,7 @@ class FDBRemoteDataHandle : public DataHandle {
 public: // methods
 
     FDBRemoteDataHandle(uint32_t requestID, Length estimate,
-                        RemoteStore::MessageQueue& queue,
+                        std::shared_ptr queue,
                         const net::Endpoint& remoteEndpoint) :
         requestID_(requestID),
         estimate_(estimate),
@@ -98,7 +98,7 @@ class FDBRemoteDataHandle : public DataHandle {
 
             // If we are in the DataHandle, then there MUST be data to read
             RemoteStore::StoredMessage msg = std::make_pair(remote::Message{}, eckit::Buffer{0});
-            ASSERT(queue_.pop(msg) != -1);
+            ASSERT(queue_->pop(msg) != -1);
 
             // Handle any remote errors communicated from the server
             if (msg.first == Message::Error) {
@@ -109,7 +109,7 @@ class FDBRemoteDataHandle : public DataHandle {
             // Are we now complete?
             if (msg.first == Message::Complete) {
                 if (overallPosition_ == eckit::Offset(0)) {
-                    ASSERT(queue_.pop(msg) != -1);
+                    ASSERT(queue_->pop(msg) != -1);
                 } else {
                     complete_ = true;
                     return total;
@@ -169,7 +169,7 @@ class FDBRemoteDataHandle : public DataHandle {
 
     uint32_t requestID_;
     Length estimate_;
-    RemoteStore::MessageQueue& queue_;
+    std::shared_ptr queue_;
     net::Endpoint remoteEndpoint_;
     size_t pos_;
     Offset overallPosition_;
@@ -197,14 +197,14 @@ std::vector> storeEndpoints(const C
 RemoteStore::RemoteStore(const Key& dbKey, const Config& config) :
     Client(storeEndpoints(config)),
     dbKey_(dbKey), config_(config),
-    retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
+    // retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
     fieldsArchived_(0), locationsReceived_(0) {}
 
 // this is used only in retrieval, with an URI already referring to an accessible Store
 RemoteStore::RemoteStore(const eckit::URI& uri, const Config& config) :
     Client(eckit::net::Endpoint(uri.hostport()), uri.hostport()),
     dbKey_(Key()), config_(config),
-    retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
+    // retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
     fieldsArchived_(0), locationsReceived_(0) { 
 
     // no need to set the local_ flag on the read path
@@ -348,7 +348,12 @@ bool RemoteStore::handle(Message message, bool control, uint32_t requestID) {
                 messageQueues_.erase(it);
 
             } else {
-                retrieveMessageQueue_.emplace(std::make_pair(message, Buffer(0)));
+                auto id = retrieveMessageQueues_.find(requestID);
+                ASSERT (id != retrieveMessageQueues_.end());
+
+                id->second->emplace(std::make_pair(message, Buffer(0)));
+
+                retrieveMessageQueues_.erase(id);
             }
             return true;
         }
@@ -403,7 +408,9 @@ bool RemoteStore::handle(Message message, bool control, uint32_t requestID, ecki
             if (it != messageQueues_.end()) {
                 it->second->emplace(message, std::move(payload));
             } else {
-                retrieveMessageQueue_.emplace(message, std::move(payload));
+                auto id = retrieveMessageQueues_.find(requestID);
+                ASSERT (id != retrieveMessageQueues_.end());
+                id->second->emplace(std::make_pair(message, std::move(payload)));
             }
             return true;
         }
@@ -425,7 +432,7 @@ bool RemoteStore::handle(Message message, bool control, uint32_t requestID, ecki
                 if (it != locations_.end()) {
 //                    archiver_->error(std::move(payload), controlEndpoint());
                 } else {
-                    retrieveMessageQueue_.emplace(message, std::move(payload));
+                    // retrieveMessageQueue_.emplace(message, std::move(payload));
                 }
             }
             return true;
@@ -450,9 +457,18 @@ eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation, c
     s << remapKey;
 
     uint32_t id = generateRequestID();
+
+    static size_t queueSize = eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200);
+                // auto id = retrieveMessageQueues_.find(requestID);
+                // ASSERT (it != retrieveMessageQueues_.end());
+                // it->second->emplace(std::make_pair(message, std::move(payload)));
+
+    auto entry = retrieveMessageQueues_.emplace(id, std::make_shared(queueSize));
+    ASSERT(entry.second);
+
     controlWriteCheckResponse(fdb5::remote::Message::Read, id, true, encodeBuffer, s.position());
 
-    return new FDBRemoteDataHandle(id, fieldLocation.length(), retrieveMessageQueue_, controlEndpoint());
+    return new FDBRemoteDataHandle(id, fieldLocation.length(), entry.first->second, controlEndpoint());
 }
 
 RemoteStore& RemoteStore::get(const eckit::URI& uri) {
diff --git a/src/fdb5/remote/client/RemoteStore.h b/src/fdb5/remote/client/RemoteStore.h
index ec6f5e4c6..810b4bfb7 100644
--- a/src/fdb5/remote/client/RemoteStore.h
+++ b/src/fdb5/remote/client/RemoteStore.h
@@ -94,7 +94,8 @@ class RemoteStore : public Store, public Client {
     // The shared_ptr allows this removal to be asynchronous with the actual task
     // cleaning up and returning to the client.
     std::map> messageQueues_;
-    MessageQueue retrieveMessageQueue_;
+    std::map> retrieveMessageQueues_;
+    // MessageQueue retrieveMessageQueue_;
 
     std::mutex locationMutex_;
     std::map fieldLocation)>> locations_;

From 9c0f629c5151481dbe13f74eb17f21230633d1cd Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 4 Jul 2024 08:31:20 +0100
Subject: [PATCH 123/186] version bump

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 97954dceb..9aaafe1cc 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.125
\ No newline at end of file
+5.11.126
\ No newline at end of file

From f90b99d52e67b264ae2a155fbd6e8a346b95ad22 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 5 Jul 2024 14:51:44 +0100
Subject: [PATCH 124/186] retrieveMessageQueue_ protected by mutex

---
 src/fdb5/remote/client/RemoteStore.cc | 15 ++++++++++++---
 src/fdb5/remote/client/RemoteStore.h  |  1 +
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/fdb5/remote/client/RemoteStore.cc b/src/fdb5/remote/client/RemoteStore.cc
index 92009439a..ea4d82912 100644
--- a/src/fdb5/remote/client/RemoteStore.cc
+++ b/src/fdb5/remote/client/RemoteStore.cc
@@ -348,6 +348,7 @@ bool RemoteStore::handle(Message message, bool control, uint32_t requestID) {
                 messageQueues_.erase(it);
 
             } else {
+                std::lock_guard lock(retrieveMessageMutex_);
                 auto id = retrieveMessageQueues_.find(requestID);
                 ASSERT (id != retrieveMessageQueues_.end());
 
@@ -408,6 +409,7 @@ bool RemoteStore::handle(Message message, bool control, uint32_t requestID, ecki
             if (it != messageQueues_.end()) {
                 it->second->emplace(message, std::move(payload));
             } else {
+                std::lock_guard lock(retrieveMessageMutex_);
                 auto id = retrieveMessageQueues_.find(requestID);
                 ASSERT (id != retrieveMessageQueues_.end());
                 id->second->emplace(std::make_pair(message, std::move(payload)));
@@ -463,12 +465,19 @@ eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation, c
                 // ASSERT (it != retrieveMessageQueues_.end());
                 // it->second->emplace(std::make_pair(message, std::move(payload)));
 
-    auto entry = retrieveMessageQueues_.emplace(id, std::make_shared(queueSize));
-    ASSERT(entry.second);
+    std::shared_ptr queue = nullptr;
+    {
+        std::lock_guard lock(retrieveMessageMutex_);
+
+        auto entry = retrieveMessageQueues_.emplace(id, std::make_shared(queueSize));
+        ASSERT(entry.second);
+
+        queue = entry.first->second;
+    }
 
     controlWriteCheckResponse(fdb5::remote::Message::Read, id, true, encodeBuffer, s.position());
 
-    return new FDBRemoteDataHandle(id, fieldLocation.length(), entry.first->second, controlEndpoint());
+    return new FDBRemoteDataHandle(id, fieldLocation.length(), queue, controlEndpoint());
 }
 
 RemoteStore& RemoteStore::get(const eckit::URI& uri) {
diff --git a/src/fdb5/remote/client/RemoteStore.h b/src/fdb5/remote/client/RemoteStore.h
index 810b4bfb7..af609b31e 100644
--- a/src/fdb5/remote/client/RemoteStore.h
+++ b/src/fdb5/remote/client/RemoteStore.h
@@ -97,6 +97,7 @@ class RemoteStore : public Store, public Client {
     std::map> retrieveMessageQueues_;
     // MessageQueue retrieveMessageQueue_;
 
+    std::mutex retrieveMessageMutex_;
     std::mutex locationMutex_;
     std::map fieldLocation)>> locations_;
     size_t fieldsArchived_;

From 6c991d759adb6603772cc932780e09f8ba4c95f2 Mon Sep 17 00:00:00 2001
From: Chris Bradley 
Date: Wed, 10 Jul 2024 12:26:23 +0000
Subject: [PATCH 125/186] Increase buffer size for api requests

---
 src/fdb5/api/RemoteFDB.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index bd7c3e4eb..f51075e08 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -24,7 +24,7 @@ struct BaseAPIHelper {
 
     typedef T ValueType;
 
-    static size_t bufferSize() { return 4096; }
+    static size_t bufferSize() { return 1024*1024; }
     static size_t queueSize() { return 100; }
     static fdb5::remote::Message message() { return msgID; }
 

From 5cd3a51666127f4cb12fd56e9c7d96ca5290ed7d Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 13 Aug 2024 10:32:16 +0200
Subject: [PATCH 126/186] fix merge

---
 src/fdb5/database/ArchiveVisitor.cc       | 6 ++++--
 src/fdb5/database/ArchiveVisitor.h        | 2 +-
 src/fdb5/database/Archiver.h              | 2 +-
 src/fdb5/remote/client/RemoteCatalogue.cc | 6 +++---
 src/fdb5/remote/client/RemoteCatalogue.h  | 2 +-
 5 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/fdb5/database/ArchiveVisitor.cc b/src/fdb5/database/ArchiveVisitor.cc
index 024c9a111..6a60a51af 100644
--- a/src/fdb5/database/ArchiveVisitor.cc
+++ b/src/fdb5/database/ArchiveVisitor.cc
@@ -23,8 +23,10 @@ ArchiveVisitor::ArchiveVisitor(Archiver& owner, const Key& initialFieldKey, cons
     callback_(callback){
 }
 
-void ArchiveVisitor::callbacks(fdb5::CatalogueWriter* catalogue, const Key& idxKey, const Key& datumKey, std::unique_ptr fieldLocation) {
-    callback_(data_, size_, *fieldLocation);
+void ArchiveVisitor::callbacks(fdb5::CatalogueWriter* catalogue, const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) {
+    std::promise> p;
+    callback_(initialFieldKey_, data_, size_, p.get_future());
+    p.set_value(fieldLocation);
     catalogue->archive(idxKey, datumKey, std::move(fieldLocation));
 }
 
diff --git a/src/fdb5/database/ArchiveVisitor.h b/src/fdb5/database/ArchiveVisitor.h
index a8c03a4b6..4decca58b 100644
--- a/src/fdb5/database/ArchiveVisitor.h
+++ b/src/fdb5/database/ArchiveVisitor.h
@@ -40,7 +40,7 @@ class ArchiveVisitor : public BaseArchiveVisitor {
 
 private: // methods
 
-    void callbacks(fdb5::CatalogueWriter* catalogue, const Key& idxKey, const Key& datumKey, std::unique_ptr fieldLocation);
+    void callbacks(fdb5::CatalogueWriter* catalogue, const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation);
 
 private: // members
 
diff --git a/src/fdb5/database/Archiver.h b/src/fdb5/database/Archiver.h
index a0e857857..d1554062d 100644
--- a/src/fdb5/database/Archiver.h
+++ b/src/fdb5/database/Archiver.h
@@ -21,7 +21,7 @@
 
 #include "eckit/memory/NonCopyable.h"
 
-#include "fdb5/api/helpers/ArchiveCallback.h"
+#include "fdb5/api/helpers/Callback.h"
 #include "fdb5/config/Config.h"
 #include "fdb5/database/Catalogue.h"
 
diff --git a/src/fdb5/remote/client/RemoteCatalogue.cc b/src/fdb5/remote/client/RemoteCatalogue.cc
index 28ada8139..e4d590637 100644
--- a/src/fdb5/remote/client/RemoteCatalogue.cc
+++ b/src/fdb5/remote/client/RemoteCatalogue.cc
@@ -58,9 +58,9 @@ void RemoteCatalogue::sendArchiveData(uint32_t id, const Key& key, std::unique_p
     dataWrite(Message::Blob, id, payloads);
 }
 
-void RemoteCatalogue::archive(const Key& idxKey, const Key& key, std::unique_ptr fieldLocation) {
+void RemoteCatalogue::archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) {
 
-    ASSERT(!key.empty());
+    ASSERT(!datumKey.empty());
     ASSERT(fieldLocation);
 
     uint32_t id = generateRequestID();
@@ -75,7 +75,7 @@ void RemoteCatalogue::archive(const Key& idxKey, const Key& key, std::unique_ptr
     Buffer buffer(8192);
     MemoryStream stream(buffer);
     stream << idxKey;
-    stream << key;
+    stream << datumKey;
     stream << *fieldLocation;
 
     std::vector> payloads;
diff --git a/src/fdb5/remote/client/RemoteCatalogue.h b/src/fdb5/remote/client/RemoteCatalogue.h
index 209a917e1..e8d1954e6 100644
--- a/src/fdb5/remote/client/RemoteCatalogue.h
+++ b/src/fdb5/remote/client/RemoteCatalogue.h
@@ -23,7 +23,7 @@ class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public C
 
     // From CatalogueWriter
     const Index& currentIndex() override;
-    void archive(const Key& idxKey, const Key& key, std::unique_ptr fieldLocation) override;
+    void archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) override;
     void overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) override;
     void index(const Key& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) override;
     void reconsolidate() override;

From ac051877154984ee6210fcabd4c57cfcfbde055a Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 16 Aug 2024 08:59:46 +0200
Subject: [PATCH 127/186] wip

---
 src/fdb5/toc/TocPurgeVisitor.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/fdb5/toc/TocPurgeVisitor.cc b/src/fdb5/toc/TocPurgeVisitor.cc
index b1058cb3f..34c73ea68 100644
--- a/src/fdb5/toc/TocPurgeVisitor.cc
+++ b/src/fdb5/toc/TocPurgeVisitor.cc
@@ -45,9 +45,9 @@ bool TocPurgeVisitor::visitDatabase(const Catalogue& catalogue) {
     }
 
     for (const auto& uri : data) {
-        if (!store.uriBelongs(uri)) {
+        if (!store_.uriBelongs(uri)) {
             Log::error() << "Catalogue is pointing to data files that do not belong to the store." << std::endl;
-            Log::error() << "Configured Store URI: " << store.uri().asString() << std::endl;
+            Log::error() << "Configured Store URI: " << store_.uri().asString() << std::endl;
             Log::error() << "Pointed Store unit URI: " << uri.asString() << std::endl;
             Log::error() << "This may occur when purging an overlayed FDB, which is not supported." << std::endl;
             NOTIMP;

From 183e257555ef37c4bc51e038c37f88a82d190d37 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 4 Sep 2024 07:13:29 +0100
Subject: [PATCH 128/186] start remoteFDB data connection earlier

---
 src/fdb5/remote/server/ServerConnection.cc | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index 3fe55f107..59769afc6 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -230,12 +230,18 @@ void ServerConnection::initialiseConnections() {
     //               server has multiple, then we use that on, whilst retaining
     //               the capacity in the protocol for the server to make a choice.
 
+    std::future accepted;
     eckit::net::Endpoint dataEndpoint;
     if (single_) {
         dataEndpoint = endpointFromClient;
     } else {
         dataSocket_.reset(new eckit::net::EphemeralTCPServer(selectDataPort()));
         dataEndpoint = eckit::net::Endpoint{endpointFromClient.hostname(), dataSocket_->localPort()};
+
+        auto socket = dataSocket_.get();
+        accepted = std::async(std::launch::async, [&socket] {
+            socket->accept();
+        });
     }
 
     eckit::Log::info() << "Sending data endpoint to client: " << dataEndpoint << std::endl;
@@ -260,7 +266,7 @@ void ServerConnection::initialiseConnections() {
     }
 
     if (!single_) {
-        dataSocket_->accept();
+        accepted.wait();
 
         // Check the response from the client.
         // Ensure that the hostname matches the original hostname, and that

From 49a2f138a4a57454d026272b432966f66617c19b Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 4 Sep 2024 07:24:04 +0100
Subject: [PATCH 129/186] version bump

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 9aaafe1cc..1b26109ca 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.126
\ No newline at end of file
+5.11.127

From b49b2a17a44e6cdd017b414502ecfa4c1b6125af Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 4 Sep 2024 11:07:14 +0100
Subject: [PATCH 130/186] fixed remoteFDB data connection binding

---
 VERSION                                    |  2 +-
 src/fdb5/remote/server/ServerConnection.cc | 10 +++-------
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/VERSION b/VERSION
index 1b26109ca..4c263eda1 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.127
+5.11.128
\ No newline at end of file
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index 59769afc6..8472c0408 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -230,18 +230,14 @@ void ServerConnection::initialiseConnections() {
     //               server has multiple, then we use that on, whilst retaining
     //               the capacity in the protocol for the server to make a choice.
 
-    std::future accepted;
     eckit::net::Endpoint dataEndpoint;
     if (single_) {
         dataEndpoint = endpointFromClient;
     } else {
         dataSocket_.reset(new eckit::net::EphemeralTCPServer(selectDataPort()));
-        dataEndpoint = eckit::net::Endpoint{endpointFromClient.hostname(), dataSocket_->localPort()};
+        ASSERT(dataSocket_->socket() != -1);
 
-        auto socket = dataSocket_.get();
-        accepted = std::async(std::launch::async, [&socket] {
-            socket->accept();
-        });
+        dataEndpoint = eckit::net::Endpoint{endpointFromClient.hostname(), dataSocket_->localPort()};
     }
 
     eckit::Log::info() << "Sending data endpoint to client: " << dataEndpoint << std::endl;
@@ -266,7 +262,7 @@ void ServerConnection::initialiseConnections() {
     }
 
     if (!single_) {
-        accepted.wait();
+        dataSocket_->accept();
 
         // Check the response from the client.
         // Ensure that the hostname matches the original hostname, and that

From b2d9c6b3eb1d127e7bd5c6c4ea5bc30462ab79db Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 5 Sep 2024 16:46:16 +0100
Subject: [PATCH 131/186] fixed unit tests (DAOS catalogue factory name +
 archive callback composition)

---
 src/fdb5/api/helpers/Callback.h           |  4 ++--
 src/fdb5/daos/DaosCatalogueReader.cc      |  2 +-
 src/fdb5/daos/DaosCatalogueWriter.cc      |  4 ++--
 src/fdb5/daos/DaosCatalogueWriter.h       |  2 +-
 src/fdb5/daos/DaosStore.cc                |  4 ++--
 src/fdb5/daos/DaosStore.h                 |  2 +-
 src/fdb5/database/ArchiveVisitor.cc       | 10 +++++-----
 src/fdb5/database/ArchiveVisitor.h        |  2 +-
 src/fdb5/database/Catalogue.h             |  2 +-
 src/fdb5/database/Field.cc                |  2 +-
 src/fdb5/database/Field.h                 |  4 ++--
 src/fdb5/database/Store.cc                |  4 ++--
 src/fdb5/database/Store.h                 |  5 ++---
 src/fdb5/rados/RadosStore.cc              |  4 ++--
 src/fdb5/rados/RadosStore.h               |  2 +-
 src/fdb5/remote/client/RemoteCatalogue.cc |  2 +-
 src/fdb5/remote/client/RemoteCatalogue.h  |  2 +-
 src/fdb5/remote/client/RemoteStore.cc     |  2 +-
 src/fdb5/remote/client/RemoteStore.h      |  4 ++--
 src/fdb5/remote/server/StoreHandler.cc    |  2 +-
 src/fdb5/toc/TocCatalogueWriter.cc        |  2 +-
 src/fdb5/toc/TocCatalogueWriter.h         |  2 +-
 src/fdb5/toc/TocStore.cc                  |  4 ++--
 src/fdb5/toc/TocStore.h                   |  2 +-
 tests/fdb/api/test_auxiliary.cc           |  4 ++--
 tests/fdb/api/test_callback.cc            |  4 ++--
 tests/fdb/daos/test_daos_catalogue.cc     |  4 ++--
 tests/fdb/daos/test_daos_store.cc         |  4 ++--
 28 files changed, 45 insertions(+), 46 deletions(-)

diff --git a/src/fdb5/api/helpers/Callback.h b/src/fdb5/api/helpers/Callback.h
index 8b9f04409..1ea128c6e 100644
--- a/src/fdb5/api/helpers/Callback.h
+++ b/src/fdb5/api/helpers/Callback.h
@@ -22,11 +22,11 @@ namespace fdb5 {
 
 class FDB;
 
-using ArchiveCallback = std::function>)>;
+using ArchiveCallback = std::function>)>;
 using FlushCallback = std::function;
 using ConstructorCallback = std::function;
 
-static const ArchiveCallback CALLBACK_NOOP = [](const Key& key, const void* data, size_t length, std::future>) {};
+static const ArchiveCallback CALLBACK_NOOP = [](const Key& key, const void* data, size_t length, std::future>) {};
 static const FlushCallback CALLBACK_FLUSH_NOOP = []() {};
 static const ConstructorCallback CALLBACK_CONSTRUCTOR_NOOP = [](FDB&) {};
 
diff --git a/src/fdb5/daos/DaosCatalogueReader.cc b/src/fdb5/daos/DaosCatalogueReader.cc
index db4bd3726..a58018d9b 100644
--- a/src/fdb5/daos/DaosCatalogueReader.cc
+++ b/src/fdb5/daos/DaosCatalogueReader.cc
@@ -133,7 +133,7 @@ bool DaosCatalogueReader::retrieve(const Key& key, Field& field) const {
 
 }
 
-static fdb5::CatalogueReaderBuilder builder("daos.reader");
+static fdb5::CatalogueReaderBuilder builder("daos");
 
 //----------------------------------------------------------------------------------------------------------------------
 
diff --git a/src/fdb5/daos/DaosCatalogueWriter.cc b/src/fdb5/daos/DaosCatalogueWriter.cc
index 928ac4263..5bb753cfe 100644
--- a/src/fdb5/daos/DaosCatalogueWriter.cc
+++ b/src/fdb5/daos/DaosCatalogueWriter.cc
@@ -228,7 +228,7 @@ const Index& DaosCatalogueWriter::currentIndex() {
 /// @todo: other writers may be simultaneously updating the axes KeyValues in DAOS. Should these
 ///        new updates be retrieved and put into in-memory axes from time to time, e.g. every
 ///        time a value is put in an axis KeyValue?
-void DaosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) {
+void DaosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) {
 
     if (current_.null()) {
         ASSERT(!currentIndexKey_.empty());
@@ -348,7 +348,7 @@ void DaosCatalogueWriter::closeIndexes() {
 
 }
 
-static fdb5::CatalogueWriterBuilder builder("daos.writer");
+static fdb5::CatalogueWriterBuilder builder("daos");
 
 //----------------------------------------------------------------------------------------------------------------------
 
diff --git a/src/fdb5/daos/DaosCatalogueWriter.h b/src/fdb5/daos/DaosCatalogueWriter.h
index cddf87257..1721886d5 100644
--- a/src/fdb5/daos/DaosCatalogueWriter.h
+++ b/src/fdb5/daos/DaosCatalogueWriter.h
@@ -56,7 +56,7 @@ class DaosCatalogueWriter : public DaosCatalogue, public CatalogueWriter {
     void clean() override;
     void close() override;
 
-    void archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) override;
+    void archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) override;
 
     virtual void print( std::ostream &out ) const override { NOTIMP; }
 
diff --git a/src/fdb5/daos/DaosStore.cc b/src/fdb5/daos/DaosStore.cc
index 48578410c..6f29a1597 100644
--- a/src/fdb5/daos/DaosStore.cc
+++ b/src/fdb5/daos/DaosStore.cc
@@ -105,7 +105,7 @@ eckit::DataHandle* DaosStore::retrieve(Field& field) const {
 
 }
 
-std::unique_ptr DaosStore::archive(const Key&, const void *data, eckit::Length length) {
+std::unique_ptr DaosStore::archive(const Key&, const void *data, eckit::Length length) {
 
     /// @note: performed RPCs:
     /// - open pool if not cached (daos_pool_connect) -- always skipped as it is cached after selectDatabase.
@@ -134,7 +134,7 @@ std::unique_ptr DaosStore::archive(const Key&, const void *data,
 
     archivedFields_++;
 
-    return std::make_unique(n.URI(), 0, length, fdb5::Key{});
+    return std::make_unique(n.URI(), 0, length, fdb5::Key{});
 
     /// @note: performed RPCs:
     /// - close (daos_array_close here) -- always performed
diff --git a/src/fdb5/daos/DaosStore.h b/src/fdb5/daos/DaosStore.h
index 2f85e085b..d842c8452 100644
--- a/src/fdb5/daos/DaosStore.h
+++ b/src/fdb5/daos/DaosStore.h
@@ -53,7 +53,7 @@ class DaosStore : public Store, public DaosCommon {
     bool exists() const override;
 
     eckit::DataHandle* retrieve(Field& field) const override;
-    std::unique_ptr archive(const Key&, const void *data, eckit::Length length) override;
+    std::unique_ptr archive(const Key&, const void *data, eckit::Length length) override;
 
     void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override;
 
diff --git a/src/fdb5/database/ArchiveVisitor.cc b/src/fdb5/database/ArchiveVisitor.cc
index 6a60a51af..b011b1180 100644
--- a/src/fdb5/database/ArchiveVisitor.cc
+++ b/src/fdb5/database/ArchiveVisitor.cc
@@ -23,10 +23,8 @@ ArchiveVisitor::ArchiveVisitor(Archiver& owner, const Key& initialFieldKey, cons
     callback_(callback){
 }
 
-void ArchiveVisitor::callbacks(fdb5::CatalogueWriter* catalogue, const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) {
-    std::promise> p;
-    callback_(initialFieldKey_, data_, size_, p.get_future());
-    p.set_value(fieldLocation);
+void ArchiveVisitor::callbacks(fdb5::CatalogueWriter* catalogue, const Key& idxKey, const Key& datumKey, std::promise>* p, std::shared_ptr fieldLocation) {
+    p->set_value(fieldLocation);
     catalogue->archive(idxKey, datumKey, std::move(fieldLocation));
 }
 
@@ -35,8 +33,10 @@ bool ArchiveVisitor::selectDatum(const TypedKey& datumKey, const TypedKey& fullC
     checkMissingKeys(fullComputedKey);
     const Key idxKey = catalogue()->currentIndexKey();
 
+    std::promise> p;
     store()->archive(idxKey, data_, size_,
-        std::bind(&ArchiveVisitor::callbacks, this, catalogue(), idxKey, datumKey.canonical(), std::placeholders::_1));
+        std::bind(&ArchiveVisitor::callbacks, this, catalogue(), idxKey, datumKey.canonical(), &p, std::placeholders::_1));
+    callback_(initialFieldKey_, data_, size_, p.get_future());
 
     return true;
 }
diff --git a/src/fdb5/database/ArchiveVisitor.h b/src/fdb5/database/ArchiveVisitor.h
index 4decca58b..f6c6466f6 100644
--- a/src/fdb5/database/ArchiveVisitor.h
+++ b/src/fdb5/database/ArchiveVisitor.h
@@ -40,7 +40,7 @@ class ArchiveVisitor : public BaseArchiveVisitor {
 
 private: // methods
 
-    void callbacks(fdb5::CatalogueWriter* catalogue, const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation);
+    void callbacks(fdb5::CatalogueWriter* catalogue, const Key& idxKey, const Key& datumKey, std::promise>* p, std::shared_ptr fieldLocation);
 
 private: // members
 
diff --git a/src/fdb5/database/Catalogue.h b/src/fdb5/database/Catalogue.h
index 7b6e006f4..339c3e3c9 100644
--- a/src/fdb5/database/Catalogue.h
+++ b/src/fdb5/database/Catalogue.h
@@ -160,7 +160,7 @@ class CatalogueWriter : virtual public Catalogue  {
 
     virtual const Index& currentIndex() = 0;
     virtual const Key currentIndexKey();
-    virtual void archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) = 0;
+    virtual void archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) = 0;
     virtual void overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) = 0;
     virtual void index(const Key& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) = 0;
     virtual void reconsolidate() = 0;
diff --git a/src/fdb5/database/Field.cc b/src/fdb5/database/Field.cc
index a29dd3b60..b8eaa1053 100644
--- a/src/fdb5/database/Field.cc
+++ b/src/fdb5/database/Field.cc
@@ -16,7 +16,7 @@ namespace fdb5 {
 
 Field::Field() {}
 
-Field::Field(std::shared_ptr location, time_t timestamp, const FieldDetails& details):
+Field::Field(std::shared_ptr location, time_t timestamp, const FieldDetails& details):
     location_(std::move(location)),
     timestamp_(timestamp),
     details_(details) {
diff --git a/src/fdb5/database/Field.h b/src/fdb5/database/Field.h
index 4ce77a3e3..5083679c1 100644
--- a/src/fdb5/database/Field.h
+++ b/src/fdb5/database/Field.h
@@ -43,7 +43,7 @@ class Field {
 
     Field();
 
-    Field(std::shared_ptr location, time_t timestamp, const FieldDetails& details = FieldDetails());
+    Field(std::shared_ptr location, time_t timestamp, const FieldDetails& details = FieldDetails());
     Field(const FieldLocation&& location, time_t timestamp, const FieldDetails& details = FieldDetails());
 
     eckit::DataHandle* dataHandle() const { return location_->dataHandle(); }
@@ -61,7 +61,7 @@ class Field {
 
 private: // members
 
-   std::shared_ptr location_;
+   std::shared_ptr location_;
 
     time_t timestamp_;
 
diff --git a/src/fdb5/database/Store.cc b/src/fdb5/database/Store.cc
index 3accae4cf..1b9615047 100644
--- a/src/fdb5/database/Store.cc
+++ b/src/fdb5/database/Store.cc
@@ -24,11 +24,11 @@ namespace fdb5 {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-void Store::archive(const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive) {
+void Store::archive(const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive) {
     catalogue_archive(archive(key, data, length));
 }
 
-std::unique_ptr Store::archive(const Key& key, const void *data, eckit::Length length) {
+std::unique_ptr Store::archive(const Key& key, const void *data, eckit::Length length) {
     NOTIMP;
 }
 
diff --git a/src/fdb5/database/Store.h b/src/fdb5/database/Store.h
index e7843f2b8..da5a3a562 100644
--- a/src/fdb5/database/Store.h
+++ b/src/fdb5/database/Store.h
@@ -35,9 +35,8 @@ class Store {
     virtual ~Store() {}
 
     virtual eckit::DataHandle* retrieve(Field& field) const = 0;
-    virtual void archive(const Key& idxKey, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive);
-    virtual std::unique_ptr archive(const Key& idxKey, const void *data, eckit::Length length);
-    // virtual std::unique_ptr archive(const Key& idxKey, const void *data, eckit::Length length) = 0;
+    virtual void archive(const Key& idxKey, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive);
+    virtual std::unique_ptr archive(const Key& idxKey, const void *data, eckit::Length length);
 
     virtual void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit = true) const = 0;
 
diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc
index 0f6df20aa..c4704e767 100644
--- a/src/fdb5/rados/RadosStore.cc
+++ b/src/fdb5/rados/RadosStore.cc
@@ -53,7 +53,7 @@ eckit::DataHandle* RadosStore::retrieve(Field& field, Key& remapKey) const {
         field.dataHandle(remapKey);
 }
 
-std::unique_ptr RadosStore::archive(const uint32_t, const Key& key, const void *data, eckit::Length length) {
+std::unique_ptr RadosStore::archive(const uint32_t, const Key& key, const void *data, eckit::Length length) {
     archivedFields_++;
 
     eckit::PathName dataPath = getDataPath(key);
@@ -67,7 +67,7 @@ std::unique_ptr RadosStore::archive(const uint32_t, const Key& ke
 
     ASSERT(len == length);
 
-    return std::unique_ptr(new RadosFieldLocation(dataUri, position, length));
+    return std::unique_ptr(new RadosFieldLocation(dataUri, position, length));
 }
 
 size_t RadosStore::flush() {
diff --git a/src/fdb5/rados/RadosStore.h b/src/fdb5/rados/RadosStore.h
index e56334838..c9d7db8b4 100644
--- a/src/fdb5/rados/RadosStore.h
+++ b/src/fdb5/rados/RadosStore.h
@@ -49,7 +49,7 @@ class RadosStore : public Store {
     bool exists() const override;
 
     eckit::DataHandle* retrieve(Field& field, Key& remapKey) const override;
-    std::unique_ptr archive(const uint32_t, const Key& key, const void *data, eckit::Length length) override;
+    std::unique_ptr archive(const uint32_t, const Key& key, const void *data, eckit::Length length) override;
 
     void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override;
 
diff --git a/src/fdb5/remote/client/RemoteCatalogue.cc b/src/fdb5/remote/client/RemoteCatalogue.cc
index e4d590637..225f2d215 100644
--- a/src/fdb5/remote/client/RemoteCatalogue.cc
+++ b/src/fdb5/remote/client/RemoteCatalogue.cc
@@ -58,7 +58,7 @@ void RemoteCatalogue::sendArchiveData(uint32_t id, const Key& key, std::unique_p
     dataWrite(Message::Blob, id, payloads);
 }
 
-void RemoteCatalogue::archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) {
+void RemoteCatalogue::archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) {
 
     ASSERT(!datumKey.empty());
     ASSERT(fieldLocation);
diff --git a/src/fdb5/remote/client/RemoteCatalogue.h b/src/fdb5/remote/client/RemoteCatalogue.h
index e8d1954e6..92b7acc0e 100644
--- a/src/fdb5/remote/client/RemoteCatalogue.h
+++ b/src/fdb5/remote/client/RemoteCatalogue.h
@@ -23,7 +23,7 @@ class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public C
 
     // From CatalogueWriter
     const Index& currentIndex() override;
-    void archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) override;
+    void archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) override;
     void overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) override;
     void index(const Key& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) override;
     void reconsolidate() override;
diff --git a/src/fdb5/remote/client/RemoteStore.cc b/src/fdb5/remote/client/RemoteStore.cc
index 41468c3a7..d56e0c444 100644
--- a/src/fdb5/remote/client/RemoteStore.cc
+++ b/src/fdb5/remote/client/RemoteStore.cc
@@ -233,7 +233,7 @@ eckit::DataHandle* RemoteStore::retrieve(Field& field) const {
     return field.dataHandle();
 }
 
-void RemoteStore::archive(const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive) {
+void RemoteStore::archive(const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive) {
 
     ASSERT(!key.empty());
     ASSERT(data);
diff --git a/src/fdb5/remote/client/RemoteStore.h b/src/fdb5/remote/client/RemoteStore.h
index b57d3b79d..a324fc8a4 100644
--- a/src/fdb5/remote/client/RemoteStore.h
+++ b/src/fdb5/remote/client/RemoteStore.h
@@ -73,7 +73,7 @@ class RemoteStore : public Store, public Client {
     bool exists() const override;
 
     eckit::DataHandle* retrieve(Field& field) const override;
-    void archive(const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive) override;
+    void archive(const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive) override;
 
     void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override;
 
@@ -103,7 +103,7 @@ class RemoteStore : public Store, public Client {
 
     std::mutex retrieveMessageMutex_;
     std::mutex locationMutex_;
-    std::map fieldLocation)>> locations_;
+    std::map fieldLocation)>> locations_;
     size_t fieldsArchived_;
     size_t locationsReceived_;
     std::promise promiseArchivalCompleted_;
diff --git a/src/fdb5/remote/server/StoreHandler.cc b/src/fdb5/remote/server/StoreHandler.cc
index c19f7678b..ca921d9f4 100644
--- a/src/fdb5/remote/server/StoreHandler.cc
+++ b/src/fdb5/remote/server/StoreHandler.cc
@@ -169,7 +169,7 @@ void StoreHandler::archiveBlob(const uint32_t clientID, const uint32_t requestID
     
     Store& ss = store(clientID, dbKey);
 
-    std::unique_ptr location = ss.archive(idxKey, charData + s.position(), length - s.position());
+    std::unique_ptr location = ss.archive(idxKey, charData + s.position(), length - s.position());
     Log::status() << "Archiving done: " << ss_key.str() << std::endl;
     
     eckit::Buffer buffer(16 * 1024);
diff --git a/src/fdb5/toc/TocCatalogueWriter.cc b/src/fdb5/toc/TocCatalogueWriter.cc
index 7c3a2a02f..253befa24 100644
--- a/src/fdb5/toc/TocCatalogueWriter.cc
+++ b/src/fdb5/toc/TocCatalogueWriter.cc
@@ -303,7 +303,7 @@ bool TocCatalogueWriter::enabled(const ControlIdentifier& controlIdentifier) con
     return TocCatalogue::enabled(controlIdentifier);
 }
 
-void TocCatalogueWriter::archive(const Key& idxKey, const Key& key, std::shared_ptr fieldLocation) {
+void TocCatalogueWriter::archive(const Key& idxKey, const Key& key, std::shared_ptr fieldLocation) {
     archivedLocations_++;
 
     if (current_.null()) {
diff --git a/src/fdb5/toc/TocCatalogueWriter.h b/src/fdb5/toc/TocCatalogueWriter.h
index 662323ff5..340de26dc 100644
--- a/src/fdb5/toc/TocCatalogueWriter.h
+++ b/src/fdb5/toc/TocCatalogueWriter.h
@@ -71,7 +71,7 @@ class TocCatalogueWriter : public TocCatalogue, public CatalogueWriter {
     void clean() override;
     void close() override;
 
-    void archive(const Key& idxKey, const Key& key, std::shared_ptr fieldLocation) override;
+    void archive(const Key& idxKey, const Key& key, std::shared_ptr fieldLocation) override;
     void reconsolidateIndexesAndTocs();
 
     virtual void print( std::ostream &out ) const override;
diff --git a/src/fdb5/toc/TocStore.cc b/src/fdb5/toc/TocStore.cc
index 8bd75dbfa..b20b9f5ea 100644
--- a/src/fdb5/toc/TocStore.cc
+++ b/src/fdb5/toc/TocStore.cc
@@ -115,7 +115,7 @@ eckit::DataHandle* TocStore::retrieve(Field& field) const {
     return field.dataHandle();
 }
 
-std::unique_ptr TocStore::archive(const Key& idxKey, const void *data, eckit::Length length) {
+std::unique_ptr TocStore::archive(const Key& idxKey, const void *data, eckit::Length length) {
     archivedFields_++;
 
     eckit::PathName dataPath = getDataPath(idxKey);
@@ -128,7 +128,7 @@ std::unique_ptr TocStore::archive(const Key& idxKey, const void *
 
     ASSERT(len == length);
 
-    return std::unique_ptr(new TocFieldLocation(dataPath, position, length, Key()));
+    return std::unique_ptr(new TocFieldLocation(dataPath, position, length, Key()));
 }
 
 size_t TocStore::flush() {
diff --git a/src/fdb5/toc/TocStore.h b/src/fdb5/toc/TocStore.h
index cd2d50736..517895ae6 100644
--- a/src/fdb5/toc/TocStore.h
+++ b/src/fdb5/toc/TocStore.h
@@ -64,7 +64,7 @@ class TocStore : public Store, public TocCommon {
     bool exists() const override;
 
     eckit::DataHandle* retrieve(Field& field) const override;
-    std::unique_ptr archive(const Key& idxKey, const void *data, eckit::Length length) override;
+    std::unique_ptr archive(const Key& idxKey, const void *data, eckit::Length length) override;
 
     void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override;
 
diff --git a/tests/fdb/api/test_auxiliary.cc b/tests/fdb/api/test_auxiliary.cc
index fe83b307c..d559c8b29 100644
--- a/tests/fdb/api/test_auxiliary.cc
+++ b/tests/fdb/api/test_auxiliary.cc
@@ -24,8 +24,8 @@ eckit::PathName writeAuxiliaryData(const eckit::PathName datapath, const std::st
 std::set setup(FDB& fdb) {
     // Setup: Write data, generating auxiliary files using the archive callback
     std::set auxPaths;
-    fdb.registerArchiveCallback([&auxPaths] (const Key& key, const void* data, size_t length, std::future> future) {
-        std::shared_ptr location = future.get();
+    fdb.registerArchiveCallback([&auxPaths] (const Key& key, const void* data, size_t length, std::future> future) {
+        std::shared_ptr location = future.get();
         for (const auto& ext : extensions) {
             auxPaths.insert(writeAuxiliaryData(location->uri().path(), ext));
         }
diff --git a/tests/fdb/api/test_callback.cc b/tests/fdb/api/test_callback.cc
index 60449c883..d27d81d73 100644
--- a/tests/fdb/api/test_callback.cc
+++ b/tests/fdb/api/test_callback.cc
@@ -26,8 +26,8 @@ CASE("Archive and flush callback") {
     std::vector keys;
     bool flushCalled = false;
 
-    fdb.registerArchiveCallback([&map] (const Key& key, const void* data, size_t length, std::future> future) {
-        std::shared_ptr location = future.get();
+    fdb.registerArchiveCallback([&map] (const Key& key, const void* data, size_t length, std::future> future) {
+        std::shared_ptr location = future.get();
         map[key] = location->fullUri();
     });
 
diff --git a/tests/fdb/daos/test_daos_catalogue.cc b/tests/fdb/daos/test_daos_catalogue.cc
index b8527ab9d..7846b35d8 100644
--- a/tests/fdb/daos/test_daos_catalogue.cc
+++ b/tests/fdb/daos/test_daos_catalogue.cc
@@ -308,7 +308,7 @@ CASE("DaosCatalogue tests") {
 
         fdb5::DaosStore dstore{db_key, config};
         fdb5::Store& store = static_cast(dstore);
-        std::unique_ptr loc(store.archive(index_key, data, sizeof(data)));
+        std::unique_ptr loc(store.archive(index_key, data, sizeof(data)));
         /// @todo: there are two cont create with label here
         /// @todo: again, daos_fini happening before cont and pool close
 
@@ -399,7 +399,7 @@ CASE("DaosCatalogue tests") {
 
         fdb5::TocStore tstore{db_key, config};
         fdb5::Store& store = static_cast(tstore);
-        std::unique_ptr loc(store.archive(index_key, data, sizeof(data)));
+        std::unique_ptr loc(store.archive(index_key, data, sizeof(data)));
         /// @todo: there are two cont create with label here
         /// @todo: again, daos_fini happening before cont and pool close
 
diff --git a/tests/fdb/daos/test_daos_store.cc b/tests/fdb/daos/test_daos_store.cc
index 49d0726d7..5ab8651a0 100644
--- a/tests/fdb/daos/test_daos_store.cc
+++ b/tests/fdb/daos/test_daos_store.cc
@@ -185,7 +185,7 @@ CASE("DaosStore tests") {
         /// DaosManager is configured with client config from the file
         fdb5::DaosStore dstore{db_key, config};
         fdb5::Store& store = dstore;
-        std::unique_ptr loc(store.archive(index_key, data, sizeof(data)));
+        std::unique_ptr loc(store.archive(index_key, data, sizeof(data)));
         /// @todo: two cont create with label happen here
         /// @todo: again, daos_fini happening before cont and pool close
 
@@ -257,7 +257,7 @@ CASE("DaosStore tests") {
 
         fdb5::DaosStore dstore{db_key, config};
         fdb5::Store& store = static_cast(dstore);
-        std::unique_ptr loc(store.archive(index_key, data, sizeof(data)));
+        std::unique_ptr loc(store.archive(index_key, data, sizeof(data)));
         /// @todo: there are two cont create with label here
         /// @todo: again, daos_fini happening before cont and pool close
 

From df121a05dce40c60679ad0e92bddde7daa540fa1 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 6 Sep 2024 08:31:33 +0100
Subject: [PATCH 132/186] improved composition of callback + async archival

---
 src/fdb5/database/ArchiveVisitor.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/fdb5/database/ArchiveVisitor.cc b/src/fdb5/database/ArchiveVisitor.cc
index b011b1180..4ad2fcce0 100644
--- a/src/fdb5/database/ArchiveVisitor.cc
+++ b/src/fdb5/database/ArchiveVisitor.cc
@@ -34,9 +34,12 @@ bool ArchiveVisitor::selectDatum(const TypedKey& datumKey, const TypedKey& fullC
     const Key idxKey = catalogue()->currentIndexKey();
 
     std::promise> p;
+    auto c = std::async(std::launch::async, [&p, this] {
+            callback_(initialFieldKey_, data_, size_, p.get_future());
+        });
     store()->archive(idxKey, data_, size_,
         std::bind(&ArchiveVisitor::callbacks, this, catalogue(), idxKey, datumKey.canonical(), &p, std::placeholders::_1));
-    callback_(initialFieldKey_, data_, size_, p.get_future());
+    c.wait();
 
     return true;
 }

From b39ee3d8d3aa5b712230ca6385662c72b409e6d0 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 11 Sep 2024 07:25:27 +0200
Subject: [PATCH 133/186] fix callback composition in archival

---
 VERSION                                    |  2 +-
 src/fdb5/database/ArchiveVisitor.cc        | 10 ++++++----
 src/fdb5/message/MessageArchiver.cc        |  2 +-
 src/fdb5/remote/Connection.cc              |  2 +-
 src/fdb5/remote/Connection.h               |  1 +
 src/fdb5/remote/client/Client.cc           | 10 +++++++---
 src/fdb5/remote/client/ClientConnection.cc |  3 +--
 src/fdb5/remote/client/RemoteStore.cc      |  2 +-
 tests/fdb/type/test_toKey.cc               |  4 ++--
 9 files changed, 21 insertions(+), 15 deletions(-)

diff --git a/VERSION b/VERSION
index 4c263eda1..706c863b3 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.128
\ No newline at end of file
+5.14.0-rc1
\ No newline at end of file
diff --git a/src/fdb5/database/ArchiveVisitor.cc b/src/fdb5/database/ArchiveVisitor.cc
index 4ad2fcce0..61340c62f 100644
--- a/src/fdb5/database/ArchiveVisitor.cc
+++ b/src/fdb5/database/ArchiveVisitor.cc
@@ -26,6 +26,7 @@ ArchiveVisitor::ArchiveVisitor(Archiver& owner, const Key& initialFieldKey, cons
 void ArchiveVisitor::callbacks(fdb5::CatalogueWriter* catalogue, const Key& idxKey, const Key& datumKey, std::promise>* p, std::shared_ptr fieldLocation) {
     p->set_value(fieldLocation);
     catalogue->archive(idxKey, datumKey, std::move(fieldLocation));
+    delete(p);
 }
 
 bool ArchiveVisitor::selectDatum(const TypedKey& datumKey, const TypedKey& fullComputedKey) {
@@ -33,12 +34,13 @@ bool ArchiveVisitor::selectDatum(const TypedKey& datumKey, const TypedKey& fullC
     checkMissingKeys(fullComputedKey);
     const Key idxKey = catalogue()->currentIndexKey();
 
-    std::promise> p;
+    std::promise>* p = new std::promise>();
+
     auto c = std::async(std::launch::async, [&p, this] {
-            callback_(initialFieldKey_, data_, size_, p.get_future());
-        });
+        callback_(initialFieldKey_, data_, size_, p->get_future());
+    });
     store()->archive(idxKey, data_, size_,
-        std::bind(&ArchiveVisitor::callbacks, this, catalogue(), idxKey, datumKey.canonical(), &p, std::placeholders::_1));
+        std::bind(&ArchiveVisitor::callbacks, this, catalogue(), idxKey, datumKey.canonical(), p, std::placeholders::_1));
     c.wait();
 
     return true;
diff --git a/src/fdb5/message/MessageArchiver.cc b/src/fdb5/message/MessageArchiver.cc
index 814b8d04c..02997630d 100644
--- a/src/fdb5/message/MessageArchiver.cc
+++ b/src/fdb5/message/MessageArchiver.cc
@@ -187,7 +187,7 @@ eckit::Length MessageArchiver::archive(eckit::DataHandle& source) {
             messageToKey(msg, key);
 
             LOG_DEBUG_LIB(LibFdb5) << "Archiving message "
-                                   << " key: " << key_ << " data: " << msg.data() << " length:" << msg.length()
+                                   << " key: " << key << " length:" << msg.length()
                                    << std::endl;
 
             ASSERT(key.match(key_));
diff --git a/src/fdb5/remote/Connection.cc b/src/fdb5/remote/Connection.cc
index f3df2dffc..2b92d891a 100644
--- a/src/fdb5/remote/Connection.cc
+++ b/src/fdb5/remote/Connection.cc
@@ -108,7 +108,7 @@ void Connection::write(remote::Message msg, bool control, uint32_t clientID, uin
 
     // std::cout << "WRITE [" << "endpoint=" << ((control || single_) ? controlSocket() : dataSocket()).remotePort() << ",message=" << message.message << ",clientID=" << message.clientID() << ",requestID=" << message.requestID << ",payload=" << message.payloadSize << "]" << std::endl;
 
-    LOG_DEBUG_LIB(LibFdb5) << "Connection::write [message=" << msg << ",clientID=" << message.clientID() << ",requestID=" << requestID << ",data=" << data.size() << ",payload=" << payloadLength << "]" << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Connection::write [message=" << msg << ",clientID=" << message.clientID() << ",control=" << control << ",requestID=" << requestID << ",data=" << data.size() << ",payload=" << payloadLength << "]" << std::endl;
 
     std::lock_guard lock((control || single_) ? controlMutex_ : dataMutex_);
     writeUnsafe(control, &message, sizeof(message));
diff --git a/src/fdb5/remote/Connection.h b/src/fdb5/remote/Connection.h
index c82347789..8f6ba872c 100644
--- a/src/fdb5/remote/Connection.h
+++ b/src/fdb5/remote/Connection.h
@@ -57,6 +57,7 @@ class Connection : eckit::NonCopyable {
 
     std::mutex controlMutex_;
     std::mutex dataMutex_;
+
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index 44386a750..e57aec747 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -55,8 +55,9 @@ void Client::controlWriteCheckResponse(Message msg, uint32_t requestID, bool dat
         data.push_back(std::make_pair(payload, payloadLength));
     }
 
-    eckit::Buffer buf = connection_.controlWrite(*this, msg, requestID, dataListener, data).get();
-    ASSERT(buf.size() == 0);
+    std::future f = connection_.controlWrite(*this, msg, requestID, dataListener, data);
+    f.wait();
+    ASSERT(f.get().size() == 0);
 }
 
 eckit::Buffer Client::controlWriteReadResponse(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) {
@@ -69,7 +70,10 @@ eckit::Buffer Client::controlWriteReadResponse(Message msg, uint32_t requestID,
     if (payloadLength) {
         data.push_back(std::make_pair(payload, payloadLength));
     }
-    return eckit::Buffer{std::move(connection_.controlWrite(*this, msg, requestID, false, data).get())};
+
+    std::future f = connection_.controlWrite(*this, msg, requestID, false, data);
+    f.wait();
+    return eckit::Buffer{f.get()};
 }
 
 void Client::dataWrite(remote::Message msg, uint32_t requestID, std::vector> data) {
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index dd82f63e4..c710f1a40 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -225,7 +225,7 @@ void ClientConnection::dataWrite(DataWriteRequest& r) {
 
 void ClientConnection::dataWrite(Client& client, remote::Message msg, uint32_t requestID, std::vector> data) {
 
-    static size_t maxQueueLength = eckit::Resource("fdbDataWriteQueueLength;$FDB_DATA_WRITE_QUEUE_LENGTH", 32);
+    static size_t maxQueueLength = 320; // eckit::Resource("fdbDataWriteQueueLength;$FDB_DATA_WRITE_QUEUE_LENGTH", 320);
     auto it = clients_.find(client.clientId());
     ASSERT(it != clients_.end());
 
@@ -402,7 +402,6 @@ void ClientConnection::listeningControlThreadLoop() {
                         promises_.erase(pp);
                         handled = true;
                     } else {
-
                         Client* client = nullptr;
                         {
                             std::lock_guard lock(clientsMutex_);
diff --git a/src/fdb5/remote/client/RemoteStore.cc b/src/fdb5/remote/client/RemoteStore.cc
index d56e0c444..ab4c882cb 100644
--- a/src/fdb5/remote/client/RemoteStore.cc
+++ b/src/fdb5/remote/client/RemoteStore.cc
@@ -460,7 +460,7 @@ eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation, c
 
     uint32_t id = generateRequestID();
 
-    static size_t queueSize = eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200);
+    static size_t queueSize = 320; // eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200);
                 // auto id = retrieveMessageQueues_.find(requestID);
                 // ASSERT (it != retrieveMessageQueues_.end());
                 // it->second->emplace(std::make_pair(message, std::move(payload)));
diff --git a/tests/fdb/type/test_toKey.cc b/tests/fdb/type/test_toKey.cc
index 122b8a11b..b6b013f91 100644
--- a/tests/fdb/type/test_toKey.cc
+++ b/tests/fdb/type/test_toKey.cc
@@ -272,8 +272,8 @@ CASE( "Date - string ctor - expansion" ) {
     eckit::Translator t;
 
     EXPECT(key.canonicalValue("date") == t(now.yyyymmdd()));
-    std::cout << key.valuesToString() << std::endl;
-    std::cout << ("od:0001:oper:ofb:"+t(now.yyyymmdd())+":0000:mhs:3001") << std::endl;
+    // std::cout << key.valuesToString() << std::endl;
+    // std::cout << ("od:0001:oper:ofb:"+t(now.yyyymmdd())+":0000:mhs:3001") << std::endl;
     
     EXPECT(key.valuesToString() == "od:0001:oper:ofb:"+t(now.yyyymmdd())+":0000:mhs:3001");
 

From b6e225f4162f03aa55285c5d243b6ca1e04d258d Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 12 Sep 2024 21:52:03 +0100
Subject: [PATCH 134/186] fix remote flush

---
 src/fdb5/database/ArchiveVisitor.cc        |  14 +--
 src/fdb5/database/ArchiveVisitor.h         |   2 +-
 src/fdb5/database/Archiver.cc              |   2 +
 src/fdb5/database/Archiver.h               |   1 +
 src/fdb5/remote/client/Client.cc           |   6 +-
 src/fdb5/remote/client/ClientConnection.cc |  57 ++++++-----
 src/fdb5/remote/client/ClientConnection.h  |   5 +-
 src/fdb5/remote/client/RemoteCatalogue.cc  |   2 +-
 src/fdb5/remote/client/RemoteStore.cc      | 114 +++++++--------------
 src/fdb5/remote/client/RemoteStore.h       |  89 ++++++++++++++--
 src/fdb5/remote/server/ServerConnection.cc |   2 +-
 src/fdb5/remote/server/StoreHandler.cc     |  12 ++-
 12 files changed, 175 insertions(+), 131 deletions(-)

diff --git a/src/fdb5/database/ArchiveVisitor.cc b/src/fdb5/database/ArchiveVisitor.cc
index 61340c62f..21d2967a8 100644
--- a/src/fdb5/database/ArchiveVisitor.cc
+++ b/src/fdb5/database/ArchiveVisitor.cc
@@ -23,10 +23,9 @@ ArchiveVisitor::ArchiveVisitor(Archiver& owner, const Key& initialFieldKey, cons
     callback_(callback){
 }
 
-void ArchiveVisitor::callbacks(fdb5::CatalogueWriter* catalogue, const Key& idxKey, const Key& datumKey, std::promise>* p, std::shared_ptr fieldLocation) {
+void ArchiveVisitor::callbacks(fdb5::CatalogueWriter* catalogue, const Key& idxKey, const Key& datumKey, std::shared_ptr>> p, std::shared_ptr fieldLocation) {
     p->set_value(fieldLocation);
     catalogue->archive(idxKey, datumKey, std::move(fieldLocation));
-    delete(p);
 }
 
 bool ArchiveVisitor::selectDatum(const TypedKey& datumKey, const TypedKey& fullComputedKey) {
@@ -34,14 +33,15 @@ bool ArchiveVisitor::selectDatum(const TypedKey& datumKey, const TypedKey& fullC
     checkMissingKeys(fullComputedKey);
     const Key idxKey = catalogue()->currentIndexKey();
 
-    std::promise>* p = new std::promise>();
+    std::shared_ptr>> p = std::make_shared>>(std::promise>());
 
-    auto c = std::async(std::launch::async, [&p, this] {
-        callback_(initialFieldKey_, data_, size_, p->get_future());
-    });
+    // auto c = std::async(std::launch::async, [p, this] {
+    //     callback_(initialFieldKey_, data_, size_, p->get_future());
+    // });
     store()->archive(idxKey, data_, size_,
         std::bind(&ArchiveVisitor::callbacks, this, catalogue(), idxKey, datumKey.canonical(), p, std::placeholders::_1));
-    c.wait();
+    callback_(initialFieldKey_, data_, size_, p->get_future());
+    // c.wait();
 
     return true;
 }
diff --git a/src/fdb5/database/ArchiveVisitor.h b/src/fdb5/database/ArchiveVisitor.h
index f6c6466f6..b2808883b 100644
--- a/src/fdb5/database/ArchiveVisitor.h
+++ b/src/fdb5/database/ArchiveVisitor.h
@@ -40,7 +40,7 @@ class ArchiveVisitor : public BaseArchiveVisitor {
 
 private: // methods
 
-    void callbacks(fdb5::CatalogueWriter* catalogue, const Key& idxKey, const Key& datumKey, std::promise>* p, std::shared_ptr fieldLocation);
+    void callbacks(fdb5::CatalogueWriter* catalogue, const Key& idxKey, const Key& datumKey, std::shared_ptr>> p, std::shared_ptr fieldLocation);
 
 private: // members
 
diff --git a/src/fdb5/database/Archiver.cc b/src/fdb5/database/Archiver.cc
index 952903f24..83a3e380e 100644
--- a/src/fdb5/database/Archiver.cc
+++ b/src/fdb5/database/Archiver.cc
@@ -40,6 +40,7 @@ void Archiver::archive(const Key& key, const void* data, size_t len) {
 
 void Archiver::archive(const Key& key, BaseArchiveVisitor& visitor) {
 
+    std::lock_guard lock(flushMutex_);
     visitor.rule(nullptr);
     
     dbConfig_.schema().expand(key, visitor);
@@ -55,6 +56,7 @@ void Archiver::archive(const Key& key, BaseArchiveVisitor& visitor) {
 }
 
 void Archiver::flush() {
+    std::lock_guard lock(flushMutex_);
     for (auto i = databases_.begin(); i != databases_.end(); ++i) {
         // flush the store, pass the number of flushed fields to the catalogue
         i->second.catalogue_->flush(i->second.store_->flush());
diff --git a/src/fdb5/database/Archiver.h b/src/fdb5/database/Archiver.h
index d1554062d..efa8e2556 100644
--- a/src/fdb5/database/Archiver.h
+++ b/src/fdb5/database/Archiver.h
@@ -83,6 +83,7 @@ class Archiver : public eckit::NonCopyable {
 
     Database* db_;
 
+    std::mutex flushMutex_;
     const ArchiveCallback& callback_;
 };
 
diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index e57aec747..4436b0113 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -13,6 +13,7 @@
 
 #include "fdb5/remote/client/Client.h"
 #include "fdb5/remote/client/ClientConnectionRouter.h"
+// #include 
 
 namespace fdb5::remote {
 
@@ -20,7 +21,10 @@ namespace fdb5::remote {
 
 void Client::setClientID() {
     static std::mutex idMutex_;
-    static uint32_t clientId_ = 0;
+    // static std::random_device rd;
+    // static std::mt19937 mt(rd());
+    // static std::uniform_int_distribution dist(0, 1000);
+    static uint32_t clientId_ = 0; // dist(mt)*1000;
 
     std::lock_guard lock(idMutex_);
     id_ = ++clientId_;
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index c710f1a40..306acba70 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -103,7 +103,7 @@ bool ClientConnection::remove(uint32_t clientID) {
             // if connection is already down, no need to escalate 
         }
 
-        ClientConnectionRouter::instance().deregister(*this);
+        // ClientConnectionRouter::instance().deregister(*this);
     }
 
     return clients_.empty();
@@ -165,19 +165,22 @@ bool ClientConnection::connect(bool singleAttempt) {
 
 void ClientConnection::disconnect() {
 
+    std::lock_guard lock(clientsMutex_);
     ASSERT(clients_.empty());
     if (connected_) {
 
-        if (dataWriteFuture_.valid()) {
-            dataWriteFuture_.wait();
-        }
-
-        if (listeningControlThread_.joinable()) {
-            listeningControlThread_.join();
+        // if (dataWriteFuture_.valid()) {
+        //     dataWriteFuture_.wait();
+        // }
+        if (dataWriteThread_.joinable()) {
+            dataWriteThread_.join();
         }
         if (listeningDataThread_.joinable()) {
             listeningDataThread_.join();
         }
+        if (listeningControlThread_.joinable()) {
+            listeningControlThread_.join();
+        }
 
         // Close both the control and data connections
         controlClient_.close();
@@ -220,30 +223,28 @@ std::future ClientConnection::controlWrite(Client& client, Messag
 }
 
 void ClientConnection::dataWrite(DataWriteRequest& r) {
+    // std::cout << "Write " << r.msg_ << ",clientID=" << r.client_->clientId() << ",requestID=" << r.id_ << std::endl;
     Connection::write(r.msg_, false, r.client_->clientId(), r.id_, r.data_.data(), r.data_.size());
 }
 
 void ClientConnection::dataWrite(Client& client, remote::Message msg, uint32_t requestID, std::vector> data) {
 
-    static size_t maxQueueLength = 320; // eckit::Resource("fdbDataWriteQueueLength;$FDB_DATA_WRITE_QUEUE_LENGTH", 320);
-    auto it = clients_.find(client.clientId());
-    ASSERT(it != clients_.end());
-
+    static size_t maxQueueLength = eckit::Resource("fdbDataWriteQueueLength;$FDB_DATA_WRITE_QUEUE_LENGTH", 320);
     {
-        std::lock_guard lock(dataWriteQueueMutex_);
-
-        if (!dataWriteFuture_.valid()) {
-
-            {
-                // Reset the queue after previous done/errors
-                // std::lock_guard lock(dataWriteQueueMutex_);
-                // TODO
-                ASSERT(!dataWriteQueue_);
-
-                dataWriteQueue_.reset(new eckit::Queue{maxQueueLength});
-            }
-
-            dataWriteFuture_ = std::async(std::launch::async, [this] { return dataWriteThreadLoop(); });
+        // retrieve or add client to the list
+        std::lock_guard lock(clientsMutex_);
+        auto it = clients_.find(client.clientId());
+        ASSERT(it != clients_.end());        
+    }
+    {
+        std::lock_guard lock(dataWriteMutex_);
+        if (!dataWriteThread_.joinable()) {
+            // Reset the queue after previous done/errors
+            ASSERT(!dataWriteQueue_);
+
+            dataWriteQueue_.reset(new eckit::Queue{maxQueueLength});
+            dataWriteThread_ = std::thread([this] { dataWriteThreadLoop(); });
+            // dataWriteFuture_ = std::async(std::launch::async, [this] { return dataWriteThreadLoop(); });
         }
     }
     uint32_t payloadLength = 0;
@@ -409,7 +410,7 @@ void ClientConnection::listeningControlThreadLoop() {
                             auto it = clients_.find(hdr.clientID());
                             if (it == clients_.end()) {
                                 std::stringstream ss;
-                                ss << "ERROR: Received [clientID="<< hdr.clientID() << ",requestID="<< hdr.requestID << ",message=" << hdr.message << ",payload=" << hdr.payloadSize << "]" << std::endl;
+                                ss << "ERROR: connection=" << controlEndpoint_ << " received [clientID="<< hdr.clientID() << ",requestID="<< hdr.requestID << ",message=" << hdr.message << ",payload=" << hdr.payloadSize << "]" << std::endl;
                                 ss << "Unexpected answer for clientID recieved (" << hdr.clientID() << "). ABORTING";
                                 eckit::Log::status() << ss.str() << std::endl;
                                 eckit::Log::error() << "Retrieving... " << ss.str() << std::endl;
@@ -428,7 +429,7 @@ void ClientConnection::listeningControlThreadLoop() {
 
                     if (!handled) {
                         std::stringstream ss;
-                        ss << "ERROR: Unexpected message recieved [message=" << hdr.message << ",clientID=" << hdr.clientID() << ",requestID=" << hdr.requestID << "]. ABORTING";
+                        ss << "ERROR: connection=" << controlEndpoint_ << "Unexpected message recieved [message=" << hdr.message << ",clientID=" << hdr.clientID() << ",requestID=" << hdr.requestID << "]. ABORTING";
                         eckit::Log::status() << ss.str() << std::endl;
                         eckit::Log::error() << "Client Retrieving... " << ss.str() << std::endl;
                         throw eckit::SeriousBug(ss.str(), Here());
@@ -493,7 +494,7 @@ void ClientConnection::listeningDataThreadLoop() {
 
                     if (!handled) {
                         std::stringstream ss;
-                        ss << "ERROR: Unexpected message recieved (" << hdr.message << "). ABORTING";
+                        ss << "ERROR: DATA connection=" << controlEndpoint_ << " Unexpected message recieved (" << hdr.message << "). ABORTING";
                         eckit::Log::status() << ss.str() << std::endl;
                         eckit::Log::error() << "Client Retrieving... " << ss.str() << std::endl;
                         throw eckit::SeriousBug(ss.str(), Here());
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index b8a772c64..f2adccd74 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -107,9 +107,10 @@ class ClientConnection : protected Connection {
     std::mutex promisesMutex_;
     std::map> promises_;
 
-    std::mutex dataWriteQueueMutex_;
+    std::mutex dataWriteMutex_;
     std::unique_ptr> dataWriteQueue_;
-    std::future dataWriteFuture_;
+    std::thread dataWriteThread_;
+    // std::future dataWriteFuture_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/client/RemoteCatalogue.cc b/src/fdb5/remote/client/RemoteCatalogue.cc
index 225f2d215..664c7ca6d 100644
--- a/src/fdb5/remote/client/RemoteCatalogue.cc
+++ b/src/fdb5/remote/client/RemoteCatalogue.cc
@@ -25,6 +25,7 @@ RemoteCatalogue::RemoteCatalogue(const Key& key, const Config& config):
     Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port")), ""),
     config_(config), schema_(nullptr), numLocations_(0) {
 
+    // std::cout << "Catalogue  " << clientId() << "  " << key << std::endl;
     loadSchema();
 }
 
@@ -109,7 +110,6 @@ const Schema& RemoteCatalogue::schema() const {
 void RemoteCatalogue::flush(size_t archivedFields) {
 
     std::lock_guard lock(archiveMutex_);
-
     ASSERT(archivedFields == numLocations_);
 
     // Flush only does anything if there is an ongoing archive();
diff --git a/src/fdb5/remote/client/RemoteStore.cc b/src/fdb5/remote/client/RemoteStore.cc
index ab4c882cb..8d068c5b3 100644
--- a/src/fdb5/remote/client/RemoteStore.cc
+++ b/src/fdb5/remote/client/RemoteStore.cc
@@ -196,17 +196,16 @@ std::vector> storeEndpoints(const C
 
 RemoteStore::RemoteStore(const Key& dbKey, const Config& config) :
     Client(storeEndpoints(config)),
-    dbKey_(dbKey), config_(config),
+    dbKey_(dbKey), config_(config)
     // retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
-    fieldsArchived_(0), locationsReceived_(0) {}
+    {}
 
 // this is used only in retrieval, with an URI already referring to an accessible Store
 RemoteStore::RemoteStore(const eckit::URI& uri, const Config& config) :
     Client(eckit::net::Endpoint(uri.hostport()), uri.hostport()),
-    dbKey_(Key()), config_(config),
+    dbKey_(Key()), config_(config)
     // retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
-    fieldsArchived_(0), locationsReceived_(0) { 
-
+    { 
     // no need to set the local_ flag on the read path
     ASSERT(uri.scheme() == "fdb");
 }
@@ -215,8 +214,9 @@ RemoteStore::~RemoteStore() {
     // If we have launched a thread with an async and we manage to get here, this is
     // an error. n.b. if we don't do something, we will block in the destructor
     // of std::future.
-    if (archivalCompleted_.valid() || !locations_.empty()) {
-        Log::error() << "Attempting to destruct RemoteStore with active archival - location to receive: " << locations_.size() << " archivalCompleted_.valid() " << archivalCompleted_.valid() << std::endl;
+    if (!locations_.complete()) {
+    // if (archivalCompleted_.valid() || !locations_.empty()) {
+        Log::error() << "Attempting to destruct RemoteStore with active archival" << std::endl; //  - location to receive: " << locations_.size() << " archivalCompleted_.valid() " << archivalCompleted_.valid() << std::endl;
         eckit::Main::instance().terminate();
     }
 }
@@ -241,19 +241,13 @@ void RemoteStore::archive(const Key& key, const void *data, eckit::Length length
 
     uint32_t id = generateRequestID();
     {   // send the archival request
-        std::lock_guard lock(archiveMutex_);
-        if (fieldsArchived_ == 0) { // if this is the first archival request, notify the server
-            ASSERT(locations_.size() == 0);
-
+        std::lock_guard lock(locations_.mutex());
+        if (locations_.archived() == 0) { // if this is the first archival request, notify the server
             controlWriteCheckResponse(Message::Store, id, true);
         }
     }
-    {   // store the callback, associated with the request id - to be done BEFORE sending the data
-        std::lock_guard lock(locationMutex_);
-        locations_[id] = catalogue_archive;
-    }
-    fieldsArchived_++;
-
+    // store the callback, associated with the request id - to be done BEFORE sending the data
+    locations_.archive(id, catalogue_archive);
 
     Buffer keyBuffer(4096);
     MemoryStream keyStream(keyBuffer);
@@ -266,7 +260,7 @@ void RemoteStore::archive(const Key& key, const void *data, eckit::Length length
 
     dataWrite(Message::Blob, id, payloads);
 
-    eckit::Log::status() << "Field " << fieldsArchived_ << " enqueued for store archival" << std::endl;
+    eckit::Log::status() << "Field " << locations_.archived() << " enqueued for store archival" << std::endl;
 }
 
 bool RemoteStore::open() {
@@ -276,45 +270,25 @@ bool RemoteStore::open() {
 size_t RemoteStore::flush() {
 
     // Flush only does anything if there is an ongoing archive();
-    if (fieldsArchived_ == 0) {
+    if (locations_.archived() == 0) {
         return 0;
     }
 
-    LOG_DEBUG_LIB(LibFdb5) << " RemoteStore::flush - fieldsArchived_ " << fieldsArchived_ << " locationsReceived_ " << locationsReceived_ << std::endl;
-
-    size_t locations;
-    bool wait = true;
-    {
-        std::lock_guard lock(locationMutex_);
-        if ((fieldsArchived_ > locationsReceived_) || !locations_.empty()) {
-            promiseArchivalCompleted_ = std::promise{};
-            archivalCompleted_ = promiseArchivalCompleted_.get_future();
-        } else {
-            wait = false;
-        }
-    }
+    // remote side reported that all fields are flushed... now wait for field locations
+    // size_t locations;
+    bool complete = locations_.complete();
 
-    if (wait) {
-        // wait for archival completion (received all fieldLocations)
-        archivalCompleted_.wait();
-        locations = archivalCompleted_.get();
-    } else {
-        locations = locationsReceived_;
-    }
+    size_t locations = complete ? locations_.archived() : locations_.wait();
 
-    ASSERT(locations_.empty());
-    if (locations > 0) {
-        Buffer sendBuf(1024);
-        MemoryStream s(sendBuf);
-        s << locations;
+    Buffer sendBuf(1024);
+    MemoryStream s(sendBuf);
+    s << locations;
 
-        LOG_DEBUG_LIB(LibFdb5) << " RemoteStore::flush - flushing " << locations << " fields" << std::endl;
-        // The flush call is blocking
-        controlWriteCheckResponse(Message::Flush, generateRequestID(), false, sendBuf, s.position());
-    }
+    LOG_DEBUG_LIB(LibFdb5) << " RemoteStore::flush - flushing " << locations << " fields" << std::endl;
+    // The flush call is blocking
+    controlWriteCheckResponse(Message::Flush, generateRequestID(), false, sendBuf, s.position());
 
-    fieldsArchived_ = 0;
-    locationsReceived_ = 0;
+    locations_.reset();
 
     return locations;
 }
@@ -381,28 +355,14 @@ bool RemoteStore::handle(Message message, bool control, uint32_t requestID, ecki
     switch (message) {
     
         case Message::Store: { // received a Field location from the remote store, can forward to the archiver for the indexing
-            std::lock_guard lock(locationMutex_);
-
-            auto it = locations_.find(requestID);
-            if (it != locations_.end()) {
-                MemoryStream s(payload);
-                std::unique_ptr location(eckit::Reanimator::reanimate(s));
-                if (defaultEndpoint().empty()) {
-                    it->second(std::move(location));
-                } else {
-                    std::unique_ptr remoteLocation = std::unique_ptr(new RemoteFieldLocation(eckit::net::Endpoint{defaultEndpoint()}, *location));
-                    it->second(std::move(remoteLocation));
-                }
-
-                locations_.erase(it);
-                locationsReceived_++;
-
-                if (archivalCompleted_.valid() && (fieldsArchived_ == locationsReceived_) && locations_.empty()) {
-                    promiseArchivalCompleted_.set_value(locationsReceived_);
-                }
-                return true;
+            MemoryStream s(payload);
+            std::unique_ptr location(eckit::Reanimator::reanimate(s));
+            if (defaultEndpoint().empty()) {
+                return locations_.location(requestID, std::move(location));
+            } else {
+                std::unique_ptr remoteLocation = std::unique_ptr(new RemoteFieldLocation(eckit::net::Endpoint{defaultEndpoint()}, *location));
+                return locations_.location(requestID, std::move(remoteLocation));
             }
-            return false;
         }
         case Message::Blob: {
             auto it = messageQueues_.find(requestID);
@@ -430,12 +390,12 @@ bool RemoteStore::handle(Message message, bool control, uint32_t requestID, ecki
                 messageQueues_.erase(it);
 
             } else {
-                auto it = locations_.find(requestID);
-                if (it != locations_.end()) {
-//                    archiver_->error(std::move(payload), controlEndpoint());
-                } else {
-                    // retrieveMessageQueue_.emplace(message, std::move(payload));
-                }
+//                 auto it = locations_.find(requestID);
+//                 if (it != locations_.end()) {
+// //                    archiver_->error(std::move(payload), controlEndpoint());
+//                 } else {
+//                     // retrieveMessageQueue_.emplace(message, std::move(payload));
+//                 }
             }
             return true;
         }
diff --git a/src/fdb5/remote/client/RemoteStore.h b/src/fdb5/remote/client/RemoteStore.h
index a324fc8a4..cedba9e50 100644
--- a/src/fdb5/remote/client/RemoteStore.h
+++ b/src/fdb5/remote/client/RemoteStore.h
@@ -24,10 +24,81 @@ namespace fdb5::remote {
 
 // class RemoteStoreArchiver;
 
+class Locations {
+
+public:
+    Locations() : fieldsArchived_(0), locationsReceived_(0) {}
+
+    std::mutex& mutex() {
+        return locationMutex_;
+    }
+
+    size_t archived() { return fieldsArchived_; }
+
+    void archive(uint32_t requestID, std::function fieldLocation)> catalogue_archive) {
+        std::lock_guard lock(locationMutex_);
+        locations_[requestID] = catalogue_archive;
+        fieldsArchived_++;
+        ASSERT(fieldsArchived_-locationsReceived_ == locations_.size());
+    }
+
+    bool location(uint32_t requestID, std::unique_ptr location) {
+
+        std::lock_guard lock(locationMutex_);
+        auto it = locations_.find(requestID);
+        if (it == locations_.end()) {
+            return false;
+        }
+        it->second(std::move(location));
+
+        locations_.erase(it);
+        locationsReceived_++;
+
+        ASSERT(fieldsArchived_-locationsReceived_ == locations_.size());
+        if (archivalCompleted_.valid() && (fieldsArchived_ == locationsReceived_)) {
+            promiseArchivalCompleted_.set_value(locationsReceived_);
+        }
+        return true;
+    }
+
+    bool complete() {
+        std::lock_guard lock(locationMutex_);
+
+        ASSERT(fieldsArchived_-locationsReceived_ == locations_.size());
+        if (fieldsArchived_ == locationsReceived_)
+            return true;
+
+        promiseArchivalCompleted_ = std::promise{};
+        archivalCompleted_ = promiseArchivalCompleted_.get_future();
+        return false;
+    }
+
+    size_t wait() {
+        archivalCompleted_.wait();
+        return archivalCompleted_.get();
+    }
+
+    void reset() {
+        std::lock_guard lock(locationMutex_);
+        ASSERT(fieldsArchived_-locationsReceived_ == locations_.size());
+        fieldsArchived_ = locations_.size();
+        locationsReceived_ = 0;
+    }
+
+private:
+
+    std::mutex locationMutex_;
+    std::map fieldLocation)>> locations_;
+    size_t fieldsArchived_;
+    size_t locationsReceived_;
+    std::promise promiseArchivalCompleted_;
+    std::future archivalCompleted_;
+
+};
+
 //----------------------------------------------------------------------------------------------------------------------
 
 /// Store that connects to a remote Store
-
 class RemoteStore : public Store, public Client {
 
 public: // types
@@ -102,13 +173,15 @@ class RemoteStore : public Store, public Client {
     // MessageQueue retrieveMessageQueue_;
 
     std::mutex retrieveMessageMutex_;
-    std::mutex locationMutex_;
-    std::map fieldLocation)>> locations_;
-    size_t fieldsArchived_;
-    size_t locationsReceived_;
-    std::promise promiseArchivalCompleted_;
-    std::future archivalCompleted_;
-    std::mutex archiveMutex_;
+
+    Locations locations_;
+    // std::recursive_mutex locationMutex_;
+    // std::map fieldLocation)>> locations_;
+    // size_t fieldsArchived_;
+    // size_t locationsReceived_;
+    // std::promise promiseArchivalCompleted_;
+    // std::future archivalCompleted_;
+    // // std::mutex archiveMutex_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index 8472c0408..4a989048e 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -66,7 +66,7 @@ ServerConnection::ServerConnection(eckit::net::TCPSocket& socket, const Config&
         Connection(), config_(config),
         dataListenHostname_(config.getString("dataListenHostname", "")),
         readLocationQueue_(eckit::Resource("fdbRetrieveQueueSize", 10000)), 
-        archiveQueue_(eckit::Resource("fdbServerMaxQueueSize", 32)),
+        archiveQueue_(eckit::Resource("fdbServerMaxQueueSize", 320)),
         controlSocket_(socket), numControlConnection_(0), numDataConnection_(0),
         dataSocket_(nullptr), dataListener_(0) {
 
diff --git a/src/fdb5/remote/server/StoreHandler.cc b/src/fdb5/remote/server/StoreHandler.cc
index ca921d9f4..a80603b7a 100644
--- a/src/fdb5/remote/server/StoreHandler.cc
+++ b/src/fdb5/remote/server/StoreHandler.cc
@@ -189,12 +189,14 @@ void StoreHandler::flush(uint32_t clientID, uint32_t requestID, const eckit::Buf
         stream >> numArchived;
     }
 
-    ASSERT(numArchived == 0 || archiveFuture_.valid());
+    if (numArchived > 0) {
+        ASSERT(archiveFuture_.valid());
 
-    std::lock_guard lock(handlerMutex_);
-    auto it = stores_.find(clientID);
-    ASSERT(it != stores_.end());
-    it->second.store->flush();
+        std::lock_guard lock(handlerMutex_);
+        auto it = stores_.find(clientID);
+        ASSERT(it != stores_.end());
+        it->second.store->flush();
+    }
 
     Log::info() << "Flush complete" << std::endl;
     Log::status() << "Flush complete" << std::endl;

From df54644fd49835f971fa43ee48e8d01f9c6ebeee Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 13 Sep 2024 14:03:53 +0100
Subject: [PATCH 135/186] improved FDB error reporting

---
 VERSION                       |  2 +-
 src/fdb5/remote/Connection.cc | 14 ++++++++++++--
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/VERSION b/VERSION
index 706c863b3..5be94ede7 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.14.0-rc1
\ No newline at end of file
+5.13.100
\ No newline at end of file
diff --git a/src/fdb5/remote/Connection.cc b/src/fdb5/remote/Connection.cc
index 2b92d891a..53d8b294e 100644
--- a/src/fdb5/remote/Connection.cc
+++ b/src/fdb5/remote/Connection.cc
@@ -27,12 +27,17 @@ class TCPException : public eckit::Exception {
 //----------------------------------------------------------------------------------------------------------------------
 
 void Connection::writeUnsafe(bool control, const void* data, size_t length) {
-    size_t written = 0;
+    long written = 0;
     if (control || single_) {
         written = controlSocket().write(data, length);
     } else {
         written = dataSocket().write(data, length);
     }
+    if (written < 0) {
+        std::stringstream ss;
+        ss << "Write error. Expected " << length << ". Error = " << eckit::Log::syserr;
+        throw TCPException(ss.str(), Here());
+    }
     if (length != written) {
         std::stringstream ss;
         ss << "Write error. Expected " << length << " bytes, wrote " << written;
@@ -41,12 +46,17 @@ void Connection::writeUnsafe(bool control, const void* data, size_t length) {
 }
 
 void Connection::readUnsafe(bool control, void* data, size_t length) {
-    size_t read = 0;
+    long read = 0;
     if (control || single_) {
         read = controlSocket().read(data, length);
     } else {
         read = dataSocket().read(data, length);
     }
+    if (read < 0) {
+        std::stringstream ss;
+        ss << "Read error. Expected " << length << ". Error = " << eckit::Log::syserr;
+        throw TCPException(ss.str(), Here());
+    }
     if (length != read) {
         std::stringstream ss;
         ss << "Read error. Expected " << length << " bytes, read " << read;

From 2a07281b92cbfac1dc7cfbaf793e17e1a7d1eb47 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 23 Sep 2024 12:38:20 +0200
Subject: [PATCH 136/186] wip

---
 src/fdb5/api/RemoteFDB.cc                |  8 ++++----
 src/fdb5/daos/DaosCatalogueWriter.cc     |  2 +-
 src/fdb5/daos/DaosCatalogueWriter.h      |  2 +-
 src/fdb5/daos/DaosStore.cc               |  2 +-
 src/fdb5/database/ArchiveVisitor.cc      |  2 +-
 src/fdb5/database/Catalogue.h            |  9 +++++++--
 src/fdb5/database/EntryVisitMechanism.cc | 12 ++++++++----
 src/fdb5/database/EntryVisitMechanism.h  |  7 ++++---
 src/fdb5/database/StatsReportVisitor.h   |  4 ++--
 src/fdb5/remote/RemoteFieldLocation.cc   |  4 ----
 src/fdb5/remote/RemoteFieldLocation.h    |  1 -
 src/fdb5/rules/MatchAlways.h             |  2 --
 src/fdb5/rules/MatchValue.h              |  2 --
 src/fdb5/toc/TocWipeVisitor.cc           |  9 +++++++++
 14 files changed, 38 insertions(+), 28 deletions(-)

diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index f51075e08..3fd798aaa 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -48,10 +48,10 @@ struct ListHelper : BaseAPIHelper remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(fieldLocationEndpoint), static_cast(elem.location())).make_shared();
+            std::shared_ptr remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(fieldLocationEndpoint), static_cast(elem.location())).make_shared();
             return fdb5::ListElement(elem.key(), remoteLocation, elem.timestamp());
         }
-        std::shared_ptr remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(), elem.location()).make_shared();
+        std::shared_ptr remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(), elem.location()).make_shared();
         return fdb5::ListElement(elem.key(), remoteLocation, elem.timestamp());
     }
 };
@@ -68,10 +68,10 @@ struct InspectHelper : BaseAPIHelper remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(fieldLocationEndpoint), static_cast(elem.location())).make_shared();
+            std::shared_ptr remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(fieldLocationEndpoint), static_cast(elem.location())).make_shared();
             return fdb5::ListElement(elem.key(), remoteLocation, elem.timestamp());
         }
-        std::shared_ptr remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(), elem.location()).make_shared();
+        std::shared_ptr remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(), elem.location()).make_shared();
         return fdb5::ListElement(elem.key(), remoteLocation, elem.timestamp());
     }
 };
diff --git a/src/fdb5/daos/DaosCatalogueWriter.cc b/src/fdb5/daos/DaosCatalogueWriter.cc
index 2135e0ad3..91b307cc7 100644
--- a/src/fdb5/daos/DaosCatalogueWriter.cc
+++ b/src/fdb5/daos/DaosCatalogueWriter.cc
@@ -228,7 +228,7 @@ const Index& DaosCatalogueWriter::currentIndex() {
 /// @todo: other writers may be simultaneously updating the axes KeyValues in DAOS. Should these
 ///        new updates be retrieved and put into in-memory axes from time to time, e.g. every
 ///        time a value is put in an axis KeyValue?
-void DaosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) {
+void DaosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) {
 
     if (current_.null()) {
         ASSERT(!currentIndexKey_.empty());
diff --git a/src/fdb5/daos/DaosCatalogueWriter.h b/src/fdb5/daos/DaosCatalogueWriter.h
index 642862bf2..0ea9f39bc 100644
--- a/src/fdb5/daos/DaosCatalogueWriter.h
+++ b/src/fdb5/daos/DaosCatalogueWriter.h
@@ -56,7 +56,7 @@ class DaosCatalogueWriter : public DaosCatalogue, public CatalogueWriter {
     void clean() override;
     void close() override;
 
-    void archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) override;
+    void archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) override;
 
     void print( std::ostream &out ) const override { NOTIMP; }
 
diff --git a/src/fdb5/daos/DaosStore.cc b/src/fdb5/daos/DaosStore.cc
index c8b97ea75..3536a1b31 100644
--- a/src/fdb5/daos/DaosStore.cc
+++ b/src/fdb5/daos/DaosStore.cc
@@ -105,7 +105,7 @@ eckit::DataHandle* DaosStore::retrieve(Field& field) const {
 
 }
 
-std::unique_ptr DaosStore::archive(const Key&, const void* data, eckit::Length length) {
+std::unique_ptr DaosStore::archive(const Key&, const void* data, eckit::Length length) {
 
     /// @note: performed RPCs:
     /// - open pool if not cached (daos_pool_connect) -- always skipped as it is cached after selectDatabase.
diff --git a/src/fdb5/database/ArchiveVisitor.cc b/src/fdb5/database/ArchiveVisitor.cc
index 6d2c9e412..338897359 100644
--- a/src/fdb5/database/ArchiveVisitor.cc
+++ b/src/fdb5/database/ArchiveVisitor.cc
@@ -37,7 +37,7 @@ bool ArchiveVisitor::selectDatum(const TypedKey& datumKey, const TypedKey& fullC
 
     store()->archive(idxKey, data_, size_,
         std::bind(&ArchiveVisitor::callbacks, this, catalogue(), idxKey, datumKey.canonical(), p, std::placeholders::_1));
-    callback_(initialFieldKey_, data_, size_, p->get_future());
+    callback_(initialFieldKey(), data_, size_, p->get_future());
 
     return true;
 }
diff --git a/src/fdb5/database/Catalogue.h b/src/fdb5/database/Catalogue.h
index e36b2509b..ae6b55bdd 100644
--- a/src/fdb5/database/Catalogue.h
+++ b/src/fdb5/database/Catalogue.h
@@ -257,7 +257,11 @@ class CatalogueWriterFactory {
 class NullCatalogue : public Catalogue {
 public:
 
-    NullCatalogue() : Catalogue(Key{}, ControlIdentifiers{}, Config{}) {}
+    const Key& key() const override { NOTIMP; }
+    const Key& indexKey() const override { NOTIMP; }
+    const Config& config() const override { NOTIMP; }
+
+    std::unique_ptr buildStore() const override { NOTIMP; }
 
     const Schema& schema() const override { NOTIMP; }
 
@@ -269,6 +273,7 @@ class NullCatalogue : public Catalogue {
     void hideContents() override { NOTIMP; }
 
     void dump(std::ostream& out, bool simple=false, const eckit::Configuration& conf = eckit::LocalConfiguration()) const override { NOTIMP; }
+    bool enabled(const ControlIdentifier& controlIdentifier) const override { NOTIMP; }
 
     StatsReportVisitor* statsReportVisitor() const override { NOTIMP; }
     PurgeVisitor* purgeVisitor(const Store& store) const override { NOTIMP; }
@@ -291,7 +296,7 @@ class NullCatalogue : public Catalogue {
 
     std::string type() const override { NOTIMP; }
     bool open() override { NOTIMP; }
-    void flush() override { NOTIMP; }
+    void flush(size_t archivedFields) override { NOTIMP; }
     void clean() override { NOTIMP; }
     void close() override { NOTIMP; }
 
diff --git a/src/fdb5/database/EntryVisitMechanism.cc b/src/fdb5/database/EntryVisitMechanism.cc
index fabbb04a5..59524a919 100644
--- a/src/fdb5/database/EntryVisitMechanism.cc
+++ b/src/fdb5/database/EntryVisitMechanism.cc
@@ -33,10 +33,14 @@ class FDBVisitException : public eckit::Exception {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-Store& EntryVisitor::store() {
+EntryVisitor::EntryVisitor() : currentCatalogue_(nullptr), currentIndex_(nullptr), currentStore_(nullptr) {}
+
+EntryVisitor::~EntryVisitor() = default;
+
+Store& EntryVisitor::store() const {
     if (!currentStore_) {
         ASSERT(currentCatalogue_);
-        currentStore_ = currentCatalogue_->buildStore();
+        currentStore_ = currentCatalogue_->buildStore().get();
         ASSERT(currentStore_);
     }
     return *currentStore_;
@@ -44,7 +48,7 @@ Store& EntryVisitor::store() {
 
 bool EntryVisitor::visitDatabase(const Catalogue& catalogue) {
     currentCatalogue_ = &catalogue;
-    currentStore_.reset();
+    currentStore_ = nullptr;
     return true;
 }
 
@@ -53,7 +57,7 @@ void EntryVisitor::catalogueComplete(const Catalogue& catalogue) {
         ASSERT(currentCatalogue_ == &catalogue);
     }
     currentCatalogue_ = nullptr;
-    currentStore_.reset();
+    currentStore_ = nullptr;
     currentIndex_ = nullptr;
 }
 
diff --git a/src/fdb5/database/EntryVisitMechanism.h b/src/fdb5/database/EntryVisitMechanism.h
index 4fe7c9e59..6c2f6b173 100644
--- a/src/fdb5/database/EntryVisitMechanism.h
+++ b/src/fdb5/database/EntryVisitMechanism.h
@@ -17,8 +17,8 @@
 #include "eckit/memory/NonCopyable.h"
 
 #include "fdb5/config/Config.h"
-#include "fdb5/database/Field.h"
 #include "fdb5/database/DatabaseNotFoundException.h"
+#include "fdb5/database/Field.h"
 
 namespace fdb5 {
 
@@ -34,13 +34,13 @@ class EntryVisitor : public eckit::NonCopyable {
 
 public:  // methods
 
+    EntryVisitor() : currentCatalogue_(nullptr), currentIndex_(nullptr), currentStore_(nullptr) {}
     virtual ~EntryVisitor() = default;
 
     // defaults
     virtual bool visitIndexes() { return true; }
     virtual bool visitEntries() { return true; }
 
-//    virtual bool visitDatabase(const Catalogue& catalogue, const Store& store);    // return true if Catalogue should be explored
     virtual bool visitDatabase(const Catalogue& catalogue);    // return true if Catalogue should be explored
     virtual bool visitIndex(const Index& index); // return true if index should be explored
     virtual void catalogueComplete(const Catalogue& catalogue);
@@ -52,7 +52,7 @@ class EntryVisitor : public eckit::NonCopyable {
 
 protected:
 
-    Store& store();
+    Store& store() const;
 
 private: // methods
 
@@ -64,6 +64,7 @@ class EntryVisitor : public eckit::NonCopyable {
     const Catalogue* currentCatalogue_ = nullptr;
     std::unique_ptr currentStore_ = nullptr;
     const Index* currentIndex_ = nullptr;
+
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/database/StatsReportVisitor.h b/src/fdb5/database/StatsReportVisitor.h
index 3d55e5549..679543504 100644
--- a/src/fdb5/database/StatsReportVisitor.h
+++ b/src/fdb5/database/StatsReportVisitor.h
@@ -15,17 +15,17 @@
 #define fdb5_StatsReportVisitor_H
 
 #include "fdb5/database/DbStats.h"
-#include "fdb5/database/EntryVisitMechanism.h"
 #include "fdb5/database/IndexStats.h"
+#include "fdb5/database/EntryVisitMechanism.h"
 
 namespace fdb5 {
-class EntryVisitor;
 
 //----------------------------------------------------------------------------------------------------------------------
 
 class StatsReportVisitor : public EntryVisitor {
 
 public: // methods
+
     using EntryVisitor::EntryVisitor;
 
     virtual IndexStats indexStatistics() const = 0;
diff --git a/src/fdb5/remote/RemoteFieldLocation.cc b/src/fdb5/remote/RemoteFieldLocation.cc
index d37ab886f..7615c3243 100644
--- a/src/fdb5/remote/RemoteFieldLocation.cc
+++ b/src/fdb5/remote/RemoteFieldLocation.cc
@@ -137,10 +137,6 @@ class FdbURIManager : public eckit::URIManager {
         return eckit::PathName{u.name()};
     }
 
-    eckit::PathName path(const eckit::URI& u) const override {
-        return eckit::PathName{u.name()};
-    }
-
     std::string asString(const eckit::URI& uri) const override {
         std::string q = uri.query();
         if (!q.empty())
diff --git a/src/fdb5/remote/RemoteFieldLocation.h b/src/fdb5/remote/RemoteFieldLocation.h
index 96de04458..b1d0b8aab 100644
--- a/src/fdb5/remote/RemoteFieldLocation.h
+++ b/src/fdb5/remote/RemoteFieldLocation.h
@@ -56,7 +56,6 @@ class RemoteFieldLocation : public FieldLocation {
 
 private: // methods
 
-    void dump(std::ostream& out) const override;
     void print(std::ostream& out) const override;
 
 private: // members
diff --git a/src/fdb5/rules/MatchAlways.h b/src/fdb5/rules/MatchAlways.h
index 61bf4fdbf..ebb80cef2 100644
--- a/src/fdb5/rules/MatchAlways.h
+++ b/src/fdb5/rules/MatchAlways.h
@@ -43,8 +43,6 @@ class MatchAlways : public Matcher {
 
     void encode(eckit::Stream&) const override;
 
-    void encode(eckit::Stream&) const override;
-
     void print( std::ostream& out ) const override;
 
 private: // members
diff --git a/src/fdb5/rules/MatchValue.h b/src/fdb5/rules/MatchValue.h
index a1199b583..bf26dbc5c 100644
--- a/src/fdb5/rules/MatchValue.h
+++ b/src/fdb5/rules/MatchValue.h
@@ -45,8 +45,6 @@ class MatchValue : public Matcher{
 
     void encode(eckit::Stream&) const override;
 
-    void encode(eckit::Stream&) const override;
-
     void print( std::ostream &out ) const override;
 
 private: // members
diff --git a/src/fdb5/toc/TocWipeVisitor.cc b/src/fdb5/toc/TocWipeVisitor.cc
index 448c9200e..3c1fe9e3b 100644
--- a/src/fdb5/toc/TocWipeVisitor.cc
+++ b/src/fdb5/toc/TocWipeVisitor.cc
@@ -164,7 +164,16 @@ bool TocWipeVisitor::visitIndex(const Index& index) {
 
     // Enumerate data files.
 
+    std::cout << index << std::endl;
     std::vector indexDataPaths(index.dataURIs());
+    store_.print(std::cout);
+    std::cout << std::endl;
+    auto paths = store_.asCollocatedDataURIs(indexDataPaths);
+    std::cout << paths.size() << std::endl;
+    for (const auto& p : paths) {
+        std::cout << "  " << p << std::endl;
+    }
+
     for (const eckit::URI& uri : store_.asCollocatedDataURIs(indexDataPaths)) {
         if (include) {
             if (!store_.uriBelongs(uri)) {

From 2d256631635fc38fd2a051c1b7a6d57c927ad29c Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 23 Sep 2024 15:52:02 +0200
Subject: [PATCH 137/186] fix merge + nuked pmem

---
 CMakeLists.txt                           |   7 -
 src/fdb5/CMakeLists.txt                  |  52 ---
 src/fdb5/database/EntryVisitMechanism.cc |  10 +-
 src/fdb5/database/EntryVisitMechanism.h  |   6 +-
 src/fdb5/pmem/DataPool.cc                | 110 -----
 src/fdb5/pmem/DataPool.h                 |  83 ----
 src/fdb5/pmem/DataPoolManager.cc         | 231 ----------
 src/fdb5/pmem/DataPoolManager.h          | 148 ------
 src/fdb5/pmem/MemoryBufferStream.cc      |  92 ----
 src/fdb5/pmem/MemoryBufferStream.h       |  67 ---
 src/fdb5/pmem/PBaseNode.cc               |  82 ----
 src/fdb5/pmem/PBaseNode.h                | 114 -----
 src/fdb5/pmem/PBranchingNode.cc          | 488 --------------------
 src/fdb5/pmem/PBranchingNode.h           | 125 ------
 src/fdb5/pmem/PDataNode.cc               |  65 ---
 src/fdb5/pmem/PDataNode.h                |  98 ----
 src/fdb5/pmem/PDataRoot.cc               |  74 ---
 src/fdb5/pmem/PDataRoot.h                |  80 ----
 src/fdb5/pmem/PIndexRoot.cc              | 186 --------
 src/fdb5/pmem/PIndexRoot.h               | 120 -----
 src/fdb5/pmem/PMemDB.cc                  | 328 --------------
 src/fdb5/pmem/PMemDB.h                   | 141 ------
 src/fdb5/pmem/PMemDBReader.cc            | 107 -----
 src/fdb5/pmem/PMemDBReader.h             |  62 ---
 src/fdb5/pmem/PMemDBWriter.cc            |  93 ----
 src/fdb5/pmem/PMemDBWriter.h             |  55 ---
 src/fdb5/pmem/PMemEngine.cc              | 183 --------
 src/fdb5/pmem/PMemEngine.h               |  74 ---
 src/fdb5/pmem/PMemFieldLocation.cc       | 106 -----
 src/fdb5/pmem/PMemFieldLocation.h        |  84 ----
 src/fdb5/pmem/PMemIndex.cc               | 177 --------
 src/fdb5/pmem/PMemIndex.h                |  81 ----
 src/fdb5/pmem/PMemIndexLocation.cc       |  83 ----
 src/fdb5/pmem/PMemIndexLocation.h        |  69 ---
 src/fdb5/pmem/PMemStats.cc               | 237 ----------
 src/fdb5/pmem/PMemStats.h                | 193 --------
 src/fdb5/pmem/PRoot.cc                   | 125 ------
 src/fdb5/pmem/PRoot.h                    | 123 -----
 src/fdb5/pmem/Pool.cc                    | 148 ------
 src/fdb5/pmem/Pool.h                     |  88 ----
 src/fdb5/pmem/PoolEntry.cc               |  49 --
 src/fdb5/pmem/PoolEntry.h                |  62 ---
 src/fdb5/pmem/PoolGroup.cc               | 103 -----
 src/fdb5/pmem/PoolGroup.h                |  74 ---
 src/fdb5/pmem/PoolManager.cc             | 317 -------------
 src/fdb5/pmem/PoolManager.h              |  68 ---
 src/fdb5/rules/Rule.cc                   |   6 +-
 src/fdb5/rules/Schema.cc                 |   4 +-
 src/fdb5/toc/TocPurgeVisitor.cc          |  20 -
 src/fdb5/types/TypesRegistry.cc          |   2 +-
 tests/fdb/CMakeLists.txt                 |   8 -
 tests/fdb/api/test_config.cc             |   4 +-
 tests/fdb/pmem/CMakeLists.txt            |  27 --
 tests/fdb/pmem/test_pbasenode.cc         | 213 ---------
 tests/fdb/pmem/test_pbranchingnode.cc    | 546 -----------------------
 tests/fdb/pmem/test_pdatanode.cc         | 107 -----
 tests/fdb/pmem/test_persistent_helpers.h | 120 -----
 tests/fdb/pmem/test_pool_manager.cc      | 300 -------------
 58 files changed, 19 insertions(+), 6806 deletions(-)
 delete mode 100644 src/fdb5/pmem/DataPool.cc
 delete mode 100644 src/fdb5/pmem/DataPool.h
 delete mode 100644 src/fdb5/pmem/DataPoolManager.cc
 delete mode 100644 src/fdb5/pmem/DataPoolManager.h
 delete mode 100644 src/fdb5/pmem/MemoryBufferStream.cc
 delete mode 100644 src/fdb5/pmem/MemoryBufferStream.h
 delete mode 100644 src/fdb5/pmem/PBaseNode.cc
 delete mode 100644 src/fdb5/pmem/PBaseNode.h
 delete mode 100644 src/fdb5/pmem/PBranchingNode.cc
 delete mode 100644 src/fdb5/pmem/PBranchingNode.h
 delete mode 100644 src/fdb5/pmem/PDataNode.cc
 delete mode 100644 src/fdb5/pmem/PDataNode.h
 delete mode 100644 src/fdb5/pmem/PDataRoot.cc
 delete mode 100644 src/fdb5/pmem/PDataRoot.h
 delete mode 100644 src/fdb5/pmem/PIndexRoot.cc
 delete mode 100644 src/fdb5/pmem/PIndexRoot.h
 delete mode 100644 src/fdb5/pmem/PMemDB.cc
 delete mode 100644 src/fdb5/pmem/PMemDB.h
 delete mode 100644 src/fdb5/pmem/PMemDBReader.cc
 delete mode 100644 src/fdb5/pmem/PMemDBReader.h
 delete mode 100644 src/fdb5/pmem/PMemDBWriter.cc
 delete mode 100644 src/fdb5/pmem/PMemDBWriter.h
 delete mode 100644 src/fdb5/pmem/PMemEngine.cc
 delete mode 100644 src/fdb5/pmem/PMemEngine.h
 delete mode 100644 src/fdb5/pmem/PMemFieldLocation.cc
 delete mode 100644 src/fdb5/pmem/PMemFieldLocation.h
 delete mode 100755 src/fdb5/pmem/PMemIndex.cc
 delete mode 100755 src/fdb5/pmem/PMemIndex.h
 delete mode 100644 src/fdb5/pmem/PMemIndexLocation.cc
 delete mode 100644 src/fdb5/pmem/PMemIndexLocation.h
 delete mode 100644 src/fdb5/pmem/PMemStats.cc
 delete mode 100644 src/fdb5/pmem/PMemStats.h
 delete mode 100644 src/fdb5/pmem/PRoot.cc
 delete mode 100644 src/fdb5/pmem/PRoot.h
 delete mode 100644 src/fdb5/pmem/Pool.cc
 delete mode 100644 src/fdb5/pmem/Pool.h
 delete mode 100644 src/fdb5/pmem/PoolEntry.cc
 delete mode 100644 src/fdb5/pmem/PoolEntry.h
 delete mode 100644 src/fdb5/pmem/PoolGroup.cc
 delete mode 100644 src/fdb5/pmem/PoolGroup.h
 delete mode 100644 src/fdb5/pmem/PoolManager.cc
 delete mode 100644 src/fdb5/pmem/PoolManager.h
 delete mode 100644 tests/fdb/pmem/CMakeLists.txt
 delete mode 100644 tests/fdb/pmem/test_pbasenode.cc
 delete mode 100644 tests/fdb/pmem/test_pbranchingnode.cc
 delete mode 100644 tests/fdb/pmem/test_pdatanode.cc
 delete mode 100644 tests/fdb/pmem/test_persistent_helpers.h
 delete mode 100644 tests/fdb/pmem/test_pool_manager.cc

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6c234eb50..dd5cfcd4b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -33,13 +33,6 @@ ecbuild_add_option( FEATURE GRIB
 
 ecbuild_find_package( NAME metkit VERSION 1.5 REQUIRED )
 
-### FDB backend in persistent memory, i.e. pmem (NVRAM)
-
-ecbuild_add_option( FEATURE PMEMFDB # option present in fdb5_config.h
-                    DEFAULT OFF
-                    DESCRIPTION "Persistent memory (NVRAM) support for FDB"
-                    REQUIRED_PACKAGES "NAME pmem" )
-
 ### FDB backend in CEPH object store (using Rados)
 
 find_package( RADOS QUIET )
diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt
index 465701964..2609214ad 100644
--- a/src/fdb5/CMakeLists.txt
+++ b/src/fdb5/CMakeLists.txt
@@ -328,58 +328,6 @@ if( HAVE_TOCFDB )
         fdb-reconsolidate-toc )
 endif()
 
-if( HAVE_PMEMFDB )
-
-    list( APPEND fdb5_srcs
-        pmem/DataPool.cc
-        pmem/DataPool.h
-        pmem/DataPoolManager.cc
-        pmem/DataPoolManager.h
-        pmem/MemoryBufferStream.h
-        pmem/MemoryBufferStream.cc
-        pmem/PBaseNode.cc
-        pmem/PBaseNode.h
-        pmem/PBranchingNode.cc
-        pmem/PBranchingNode.h
-        pmem/PDataNode.cc
-        pmem/PDataNode.h
-        pmem/PDataRoot.cc
-        pmem/PDataRoot.h
-        pmem/PIndexRoot.cc
-        pmem/PIndexRoot.h
-        pmem/PMemDB.cc
-        pmem/PMemDB.h
-        pmem/PMemDBReader.cc
-        pmem/PMemDBReader.h
-        pmem/PMemDBWriter.cc
-        pmem/PMemDBWriter.h
-        pmem/PMemFieldLocation.cc
-        pmem/PMemFieldLocation.h
-        pmem/PMemIndex.cc
-        pmem/PMemIndex.h
-        pmem/PMemIndexLocation.cc
-        pmem/PMemIndexLocation.h
-        pmem/PRoot.cc
-        pmem/PRoot.h
-        pmem/Pool.cc
-        pmem/Pool.h
-        pmem/PoolEntry.cc
-        pmem/PoolEntry.h
-        pmem/PoolGroup.cc
-        pmem/PoolGroup.h
-        pmem/PoolManager.cc
-        pmem/PoolManager.h
-        pmem/PMemStats.cc
-        pmem/PMemStats.h
-        pmem/PMemEngine.cc
-        pmem/PMemEngine.h
-    )
-
-else()
-    set( PMEM_LIBRARIES "" )
-    set( PMEM_INCLUDE_DIRS "" )
-endif()
-
 if( HAVE_RADOSFDB )
     list( APPEND fdb5_srcs
         rados/RadosFieldLocation.cc
diff --git a/src/fdb5/database/EntryVisitMechanism.cc b/src/fdb5/database/EntryVisitMechanism.cc
index d718c485c..ccecc6c1c 100644
--- a/src/fdb5/database/EntryVisitMechanism.cc
+++ b/src/fdb5/database/EntryVisitMechanism.cc
@@ -33,14 +33,17 @@ class FDBVisitException : public eckit::Exception {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-EntryVisitor::EntryVisitor() : currentCatalogue_(nullptr), currentIndex_(nullptr), currentStore_(nullptr) {}
+EntryVisitor::EntryVisitor() : currentCatalogue_(nullptr), currentStore_(nullptr), currentIndex_(nullptr) {}
 
-EntryVisitor::~EntryVisitor() = default;
+EntryVisitor::~EntryVisitor() {
+    if (currentStore_)
+        delete currentStore_;
+}
 
 Store& EntryVisitor::store() const {
     if (!currentStore_) {
         ASSERT(currentCatalogue_);
-        currentStore_ = currentCatalogue_->buildStore().get();
+        currentStore_ = currentCatalogue_->buildStore().release();
         ASSERT(currentStore_);
     }
     return *currentStore_;
@@ -57,6 +60,7 @@ void EntryVisitor::catalogueComplete(const Catalogue& catalogue) {
         ASSERT(currentCatalogue_ == &catalogue);
     }
     currentCatalogue_ = nullptr;
+    delete currentStore_;
     currentStore_ = nullptr;
     currentIndex_ = nullptr;
 }
diff --git a/src/fdb5/database/EntryVisitMechanism.h b/src/fdb5/database/EntryVisitMechanism.h
index 6c2f6b173..21aff6dc9 100644
--- a/src/fdb5/database/EntryVisitMechanism.h
+++ b/src/fdb5/database/EntryVisitMechanism.h
@@ -34,8 +34,8 @@ class EntryVisitor : public eckit::NonCopyable {
 
 public:  // methods
 
-    EntryVisitor() : currentCatalogue_(nullptr), currentIndex_(nullptr), currentStore_(nullptr) {}
-    virtual ~EntryVisitor() = default;
+    EntryVisitor();
+    virtual ~EntryVisitor();
 
     // defaults
     virtual bool visitIndexes() { return true; }
@@ -62,7 +62,7 @@ class EntryVisitor : public eckit::NonCopyable {
 
     // n.b. non-owning
     const Catalogue* currentCatalogue_ = nullptr;
-    std::unique_ptr currentStore_ = nullptr;
+    mutable Store* currentStore_ = nullptr;
     const Index* currentIndex_ = nullptr;
 
 };
diff --git a/src/fdb5/pmem/DataPool.cc b/src/fdb5/pmem/DataPool.cc
deleted file mode 100644
index 08351f259..000000000
--- a/src/fdb5/pmem/DataPool.cc
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-3.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation
- * nor does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Simon Smart
-/// @date   Feb 2016
-
-
-#include "eckit/log/Log.h"
-#include "eckit/log/TimeStamp.h"
-
-#include "pmem/PersistentPtr.h"
-
-#include "fdb5/pmem/DataPool.h"
-#include "fdb5/pmem/PDataRoot.h"
-#include "fdb5/pmem/PRoot.h"
-#include "fdb5/LibFdb5.h"
-
-#include 
-
-
-using namespace eckit;
-using namespace pmem;
-
-
-namespace fdb5 {
-namespace pmem {
-
-
-// ---------------------------------------------------------------------------------------------------------------------
-
-std::string data_pool_name(size_t index) {
-    std::stringstream ss;
-
-    ss << "data-" << index;
-    return ss.str();
-}
-
-std::string data_pool_path(const PathName& poolDir, size_t index) {
-    return poolDir / data_pool_name(index);
-}
-
-
-DataPool::DataPool(const PathName& poolDir, size_t index) :
-    PersistentPool(data_pool_path(poolDir, index), data_pool_name(index)) {
-
-    ASSERT(root().valid());
-
-    LOG_DEBUG_LIB(LibFdb5) << "Opened persistent pool created at: " << TimeStamp(root().created()) << std::endl;
-}
-
-
-DataPool::DataPool(const PathName& poolDir, size_t index, const size_t size) :
-    PersistentPool(data_pool_path(poolDir, index), size, data_pool_name(index), PRoot::Constructor(PRoot::DataClass)) {}
-
-
-DataPool::~DataPool() {}
-
-
-void DataPool::buildRoot() {
-    // n.b. cannot use baseRoot yet, as not yet valid...
-    PersistentPtr rt = getRoot();
-    ASSERT(rt.valid());
-
-    // n.b. null arguments. Arguments only apply for index root.
-    rt->buildRoot(Key(), "");
-}
-
-
-bool DataPool::finalised() const {
-    return root().finalised();
-}
-
-void DataPool::finalise() {
-    root().finalise();
-}
-
-
-PersistentPtr DataPool::baseRoot() const {
-    PersistentPtr rt = getRoot();
-
-    ASSERT(rt.valid());
-    ASSERT(rt->valid());
-    ASSERT(rt->root_class() == PRoot::DataClass);
-
-    return rt;
-}
-
-
-PDataRoot& DataPool::root() const {
-
-    PDataRoot& rt = baseRoot()->dataRoot();
-
-    ASSERT(rt.valid());
-    return rt;
-}
-
-// ---------------------------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace tree
diff --git a/src/fdb5/pmem/DataPool.h b/src/fdb5/pmem/DataPool.h
deleted file mode 100644
index e84870933..000000000
--- a/src/fdb5/pmem/DataPool.h
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-3.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation
- * nor does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Simon Smart
-/// @date   Feb 2016
-
-
-#ifndef fdb5_pmem_DataPool_H
-#define fdb5_pmem_DataPool_H
-
-#include "pmem/PersistentPool.h"
-
-
-// -------------------------------------------------------------------------------------------------
-
-// Forward declaration
-namespace pmem {
-    template  class PersistentPtr;
-}
-
-// -------------------------------------------------------------------------------------------------
-
-namespace fdb5 {
-namespace pmem {
-
-class PRoot;
-class PDataRoot;
-
-// -------------------------------------------------------------------------------------------------
-
-/*
- * Here we define the persistent pool used in this project
- *
- * --> The TreePool needs to know:
- *
- *       i) The data type of the root
- *      ii) How to construct the root object if required
- *     iii) How to map type_ids to actual types
- */
-
-class DataPool : public ::pmem::PersistentPool {
-
-public: // methods
-
-    DataPool(const eckit::PathName& poolDir, size_t poolIndex);
-
-    DataPool(const eckit::PathName& poolDir, size_t poolIndex, size_t size);
-
-    ~DataPool();
-
-    /// Normally we allocate persistent objects by passing in their subcomponents. We cannot do
-    /// that with a root object, as it is allocated at pool creation time.
-    ///
-    /// --> buildRoot() should be called immediately after pool creation to initialise the root.
-    void buildRoot();
-
-    bool finalised() const;
-    void finalise();
-
-private: // methods
-
-    ::pmem::PersistentPtr baseRoot() const;
-    PDataRoot& root() const;
-};
-
-
-// -------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
-
-
-#endif // fdb5_pmem_DataPool_H
diff --git a/src/fdb5/pmem/DataPoolManager.cc b/src/fdb5/pmem/DataPoolManager.cc
deleted file mode 100644
index a60a70c6d..000000000
--- a/src/fdb5/pmem/DataPoolManager.cc
+++ /dev/null
@@ -1,231 +0,0 @@
-/*
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-3.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation
- * nor does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Simon Smart
-/// @date   Sept 2016
-
-
-#include "eckit/filesystem/PathName.h"
-#include "eckit/thread/AutoLock.h"
-#include "eckit/config/Resource.h"
-
-#include "pmem/PersistentMutex.h"
-
-#include "fdb5/pmem/DataPoolManager.h"
-#include "fdb5/pmem/DataPool.h"
-#include "fdb5/pmem/Pool.h"
-#include "fdb5/pmem/PIndexRoot.h"
-#include "fdb5/LibFdb5.h"
-
-using namespace eckit;
-using namespace pmem;
-
-
-namespace fdb5 {
-namespace pmem {
-
-// ---------------------------------------------------------------------------------------------------------------------
-
-DataPoolManager::DataPoolManager(const PathName& poolDir, PIndexRoot& masterRoot, uint64_t rootUUID) :
-    poolDir_(poolDir),
-    masterRoot_(masterRoot),
-    masterUUID_(rootUUID),
-    currentPool_(0) {}
-
-
-DataPoolManager::~DataPoolManager() {
-
-    // Clean up the pools
-
-    std::map::iterator it = pools_.begin();
-
-    for (; it != pools_.end(); ++it) {
-        delete it->second;
-    }
-}
-
-
-DataPool& DataPoolManager::currentWritePool() {
-
-    LOG_DEBUG_LIB(LibFdb5) << "[" << *this << "]: " << "Requesting current pool" << std::endl;
-
-    if (currentPool_ == 0) {
-
-        // If there isn't a currently active pool, we are potentially going to be modifying the root object,
-        // so lock things.
-
-        AutoLock lock(masterRoot_.mutex_);
-
-        size_t pool_count = masterRoot_.dataPoolUUIDs_.size();
-
-        // If there are any pools in the pool list, open the last one. If this is not finalised, then use it.
-
-        if (pool_count > 0) {
-
-            // TODO: Given index, get UUID
-            // TODO: Given UUID, have function that checks if it is in the pools_ list, and if not, open it.
-            size_t idx = pool_count - 1;
-            size_t uuid = masterRoot_.dataPoolUUIDs_[idx];
-            DataPool* pool;
-
-            if (pools_.find(uuid) == pools_.end()) {
-
-                LOG_DEBUG_LIB(LibFdb5) << "[" << *this << "]: " << "Opening pool index: " << idx << std::endl;
-                pool = new DataPool(poolDir_, idx);
-
-                uuid = pool->uuid();
-                pools_[uuid] = pool;
-            } else {
-                pool = pools_[uuid];
-            }
-
-            if (!pool->finalised())
-                currentPool_ = pool;
-        }
-
-        // If we still don't have a workable pool, we need to open a new one.
-        if (currentPool_ == 0) {
-
-            size_t idx = pool_count;
-
-            LOG_DEBUG_LIB(LibFdb5) << "[" << *this << "]: " << "Creating a new data pool with index: " << idx << std::endl;
-            currentPool_ = new DataPool(poolDir_, idx, Resource("fdbPMemDataPoolSize;$fdbPMemDataPoolSize", 1024 * 1024 * 1024));
-            currentPool_->buildRoot();
-
-            // Add pool to the list of opened pools
-
-            size_t uuid = currentPool_->uuid();
-            ASSERT(pools_.find(uuid) == pools_.end());
-            pools_[uuid] = currentPool_;
-
-            masterRoot_.dataPoolUUIDs_.push_back(uuid);
-        }
-    }
-
-    return *currentPool_;
-}
-
-
-void DataPoolManager::invalidateCurrentPool(DataPool& pool) {
-
-    // If the supplied pool does not equal currentPool_, then another thread/process has already done this.
-    if (&pool != currentPool_)
-        return;
-
-    // We don't worry too much about other threads/processors. They will also hit out-of-space allocation errors.
-    // This finalisation is only for informational purposes.
-
-    LOG_DEBUG_LIB(LibFdb5) << "[" << *this << "]: " << "Finalising current pool" << std::endl;
-
-    currentPool_->finalise();
-    currentPool_ = 0;
-}
-
-
-PathName DataPoolManager::dataPoolPath(uint64_t uuid) {
-
-    ensurePoolLoaded(uuid);
-
-    if (uuid == masterUUID_) {
-        NOTIMP;
-        return PathName();
-    }
-
-    return pools_[uuid]->path();
-}
-
-
-DataPool& DataPoolManager::getPool(uint64_t uuid) {
-
-    ensurePoolLoaded(uuid);
-
-    if (uuid == masterUUID_) {
-        NOTIMP;
-    }
-
-    return *pools_[uuid];
-}
-
-
-void DataPoolManager::ensurePoolLoaded(uint64_t uuid) {
-
-    if (uuid != masterUUID_ && pools_.find(uuid) == pools_.end()) {
-
-        LOG_DEBUG_LIB(LibFdb5) << "Data pool with UUID=" << uuid << " not yet opened" << std::endl;
-
-        size_t pool_count = masterRoot_.dataPoolUUIDs_.size();
-
-        for (size_t idx = 0; idx < pool_count; idx++) {
-            if (uuid == masterRoot_.dataPoolUUIDs_[idx]) {
-                LOG_DEBUG_LIB(LibFdb5) << "Opening data pool, UUID=" << uuid << ", index=" << idx << std::endl;
-                pools_[uuid] = new DataPool(poolDir_, idx);
-                return;
-            }
-        }
-
-        // If none are found, then we have an issue.
-
-        std::stringstream err;
-        err << "Data pool " << uuid << " not found";
-        throw SeriousBug(err.str(), Here());
-    }
-}
-
-
-const std::map& DataPoolManager::pools() const {
-    return pools_;
-}
-
-
-size_t DataPoolManager::dataPoolsCount() const {
-
-    return masterRoot_.dataPoolUUIDs().size();
-}
-
-
-
-size_t DataPoolManager::dataSize() {
-
-    // Don't need to lock access to master pools list, as it is append only and always valid for read.
-
-    size_t data_size = 0;
-
-    for (size_t idx = 0; idx < masterRoot_.dataPoolUUIDs().size(); idx++) {
-
-        data_size += getPool(masterRoot_.dataPoolUUIDs()[idx]).size();
-    }
-
-    return data_size;
-}
-
-std::vector DataPoolManager::dataPoolPaths() {
-
-    std::vector paths;
-
-    for (size_t idx = 0; idx < masterRoot_.dataPoolUUIDs().size(); idx++) {
-        paths.push_back(getPool(masterRoot_.dataPoolUUIDs()[idx]).path());
-    }
-
-    return paths;
-}
-
-
-void DataPoolManager::print(std::ostream& s) const {
-    s << "PMemDataPoolManager(" << poolDir_ << ")";
-}
-
-
-// ---------------------------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
diff --git a/src/fdb5/pmem/DataPoolManager.h b/src/fdb5/pmem/DataPoolManager.h
deleted file mode 100644
index c0cc4fbd8..000000000
--- a/src/fdb5/pmem/DataPoolManager.h
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation
- * nor does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Simon Smart
-/// @date   Sep 2016
-
-
-#ifndef fdb5_pmem_DataPoolManager_H
-#define fdb5_pmem_DataPoolManager_H
-
-#include 
-#include 
-#include 
-
-#include "eckit/memory/NonCopyable.h"
-#include "eckit/filesystem/PathName.h"
-
-#include "pmem/AtomicConstructor.h"
-#include "pmem/PersistentPtr.h"
-
-#include "fdb5/pmem/DataPool.h"
-
-
-
-namespace eckit {
-    class PathName;
-}
-
-
-namespace fdb5 {
-namespace pmem {
-
-class PIndexRoot;
-
-// ---------------------------------------------------------------------------------------------------------------------
-
-/*
- * This class needs to:
- *
- * - Open data pools for reading when they are required. Given all the memory is there. Maintain
- *   the mapping with uuids.
- * - When a pool is filled up (writing), open a new one
- * - Generate the pool filenames based off the filename of the DB pool, + a counting integer
- * - Manage atomic allocations, with retries.
- */
-
-class DataPoolManager : private eckit::NonCopyable {
-
-public: // methods
-
-    DataPoolManager(const eckit::PathName& poolDir, PIndexRoot& masterRoot, uint64_t rootUUID);
-
-    ~DataPoolManager();
-
-    /// Allocate data into the currently active pool
-    template 
-    void allocate(::pmem::PersistentPtr& ptr, const ::pmem::AtomicConstructor& ctr);
-
-    /// Ensure that the pool associated with the specified UUID is loaded. If it is not, then load
-    /// it. If it is unavailable, then throw an error.
-    void ensurePoolLoaded(uint64_t uuid);
-
-    /// For output/debugging purposes
-    eckit::PathName dataPoolPath(uint64_t uuid);
-
-    DataPool& getPool(uint64_t uuid);
-
-    /// How many data pools are under management?
-    size_t dataPoolsCount() const;
-
-    /// The total size of the data pools (does not include the master pool).
-    size_t dataSize();
-
-    std::vector dataPoolPaths();
-
-protected: // methods
-
-    /// Obtain the current pool for writing. If no pool is opened, then open/create the latest one.
-    DataPool& currentWritePool();
-
-    /// The current pool is full! Finalise it, and create a new one. Check for the obvious complications (e.g. another
-    /// process, or thread, has already created a new pool, or the pool being used is already out of date).
-    void invalidateCurrentPool(DataPool& pool);
-
-    void print(std::ostream& out) const;
-
-    const std::map& pools() const;
-
-private: // members
-
-    eckit::PathName poolDir_;
-
-    /// A mapping of pools' UUIDs to the opened pool objects.
-    std::map pools_;
-
-    PIndexRoot& masterRoot_;
-    uint64_t masterUUID_;
-
-    DataPool* currentPool_;
-
-private: // friends
-
-    friend std::ostream& operator<<(std::ostream& s,const DataPoolManager& p) { p.print(s); return s; }
-};
-
-
-// ---------------------------------------------------------------------------------------------------------------------
-
-// TODO: Deprecate and remove.
-template 
-void DataPoolManager::allocate(::pmem::PersistentPtr& ptr, const ::pmem::AtomicConstructor& ctr) {
-
-    // n.b. Remember to lock the PMemRoot whilst updating the list of pools.
-
-    while (true) {
-
-        DataPool& pool(currentWritePool());
-
-        try {
-
-            ptr.allocate_ctr(pool, ctr);
-            break;
-
-        } catch (::pmem::AtomicConstructorBase::AllocationError& e) {
-            // TODO: Check errno
-            // If the allocation fails due to lack of space, this is fine. We just need to retry.
-        }
-
-        invalidateCurrentPool(pool);
-    };
-}
-
-} // namespace pmem
-} // namespace fdb5
-
-#endif // fdb5_pmem_PMemDataPoolManager_H
diff --git a/src/fdb5/pmem/MemoryBufferStream.cc b/src/fdb5/pmem/MemoryBufferStream.cc
deleted file mode 100644
index f7c048fa7..000000000
--- a/src/fdb5/pmem/MemoryBufferStream.cc
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-3.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation
- * nor does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Simon Smart
-/// @date   Dec 2016
-
-#include 
-
-#include "eckit/exception/Exceptions.h"
-
-#include "fdb5/pmem/MemoryBufferStream.h"
-
-using namespace eckit;
-
-namespace fdb5 {
-namespace pmem {
-
-// -------------------------------------------------------------------------------------------------
-
-
-MemoryBufferStream::MemoryBufferStream() :
-    size_(4096),
-    position_(0),
-    buffer_(size_) {}
-
-
-MemoryBufferStream::~MemoryBufferStream() {}
-
-
-long MemoryBufferStream::read(void* buffer, long length) {
-
-    // Intentionally not implemented. This is a write buffer.
-    NOTIMP;
-
-    (void) buffer;
-    (void) length;
-}
-
-
-long MemoryBufferStream::write(const void* buffer, long length) {
-
-    // If the buffer is full, then reallocate it to be bigger.
-
-    Length remaining = size_ - position_;
-    if (remaining < Length(length)) {
-        MemoryBuffer tmp(size_ * 2);
-        ::memcpy(tmp, buffer_, size_);
-        buffer_.swap(tmp);
-        size_ = size_ * 2;
-    }
-
-    ::memcpy(buffer_ + position_, buffer, length);
-    position_ += length;
-
-    return length;
-}
-
-
-void MemoryBufferStream::rewind()
-{
-    position_ = 0;
-}
-
-
-std::string MemoryBufferStream::name() const {
-    return "MemoryBufferStream";
-}
-
-
-size_t MemoryBufferStream::position() const {
-    return position_;
-}
-
-
-const MemoryBuffer& MemoryBufferStream::buffer() const {
-    return buffer_;
-}
-
-// -------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
diff --git a/src/fdb5/pmem/MemoryBufferStream.h b/src/fdb5/pmem/MemoryBufferStream.h
deleted file mode 100644
index 1cc99f86d..000000000
--- a/src/fdb5/pmem/MemoryBufferStream.h
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Simon Smart
-/// @date   Dec 2016
-
-
-#ifndef fdb5_pmem_MemoryBufferStream_H
-#define fdb5_pmem_MemoryBufferStream_H
-
-#include "eckit/serialisation/Stream.h"
-#include "eckit/memory/MemoryBuffer.h"
-#include "eckit/io/Length.h"
-#include "eckit/io/Offset.h"
-
-#include 
-
-
-namespace fdb5 {
-namespace pmem {
-
-// -------------------------------------------------------------------------------------------------
-
-// A little helper class that might end up somewhere else
-
-class MemoryBufferStream : public eckit::Stream {
-
-public: // methods
-
-    MemoryBufferStream();
-    ~MemoryBufferStream();
-
-    virtual long read(void*,long);
-    virtual long write(const void*,long);
-    virtual void rewind();
-    virtual std::string name() const;
-
-    size_t position() const;
-    const eckit::MemoryBuffer& buffer() const;
-
-private: // members
-
-    eckit::Length size_;
-    eckit::Offset position_;
-
-    eckit::MemoryBuffer buffer_;
-};
-
-
-// -------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
-
-#endif // fdb5_pmem_MemoryBufferStream_H
diff --git a/src/fdb5/pmem/PBaseNode.cc b/src/fdb5/pmem/PBaseNode.cc
deleted file mode 100644
index 8a627bbb7..000000000
--- a/src/fdb5/pmem/PBaseNode.cc
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-3.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation
- * nor does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Simon Smart
-/// @date   Sep 2016
-
-#include "eckit/log/Log.h"
-
-#include "fdb5/pmem/PBaseNode.h"
-#include "fdb5/pmem/PDataNode.h"
-#include "fdb5/pmem/PBranchingNode.h"
-
-
-namespace fdb5 {
-namespace pmem {
-
-// -------------------------------------------------------------------------------------------------
-
-PBaseNode::PBaseNode(NodeType type, const KeyType &key, const ValueType &value) :
-    type_(type),
-    idKey_(key),
-    idValue_(value) {}
-
-
-bool PBaseNode::isNull() const {
-    return type_ == PBaseNode::NULL_NODE;
-}
-
-bool PBaseNode::isBranchingNode() const {
-    return type_ == PBaseNode::BRANCHING_NODE;
-}
-
-bool PBaseNode::isDataNode() const {
-    return type_ == PBaseNode::DATA_NODE;
-}
-
-PBranchingNode& PBaseNode::asBranchingNode() {
-    ASSERT(isBranchingNode());
-
-    return *(static_cast(this));
-}
-
-PDataNode& PBaseNode::asDataNode() {
-    ASSERT(isDataNode());
-
-    return *(static_cast(this));
-}
-
-bool PBaseNode::matches(const KeyType& key, const ValueType& value) const {
-    return key == idKey_ && value == idValue_;
-}
-
-std::string PBaseNode::key() const {
-    return idKey_;
-}
-
-std::string PBaseNode::value() const {
-    return idValue_;
-}
-
-std::ostream& operator<< (std::ostream& s, const PBaseNode& n) {
-
-    // TODO: Include the node _type_ here
-
-    s << "Node(" << n.idKey_ << ":" << n.idValue_ << ")";
-    return s;
-}
-
-// -------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace tree
diff --git a/src/fdb5/pmem/PBaseNode.h b/src/fdb5/pmem/PBaseNode.h
deleted file mode 100644
index 7fde3b7a7..000000000
--- a/src/fdb5/pmem/PBaseNode.h
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation
- * nor does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Simon Smart
-/// @date   Sep 2016
-
-
-#ifndef fdb5_pmem_PBaseNode_H
-#define fdb5_pmem_PBaseNode_H
-
-#include "eckit/types/FixedString.h"
-
-#include "pmem/PersistentPtr.h"
-
-#include 
-
-
-namespace fdb5 {
-namespace pmem {
-
-// -------------------------------------------------------------------------------------------------
-
-class PBranchingNode;
-class PDataNode;
-
-// N.B. This is to be stored in PersistentPtr --> NO virtual behaviour.
-
-class PBaseNode {
-
-public: // Types
-
-    typedef eckit::FixedString<12> KeyType;
-    typedef eckit::FixedString<12> ValueType;
-
-protected: // Types
-
-    enum NodeType {
-        NULL_NODE,
-        DATA_NODE,
-        BRANCHING_NODE
-    };
-
-protected: // Construction objects
-
-    // This object MUST NOT be constructed manually. Only as part of a derived base
-
-    PBaseNode(NodeType type, const KeyType& key, const ValueType& value);
-
-public: // methods
-
-    bool isNull() const;
-    bool isBranchingNode() const;
-    bool isDataNode() const;
-
-    PBranchingNode& asBranchingNode();
-    PDataNode& asDataNode();
-
-    bool matches(const KeyType& key, const ValueType& value) const;
-
-    std::string key() const;
-    std::string value() const;
-
-protected: // members
-
-    unsigned int type_;
-
-    /// The key:value pair that identifies which node we have reached.
-    /// @note Fixed with of 12 characters. There is no reason that this could not be updated.
-    KeyType idKey_;
-    ValueType idValue_;
-
-private: // friends
-
-    friend std::ostream& operator<< (std::ostream&, const PBaseNode&);
-};
-
-
-// -------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
-
-// -------------------------------------------------------------------------------------------------
-
-// Add a specialisation of valid for PBaseNode, to facilitate the derived types.
-
-namespace pmem {
-
-template <>
-inline bool PersistentType::validate_type_id(uint64_t id) {
-
-    // The Base node class should NEVER exist on its own. So exclude it from the possibilites.
-    return PersistentType::validate_type_id(id) ||
-           PersistentType::validate_type_id(id);
-}
-
-}
-
-// -------------------------------------------------------------------------------------------------
-
-
-#endif // fdb5_pmem_PBaseNode_H
diff --git a/src/fdb5/pmem/PBranchingNode.cc b/src/fdb5/pmem/PBranchingNode.cc
deleted file mode 100644
index 1b2b2175d..000000000
--- a/src/fdb5/pmem/PBranchingNode.cc
+++ /dev/null
@@ -1,488 +0,0 @@
-/*
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-3.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation
- * nor does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Simon Smart
-/// @date   Feb 2016
-
-#include "eckit/log/Log.h"
-#include "eckit/thread/AutoLock.h"
-#include "eckit/types/Types.h"
-
-#include "fdb5/database/Index.h"
-#include "fdb5/database/Key.h"
-#include "fdb5/database/EntryVisitMechanism.h"
-#include "fdb5/pmem/DataPoolManager.h"
-#include "fdb5/pmem/PBranchingNode.h"
-#include "fdb5/pmem/PDataNode.h"
-#include "fdb5/pmem/PMemIndex.h"
-#include "fdb5/pmem/PMemFieldLocation.h"
-#include "fdb5/pmem/PRoot.h"
-#include "fdb5/rules/Schema.h"
-
-#include "pmem/PoolRegistry.h"
-
-#include 
-
-using namespace eckit;
-using namespace pmem;
-
-
-namespace fdb5 {
-namespace pmem {
-
-// -------------------------------------------------------------------------------------------------
-
-
-PBranchingNode::PBranchingNode(const KeyType &key, const ValueType &value) :
-    PBaseNode(BRANCHING_NODE, key, value) {}
-
-
-// -------------------------------------------------------------------------------------------------
-
-
-PBranchingNode& PBranchingNode::getCreateBranchingNode(const Key& key) {
-
-    // Extract an _ordered_ sequence of key-values pairs to use to identify the node.
-    // Order comes (originally) from the schema.
-    // The vector names() is constructed as the Rules ar matched (see Rule.cc:216)
-
-    const StringList& ordered_keys(key.names());
-    KeyValueVector identifier;
-
-    identifier.reserve(ordered_keys.size());
-    for (StringList::const_iterator it = ordered_keys.begin(); it != ordered_keys.end(); ++it)
-        identifier.push_back(std::make_pair(*it, key.value(*it)));
-
-    return getCreateBranchingNode(identifier);
-}
-
-
-PBranchingNode& PBranchingNode::getCreateBranchingNode(const KeyValueVector& identifier) {
-
-    PBranchingNode* current = this;
-
-    for (KeyValueVector::const_iterator it = identifier.begin(); it != identifier.end(); ++it) {
-
-        // We don't want two processes to simultaneously create two same-named branches. So one processs cannot be
-        // checking if a branch exists whilst the other might be creating it.
-        AutoLock lock(current->mutex_);
-
-        // Search _backwards_ through the subnodes to find the element we are looking for (this permits newly written
-        // fields to mask existing ones without actually overwriting the data and making it irretrievable).
-
-        int i = current->nodes_.size() - 1;
-        for (; i >= 0; --i) {
-
-            PersistentPtr subnode = current->nodes_[i];
-
-            if (subnode->matches(it->first, it->second)) {
-
-                // Given that we are operating inside a schema, if this matches then it WILL be of the
-                // correct type --> we can request it directly.
-                ASSERT(subnode->isBranchingNode());
-                current = &subnode->asBranchingNode();
-                break;
-            }
-        }
-
-        // Create the node (and all contained sub-nodes) if it hasn't been found. Note that we
-        // start at the first
-
-        if (i < 0) {
-
-            PersistentPool& pool(::pmem::PoolRegistry::instance().poolFromPointer(this));
-
-            for (; it != identifier.end(); it++) {
-                PersistentPtr newNode;
-                newNode.allocate_ctr(pool, BaseConstructor(PBranchingNode::NodeConstructor(it->first, it->second)));
-                current->nodes_.push_back_elem(newNode);
-                current = &newNode->asBranchingNode();
-            }
-
-            break;
-        }
-    }
-
-    return *current;
-}
-
-
-::pmem::PersistentPtr PBranchingNode::getDataNode(const Key& key, DataPoolManager& mgr) const {
-
-    // Extract an _ordered_ sequence of key-values pairs to use to identify the node.
-    // Order comes (originally) from the schema.
-    // The vector names() is constructed as the Rules ar matched (see Rule.cc:216)
-
-    const StringList& ordered_keys(key.names());
-    KeyValueVector identifier;
-
-    identifier.reserve(ordered_keys.size());
-    for (StringList::const_iterator it = ordered_keys.begin(); it != ordered_keys.end(); ++it)
-        identifier.push_back(std::make_pair(*it, key.value(*it)));
-
-    // And find and return the node!
-
-    PersistentPtr ret;
-    const PBranchingNode* current = this;
-
-    for (KeyValueVector::const_iterator it = identifier.begin(); it != identifier.end(); ++it) {
-
-        AutoLock lock(current->mutex_);
-
-        // Search _backwards_ through the subnodes to find the element we are looking for (this permits newly written
-        // fields to mask existing ones without actually overwriting the data and making it irretrievable).
-
-        int i = current->nodes_.size() - 1;
-        for (; i >= 0; --i) {
-
-            PersistentPtr subnode = current->nodes_[i];
-
-            // This breaks a bit of the encapsulation, but hook in here to check that the relevant
-            // pools are loaded. We can't do this earlier, as the data is allowed to be changing
-            // up to this point...
-
-            mgr.ensurePoolLoaded(subnode.uuid());
-
-            if (subnode->matches(it->first, it->second)) {
-
-                // The last element in the chain is a data node, otherwise a branching node
-
-                KeyValueVector::const_iterator next = it;
-                ++next;
-                if (next == identifier.end()) {
-                    ASSERT(subnode->isDataNode());
-                    ret = subnode.as();
-                } else {
-                    ASSERT(subnode->isBranchingNode());
-                    current = &subnode->asBranchingNode();
-                }
-
-                break;
-            }
-        }
-
-        // We have failed to find the relevant node. Oops.
-        if (i < 0) {
-            ret.nullify();
-            break;
-        }
-    }
-
-    return ret;
-}
-
-
-::pmem::PersistentPtr PBranchingNode::getBranchingNode(const Key& key) const {
-
-    // Extract an _ordered_ sequence of key-values pairs to use to identify the node.
-    // Order comes (originally) from the schema.
-    // The vector names() is constructed as the Rules ar matched (see Rule.cc:216)
-
-    const StringList& ordered_keys(key.names());
-    KeyValueVector identifier;
-
-    identifier.reserve(ordered_keys.size());
-    for (StringList::const_iterator it = ordered_keys.begin(); it != ordered_keys.end(); ++it)
-        identifier.push_back(std::make_pair(*it, key.value(*it)));
-
-    // need to ensure that the underlying vector doesn't change too much while we are
-    // reading it (there might be a purge!)
-
-    PersistentPtr ret;
-    const PBranchingNode* current = this;
-
-    for (KeyValueVector::const_iterator it = identifier.begin(); it != identifier.end(); ++it) {
-
-        //TODO: Is this locking overzealous?
-
-        AutoLock lock(current->mutex_);
-
-        // Search _backwards_ through the subnodes to find the element we are looking for (this permits newly written
-        // fields to mask existing ones without actually overwriting the data and making it irretrievable).
-
-        int i = current->nodes_.size() - 1;
-        for (; i >= 0; --i) {
-
-            PersistentPtr subnode = current->nodes_[i];
-
-            if (subnode->matches(it->first, it->second)) {
-
-                // Given that we are operating inside a schema, if this matches then it WILL be of the
-                // correct type --> we can request it directly.
-                ASSERT(subnode->isBranchingNode());
-
-                ret = subnode.as();
-                current = &subnode->asBranchingNode();
-                break;
-            }
-        }
-
-        // We have failed to find the relevant node. Oops.
-        if (i < 0) {
-            ret.nullify();
-            break;
-        }
-    }
-
-    return ret;
-}
-
-
-void PBranchingNode::insertDataNode(const Key& key, const PersistentPtr& dataNode, DataPoolManager& mgr) {
-
-    // Obtain the _parent_ branching node - the final element in the chain identifies the
-    // specific data node, and so should be excluded from this first operation.
-
-    const StringList& ordered_keys(key.names());
-    KeyValueVector parentIdentifier;
-
-    StringList::const_iterator end = ordered_keys.end();
-    --end;
-
-    parentIdentifier.reserve(ordered_keys.size());
-    for (StringList::const_iterator it = ordered_keys.begin(); it != end; ++it)
-        parentIdentifier.push_back(std::make_pair(*it, key.value(*it)));
-
-    PBranchingNode& dataParent(getCreateBranchingNode(parentIdentifier));
-
-    // Some sanity checking
-
-    std::string k = ordered_keys[ordered_keys.size()-1];
-    std::string v = key.value(k);
-    ASSERT(dataNode->matches(k, v));
-
-    // Lock the parent node for editing.
-
-    AutoLock lock(dataParent.mutex_);
-
-
-    // Check that we will only mask PDataNodes if there is a matching one. We don't want to block
-    // off whole sections of the tree because someone has gone off-schema.
-
-    for (size_t i = 0; i < dataParent.nodes_.size(); i++) {
-
-        // This breaks a bit of the encapsulation, but hook in here to check that the relevant
-        // pools are loaded. We can't do this earlier, as the data is allowed to be changing
-        // up to this point...
-
-        mgr.ensurePoolLoaded(dataParent.nodes_[i].uuid());
-
-        if (dataParent.nodes_[i]->matches(k, v))
-            ASSERT(dataParent.nodes_[i]->isDataNode());
-    }
-
-    // And then append the data node to that one.
-
-    dataParent.nodes_.push_back_elem(dataNode.as());
-}
-
-#if 0
-
-/// visitLeaves should NOT skip out masked data. It is used for enumerating fdb-list, and listing dupliactes, etc.
-
-void PBranchingNode::visitLeaves(EntryVisitor &visitor,
-                                 DataPoolManager& mgr,
-                                 std::vector& keys,
-                                 size_t depth,
-                                 Index index) {
-
-    // Get a list of sub-nodes in memory before starting to visit them, so that we can
-    // release the lock for further writers.
-    // Use this opportunity to de-deplicate the entries.
-
-    if (!(key().empty() && value().empty())) {
-        keys.back().push(key(), value());
-        if (isIndex()) {
-
-            index = Index(new PMemIndex(keys.back(), *this, mgr));
-            keys.push_back(Key());
-            depth++;
-        }
-    }
-
-    std::map subtrees;
-    std::map > leaves;
-
-    {
-        AutoLock lock(mutex_);
-
-        int i = nodes_.size() - 1;
-        for (; i >= 0; --i) {
-
-            PersistentPtr subnode = nodes_[i];
-
-            // This breaks a bit of the encapsulation, but hook in here to check that the relevant
-            // pools are loaded. We can't do this earlier, as the data is allowed to be changing
-            // up to this point...
-
-            mgr.ensurePoolLoaded(subnode.uuid());
-
-            std::string kv = subnode->key() + ":" + subnode->value();
-
-            if (subnode->isDataNode()) {
-
-                if (leaves.find(kv) == leaves.end())
-                    leaves[kv] = subnode.as();
-
-            } else {
-
-                // n.b. Should be no masked subtrees, and no mixed data/trees
-                ASSERT(subnode->isBranchingNode());
-                ASSERT(subtrees.find(kv) == subtrees.end());
-                ASSERT(leaves.find(kv) == leaves.end());
-                subtrees[kv] = &(subnode->asBranchingNode());
-            }
-        }
-    }
-
-    // Visit the leaves
-    std::map >::const_iterator it_dt = leaves.begin();
-    for (; it_dt != leaves.end(); ++it_dt) {
-
-        ASSERT(!index.null());
-
-        // we do the visitation from here, not from PDataNode::visit, as the PMemFieldLocation needs
-        // the PersistentPtr, not the "this" pointer.
-        keys.back().push(it_dt->second->key(), it_dt->second->value());
-        Field field(PMemFieldLocation(it_dt->second, mgr.getPool(it_dt->second.uuid())));
-
-        visitor.visit(index, field,"Index fingerprint unused", keys.back().valuesToString());
-
-        keys.back().pop(it_dt->second->key());
-    }
-
-    // Recurse down the trees
-    std::map::const_iterator it_br = subtrees.begin();
-    for (; it_br != subtrees.end(); ++it_br) {
-        it_br->second->visitLeaves(visitor, mgr, keys, depth, index);
-    }
-
-    if (!(key().empty() && value().empty())) {
-        if (isIndex())
-            keys.pop_back();
-        keys.back().pop(key());
-    }
-}
-#endif
-
-
-void PBranchingNode::visitLeaves(EntryVisitor &visitor,
-                                 DataPoolManager& mgr,
-                                 std::vector& keys,
-                                 size_t depth,
-                                 Index index) {
-
-    // Get a list of sub-nodes in memory before starting to visit them, so that we can
-    // release the lock for further writers.
-    // Use this opportunity to de-deplicate the entries.
-
-    if (!(key().empty() && value().empty())) {
-        keys.back().push(key(), value());
-        if (isIndex()) {
-
-            index = Index(new PMemIndex(keys.back(), *this, mgr));
-
-            visitor.visitIndex(index);
-
-            // If we are not visiting right to the leaves, there is no need to go
-            // any further
-            if (!visitor.visitEntries()) {
-                keys.back().pop(key());
-                return;
-            }
-
-            keys.push_back(Key());
-            depth++;
-        }
-    }
-
-    std::vector subtrees;
-    std::vector > leaves;
-
-    {
-        AutoLock lock(mutex_);
-
-        int i = nodes_.size() - 1;
-        for (; i >= 0; --i) {
-
-            PersistentPtr subnode = nodes_[i];
-
-            // This breaks a bit of the encapsulation, but hook in here to check that the relevant
-            // pools are loaded. We can't do this earlier, as the data is allowed to be changing
-            // up to this point...
-
-            mgr.ensurePoolLoaded(subnode.uuid());
-
-            std::string kv = subnode->key() + ":" + subnode->value();
-
-            if (subnode->isDataNode()) {
-
-                // This would do masking...
-                //if (leaves.find(kv) == leaves.end())
-                //    leaves[kv] = subnode.as();
-                leaves.push_back(subnode.as());
-
-            } else {
-
-                // n.b. Should be no masked subtrees, and no mixed data/trees
-                ASSERT(subnode->isBranchingNode());
-//                ASSERT(subtrees.find(kv) == subtrees.end());
-//                ASSERT(leaves.find(kv) == leaves.end());
-                subtrees.push_back(&subnode->asBranchingNode());
-//                subtrees[kv] = &(subnode->asBranchingNode());
-            }
-        }
-    }
-
-    // Visit the leaves
-    std::vector >::const_iterator it_dt = leaves.begin();
-    for (; it_dt != leaves.end(); ++it_dt) {
-
-        ASSERT(!index.null());
-
-        // we do the visitation from here, not from PDataNode::visit, as the PMemFieldLocation needs
-        // the PersistentPtr, not the "this" pointer.
-        keys.back().push((*it_dt)->key(), (*it_dt)->value());
-        Field field(PMemFieldLocation(*it_dt, mgr.getPool(it_dt->uuid())));
-
-        visitor.visitDatum(field, keys.back().valuesToString());
-
-        keys.back().pop((*it_dt)->key());
-    }
-
-    // Recurse down the trees
-    std::vector::const_iterator it_br = subtrees.begin();
-    for (; it_br != subtrees.end(); ++it_br) {
-        (*it_br)->visitLeaves(visitor, mgr, keys, depth, index);
-    }
-
-    if (!(key().empty() && value().empty())) {
-        if (isIndex()) {
-            keys.pop_back();
-        }
-        keys.back().pop(key());
-    }
-}
-
-
-bool PBranchingNode::isIndex() const {
-
-    // An index will have an axis object. Couple these concepts
-    return !axis_.null();
-}
-
-
-// -------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace tree
diff --git a/src/fdb5/pmem/PBranchingNode.h b/src/fdb5/pmem/PBranchingNode.h
deleted file mode 100644
index a2e3abaf5..000000000
--- a/src/fdb5/pmem/PBranchingNode.h
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation
- * nor does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Simon Smart
-/// @date   Sep 2016
-
-
-#ifndef fdb5_pmem_PBranchingNode_H
-#define fdb5_pmem_PBranchingNode_H
-
-#include "eckit/types/Types.h"
-
-#include "pmem/AtomicConstructor.h"
-#include "pmem/AtomicConstructorCast.h"
-#include "pmem/PersistentVector.h"
-#include "pmem/PersistentMutex.h"
-#include "pmem/PersistentBuffer.h"
-
-#include "fdb5/database/Index.h"
-#include "fdb5/database/Key.h"
-#include "fdb5/pmem/PBaseNode.h"
-
-
-namespace fdb5 {
-
-class EntryVisitor;
-class Schema;
-class IndexBase;
-
-namespace pmem {
-
-class DataPoolManager;
-
-// -------------------------------------------------------------------------------------------------
-
-// N.B. This is to be stored in PersistentPtr --> NO virtual behaviour.
-
-class PBranchingNode : public PBaseNode {
-
-public: // types
-
-    typedef std::vector > KeyValueVector;
-
-public: // Construction objects
-
-    /// The Index constructor creates a BranchingNode, and then recursively creates required
-    /// subnodes until the key is exhausted with a BranchingNode.
-
-    typedef ::pmem::AtomicConstructor2 NodeConstructor;
-
-    typedef ::pmem::AtomicConstructorCast BaseConstructor;
-
-public: // methods
-
-    PBranchingNode(const KeyType& key, const ValueType& value);
-
-    /// Recursively create a chain of nodes, returning a pointer to the tail of the chain
-    PBranchingNode(KeyValueVector::const_iterator start,
-                   KeyValueVector::const_iterator end,
-                   PBranchingNode** const tailNode);
-
-    /// Obtain a branching (intermediate) node in the tree. If it doesn't exist, then
-    /// create it.
-    /// @arg key - The key of the node to create or get _relative_to_the_current_node_
-    /// @return A reference to the newly created, or found, node
-    /// @note This is primarily in place to support the getIndex functionality.
-    PBranchingNode& getCreateBranchingNode(const Key& key);
-
-    void insertDataNode(const Key& key, const ::pmem::PersistentPtr& dataNode, DataPoolManager& mgr);
-
-    ::pmem::PersistentPtr getDataNode(const Key& key, DataPoolManager& mgr) const;
-
-    /// Obtain a branching (intermediate) node in the tree. If it doesn't exist, then
-    /// a null pointer is returned.
-    /// @arg key - The key of the node to create or get _relative_to_the_current_node_
-    /// @return A PersistentPtr to the node, or a null() pointer.
-    ::pmem::PersistentPtr getBranchingNode(const Key& key) const;
-
-    void visitLeaves(EntryVisitor &visitor,
-                     DataPoolManager& mgr,
-                     std::vector& keys,
-                     size_t depth,
-                     Index index = Index());
-
-    /// Returns true if this node is (or at least has been used as) an Index.
-    bool isIndex() const;
-
-private: // methods
-
-    PBranchingNode& getCreateBranchingNode(const KeyValueVector& identifier);
-
-protected: // members
-
-    ::pmem::PersistentVector nodes_;
-
-    mutable ::pmem::PersistentMutex mutex_;
-
-    // TODO: Do we want to have a separate IndexNode?
-    ::pmem::PersistentPtr< ::pmem::PersistentBuffer> axis_;
-
-private: // friends
-
-    friend class PMemIndex;
-};
-
-
-// -------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
-
-
-#endif // fdb5_pmem_PBranchingNode_H
diff --git a/src/fdb5/pmem/PDataNode.cc b/src/fdb5/pmem/PDataNode.cc
deleted file mode 100644
index d4252bc82..000000000
--- a/src/fdb5/pmem/PDataNode.cc
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-3.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation
- * nor does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Simon Smart
-/// @date   Feb 2016
-
-#include "eckit/io/Length.h"
-#include "eckit/log/Log.h"
-#include "eckit/types/Types.h"
-
-#include "fdb5/database/Key.h"
-#include "fdb5/pmem/PDataNode.h"
-#include "fdb5/pmem/PMemIndex.h"
-#include "fdb5/pmem/PMemFieldLocation.h"
-#include "fdb5/pmem/PBranchingNode.h"
-
-#include 
-
-using namespace eckit;
-using namespace pmem;
-
-
-namespace fdb5 {
-namespace pmem {
-
-// ---------------------------------------------------------------------------------------------------------------------
-
-
-PDataNode::PDataNode(const KeyType& key, const ValueType& value, const void* data, eckit::Length length) :
-    PBaseNode(DATA_NODE, key, value),
-    length_(length) {
-
-    ::memcpy(data_, data, length_);
-}
-
-
-size_t PDataNode::data_size(eckit::Length length) {
-    return sizeof(PDataNode) - sizeof(PDataNode::data_) + length;
-}
-
-
-const void* PDataNode::data() const {
-    return data_;
-}
-
-
-eckit::Length PDataNode::length() const {
-    return length_;
-}
-
-
-// ---------------------------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace tree
diff --git a/src/fdb5/pmem/PDataNode.h b/src/fdb5/pmem/PDataNode.h
deleted file mode 100644
index 152a58235..000000000
--- a/src/fdb5/pmem/PDataNode.h
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation
- * nor does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Simon Smart
-/// @date   Sep 2016
-
-
-#ifndef fdb5_pmem_PDataNode_H
-#define fdb5_pmem_PDataNode_H
-
-#include "eckit/types/Types.h"
-
-#include "pmem/AtomicConstructor.h"
-#include "pmem/AtomicConstructorCast.h"
-#include "pmem/PersistentVector.h"
-
-#include "fdb5/pmem/PBaseNode.h"
-
-
-namespace fdb5 {
-
-class Key;
-class EntryVisitor;
-
-namespace pmem {
-
-class DataPoolManager;
-
-// -------------------------------------------------------------------------------------------------
-
-// N.B. This is to be stored in PersistentPtr --> NO virtual behaviour.
-
-class PDataNode : public PBaseNode {
-
-public: // types
-
-    typedef ::pmem::AtomicConstructor4 Constructor;
-
-    typedef ::pmem::AtomicConstructorCast BaseConstructor;
-
-public: // methods
-
-    PDataNode(const KeyType& key, const ValueType& value, const void* data, eckit::Length length);
-
-    const void* data() const;
-
-    eckit::Length length() const;
-
-    static size_t data_size(eckit::Length length);
-
-private: // members
-
-    eckit::Length length_;
-
-    // We use a length of 8 to ensure (a) this is legal c++, and (b) there is no possibility of the complier
-    // adding unecessary padding due to an unreasonable desire for structure alignment.
-    char data_[8];
-
-};
-
-// -------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
-
-// -------------------------------------------------------------------------------------------------
-
-namespace pmem {
-
-template<>
-inline size_t AtomicConstructor4Base::size() const {
-    return fdb5::pmem::PDataNode::data_size(x4_);
-
-}
-
-}
-
-
-#endif // fdb5_pmem_PDataNode_H
diff --git a/src/fdb5/pmem/PDataRoot.cc b/src/fdb5/pmem/PDataRoot.cc
deleted file mode 100644
index 310983c69..000000000
--- a/src/fdb5/pmem/PDataRoot.cc
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-3.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation
- * nor does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Simon Smart
-/// @date   Sept 2016
-
-#include "fdb5/pmem/PDataRoot.h"
-
-#include "pmem/PersistentPtr.h"
-
-#include 
-
-using namespace eckit;
-
-namespace fdb5 {
-namespace pmem {
-
-// -------------------------------------------------------------------------------------------------
-
-PDataRoot::PDataRoot() :
-    tag_(PDataRootTag),
-    version_(PDataRootVersion),
-    created_(time(0)),
-    createdBy_(getuid()),
-    finalised_(false) {}
-
-
-bool PDataRoot::valid() const {
-
-    if (tag_ != PDataRootTag) {
-        Log::error() << "Persistent root tag does not match" << std::endl;
-        return false;
-    }
-
-    if (version_ != PDataRootVersion) {
-        Log::error() << "Invalid persistent root version" << std::endl;
-        return false;
-    }
-
-    return true;
-}
-
-const time_t& PDataRoot::created() const {
-    return created_;
-}
-
-bool PDataRoot::finalised() const {
-    return finalised_;
-}
-
-void PDataRoot::finalise() {
-
-    finalised_ = true;
-    ::pmemobj_persist(::pmemobj_pool_by_ptr(&finalised_), &finalised_, sizeof(finalised_));
-}
-
-void PDataRoot::print(std::ostream& s) const {
-    s << "PDataRoot(0x" << this << ")";
-}
-
-// -------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
diff --git a/src/fdb5/pmem/PDataRoot.h b/src/fdb5/pmem/PDataRoot.h
deleted file mode 100644
index 05d29c66a..000000000
--- a/src/fdb5/pmem/PDataRoot.h
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation
- * nor does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Simon Smart
-/// @date   Sep 2016
-
-
-#ifndef fdb5_pmem_PDataRoot_H
-#define fdb5_pmem_PDataRoot_H
-
-#include "eckit/memory/NonCopyable.h"
-#include "eckit/types/FixedString.h"
-
-namespace fdb5 {
-namespace pmem {
-
-// -------------------------------------------------------------------------------------------------
-
-/// A root object for a persistent data pool
-/// @note Unlike all other root objects (in the examples) this does NOT contain PersistentPtrs to
-///       the data - and as such the data is inaccessible directly within the root. The data is
-///       pointed to with PersistentPtrs from another pool (linked from a PIndexRoot).
-
-class PDataRoot : public eckit::NonCopyable {
-
-public: // methods
-
-    PDataRoot();
-
-    bool valid() const;
-
-    const time_t& created() const;
-
-    bool finalised() const;
-
-    void finalise();
-
-    void print(std::ostream& s) const;
-
-private: // members
-
-    eckit::FixedString<8> tag_;
-
-    unsigned short int version_;
-
-    time_t created_;
-
-    long createdBy_;
-
-    bool finalised_;
-
-private: // friends
-
-    friend std::ostream& operator<<(std::ostream& s, const PDataRoot& r) { r.print(s); return s; }
-};
-
-
-// A consistent definition of the tag for comparison purposes.
-const eckit::FixedString<8> PDataRootTag = "66FDB566";
-const unsigned short int PDataRootVersion = 2;
-
-
-// -------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
-
-#endif // fdb5_pmem_PDataRoot_H
diff --git a/src/fdb5/pmem/PIndexRoot.cc b/src/fdb5/pmem/PIndexRoot.cc
deleted file mode 100644
index 506a3369b..000000000
--- a/src/fdb5/pmem/PIndexRoot.cc
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-3.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation
- * nor does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Simon Smart
-/// @author Tiago Quintino
-/// @date   Sept 2016
-
-#include 
-
-#include "eckit/log/Log.h"
-#include "eckit/serialisation/MemoryStream.h"
-
-#include "fdb5/LibFdb5.h"
-#include "fdb5/pmem/PIndexRoot.h"
-#include "fdb5/pmem/MemoryBufferStream.h"
-#include "fdb5/config/Config.h"
-
-#include "pmem/PoolRegistry.h"
-
-using namespace eckit;
-using namespace pmem;
-
-
-namespace fdb5 {
-namespace pmem {
-
-//----------------------------------------------------------------------------------------------------------------------
-
-PIndexRoot::PIndexRoot(const PersistentPtr& key,
-                       const PersistentPtr& schema,
-                       const PersistentPtr& rootNode) :
-    tag_(PIndexRootTag),
-    version_(PIndexRootVersion),
-    rootSize_(sizeof(PIndexRoot)),
-    created_(time(0)),
-    createdBy_(getuid()),
-    rootNode_(rootNode),
-    schema_(schema),
-    dbKey_(key) {
-
-    dataPoolUUIDs_.nullify();
-}
-
-
-void PIndexRoot::build(PersistentPtr& ptr, const Key& dbKey, const eckit::PathName& schema) {
-
-    PersistentPool& pool(PoolRegistry::instance().poolFromPointer(&ptr));
-
-    // The root node of the tree does not have an associated key/value.
-
-    PersistentPtr rootNode = pool.allocate(std::string(), std::string());
-
-    // Store the currently loaded master schema, so it can be recovered later
-
-    eckit::PathName schemaPath(schema.exists() ? schema : LibFdb5::instance().defaultConfig().schemaPath());
-    std::unique_ptr schemaFile(schemaPath.fileHandle());
-    std::string buf(static_cast(schemaFile->openForRead()), '\0');
-    schemaFile->read(&buf[0], buf.size());
-
-    PersistentPtr schemaObj = pool.allocate(buf);
-
-    // Store the current DB key
-
-    MemoryBufferStream s;
-    s << dbKey;
-    const void* key_data = s.buffer();
-
-    PersistentPtr key = pool.allocate(key_data, s.position());
-
-    ptr.allocate(key, schemaObj, rootNode);
-}
-
-//----------------------------------------------------------------------------------------------------------------------
-
-/*
- * We can use whatever knowledge we have to test the validity of the structure.
- *
- * For now, we just check that the tag is set (i.e. it is initialised).
- */
-bool PIndexRoot::valid() const {
-
-    if (tag_ != PIndexRootTag) {
-        Log::error() << "Persistent root tag does not match" << std::endl;
-        return false;
-    }
-
-    if (rootSize_ != sizeof(PIndexRoot)) {
-        Log::error() << "Invalid (old) structure size for persistent root" << std::endl;
-        return false;
-    }
-
-    if (version_ != PIndexRootVersion) {
-        Log::error() << "Invalid persistent root version" << std::endl;
-        return false;
-    }
-
-    if (rootNode_.null() || !rootNode_.valid()) {
-        Log::error() << "Inconsistent tree root node" << std::endl;
-        return false;
-    }
-
-    if (schema_.null()) {
-        Log::error() << "Schema missing from tree root node" << std::endl;
-        return false;
-    }
-
-    if (dbKey_.null()) {
-        Log::error() << "(Root) database key missing from tree root node" << std::endl;
-        return false;
-    }
-
-    return true;
-}
-
-const time_t& PIndexRoot::created() const {
-    return created_;
-}
-
-unsigned short int PIndexRoot::version() const {
-    return version_;
-}
-
-long PIndexRoot::uid() const {
-    return createdBy_;
-}
-
-::pmem::PersistentPtr PIndexRoot::getBranchingNode(const Key& key) const {
-
-    // Get the relevant index.
-
-    return rootNode_->getBranchingNode(key);
-}
-
-PBranchingNode& PIndexRoot::getCreateBranchingNode(const Key& key) {
-
-    // Get the relevant index. If the system is open for writing then we should create
-    // a new index if it doesn't exist.
-
-    return rootNode_->getCreateBranchingNode(key);
-}
-
-void PIndexRoot::visitLeaves(EntryVisitor& visitor, DataPoolManager& mgr, const Schema& schema) const {
-
-    std::vector keys;
-    keys.push_back(databaseKey());
-    keys.push_back(Key());
-    return rootNode_->visitLeaves(visitor, mgr, keys, 1);
-}
-
-const ::pmem::PersistentString& PIndexRoot::schema() const {
-    return *schema_;
-}
-
-const ::pmem::PersistentPODVector& PIndexRoot::dataPoolUUIDs() const {
-    return dataPoolUUIDs_;
-}
-
-Key PIndexRoot::databaseKey() const {
-
-    ASSERT(!dbKey_.null());
-    const PersistentBuffer& buf(*dbKey_);
-    MemoryStream s(buf.data(), buf.size());
-
-    Key k(s);
-    return k;
-}
-
-void PIndexRoot::print(std::ostream& s) const {
-    s << "PIndexRoot(0x" << this << ")";
-}
-
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
diff --git a/src/fdb5/pmem/PIndexRoot.h b/src/fdb5/pmem/PIndexRoot.h
deleted file mode 100644
index 692acad18..000000000
--- a/src/fdb5/pmem/PIndexRoot.h
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation
- * nor does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Simon Smart
-/// @date   Sep 2016
-
-
-#ifndef fdb5_pmem_PIndexRoot_H
-#define fdb5_pmem_PIndexRoot_H
-
-#include "eckit/memory/NonCopyable.h"
-#include "eckit/types/FixedString.h"
-#include "eckit/types/Types.h"
-
-#include "pmem/AtomicConstructor.h"
-#include "pmem/PersistentPtr.h"
-#include "pmem/PersistentPODVector.h"
-#include "pmem/PersistentString.h"
-
-#include "fdb5/pmem/PBranchingNode.h"
-#include "fdb5/pmem/PDataNode.h"
-
-#include 
-
-
-namespace fdb5 {
-
-class EntryVisitor;
-
-namespace pmem {
-
-// -------------------------------------------------------------------------------------------------
-
-// N.B. This is to be stored in PersistentPtr --> NO virtual behaviour.
-
-class PIndexRoot : public eckit::NonCopyable {
-
-public: // methods
-
-    PIndexRoot(const ::pmem::PersistentPtr< ::pmem::PersistentBuffer>& key,
-               const ::pmem::PersistentPtr< ::pmem::PersistentString>& schema,
-               const ::pmem::PersistentPtr& rootNode);
-
-    static void build(::pmem::PersistentPtr& ptr, const Key& dbKey, const eckit::PathName& schema);
-
-    bool valid() const;
-
-    const time_t& created() const;
-    unsigned short int version() const;
-    long uid() const;
-
-    ::pmem::PersistentPtr getBranchingNode(const Key& key) const;
-    PBranchingNode& getCreateBranchingNode(const Key& key);
-
-    void visitLeaves(EntryVisitor& visitor, DataPoolManager& mgr, const Schema& schema) const;
-
-    void print(std::ostream& s) const;
-
-    const ::pmem::PersistentString& schema() const;
-
-    const ::pmem::PersistentPODVector& dataPoolUUIDs() const;
-
-    Key databaseKey() const;
-
-private: // members
-
-    eckit::FixedString<8> tag_;
-
-    unsigned short int version_;
-
-    /// Store the size of the PMemRoot object. Allows some form of sanity check
-    /// when it is being opened, that everything is safe.
-    unsigned short int rootSize_;
-
-    time_t created_;
-
-    long createdBy_;
-
-    ::pmem::PersistentPtr rootNode_;
-
-    /// Keep track of how many data pools are in use. Ensure locking whenever updating this variable.
-
-    ::pmem::PersistentMutex mutex_;
-    ::pmem::PersistentPODVector dataPoolUUIDs_;
-
-    ::pmem::PersistentPtr< ::pmem::PersistentString> schema_;
-
-    ::pmem::PersistentPtr< ::pmem::PersistentBuffer> dbKey_;
-
-private: // friends
-
-    friend class DataPoolManager;
-
-    friend std::ostream& operator<<(std::ostream& s, const PIndexRoot& r) { r.print(s); return s; }
-};
-
-
-// A consistent definition of the tag for comparison purposes.
-const eckit::FixedString<8> PIndexRootTag = "77FDB577";
-const unsigned short int PIndexRootVersion = 2;
-
-
-// -------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
-
-#endif // fdb5_pmem_PIndexRoot_H
diff --git a/src/fdb5/pmem/PMemDB.cc b/src/fdb5/pmem/PMemDB.cc
deleted file mode 100644
index 460a418fb..000000000
--- a/src/fdb5/pmem/PMemDB.cc
+++ /dev/null
@@ -1,328 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-#include "eckit/config/Resource.h"
-#include "eckit/log/Timer.h"
-
-#include "fdb5/LibFdb5.h"
-#include "fdb5/database/Key.h"
-#include "fdb5/rules/Rule.h"
-
-#include "fdb5/pmem/PMemDB.h"
-#include "fdb5/pmem/PMemFieldLocation.h"
-#include "fdb5/pmem/PoolManager.h"
-#include "fdb5/pmem/PMemStats.h"
-
-#include "pmem/PersistentString.h"
-
-#include 
-#include 
-#include 
-
-using namespace eckit;
-using namespace pmem;
-
-namespace fdb5 {
-namespace pmem {
-
-static std::string userName(long id) {
-
-  struct passwd *p = getpwuid(id);
-
-  if (p) {
-    return p->pw_name;
-  } else {
-    return eckit::Translator()(id);
-  }
-}
-
-//----------------------------------------------------------------------------------------------------------------------
-
-PMemDB::PMemDB(const Key& key, const Config& config) :
-    DB(key),
-    poolDir_(PoolManager(config).pool(key)),
-    dbConfig_(config),
-    init_(false)
-{
-
-    // If opened with a key in this manner, it is for write, so should open up fully.
-    initialisePool();
-}
-
-
-PMemDB::PMemDB(const PathName& poolDir, const Config& config) :
-    DB(Key()),
-    poolDir_(poolDir),
-    dbConfig_(config),
-    init_(false)
-{
-    initialisePool();
-}
-
-PMemDB::~PMemDB() {
-    close();
-}
-
-void PMemDB::initialisePool() {
-
-    ASSERT(init_ == false);
-
-    // Get (or create) the pool
-    pool_.reset(Pool::obtain(poolDir_, Resource("fdbPMemPoolSize", 1024 * 1024 * 1024), dbKey_, dbConfig_.schemaPath()));
-
-    root_ = &pool_->root();
-
-    dataPoolMgr_.reset(new DataPoolManager(poolDir_, *root_, pool_->baseRoot().uuid()));
-
-    // Initialise the schema object for comparison against the global schema.
-
-    const PersistentString& schemaBuf(root_->schema());
-    std::string s(schemaBuf.c_str(), schemaBuf.length());
-    std::istringstream iss(s);
-    schema_.load(iss);
-
-    if (dbKey_.empty())
-        dbKey_ = root_->databaseKey();
-}
-
-//----------------------------------------------------------------------------------------------------------------------
-
-bool PMemDB::open() {
-
-    if (!pool_)
-        initialisePool();
-
-    return true;
-}
-
-bool PMemDB::exists() const {
-    return Pool::exists(poolDir_);
-}
-
-void PMemDB::archive(const Key& key, const void *data, Length length) {
-    Log::error() << "archive not implemented for " << *this << std::endl;
-    NOTIMP;
-}
-
-void PMemDB::visitEntries(EntryVisitor& visitor, bool sorted) {
-
-    // There is no meaningfully more-efficient way or sorting the indexes!
-    (void) sorted;
-
-    LOG_DEBUG_LIB(LibFdb5) << "Visiting entries in DB with key " << dbKey_ << std::endl;
-
-    visitor.visitDatabase(*this);
-
-    if (visitor.visitIndexes()) {
-        ASSERT(pool_ && root_);
-        root_->visitLeaves(visitor, *dataPoolMgr_, schema());
-    }
-
-    visitor.databaseComplete(*this);
-}
-
-eckit::DataHandle * PMemDB::retrieve(const Key& key) const {
-    Log::error() << "retrieve not implemented for " << *this << std::endl;
-    NOTIMP;
-}
-
-void PMemDB::flush() {
-    // Intentionally left blank.
-    // The libpmemobj functionality works with atomic writes, and everything is flushed before it returns success.
-}
-
-void PMemDB::close() {
-
-    // Close any open indices
-
-    for (IndexStore::iterator it = indexes_.begin(); it != indexes_.end(); ++it) {
-        Index& idx = it->second;
-        idx.close();
-    }
-    indexes_.clear();
-}
-
-void PMemDB::axis(const std::string &keyword, eckit::StringSet &s) const {
-    Log::error() << "axis not implemented for " << *this << std::endl;
-    NOTIMP;
-}
-
-bool PMemDB::selectIndex(const Key& idxKey) {
-    NOTIMP;
-}
-
-void PMemDB::dump(std::ostream& out, bool simple) const {
-
-    // Check that things are open
-    ASSERT(pool_);
-
-    // Output details of the DB itself
-
-    out << std::endl << "PMEM_DB" << std::endl;
-    out << "  Version: " << root_->version() << std::endl;
-    out << "  Created: " << root_->created() << std::endl;
-
-    out << "  uid: ";
-    struct passwd* p = ::getpwuid(root_->uid());
-    if (p)
-        out << p->pw_name;
-    else
-        out << root_->uid();
-    out << std::endl;
-
-    Log::info() << "  Key: " << dbKey_ << std::endl << std::endl;
-
-    // And dump the rest of the stuff
-
-    struct DumpVisitor : EntryVisitor {
-        DumpVisitor(std::ostream& out) : out_(out) {}
-        void visitDatum(const Field& field, const TypedKey& datumKey) override {
-            out_ << "ENTRY" << std::endl;
-            out_ << "  " << datumKey << std::endl;
-            field.location().dump(out_);
-        }
-        std::ostream& out_;
-    };
-
-    DumpVisitor visitor(out);
-    const_cast(this)->visitEntries(visitor);
-}
-
-std::string PMemDB::owner() const {
-
-    ASSERT(pool_);
-    ASSERT(root_);
-
-    return userName(root_->uid());
-}
-
-void PMemDB::visit(DBVisitor &visitor) {
-    visitor(*this);
-}
-
-const Schema& PMemDB::schema() const {
-
-    // We must have opened the DB for the schema to have been loaded.
-    ASSERT(pool_);
-    return schema_;
-}
-
-void PMemDB::deselectIndex() {
-    currentIndex_ = Index(); // essentially a no-op, as we don't have any files to open, etc.
-}
-
-DbStats PMemDB::statistics() const
-{
-    PMemDbStats* stats = new PMemDbStats();
-
-    stats->dataPoolsCount_ += dataPoolsCount();
-    stats->indexPoolsCount_ += 1; // FTM we use a single pool, but we may support overspill in the future
-    stats->indexesCount_ += 1;    // FTM we use a single index, but we may support overspill in the future
-
-    stats->schemaSize_ += schemaSize();
-    stats->indexPoolsSize_ += pool_->size();
-    stats->dataPoolsSize_  += dataPoolsSize();
-
-    return DbStats(stats);
-}
-
-StatsReportVisitor* PMemDB::statsReportVisitor() const {
-    return new PMemStatsReportVisitor(*this);
-}
-
-PurgeVisitor *PMemDB::purgeVisitor() const {
-    NOTIMP;
-    //    return new TocPurgeVisitor(*this);
-}
-
-void PMemDB::maskIndexEntry(const Index &index) const {
-    NOTIMP;
-}
-
-
-std::vector PMemDB::indexes(bool sorted) const {
-    throw eckit::NotImplemented("TocDB::indexes() isn't implemented for this DB type "
-                                "-- perhaps this is a writer?", Here());
-}
-
-eckit::PathName PMemDB::basePath() const {
-    return poolDir_;
-}
-
-std::vector PMemDB::metadataPaths() const {
-
-    ASSERT(pool_);
-    ASSERT(root_);
-    ASSERT(dataPoolMgr_);
-
-    return { pool_->path() };
-}
-
-
-size_t PMemDB::dataPoolsCount() const {
-    return dataPoolMgr_->dataPoolsCount();
-}
-
-size_t PMemDB::dataPoolsSize() const {
-
-    return dataPoolMgr_->dataSize();
-}
-
-std::string PMemDB::dbType() const
-{
-    return PMemDB::dbTypeName();
-}
-
-void PMemDB::checkUID() const {
-
-    ASSERT(pool_);
-    ASSERT(root_);
-
-    static bool fdbOnlyCreatorCanWrite = eckit::Resource("fdbOnlyCreatorCanWrite", true);
-    if (!fdbOnlyCreatorCanWrite) {
-        return;
-    }
-
-    static std::vector fdbSuperUsers = eckit::Resource >("fdbSuperUsers", "", true);
-
-    long uid = ::getuid();
-
-    if (root_->uid() != uid) {
-
-        if(std::find(fdbSuperUsers.begin(), fdbSuperUsers.end(), userName(uid)) == fdbSuperUsers.end()) {
-
-            std::ostringstream oss;
-            oss << "Only user '"
-                << userName(root_->uid())
-                << "' can write to FDB "
-                << basePath()
-                << ", current user is '"
-                << userName(uid)
-                << "'";
-
-            throw eckit::UserError(oss.str());
-        }
-    }
-}
-
-size_t PMemDB::schemaSize() const {
-    ASSERT(pool_);
-    return root_->schema().length();
-}
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
diff --git a/src/fdb5/pmem/PMemDB.h b/src/fdb5/pmem/PMemDB.h
deleted file mode 100644
index 6a33185ab..000000000
--- a/src/fdb5/pmem/PMemDB.h
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @file   PMemDB.h
-/// @author Baudouin Raoult
-/// @author Tiago Quintino
-/// @date   Mar 2016
-
-#ifndef fdb5_PMemDB_H
-#define fdb5_PMemDB_H
-
-#include 
-
-#include "fdb5/config/Config.h"
-#include "fdb5/database/DB.h"
-#include "fdb5/database/Index.h"
-#include "fdb5/rules/Schema.h"
-
-#include "fdb5/pmem/DataPoolManager.h"
-#include "fdb5/pmem/PIndexRoot.h"
-#include "fdb5/pmem/PMemEngine.h"
-#include "fdb5/pmem/Pool.h"
-
-namespace fdb5 {
-namespace pmem {
-
-//----------------------------------------------------------------------------------------------------------------------
-
-/// DB that implements the FDB on POSIX filesystems
-
-class PMemDB : public DB {
-
-public: // methods
-
-    PMemDB(const Key& key, const Config& config);
-    PMemDB(const eckit::PathName& directory, const Config& config);
-
-    ~PMemDB() override;
-
-    static const char* dbTypeName() { return PMemEngine::typeName(); }
-
-    size_t schemaSize() const;
-
-    /// How many data pools are under management?
-    size_t dataPoolsCount() const;
-
-    /// The total size of all of the pools involved in this database
-    size_t dataPoolsSize() const;
-
-protected: // methods
-
-    virtual std::string dbType() const;
-
-    void checkUID() const override;
-
-    bool open() override;
-    void close() override;
-    void flush() override;
-    bool exists() const override;
-    void visitEntries(EntryVisitor& visitor, bool sorted=false) override;
-    void visit(DBVisitor& visitor) override;
-    void dump(std::ostream& out, bool simple=false) const override;
-    std::string owner() const override;
-    eckit::PathName basePath() const override;
-    std::vector metadataPaths() const;
-    const Schema& schema() const override;
-
-    virtual eckit::DataHandle *retrieve(const Key& key) const;
-    virtual void archive(const Key& key, const void *data, eckit::Length length);
-    virtual void axis(const std::string &keyword, eckit::StringSet &s) const;
-
-    StatsReportVisitor* statsReportVisitor() const override;
-    PurgeVisitor* purgeVisitor() const override;
-    void maskIndexEntry(const Index& index) const override;
-
-    // void loadSchema();
-
-    virtual bool selectIndex(const Key& idxKey);
-    virtual void deselectIndex();
-
-    virtual DbStats statistics() const;
-
-    virtual std::vector indexes(bool sorted=false) const;
-
-    // Control access properties of the DB
-
-    void control(const ControlAction& action, const ControlIdentifiers& identifiers) override { NOTIMP; }
-
-    bool retrieveLocked() const override { NOTIMP; }
-    bool archiveLocked() const override { NOTIMP; }
-    bool listLocked() const override { NOTIMP; }
-    bool wipeLocked() const override { NOTIMP; }
-
-private: // methods
-
-    /// Initialise or open the peristent pool. Worker function for the construtor
-    void initialisePool();
-
-protected: // types
-
-    typedef std::map IndexStore;
-
-protected: // members
-
-    eckit::PathName poolDir_;
-    Config dbConfig_;
-
-    std::unique_ptr pool_;
-
-    // Not owned by PMemDB but by pool_. This is only a pointer not a reference so it can
-    // be initialised later than the PMemDB is constructed.
-    PIndexRoot* root_;
-
-    std::unique_ptr dataPoolMgr_;
-
-    IndexStore  indexes_;
-    Index currentIndex_;
-
-    Schema schema_;
-
-    bool init_;
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
-
-#endif
diff --git a/src/fdb5/pmem/PMemDBReader.cc b/src/fdb5/pmem/PMemDBReader.cc
deleted file mode 100644
index 1e975936c..000000000
--- a/src/fdb5/pmem/PMemDBReader.cc
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-#include "fdb5/pmem/PMemDBReader.h"
-#include "fdb5/pmem/PMemIndex.h"
-#include "fdb5/pmem/PMemStats.h"
-#include "fdb5/LibFdb5.h"
-
-using namespace eckit;
-
-namespace fdb5 {
-namespace pmem {
-
-//----------------------------------------------------------------------------------------------------------------------
-
-
-PMemDBReader::PMemDBReader(const Key& key, const eckit::Configuration& config) :
-    PMemDB(key, config) {}
-
-
-PMemDBReader::PMemDBReader(const eckit::PathName& directory, const eckit::Configuration& config) :
-    PMemDB(directory, config) {}
-
-
-PMemDBReader::~PMemDBReader() {
-}
-
-
-bool PMemDBReader::open() {
-
-    // We don't want to create a DB if it doesn't exist when opening for read.
-    if (!exists())
-        return false;
-
-    return PMemDB::open();
-}
-
-
-bool PMemDBReader::selectIndex(const Key& idxKey) {
-
-    if (indexes_.find(idxKey) == indexes_.end()) {
-
-        ::pmem::PersistentPtr node = root_->getBranchingNode(idxKey);
-        if (!node.null())
-            indexes_[idxKey] = new PMemIndex(idxKey, *node, *dataPoolMgr_);
-        else
-            return false;
-    }
-
-    currentIndex_ = indexes_[idxKey];
-
-    LOG_DEBUG_LIB(LibFdb5) << "PMemDBReader::selectIndex " << idxKey
-                                << ", found match" << std::endl;
-
-    return true;
-}
-
-
-void PMemDBReader::axis(const std::string& keyword, eckit::StringSet& s) const {
-    s = currentIndex_.axes().values(keyword);
-}
-
-
-eckit::DataHandle* PMemDBReader::retrieve(const Key& key) const {
-
-    LOG_DEBUG_LIB(LibFdb5) << "Trying to retrieve key " << key << std::endl;
-    LOG_DEBUG_LIB(LibFdb5) << "From index " << currentIndex_ << std::endl;
-
-    Field field;
-    if (currentIndex_.get(key, field))
-        return field.dataHandle();
-
-    return 0;
-}
-
-std::vector PMemDBReader::indexes(bool sorted) const {
-    NOTIMP;
-}
-
-StatsReportVisitor *PMemDBReader::statsReportVisitor() {
-    return new PMemStatsReportVisitor(*this);
-}
-
-void PMemDBReader::print(std::ostream &out) const {
-    out << "PMemDBReader["
-        /// @todo should print more here
-        << "]";
-}
-
-static DBBuilder builder("pmem.reader", true, false);
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
diff --git a/src/fdb5/pmem/PMemDBReader.h b/src/fdb5/pmem/PMemDBReader.h
deleted file mode 100644
index b55cb9958..000000000
--- a/src/fdb5/pmem/PMemDBReader.h
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @file   PMemDBWriter.h
-/// @author Simon Smart
-/// @date   Mar 2016
-
-#ifndef fdb5_PMemDBReader_H
-#define fdb5_PMemDBReader_H
-
-#include "fdb5/pmem/PMemDB.h"
-
-namespace fdb5 {
-namespace pmem {
-
-//----------------------------------------------------------------------------------------------------------------------
-
-/// DB that implements the FDB bases on persistent memory devices
-
-class PMemDBReader : public PMemDB {
-
-public: // methods
-
-    PMemDBReader(const Key& key, const eckit::Configuration& config);
-    PMemDBReader(const eckit::PathName& directory, const eckit::Configuration& config);
-
-    virtual bool open();
-    virtual void axis(const std::string& keyword, eckit::StringSet& s) const;
-
-    ~PMemDBReader() override;
-
-    virtual bool selectIndex(const Key& idxKey);
-    virtual eckit::DataHandle* retrieve(const Key& key) const;
-
-    virtual std::vector indexes(bool sorted) const;
-
-    virtual StatsReportVisitor* statsReportVisitor();
-
-private: // methods
-
-    void print( std::ostream &out ) const override;
-
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
-
-#endif // fdb5_PMemDBReader_H
diff --git a/src/fdb5/pmem/PMemDBWriter.cc b/src/fdb5/pmem/PMemDBWriter.cc
deleted file mode 100644
index e36114c7d..000000000
--- a/src/fdb5/pmem/PMemDBWriter.cc
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-#include "fdb5/pmem/PMemDBWriter.h"
-#include "fdb5/pmem/PMemIndex.h"
-#include "fdb5/pmem/PMemFieldLocation.h"
-#include "fdb5/pmem/PDataNode.h"
-#include "fdb5/LibFdb5.h"
-
-using namespace eckit;
-
-namespace fdb5 {
-namespace pmem {
-
-//----------------------------------------------------------------------------------------------------------------------
-
-
-PMemDBWriter::PMemDBWriter(const Key& key, const eckit::Configuration& config) :
-    PMemDB(key, config) {}
-
-
-PMemDBWriter::PMemDBWriter(const PathName &directory, const eckit::Configuration& config) :
-    PMemDB(directory, config) {}
-
-PMemDBWriter::~PMemDBWriter() {
-}
-
-bool PMemDBWriter::selectIndex(const Key& idxKey) {
-
-    if (indexes_.find(key) == indexes_.end()) {
-        indexes_[key] = Index(new PMemIndex(key, root_->getCreateBranchingNode(key), *dataPoolMgr_));
-    }
-
-    currentIndex_ = indexes_[key];
-
-    LOG_DEBUG_LIB(LibFdb5) << "PMemDBWriter::selectIndex " << key << ", found match" << std::endl;
-
-    return true;
-}
-
-void PMemDBWriter::close() {
-
-    // Close any open indices
-
-    for (IndexStore::iterator it = indexes_.begin(); it != indexes_.end(); ++it) {
-        Index& idx = it->second;
-        idx.close();
-    }
-    indexes_.clear();
-}
-
-void PMemDBWriter::archive(const Key& key, const void *data, Length length) {
-
-    // Get the key:value identifier associated with this key
-
-    std::string data_key = key.names().back();
-    std::string data_value = key.value(data_key);
-
-    // Note that this pointer is NOT inside persistent space. The craeated object will be
-    // orphaned if anything goes wrong before it is added to a tree structure.
-
-    ::pmem::PersistentPtr ptr;
-    dataPoolMgr_->allocate(ptr, PDataNode::Constructor(data_key, data_value, data, length));
-
-    Field field( (PMemFieldLocation(ptr, dataPoolMgr_->getPool(ptr.uuid()))) );
-
-    currentIndex_.put(key, field);
-}
-
-void PMemDBWriter::print(std::ostream &out) const {
-    out << "PMemDBWriter["
-        /// @todo should print more here
-        << "]";
-}
-
-static DBBuilder builder("pmem.writer", false, true);
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
diff --git a/src/fdb5/pmem/PMemDBWriter.h b/src/fdb5/pmem/PMemDBWriter.h
deleted file mode 100644
index f99eef04f..000000000
--- a/src/fdb5/pmem/PMemDBWriter.h
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @file   PMemDBWriter.h
-/// @author Simon Smart
-/// @date   Mar 2016
-
-#ifndef fdb5_PMemDBWriter_H
-#define fdb5_PMemDBWriter_H
-
-#include "fdb5/pmem/PMemDB.h"
-
-namespace fdb5 {
-namespace pmem {
-
-//----------------------------------------------------------------------------------------------------------------------
-
-/// DB that implements the FDB bases on persistent memory devices
-
-class PMemDBWriter : public PMemDB {
-
-public: // methods
-
-    PMemDBWriter(const Key& key, const eckit::Configuration& config);
-    PMemDBWriter(const eckit::PathName& directory, const eckit::Configuration& config);
-
-    ~PMemDBWriter() override;
-
-    virtual bool selectIndex(const Key& idxKey);
-    virtual void close();
-    virtual void archive(const Key& key, const void *data, eckit::Length length);
-
-private: // methods
-
-    void print( std::ostream &out ) const override;
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
-
-#endif
diff --git a/src/fdb5/pmem/PMemEngine.cc b/src/fdb5/pmem/PMemEngine.cc
deleted file mode 100644
index 80a9fd3ac..000000000
--- a/src/fdb5/pmem/PMemEngine.cc
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-#include 
-
-#include "eckit/utils/Regex.h"
-#include "eckit/io/AutoCloser.h"
-
-#include "fdb5/LibFdb5.h"
-
-#include "fdb5/pmem/PMemEngine.h"
-#include "fdb5/pmem/PoolManager.h"
-#include "fdb5/pmem/PMemDBReader.h"
-#include "fdb5/rules/Schema.h"
-
-using eckit::Regex;
-using eckit::Log;
-
-namespace fdb5 {
-
-//----------------------------------------------------------------------------------------------------------------------
-
-std::string PMemEngine::name() const {
-    return PMemEngine::typeName();
-}
-
-std::string PMemEngine::dbType() const {
-    return PMemEngine::typeName();
-}
-
-eckit::PathName PMemEngine::location(const Key& key, const Config& config) const
-{
-    return PoolManager(config).pool(key);
-}
-
-static bool isPMemDB(const eckit::PathName& path) {
-    eckit::PathName master = path / "master";
-    return path.isDir() && master.exists();
-}
-
-
-bool PMemEngine::canHandle(const eckit::PathName& path) const
-{
-    return isPMemDB(path);
-}
-
-
-static void matchKeyToDB(const Key& key, std::set& keys, const char* missing, const Config& config)
-{
-    const Schema& schema = config.schema();
-    schema.matchFirstLevel(key, keys, missing);
-}
-
-static void matchRequestToDB(const metkit::mars::MarsRequest& rq, std::set& keys, const char* missing, const Config& config)
-{
-    const Schema& schema = config.schema();
-    schema.matchFirstLevel(rq, keys, missing);
-}
-
-std::vector PMemEngine::databases(const Key& key,
-                                                   const std::vector& dirs,
-                                                   const Config& config) {
-
-    std::set keys;
-
-    const char* regexForMissingValues = "[^:/]*";
-
-    matchKeyToDB(key, keys, regexForMissingValues, config);
-
-    LOG_DEBUG_LIB(LibFdb5) << "Matched DB keys " << keys << std::endl;
-
-    return databases(keys, dirs, config);
-}
-
-std::vector PMemEngine::databases(const metkit::mars::MarsRequest& request,
-                                                   const std::vector& dirs,
-                                                   const Config& config) {
-
-    std::set keys;
-
-    const char* regexForMissingValues = "[^:/]*";
-
-    matchRequestToDB(request, keys, regexForMissingValues, config);
-
-    LOG_DEBUG_LIB(LibFdb5) << "Matched DB keys " << keys << std::endl;
-
-    std::vector result;
-    for (const auto& path : databases(keys, dirs, config)) {
-        try {
-            pmem::PMemDBReader db(path, config);
-            if (db.key().partialMatch(request)) {
-                result.push_back(path);
-            }
-        } catch (eckit::Exception& e) {
-            eckit::Log::error() <<  "Error loading PMem FDB database from " << path << std::endl;
-            eckit::Log::error() << e.what() << std::endl;
-        }
-    }
-    return result;
-}
-
-std::vector PMemEngine::databases(const std::set& keys,
-                                                   const std::vector& dirs,
-                                                   const Config& config) {
-
-    std::vector result;
-    std::set seen;
-
-    for (std::vector::const_iterator j = dirs.begin(); j != dirs.end(); ++j) {
-
-        LOG_DEBUG_LIB(LibFdb5) << "Trying dir " << *j << std::endl;
-
-        std::vector subdirs;
-        eckit::PathName::match((*j) / "*:*", subdirs, false);
-
-        LOG_DEBUG_LIB(LibFdb5) << "Subdirs " << subdirs << std::endl;
-
-        for (std::set::const_iterator i = keys.begin(); i != keys.end(); ++i) {
-
-            Regex re("^" + (*i).valuesToString() + "$");
-
-            for (std::vector::const_iterator k = subdirs.begin(); k != subdirs.end(); ++k) {
-
-                if(seen.find(*k) != seen.end()) {
-                    continue;
-                }
-
-                if (re.match((*k).baseName())) {
-                    try {
-                        if(isPMemDB(*k)) {
-                            result.push_back(*k);
-                        }
-                    } catch (eckit::Exception& e) {
-                        Log::error() <<  "Error loading FDB database from " << *k << std::endl;
-                        Log::error() << e.what() << std::endl;
-                    }
-                    seen.insert(*k);;
-                }
-
-            }
-        }
-    }
-
-    return result;
-}
-
-std::vector PMemEngine::visitableLocations(const Key& key, const Config& config) const
-{
-    return databases(key, PoolManager(config).visitablePools(key), config);
-}
-
-std::vector PMemEngine::visitableLocations(const metkit::mars::MarsRequest& rq, const Config& config) const
-{
-    return databases(rq, PoolManager(config).visitablePools(rq), config);
-}
-
-std::vector PMemEngine::writableLocations(const Key& key, const Config& config) const
-{
-    return databases(key, PoolManager(config).writablePools(key), config);
-}
-
-void PMemEngine::print(std::ostream& out) const
-{
-    out << "PMemEngine()";
-}
-
-static EngineBuilder pmem_builder;
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace fdb5
diff --git a/src/fdb5/pmem/PMemEngine.h b/src/fdb5/pmem/PMemEngine.h
deleted file mode 100644
index ff356cb21..000000000
--- a/src/fdb5/pmem/PMemEngine.h
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Baudouin Raoult
-/// @author Tiago Quintino
-/// @author Simon Smart
-/// @date   Jan 2017
-
-#ifndef fdb5_pmem_PMemEngine_H
-#define fdb5_pmem_PMemEngine_H
-
-#include "fdb5/database/Engine.h"
-
-namespace fdb5 {
-
-//----------------------------------------------------------------------------------------------------------------------
-
-class PMemEngine : public fdb5::Engine {
-
-public: // methods
-
-    static const char* typeName() { return "pmem"; }
-
-    static std::vector databases(const Key& key,
-                                                  const std::vector& dirs,
-                                                  const Config& config);
-
-    static std::vector databases(const metkit::mars::MarsRequest& request,
-                                                  const std::vector& dirs,
-                                                  const Config& config);
-
-private: // methods
-
-    static std::vector databases(const std::set& keys,
-                                                  const std::vector& dirs,
-                                                  const Config& config);
-
-protected: // methods
-
-    virtual std::string name() const;
-
-    virtual std::string dbType() const;
-
-    virtual eckit::PathName location(const Key& key, const Config& config) const;
-
-    virtual bool canHandle(const eckit::PathName& path) const;
-
-    virtual std::vector visitableLocations(const Key& key, const Config& config) const;
-    virtual std::vector visitableLocations(const metkit::mars::MarsRequest& rq, const Config& config) const;
-
-    virtual std::vector writableLocations(const Key& key, const Config& config) const;
-
-    void print( std::ostream &out ) const override;
-
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-
-} // namespace fdb5
-
-#endif
diff --git a/src/fdb5/pmem/PMemFieldLocation.cc b/src/fdb5/pmem/PMemFieldLocation.cc
deleted file mode 100644
index d983d2e39..000000000
--- a/src/fdb5/pmem/PMemFieldLocation.cc
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-#include "eckit/io/MemoryHandle.h"
-
-#include "fdb5/pmem/PMemFieldLocation.h"
-#include "fdb5/pmem/PDataNode.h"
-#include "fdb5/pmem/DataPool.h"
-#include "fdb5/toc/TocFieldLocation.h"
-
-namespace fdb5 {
-namespace pmem {
-
-// PMemFieldLocation cannot be sensibly reconstructed on a remote.
-// Create something that gives info without needing the pmem library, that
-// could be remapped into a PMemFieldLocation if we later so chose.
-
-// --> For info purposes we return a TocFieldLocation which has the required
-//     components.
-// --> Obviously, if this needs to be reconstructed, then we need to do
-//     something else magical.
-
-//::eckit::ClassSpec PMemFieldLocation::classSpec_ = {&FieldLocation::classSpec(), "PMemFieldLocation",};
-//::eckit::Reanimator PMemFieldLocation::reanimator_;
-
-//----------------------------------------------------------------------------------------------------------------------
-
-
-PMemFieldLocation::PMemFieldLocation(const PMemFieldLocation& rhs) :
-    FieldLocation(rhs.length()),
-    dataNode_(rhs.dataNode_),
-    dataPool_(rhs.dataPool_) {}
-
-
-PMemFieldLocation::PMemFieldLocation(const ::pmem::PersistentPtr& dataNode, DataPool& pool) :
-    FieldLocation(dataNode->length()),
-    dataNode_(dataNode),
-    dataPool_(pool) {}
-
-
-std::shared_ptr PMemFieldLocation::make_shared() const {
-    return std::make_shared(*this);
-}
-
-std::shared_ptr PMemFieldLocation::stableLocation() const {
-    return std::make_shared(url(), node().offset(), node()->length());
-}
-
-
-eckit::DataHandle *PMemFieldLocation::dataHandle() const {
-    const PDataNode& node(*dataNode_);
-    return new eckit::MemoryHandle(node.data(), node.length());
-}
-
-eckit::DataHandle *PMemFieldLocation::dataHandle(const Key& remapKey) const {
-    throw eckit::NotImplemented("fdb-mount functionality not implemented in pmem backend (yet)", Here());
-}
-
-void PMemFieldLocation::print(std::ostream& out) const {
-    out << "(" << node().uuid() << "," << node().offset() << "," << length_ << ")";
-}
-
-void PMemFieldLocation::visit(FieldLocationVisitor& visitor) const {
-    visitor(*this);
-}
-
-::pmem::PersistentPtr PMemFieldLocation::node() const {
-    return dataNode_;
-}
-
-DataPool& PMemFieldLocation::pool() const {
-    return dataPool_;
-}
-
-void PMemFieldLocation::encode(eckit::Stream& s) const {
-    NOTIMP; // See comment
-}
-
-void PMemFieldLocation::dump(std::ostream& out) const
-{
-    out << "  pool_uuid: " << node().uuid() << std::endl;
-    out << "  data_pool: " << pool().path() << std::endl;
-    out << "  offset: "    << node().offset() << std::endl;
-
-}
-
-eckit::PathName fdb5::pmem::PMemFieldLocation::url() const {
-    return pool().path();
-}
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
diff --git a/src/fdb5/pmem/PMemFieldLocation.h b/src/fdb5/pmem/PMemFieldLocation.h
deleted file mode 100644
index 930658525..000000000
--- a/src/fdb5/pmem/PMemFieldLocation.h
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Tiago Quintino
-/// @author Simon Smart
-/// @date Nov 2016
-
-#ifndef fdb5_pmem_PMemFieldLocation_H
-#define fdb5_pmem_PMemFieldLocation_H
-
-#include "pmem/PersistentPtr.h"
-
-#include "fdb5/database/FieldLocation.h"
-
-#include 
-
-namespace fdb5 {
-namespace pmem {
-
-class PDataNode;
-class DataPool;
-
-//----------------------------------------------------------------------------------------------------------------------
-
-class PMemFieldLocation : public FieldLocation {
-public:
-
-    PMemFieldLocation(const PMemFieldLocation& rhs);
-    PMemFieldLocation(const ::pmem::PersistentPtr& dataNode, DataPool& pool);
-
-    ::pmem::PersistentPtr node() const;
-
-    virtual eckit::PathName url() const;
-
-    virtual eckit::DataHandle *dataHandle() const;
-    virtual eckit::DataHandle *dataHandle(const Key& remapKey) const;
-
-    virtual std::shared_ptr make_shared() const;
-
-    // The PMemFieldLocation only has validity equal to that of the PMemDB.
-    // This is a problem with any async API functions.
-    // --> For visibility purposes return something more stable (currently a
-    //     TocFieldLocation....)
-    virtual std::shared_ptr stableLocation() const;
-
-    virtual void visit(FieldLocationVisitor& visitor) const;
-
-    DataPool& pool() const;
-
-protected: // For Streamable (see comments. This is a bit odd).
-
-    void encode(eckit::Stream&) const override;
-
-private: // methods
-
-    virtual void dump(std::ostream &out) const;
-
-    void print(std::ostream &out) const override;
-
-private: // members
-
-    ::pmem::PersistentPtr dataNode_;
-
-    DataPool& dataPool_;
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
-
-#endif // fdb5_pmem_PMemFieldLocation_H
diff --git a/src/fdb5/pmem/PMemIndex.cc b/src/fdb5/pmem/PMemIndex.cc
deleted file mode 100755
index fffec3e93..000000000
--- a/src/fdb5/pmem/PMemIndex.cc
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-#include "eckit/io/Buffer.h"
-#include "eckit/serialisation/MemoryStream.h"
-
-#include "fdb5/LibFdb5.h"
-#include "fdb5/pmem/PMemStats.h"
-#include "fdb5/pmem/PMemIndex.h"
-#include "fdb5/pmem/PBranchingNode.h"
-#include "fdb5/pmem/PMemFieldLocation.h"
-#include "fdb5/pmem/MemoryBufferStream.h"
-
-using namespace eckit;
-
-
-namespace fdb5 {
-namespace pmem {
-
-//-----------------------------------------------------------------------------
-
-
-PMemIndex::PMemIndex(const Key& key, PBranchingNode& node, DataPoolManager& mgr, const std::string& type) :
-    IndexBase(key, type),
-    location_(node, mgr) {
-
-    if (!location_.node().axis_.null()) {
-
-        LOG_DEBUG_LIB(LibFdb5) << "PMemIndex Loading axes from buffer" << std::endl;
-        const ::pmem::PersistentBuffer& buf(*location_.node().axis_);
-        MemoryStream s(buf.data(), buf.size());
-        axes_.decode(s);
-        LOG_DEBUG_LIB(LibFdb5) << "PMemIndex axes = " << axes_ << std::endl;
-    }
-}
-
-
-PMemIndex::~PMemIndex() {}
-
-
-void PMemIndex::visit(IndexLocationVisitor& visitor) const {
-    visitor(location_);
-}
-
-
-bool PMemIndex::get(const Key& key, Field &field) const {
-
-    ::pmem::PersistentPtr node = location_.node().getDataNode(key, location_.pool_manager());
-
-    if (!node.null()) {
-        field = Field(PMemFieldLocation(node, location_.pool_manager().getPool(node.uuid())));
-        return true;
-    }
-
-    return false;
-}
-
-
-void PMemIndex::open() {
-    // Intentionally left blank. Indices neither opened nor closed (part of open DB).
-}
-
-void PMemIndex::reopen() {
-    // Intentionally left blank. Indices neither opened nor closed (part of open DB).
-}
-
-void PMemIndex::close() {
-    // Intentionally left blank. Indices neither opened nor closed (part of open DB).
-}
-
-void PMemIndex::add(const Key& key, const Field &field) {
-
-    struct Inserter : FieldLocationVisitor {
-        Inserter(PBranchingNode& indexNode, const Key& key, DataPoolManager& poolManager) :
-            indexNode_(indexNode),
-            key_(key),
-            poolManager_(poolManager) {}
-
-        virtual void operator() (const FieldLocation& location) {
-
-            const PMemFieldLocation& ploc = dynamic_cast( location);
-            indexNode_.insertDataNode(key_, ploc.node(), poolManager_);
-        }
-
-    private:
-        PBranchingNode& indexNode_;
-        const Key& key_;
-        DataPoolManager& poolManager_;
-    };
-
-    Inserter inserter(location_.node(), key, location_.pool_manager());
-    field.location().visit(inserter);
-
-    // Keep track of the axes()
-
-    if (axes().dirty()) {
-
-        MemoryBufferStream s;
-        axes().encode(s);
-
-        const void* data = s.buffer();
-        if (location_.node().axis_.null())
-            location_.node().axis_.allocate(data, s.position());
-        else
-            location_.node().axis_.replace(data, s.position());
-
-        axes_.clean();
-    }
-}
-
-void PMemIndex::flush() {
-    // Intentionally left blank. Flush not used in PMem case.
-}
-
-void PMemIndex::encode(Stream& s) const {
-    NOTIMP;
-}
-
-void PMemIndex::entries(EntryVisitor &visitor) const {
-    NOTIMP;
-}
-
-void PMemIndex::print(std::ostream &out) const {
-    out << "PMemIndex[]";
-}
-
-
-std::string PMemIndex::defaulType() {
-    return "PMemIndex";
-}
-
-const std::vector PMemIndex::dataURIs() const {
-    // n.b. this lists the pools that _could_ be referenced, not those that
-    // necessarily are. That would need proper enumeration of all contents.
-
-    std::vector result;
-    for (auto path : location_.pool_manager().dataPoolPaths())
-        result.push_back(eckit::URI{"pmem", path});
-
-    return result;
-}
-
-void PMemIndex::dump(std::ostream& out, const char* indent, bool simple, bool dumpFields) const {
-    NOTIMP;
-}
-
-IndexStats PMemIndex::statistics() const
-{
-    IndexStats s(new PMemIndexStats());
-
-    NOTIMP;
-
-    return s;
-}
-
-bool PMemIndex::dirty() const {
-    // data is always kept in sync. never dirty.
-    return false;
-}
-
-
-//-----------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
diff --git a/src/fdb5/pmem/PMemIndex.h b/src/fdb5/pmem/PMemIndex.h
deleted file mode 100755
index bc50b158e..000000000
--- a/src/fdb5/pmem/PMemIndex.h
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Baudouin Raoult
-/// @author Tiago Quintino
-/// @date Sep 2012
-
-#ifndef fdb5_pmem_PMemIndex_H
-#define fdb5_pmem_PMemIndex_H
-
-#include "eckit/eckit.h"
-
-#include "fdb5/database/Index.h"
-
-#include "fdb5/pmem/PMemIndexLocation.h"
-
-namespace fdb5 {
-namespace pmem {
-
-class PBranchingNode;
-
-//----------------------------------------------------------------------------------------------------------------------
-
-
-class PMemIndex : public IndexBase {
-
-public: // methods
-
-    PMemIndex(const Key& key, PBranchingNode& node, DataPoolManager& mgr, const std::string& type=defaulType());
-
-    ~PMemIndex() override;
-
-    virtual void visit(IndexLocationVisitor& visitor) const;
-
-    static std::string defaulType();
-
-protected: // methods
-
-    virtual const IndexLocation& location() const { return location_; }
-    const std::vector dataURIs() const override;
-
-    virtual bool dirty() const;
-
-    virtual void open();
-    virtual void close();
-    virtual void reopen();
-
-    virtual bool get(const Key& key, Field &field ) const;
-    virtual void add(const Key& key, const Field &field );
-    virtual void flush();
-    void encode(eckit::Stream &s) const override;
-    virtual void entries(EntryVisitor &visitor) const;
-
-    void print( std::ostream &out ) const override;
-    virtual void dump(std::ostream& out, const char* indent, bool simple = false, bool dumpFields = false) const;
-
-    virtual IndexStats statistics() const;
-
-private: // members
-
-    PMemIndexLocation location_;
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
-
-#endif
diff --git a/src/fdb5/pmem/PMemIndexLocation.cc b/src/fdb5/pmem/PMemIndexLocation.cc
deleted file mode 100644
index bbe077c3c..000000000
--- a/src/fdb5/pmem/PMemIndexLocation.cc
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Simon Smart
-/// @date Nov 2016
-
-#include "fdb5/pmem/PMemIndexLocation.h"
-#include "fdb5/pmem/PBranchingNode.h"
-
-#include "pmem/PoolRegistry.h"
-
-using namespace eckit;
-
-
-namespace fdb5 {
-namespace pmem {
-
-// PMemIndexLocation cannot be sensibly reconstructed on a remote.
-// Create something that gives info without needing the pmem library, that
-// could be remapped into a PMemIndexLocation if we later so chose.
-
-// --> For info purposes we return a TocIndexLocation which has the required
-//     components.
-// --> Obviously, if this needs to be reconstructed, then we need to do
-//     something else magical.
-
-//::eckit::ClassSpec PMemFieldLocation::classSpec_ = {&FieldLocation::classSpec(), "PMemFieldLocation",};
-//::eckit::Reanimator PMemFieldLocation::reanimator_;
-
-//----------------------------------------------------------------------------------------------------------------------
-
-
-PMemIndexLocation::PMemIndexLocation(PBranchingNode& node, DataPoolManager& mgr) :
-    node_(node),
-    poolManager_(mgr) {}
-
-
-PBranchingNode& PMemIndexLocation::node() const {
-    return node_;
-}
-
-
-DataPoolManager& PMemIndexLocation::pool_manager() const {
-    return poolManager_;
-}
-
-PathName PMemIndexLocation::url() const
-{
-    ::pmem::PersistentPool& pool(::pmem::PoolRegistry::instance().poolFromPointer(&node_));
-
-    return pool.path();
-}
-
-IndexLocation* PMemIndexLocation::clone() const {
-    return new PMemIndexLocation(node_, poolManager_);
-}
-
-void PMemIndexLocation::encode(Stream &) const {
-    NOTIMP; // See comment at top of file
-}
-
-void PMemIndexLocation::print(std::ostream &out) const
-{
-    out << "(" << url() << ")";
-
-}
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
diff --git a/src/fdb5/pmem/PMemIndexLocation.h b/src/fdb5/pmem/PMemIndexLocation.h
deleted file mode 100644
index e878f463a..000000000
--- a/src/fdb5/pmem/PMemIndexLocation.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Simon Smart
-/// @date Nov 2016
-
-#ifndef fdb5_pmem_PMemIndexLocation_H
-#define fdb5_pmem_PMemIndexLocation_H
-
-#include "pmem/PersistentPtr.h"
-
-#include "fdb5/database/IndexLocation.h"
-#include "fdb5/pmem/DataPoolManager.h"
-
-
-namespace fdb5 {
-namespace pmem {
-
-//----------------------------------------------------------------------------------------------------------------------
-
-class PBranchingNode;
-
-
-class PMemIndexLocation : public IndexLocation {
-
-public: // methods
-
-    PMemIndexLocation(PBranchingNode& node, DataPoolManager& mgr);
-
-    PBranchingNode& node() const;
-    DataPoolManager& pool_manager() const;
-
-    IndexLocation* clone() const override;
-
-    eckit::PathName url() const override;
-
-protected: // For Streamable (see comments. This is a bit odd).
-
-    void encode(eckit::Stream&) const override;
-
-private: // methods
-
-    void print(std::ostream &out) const override;
-
-private: // friends
-
-    PBranchingNode& node_;
-    DataPoolManager& poolManager_;
-};
-
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
-
-#endif // fdb5_pmem_PMemIndexLocation_H
diff --git a/src/fdb5/pmem/PMemStats.cc b/src/fdb5/pmem/PMemStats.cc
deleted file mode 100644
index d72a805c6..000000000
--- a/src/fdb5/pmem/PMemStats.cc
+++ /dev/null
@@ -1,237 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-#include "fdb5/pmem/PMemStats.h"
-#include "fdb5/pmem/PMemDB.h"
-
-namespace fdb5 {
-namespace pmem {
-
-::eckit::ClassSpec PMemDbStats::classSpec_ = {&FieldLocation::classSpec(), "PMemDbStats",};
-::eckit::Reanimator PMemDbStats::reanimator_;
-
-::eckit::ClassSpec PMemIndexStats::classSpec_ = {&FieldLocation::classSpec(), "PMemIndexStats",};
-::eckit::Reanimator PMemIndexStats::reanimator_;
-
-//----------------------------------------------------------------------------------------------------------------------
-
-
-PMemDbStats::PMemDbStats():
-    dataPoolsSize_(0),
-    indexPoolsSize_(0),
-    schemaSize_(0),
-    dataPoolsCount_(0),
-    indexPoolsCount_(0),
-    indexesCount_(0) {}
-
-PMemDbStats::PMemDbStats(eckit::Stream& s) {
-    s >> dataPoolsSize_;
-    s >> indexPoolsSize_;
-    s >> schemaSize_;
-
-    s >> dataPoolsCount_;
-    s >> indexPoolsCount_;
-    s >> indexesCount_;
-}
-
-
-PMemDbStats& PMemDbStats::operator+=(const PMemDbStats &rhs) {
-    dataPoolsSize_ += rhs.dataPoolsSize_;
-    indexPoolsSize_ += rhs.indexPoolsSize_;
-    schemaSize_ += rhs.schemaSize_;
-    dataPoolsCount_ += rhs.dataPoolsCount_;
-    indexPoolsCount_ += rhs.indexPoolsCount_;
-    indexesCount_ += rhs.indexesCount_;
-    return *this;
-}
-
-
-void PMemDbStats::add(const DbStatsContent& rhs)
-{
-    const PMemDbStats& stats = dynamic_cast(rhs);
-    *this += stats;
-}
-
-void PMemDbStats::report(std::ostream &out, const char *indent) const {
-    reportCount(out, "Data Pools ", dataPoolsCount_, indent);
-    reportBytes(out, "Size of data pools", dataPoolsSize_, indent);
-    reportCount(out, "Index Pools ", indexPoolsCount_, indent);
-    reportBytes(out, "Size of index pools", indexPoolsSize_, indent);
-    reportCount(out, "Indexes", indexesCount_, indent);
-    reportBytes(out, "Size of schemas", schemaSize_, indent);
-}
-
-void PMemDbStats::encode(eckit::Stream& s) const {
-    s << dataPoolsSize_;
-    s << indexPoolsSize_;
-    s << schemaSize_;
-
-    s << dataPoolsCount_;
-    s << indexPoolsCount_;
-    s << indexesCount_;
-}
-
-
-//----------------------------------------------------------------------------------------------------------------------
-
-
-PMemIndexStats::PMemIndexStats():
-    fieldsCount_(0),
-    duplicatesCount_(0),
-    fieldsSize_(0),
-    duplicatesSize_(0) {}
-
-PMemIndexStats::PMemIndexStats(eckit::Stream& s) {
-    s >> fieldsCount_;
-    s >> duplicatesCount_;
-    s >> fieldsSize_;
-    s >> duplicatesSize_;
-}
-
-
-PMemIndexStats &PMemIndexStats::operator+=(const PMemIndexStats &rhs) {
-    fieldsCount_ += rhs.fieldsCount_;
-    duplicatesCount_ += rhs.duplicatesCount_;
-    fieldsSize_ += rhs.fieldsSize_;
-    duplicatesSize_ += rhs.duplicatesSize_;
-
-    return *this;
-}
-
-void PMemIndexStats::add(const IndexStatsContent& rhs)
-{
-    const PMemIndexStats& stats = dynamic_cast(rhs);
-    *this += stats;
-}
-
-void PMemIndexStats::report(std::ostream &out, const char *indent) const {
-    reportCount(out, "Fields", fieldsCount_, indent);
-    reportBytes(out, "Size of fields", fieldsSize_, indent);
-    reportCount(out, "Duplicated fields ", duplicatesCount_, indent);
-    reportBytes(out, "Size of duplicates", duplicatesSize_, indent);
-    reportCount(out, "Reacheable fields ", fieldsCount_ - duplicatesCount_, indent);
-    reportBytes(out, "Reachable size", fieldsSize_ - duplicatesSize_, indent);
-}
-
-void PMemIndexStats::encode(eckit::Stream& s) const {
-    s << fieldsCount_;
-    s << duplicatesCount_;
-    s << fieldsSize_;
-    s << duplicatesSize_;
-}
-
-
-//----------------------------------------------------------------------------------------------------------------------
-
-
-PMemDataStats::PMemDataStats() {
-}
-
-
-PMemDataStats &PMemDataStats::operator+=(const PMemDataStats &rhs) {
-
-    NOTIMP;
-
-    return *this;
-}
-
-void PMemDataStats::add(const DataStatsContent& rhs)
-{
-    const PMemDataStats& stats = dynamic_cast(rhs);
-    *this += stats;
-}
-
-void PMemDataStats::report(std::ostream &out, const char *indent) const {
-    NOTIMP;
-}
-
-
-//----------------------------------------------------------------------------------------------------------------------
-
-
-PMemStatsReportVisitor::PMemStatsReportVisitor(const PMemDB& db) :
-//    directory_(reader.directory()),
-    dbStats_(new PMemDbStats()) {
-
-    currentDatabase_ = &db;
-    dbStats_ = currentDatabase_->statistics();
-}
-
-PMemStatsReportVisitor::~PMemStatsReportVisitor() {
-}
-
-bool PMemStatsReportVisitor::visitDatabase(const DB& db) {
-    ASSERT(&db == currentDatabase_);
-    return true;
-}
-
-void PMemStatsReportVisitor::visitDatum(const Field& field, const std::string& fieldFingerprint) {
-
-//    ASSERT(currIndex_ != 0);
-
-    // If this index is not yet in the map, then create an entry
-
-    std::map::iterator stats_it = indexStats_.find(*currentIndex_);
-
-    if (stats_it == indexStats_.end()) {
-        stats_it = indexStats_.insert(std::make_pair(*currentIndex_, IndexStats(new PMemIndexStats()))).first;
-    }
-
-    IndexStats& stats(stats_it->second);
-
-    eckit::Length len = field.location().length();
-
-    stats.addFieldsCount(1);
-    stats.addFieldsSize(len);
-
-    const eckit::PathName& dataPath  = field.location().url();
-    const eckit::PathName& indexPath = currentIndex_->location().url();
-
-    // n.b. Unlike in the Toc case, we don't need to track the index and data pools here. They are stored and
-    //      referenced centrally in the master pool, so the data about them is ALREADY located in the global
-    //      dbStats!
-
-    std::string unique = currentIndex_->key().valuesToString() + "+" + fieldFingerprint;
-
-    if (active_.insert(unique).second) {
-        indexUsage_[indexPath]++;
-        dataUsage_[dataPath]++;
-    } else {
-        stats.addDuplicatesCount(1);
-        stats.addDuplicatesSize(len);
-    }
-}
-
-void PMemStatsReportVisitor::databaseComplete(const DB &db) {}
-
-DbStats PMemStatsReportVisitor::dbStatistics() const {
-    return dbStats_;
-}
-
-IndexStats PMemStatsReportVisitor::indexStatistics() const {
-
-    IndexStats total(new PMemIndexStats());
-    for (const auto& it : indexStats_) {
-        total += it.second;
-    }
-    return total;
-}
-
-
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
diff --git a/src/fdb5/pmem/PMemStats.h b/src/fdb5/pmem/PMemStats.h
deleted file mode 100644
index 20c90a31b..000000000
--- a/src/fdb5/pmem/PMemStats.h
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Baudouin Raoult
-/// @author Tiago Quintino
-/// @date   April 2016
-
-#ifndef fdb5_PMemDBStats_H
-#define fdb5_PMemDBStats_H
-
-#include 
-
-#include "fdb5/database/DataStats.h"
-#include "fdb5/database/DbStats.h"
-#include "fdb5/database/IndexStats.h"
-#include "fdb5/database/StatsReportVisitor.h"
-
-#include "eckit/exception/Exceptions.h"
-#include "eckit/filesystem/PathName.h"
-
-#include 
-#include 
-
-namespace fdb5 {
-
-class Index;
-class Field;
-
-namespace pmem {
-
-class PMemDB;
-
-//----------------------------------------------------------------------------------------------------------------------
-
-
-class PMemDbStats : public DbStatsContent {
-public:
-
-    PMemDbStats();
-    PMemDbStats(eckit::Stream& s);
-
-    static DbStats make() override { return DbStats(new PMemDbStats()); }
-
-    unsigned long long dataPoolsSize_;
-    unsigned long long indexPoolsSize_;
-    unsigned long long schemaSize_;
-
-    size_t dataPoolsCount_;
-    size_t indexPoolsCount_;
-    size_t indexesCount_;
-
-    PMemDbStats& operator+= (const PMemDbStats &rhs) ;
-
-    virtual void add(const DbStatsContent&);
-    virtual void report(std::ostream &out, const char* indent = "") const;
-
-public: // For Streamable
-
-    static const eckit::ClassSpec&  classSpec() { return classSpec_;}
-
-protected: // For Streamable
-
-    void encode(eckit::Stream&) const override;
-    virtual const eckit::ReanimatorBase& reanimator() const { return reanimator_; }
-
-    static eckit::ClassSpec                 classSpec_;
-    static eckit::Reanimator   reanimator_;
-};
-
-
-//----------------------------------------------------------------------------------------------------------------------
-
-
-class PMemIndexStats : public IndexStatsContent {
-public:
-
-    PMemIndexStats();
-    PMemIndexStats(eckit::Stream& s);
-
-    size_t fieldsCount_;
-    size_t duplicatesCount_;
-
-    unsigned long long fieldsSize_;
-    unsigned long long duplicatesSize_;
-
-    PMemIndexStats& operator+= (const PMemIndexStats& rhs);
-
-    virtual size_t fieldsCount() const { return fieldsCount_; }
-    virtual size_t duplicatesCount() const { return duplicatesCount_; }
-
-    virtual size_t fieldsSize() const { return fieldsSize_; }
-    virtual size_t duplicatesSize() const { return duplicatesSize_; }
-
-    virtual size_t addFieldsCount(size_t i) { fieldsCount_ += i; return fieldsCount_; }
-    virtual size_t addDuplicatesCount(size_t i) { duplicatesCount_ += i; return duplicatesCount_; }
-
-    virtual size_t addFieldsSize(size_t i) { fieldsSize_ += i; return fieldsSize_; }
-    virtual size_t addDuplicatesSize(size_t i) { duplicatesSize_ += i; return duplicatesSize_; }
-
-    virtual void add(const IndexStatsContent&);
-
-    virtual void report(std::ostream &out, const char* indent) const;
-
-public: // For Streamable
-
-    static const eckit::ClassSpec&  classSpec() { return classSpec_;}
-
-protected: // For Streamable
-
-    void encode(eckit::Stream&) const override;
-    virtual const eckit::ReanimatorBase& reanimator() const { return reanimator_; }
-
-    static eckit::ClassSpec                  classSpec_;
-    static eckit::Reanimator reanimator_;
-};
-
-
-//----------------------------------------------------------------------------------------------------------------------
-
-
-class PMemDataStats : public DataStatsContent {
-public:
-
-    PMemDataStats();
-
-    PMemDataStats& operator+= (const PMemDataStats& rhs);
-
-    virtual void add(const DataStatsContent&);
-
-    virtual void report(std::ostream &out, const char* indent) const;
-};
-
-
-//----------------------------------------------------------------------------------------------------------------------
-
-
-class PMemStatsReportVisitor : public virtual StatsReportVisitor {
-public:
-
-    PMemStatsReportVisitor(const PMemDB& db);
-    ~PMemStatsReportVisitor() override;
-
-    virtual IndexStats indexStatistics() const;
-    virtual DbStats    dbStatistics() const;
-
-private: // methods
-
-
-private: // methods
-
-    bool visitDatabase(const DB& db) override;
-    void visitDatum(const Field& field, const std::string& keyFingerprint) override;
-    void visitDatum(const Field& field, const TypedKey& datumKey) override { NOTIMP; }
-
-    // This visitor is only legit for one DB - so don't reset database
-    void databaseComplete(const DB& db) override;
-
-protected: // members
-
-    std::unordered_set allDataPools_;
-    std::unordered_set allIndexPools_;
-
-    std::unordered_map indexUsage_;
-    std::unordered_map dataUsage_;
-
-    std::unordered_set active_;
-
-    std::map indexStats_;
-
-    DbStats dbStats_;
-
-    eckit::PathName lastDataPath_;
-    eckit::PathName lastIndexPath_;
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
-
-#endif
diff --git a/src/fdb5/pmem/PRoot.cc b/src/fdb5/pmem/PRoot.cc
deleted file mode 100644
index ac30c06e9..000000000
--- a/src/fdb5/pmem/PRoot.cc
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-3.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation
- * nor does it submit to any jurisdiction.
- */
-
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Simon Smart
-/// @date   Sept 2016
-
-#include "eckit/log/Log.h"
-#include "eckit/thread/AutoLock.h"
-
-#include "fdb5/database/Key.h"
-#include "fdb5/pmem/PRoot.h"
-#include "fdb5/pmem/PIndexRoot.h"
-#include "fdb5/pmem/PDataRoot.h"
-
-#include 
-
-using namespace eckit;
-using namespace pmem;
-
-
-namespace fdb5 {
-namespace pmem {
-
-// -------------------------------------------------------------------------------------------------
-
-
-PRoot::PRoot(RootClass cls) :
-    tag_(PRootTag),
-    version_(PRootVersion),
-    rootSize_(sizeof(PRoot)),
-    created_(time(0)),
-    createdBy_(getuid()),
-    class_(cls),
-    indexRoot_(),
-    dataRoot_() {}
-
-
-void PRoot::buildRoot(const Key& dbKey, const eckit::PathName& schemaPath) {
-
-    AutoLock lock(rootMutex_);
-
-    ASSERT(indexRoot_.null());
-    ASSERT(dataRoot_.null());
-
-    if (class_ == IndexClass) {
-        PIndexRoot::build(indexRoot_, dbKey, schemaPath);
-    } else {
-        ASSERT(class_ == DataClass);
-        dataRoot_.allocate();
-    }
-}
-
-
-/*
- * We can use whatever knowledge we have to test the validity of the structure.
- *
- * For now, we just check that the tag is set (i.e. it is initialised).
- */
-bool PRoot::valid() const {
-
-    if (tag_ != PRootTag) {
-        Log::error() << "Persistent root tag does not match" << std::endl;
-        return false;
-    }
-
-    if (rootSize_ != sizeof(PRoot)) {
-        Log::error() << "Invalid (old) structure size for persistent root" << std::endl;
-        return false;
-    }
-
-    if (version_ != PRootVersion) {
-        Log::error() << "Invalid persistent root version" << std::endl;
-        return false;
-    }
-
-    if ((class_ == IndexClass && ( indexRoot_.null() || !dataRoot_.null())) ||
-        (class_ == DataClass  && (!indexRoot_.null() ||  dataRoot_.null())) ||
-        (class_ != IndexClass && class_ != DataClass)) {
-
-        Log::error() << "Inconsistent root node" << std::endl;
-        return false;
-    }
-
-    return true;
-}
-
-const time_t& PRoot::created() const {
-    return created_;
-}
-
-
-void PRoot::print(std::ostream& s) const {
-    s << "PRoot(0x" << this << ")";
-}
-
-
-PRoot::RootClass PRoot::root_class() const {
-    return class_;
-}
-
-
-PIndexRoot& PRoot::indexRoot() const {
-    return *indexRoot_;
-}
-
-
-PDataRoot& PRoot::dataRoot() const {
-    return *dataRoot_;
-}
-
-// -------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
diff --git a/src/fdb5/pmem/PRoot.h b/src/fdb5/pmem/PRoot.h
deleted file mode 100644
index 0feda0eb6..000000000
--- a/src/fdb5/pmem/PRoot.h
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation
- * nor does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Simon Smart
-/// @date   Sep 2016
-
-#ifndef fdb5_pmem_PRoot_H
-#define fdb5_pmem_PRoot_H
-
-#include "eckit/memory/NonCopyable.h"
-#include "eckit/types/FixedString.h"
-
-#include "pmem/PersistentPtr.h"
-#include "pmem/PersistentMutex.h"
-
-#include "fdb5/database/Key.h"
-
-#include 
-
-
-namespace fdb5 {
-
-class Key;
-
-namespace pmem {
-
-class PIndexRoot;
-class PDataRoot;
-
-// -------------------------------------------------------------------------------------------------
-
-/// The primary root object
-/// @note We have a "primary" root object, that points to either the real index or data data
-///       roots depending on the pool we are in. This permits having unique mapping between
-///       the type_ids (and correct versioning) for all objects, including the two differing
-///       root objects (as the root object MUST have the type_id POBJ_ROOT_TYPE_NUM).
-
-class PRoot : public eckit::NonCopyable {
-
-public: // types
-
-    enum RootClass {
-        IndexClass = 0,
-        DataClass
-    };
-
-    typedef ::pmem::AtomicConstructor1 Constructor;
-
-public: // methods
-
-    PRoot(RootClass cls);
-
-    /// Normally we allocate persistent objects by passing in their subcomponents. We cannot do
-    /// that with a root object, as it is allocated at pool creation time.
-    ///
-    /// --> buildRoot() should be called immediately after pool creation to initialise the root.
-    void buildRoot(const Key& dbKey, const eckit::PathName& schemaPath);
-
-    bool valid() const;
-
-    const time_t& created() const;
-
-    void print(std::ostream& s) const;
-
-    RootClass root_class() const;
-
-    PIndexRoot& indexRoot() const;
-    PDataRoot& dataRoot() const;
-
-private: // members
-
-    eckit::FixedString<8> tag_;
-
-    unsigned short int version_;
-
-    /// Store the size of the PMemRoot object. Allows some form of sanity check
-    /// when it is being opened, that everything is safe.
-    unsigned short int rootSize_;
-
-    time_t created_;
-
-    long createdBy_;
-
-    RootClass class_;
-
-    /// Access to the "real" root objects.
-
-    ::pmem::PersistentPtr indexRoot_;
-    ::pmem::PersistentPtr dataRoot_;
-
-    /// Ensure that only one thread at a time tries to allocate a root!
-    ::pmem::PersistentMutex rootMutex_;
-
-private: // friends
-
-    friend std::ostream& operator<<(std::ostream& s, const PRoot& r) { r.print(s); return s; }
-};
-
-
-// A consistent definition of the tag for comparison purposes.
-
-const eckit::FixedString<8> PRootTag = "99FDB599";
-const unsigned short int PRootVersion = 2;
-
-
-// -------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
-
-#endif // fdb5_pmem_PRoot_H
diff --git a/src/fdb5/pmem/Pool.cc b/src/fdb5/pmem/Pool.cc
deleted file mode 100644
index f8dc3e494..000000000
--- a/src/fdb5/pmem/Pool.cc
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-3.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation
- * nor does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Simon Smart
-/// @date   Feb 2016
-
-#include "eckit/log/Log.h"
-#include "eckit/log/TimeStamp.h"
-
-#include "pmem/PersistentPtr.h"
-#include "pmem/Exceptions.h"
-
-#include "fdb5/pmem/PBaseNode.h"
-#include "fdb5/pmem/PBranchingNode.h"
-#include "fdb5/pmem/PDataRoot.h"
-#include "fdb5/pmem/PIndexRoot.h"
-#include "fdb5/pmem/PRoot.h"
-#include "fdb5/pmem/Pool.h"
-#include "fdb5/LibFdb5.h"
-
-using namespace eckit;
-using namespace pmem;
-
-
-template<> uint64_t pmem::PersistentType::type_id = POBJ_ROOT_TYPE_NUM;
-
-template<> uint64_t pmem::PersistentType::type_id = 1;
-template<> uint64_t pmem::PersistentType::type_id = 2;
-template<> uint64_t pmem::PersistentType::type_id = 3;
-template<> uint64_t pmem::PersistentType >::type_id = 4;
-template<> uint64_t pmem::PersistentType >::type_id = 5;
-template<> uint64_t pmem::PersistentType::type_id = 6;
-template<> uint64_t pmem::PersistentType::type_id = 7;
-template<> uint64_t pmem::PersistentType::type_id = 8;
-template<> uint64_t pmem::PersistentType::type_id = 9;
-
-
-// --------------------------------------------------------------------------------------------------
-
-namespace fdb5 {
-namespace pmem {
-
-// -------------------------------------------------------------------------------------------------
-
-Pool::Pool(const PathName& path, const std::string& name) :
-    PersistentPool(path, name) {
-
-    ASSERT(baseRoot().valid());
-    ASSERT(baseRoot()->valid());
-    ASSERT(root().valid());
-
-    LOG_DEBUG_LIB(LibFdb5) << "Opened persistent pool created at: " << TimeStamp(root().created()) << std::endl;
-}
-
-
-
-
-Pool::Pool(const PathName& path, const size_t size, const std::string& name,
-                   const AtomicConstructor& constructor) :
-    PersistentPool(path, size, name, constructor) {}
-
-
-Pool::~Pool() {}
-
-
-void Pool::buildRoot(const Key& dbKey, const eckit::PathName& schemaPath) {
-    // n.b. cannot use baseRoot yet, as not yet valid...
-    PersistentPtr rt = getRoot();
-    ASSERT(rt.valid());
-    rt->buildRoot(dbKey, schemaPath);
-}
-
-
-/// Open an existing persistent pool, if it exists. If it does _not_ exist, then create it.
-/// If open/create fail for other reasons, then the appropriate error is thrown.
-///
-/// @arg path - Specifies the directory to open, rather than the pool size itself.
-Pool* Pool::obtain(const PathName& poolDir, const size_t size, const Key& dbKey, const eckit::PathName& schemaPath) {
-
-    // The pool must exist as a file within a directory.
-    // n.b. PathName::mkdir uses mkdir_if_not_exists internally, so works OK if another process gets ahead.
-    if (!poolDir.exists())
-        poolDir.mkdir();
-    ASSERT(poolDir.isDir());
-
-    Pool* pool = 0;
-
-
-    if(exists(poolDir)) {
-        LOG_DEBUG_LIB(LibFdb5) << "Opening FDB PMem master pool  " << poolDir << std::endl;
-        pool = new Pool(poolMaster(poolDir), "pmem-pool");
-    }
-    else {
-        LOG_DEBUG_LIB(LibFdb5) << "Creating FDB PMem master pool " << poolDir << std::endl;
-        pool = new Pool(poolMaster(poolDir), size, "pmem-pool", PRoot::Constructor(PRoot::IndexClass));
-        pool->buildRoot(dbKey, schemaPath);
-    }
-
-    ASSERT(pool != 0);
-    return pool;
-}
-
-
-bool Pool::exists(const PathName &poolDir) {
-    return poolMaster(poolDir).exists();
-}
-
-
-eckit::PathName Pool::poolMaster(const PathName &poolDir) {
-
-    return poolDir / "master";
-}
-
-
-PersistentPtr Pool::baseRoot() const {
-
-    PersistentPtr rt = getRoot();
-    ASSERT(rt.valid());
-    ASSERT(rt->valid());
-    ASSERT(rt->root_class() == PRoot::IndexClass);
-
-    return rt;
-}
-
-
-PIndexRoot& Pool::root() const {
-
-    PIndexRoot& rt = baseRoot()->indexRoot();
-
-    ASSERT(rt.valid());
-    return rt;
-}
-
-
-// -------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace tree
diff --git a/src/fdb5/pmem/Pool.h b/src/fdb5/pmem/Pool.h
deleted file mode 100644
index 2bb407d46..000000000
--- a/src/fdb5/pmem/Pool.h
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-3.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation
- * nor does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Simon Smart
-/// @date   Feb 2016
-
-#ifndef fdb5_pmem_Pool_H
-#define fdb5_pmem_Pool_H
-
-#include "pmem/PersistentPool.h"
-
-#include "fdb5/database/Key.h"
-
-
-// -------------------------------------------------------------------------------------------------
-
-// Forward declaration
-namespace pmem {
-    template  class PersistentPtr;
-}
-
-// -------------------------------------------------------------------------------------------------
-
-namespace fdb5 {
-
-namespace pmem {
-
-class PRoot;
-class PIndexRoot;
-
-// -------------------------------------------------------------------------------------------------
-
-/*
- * Here we define the persistent pool used in this project
- *
- * --> The TreePool needs to know:
- *
- *       i) The data type of the root
- *      ii) How to construct the root object if required
- *     iii) How to map type_ids to actual types
- */
-
-class Pool : public ::pmem::PersistentPool {
-
-public: // methods
-
-    Pool(const eckit::PathName& path, const std::string& name);
-
-    Pool(const eckit::PathName& path, const size_t size, const std::string& name,
-             const ::pmem::AtomicConstructor& constructor);
-
-    ~Pool();
-
-    /// Normally we allocate persistent objects by passing in their subcomponents. We cannot do
-    /// that with a root object, as it is allocated at pool creation time.
-    ///
-    /// --> buildRoot() should be called immediately after pool creation to initialise the root.
-    void buildRoot(const Key& dbKey, const eckit::PathName& schemaPath);
-
-    static bool exists(const eckit::PathName& poolDir);
-
-    static Pool* obtain(const eckit::PathName& poolDir, const size_t size, const Key& dbKey, const eckit::PathName& schemaPath);
-
-    ::pmem::PersistentPtr baseRoot() const;
-    PIndexRoot& root() const;
-
-private: // methods
-
-    static eckit::PathName poolMaster(const eckit::PathName& poolDir);
-};
-
-// -------------------------------------------------------------------------------------------------
-
-} // namespace pmem
-} // namespace fdb5
-
-
-#endif // fdb5_pmem_Pool_H
diff --git a/src/fdb5/pmem/PoolEntry.cc b/src/fdb5/pmem/PoolEntry.cc
deleted file mode 100644
index e64a96834..000000000
--- a/src/fdb5/pmem/PoolEntry.cc
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-#include "fdb5/pmem/PoolEntry.h"
-
-namespace fdb5 {
-
-//----------------------------------------------------------------------------------------------------------------------
-
-PoolEntry::PoolEntry(const std::string &path, const std::string& poolgroup, bool active, bool visit):
-    path_(path),
-    poolgroup_(poolgroup),
-    writable_(active),
-    visit_(visit) {
-
-}
-
-const eckit::PathName& PoolEntry::path() const {
-    return path_;
-}
-
-bool PoolEntry::writable() const {
-    return writable_;
-}
-
-bool PoolEntry::visit() const {
-    return visit_;
-}
-
-const std::string& PoolEntry::poolgroup() const
-{
-    return poolgroup_;
-}
-
-//----------------------------------------------------------------------------------------------------------------------
-
-}  // namespace fdb5
diff --git a/src/fdb5/pmem/PoolEntry.h b/src/fdb5/pmem/PoolEntry.h
deleted file mode 100644
index 344c2f14e..000000000
--- a/src/fdb5/pmem/PoolEntry.h
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @file   PoolEntry.h
-/// @author Baudouin Raoult
-/// @author Tiago Quintino
-/// @date   Mar 2016
-
-#ifndef fdb5_PoolEntry_H
-#define fdb5_PoolEntry_H
-
-#include "eckit/filesystem/PathName.h"
-
-namespace fdb5 {
-
-class Key;
-
-//----------------------------------------------------------------------------------------------------------------------
-
-class PoolEntry  {
-
-public: // methods
-
-    PoolEntry(const std::string& path,
-         const std::string& poolgroup,
-         bool writable = true,
-         bool visit  = true);
-
-    const eckit::PathName &path() const;
-
-    bool writable() const; ///< PoolEntry is in use, when archiving
-    bool visit() const;    ///< PoolEntry is visited, when retrievind
-
-    const std::string& poolgroup() const;
-
-private: // members
-
-    eckit::PathName path_;
-
-    std::string poolgroup_;
-
-    bool writable_;
-    bool visit_;
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace fdb5
-
-#endif
diff --git a/src/fdb5/pmem/PoolGroup.cc b/src/fdb5/pmem/PoolGroup.cc
deleted file mode 100644
index b04dc66e4..000000000
--- a/src/fdb5/pmem/PoolGroup.cc
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-#include "fdb5/pmem/PoolGroup.h"
-
-#include "eckit/filesystem/FileSpaceStrategies.h"
-#include "eckit/exception/Exceptions.h"
-
-#include "fdb5/database/Key.h"
-
-using eckit::Log;
-
-namespace fdb5 {
-
-//----------------------------------------------------------------------------------------------------------------------
-
-PoolGroup::PoolGroup(const std::string& name,
-                     const std::string& re,
-                     const std::string& handler,
-                     const std::vector& pools) :
-    name_(name),
-    handler_(handler),
-    re_(re),
-    pools_(pools) {
-}
-
-eckit::PathName PoolGroup::pool(const Key& key) const
-{
-    /// TEMPORARY
-    /// We return the first entry in the PoolGroup
-    /// We must understand what it means to manage groups of pools
-
-    ASSERT(pools_.size());
-
-    return pools_.front().path();
-}
-
-std::vector PoolGroup::writable() const
-{
-    std::vector result;
-    for (PoolVec::const_iterator i = pools_.begin(); i != pools_.end() ; ++i) {
-        if(i->writable()) {
-            result.push_back(i->path());
-        }
-    }
-    return result;
-}
-
-std::vector PoolGroup::visitable() const
-{
-    std::vector result;
-    for (PoolVec::const_iterator i = pools_.begin(); i != pools_.end() ; ++i) {
-        if(i->visit()) {
-            result.push_back(i->path());
-        }
-    }
-    return result;
-}
-
-void PoolGroup::all(eckit::StringSet& pools) const
-{
-    for (PoolVec::const_iterator i = pools_.begin(); i != pools_.end() ; ++i) {
-        pools.insert(i->path());
-    }
-}
-
-void PoolGroup::writable(eckit::StringSet& pools) const
-{
-    for (PoolVec::const_iterator i = pools_.begin(); i != pools_.end() ; ++i) {
-        if(i->writable()) {
-            pools.insert(i->path());
-        }
-    }
-}
-
-void PoolGroup::visitable(eckit::StringSet& pools) const
-{
-    for (PoolVec::const_iterator i = pools_.begin(); i != pools_.end() ; ++i) {
-        if(i->visit()) {
-            pools.insert(i->path());
-        }
-    }
-}
-
-bool PoolGroup::match(const std::string& s) const {
-    return re_.match(s);
-}
-
-//----------------------------------------------------------------------------------------------------------------------
-
-}  // namespace fdb5
diff --git a/src/fdb5/pmem/PoolGroup.h b/src/fdb5/pmem/PoolGroup.h
deleted file mode 100644
index 76b952762..000000000
--- a/src/fdb5/pmem/PoolGroup.h
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @file   PoolGroup.h
-/// @author Tiago Quintino
-/// @date   Dec 2016
-
-#ifndef fdb5_PoolGroup_H
-#define fdb5_PoolGroup_H
-
-#include 
-#include 
-
-#include "eckit/utils/Regex.h"
-#include "eckit/types/Types.h"
-
-#include "fdb5/pmem/PoolEntry.h"
-
-namespace fdb5 {
-
-//----------------------------------------------------------------------------------------------------------------------
-
-class PoolGroup {
-
-public: // methods
-
-    PoolGroup(const std::string& name,
-              const std::string& re,
-              const std::string& handler,
-              const std::vector& pools);
-
-    /// Selects the pool where this Key will be inserted
-    /// @note This method must be idempotent -- it returns always the same value after the first call
-    eckit::PathName pool(const Key& key) const;
-
-    void all(eckit::StringSet&) const;
-    void writable(eckit::StringSet&) const;
-    void visitable(eckit::StringSet&) const;
-
-    bool match(const std::string& s) const;
-
-    std::vector writable() const;
-    std::vector visitable() const;
-
-private: // members
-
-    typedef std::vector PoolVec;
-
-    std::string name_;
-
-    std::string handler_;
-
-    eckit::Regex re_;
-
-    PoolVec pools_;
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace fdb5
-
-#endif
diff --git a/src/fdb5/pmem/PoolManager.cc b/src/fdb5/pmem/PoolManager.cc
deleted file mode 100644
index 7185565f3..000000000
--- a/src/fdb5/pmem/PoolManager.cc
+++ /dev/null
@@ -1,317 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-#include "fdb5/pmem/PoolManager.h"
-
-#include "eckit/config/Resource.h"
-#include "eckit/utils/Tokenizer.h"
-#include "eckit/utils/Translator.h"
-
-#include "metkit/mars/MarsRequest.h"
-
-#include "fdb5/LibFdb5.h"
-#include "fdb5/database/Key.h"
-#include "fdb5/pmem/PoolEntry.h"
-#include "fdb5/pmem/PoolGroup.h"
-#include "fdb5/rules/Schema.h"
-
-#include 
-#include 
-
-using namespace eckit;
-
-namespace fdb5 {
-
-//----------------------------------------------------------------------------------------------------------------------
-
-typedef std::vector PoolGroupTable;
-typedef std::map PoolGroupMap;
-
-std::mutex poolGroupMutex;
-static PoolGroupMap poolGroupTables;
-
-
-static std::vector readPools(const eckit::PathName& path) {
-
-    std::vector result;
-
-    std::ifstream in(path.localPath());
-
-    LOG_DEBUG_LIB(LibFdb5) << "Loading FDB pools from " << path << std::endl;
-
-    if (!in) {
-        eckit::Log::error() << path << eckit::Log::syserr << std::endl;
-        return result;
-    }
-
-    eckit::Translator str2bool;
-
-    eckit::Tokenizer parse(" ");
-
-    char line[1024];
-    while (in.getline(line, sizeof(line))) {
-
-        std::vector s;
-        parse(line, s);
-
-        size_t i = 0;
-        while (i < s.size()) {
-            if (s[i].length() == 0) {
-                s.erase(s.begin() + i);
-            } else {
-                i++;
-            }
-        }
-
-        if (s.size() == 0 || s[0][0] == '#') {
-            continue;
-        }
-
-        switch (s.size()) {
-            case 4: {
-                const std::string& path       = s[0];
-                const std::string& poolgroup  = s[1];
-                bool writable        = str2bool(s[2]);
-                bool visit           = str2bool(s[3]);
-
-                result.push_back(PoolEntry(path, poolgroup, writable, visit));
-                break;
-            }
-
-        default:
-            eckit::Log::warning() << "FDB PoolManager: Invalid line ignored: " << line << std::endl;
-            break;
-        }
-    }
-
-    return result;
-}
-
-static std::vector poolsInGroup(const std::vector& all, const std::string& poolgroup) {
-
-    std::vector pools;
-
-    for (std::vector::const_iterator i = all.begin(); i != all.end() ; ++i) {
-        if (i->poolgroup() == poolgroup) {
-            pools.push_back(*i);
-        }
-    }
-    return pools;
-}
-
-
-static PoolGroupTable readPoolGroups(const eckit::PathName& fdbHome) {
-
-    std::lock_guard lock(poolGroupMutex);
-
-    // Pool groups are memoised, so only read it once
-
-    PoolGroupMap::const_iterator it = poolGroupTables.find(fdbHome);
-    if (it != poolGroupTables.end()) {
-        return it->second;
-    }
-
-    eckit::PathName fdbPMemPoolsFile = eckit::Resource("fdbPMemPoolsFile;$FDB_PMEM_POOLS_FILE", fdbHome / "pools");
-    std::vector allPools = readPools(fdbPMemPoolsFile);
-
-    eckit::PathName fdbPMemPoolGroupsFile = eckit::Resource("fdbPMemPoolGroupsFile;$FDB_PMEM_POOLGROUPS_FILE", fdbHome / "poolgroups");
-    std::ifstream in(fdbPMemPoolGroupsFile.localPath());
-
-    LOG_DEBUG_LIB(LibFdb5) << "Loading FDB file poolgroups from " << fdbPMemPoolGroupsFile << std::endl;
-
-    if (!in) {
-        throw eckit::ReadError(fdbPMemPoolGroupsFile, Here());
-    }
-
-    PoolGroupTable& poolgroupsTable(poolGroupTables[fdbHome]);
-
-    eckit::Tokenizer parse(" ");
-
-    char line[1024];
-    while (in.getline(line, sizeof(line))) {
-
-        std::vector s;
-        parse(line, s);
-
-        size_t i = 0;
-        while (i < s.size()) {
-            if (s[i].length() == 0) {
-                s.erase(s.begin() + i);
-            } else {
-                i++;
-            }
-        }
-
-        if (s.size() == 0 || s[0][0] == '#') {
-            continue;
-        }
-
-        switch (s.size()) {
-            case 3: {
-                const std::string& regex     = s[0];
-                const std::string& poolgroup = s[1];
-                const std::string& handler   = s[2];
-
-                std::vector pools = poolsInGroup(allPools, poolgroup);
-
-                if(!pools.size()) {
-                    std::ostringstream oss;
-                    oss << "No pools found for poolgroup " << poolgroup;
-                    throw UserError(oss.str(), Here());
-                }
-
-                poolgroupsTable.push_back(PoolGroup(poolgroup, regex, handler, pools));
-                break;
-            }
-
-        default:
-            eckit::Log::warning() << "FDB PoolManager: Invalid line ignored: " << line << std::endl;
-            break;
-
-        }
-    }
-
-    return poolgroupsTable;
-}
-
-static PoolGroupTable poolGroups(const Config& config) {
-
-    static std::string poolGroupFile = eckit::Resource("fdbPMemPoolsFile;$FDB_PMEM_POOLS_FILE", "~fdb/etc/fdb/pools");
-
-    if (config.has("groups")) {
-        PoolGroupTable table;
-        std::vector groupsConfigs(config.getSubConfigurations("groups"));
-        for (const auto& group : groupsConfigs) {
-
-            std::vector poolEntries;
-            std::vector pools(group.getSubConfigurations("pools"));
-            for (const auto& pool : pools) {
-                poolEntries.emplace_back(
-                    PoolEntry(
-                        pool.getString("path"),
-                        "",
-                        pool.getBool("writable", true),
-                        pool.getBool("visit", true)
-                    )
-                );
-            }
-
-            table.emplace_back(
-                PoolGroup(
-                    group.getString("name", ""),
-                    group.getString("regex", ".*"),
-                    group.getString("handler", "Default"),
-                    poolEntries
-                )
-            );
-        }
-        return table;
-    } else {
-        return readPoolGroups(config.expandPath("~fdb/"));
-    }
-}
-
-
-//----------------------------------------------------------------------------------------------------------------------
-
-PoolManager::PoolManager(const Config& config) :
-    poolGroupTable_(poolGroups(config)),
-    config_(config) {}
-
-
-eckit::PathName PoolManager::pool(const Key& key) {
-
-    std::string name(key.valuesToString());
-
-    /// @note returns the first PoolGroup that matches
-
-    for (const PoolGroup& group : poolGroupTable_) {
-        if(group.match(name)) {
-            PathName path = group.pool(key);
-            return path / name;
-        }
-    }
-
-    std::ostringstream oss;
-    oss << "No FDB memory pool for " << key << " (" << name << ")";
-    throw eckit::SeriousBug(oss.str());
-}
-
-std::vector PoolManager::allPools(const Key& key)
-{
-    eckit::StringSet pools;
-
-    std::string k = key.valuesToString();
-
-    for (const PoolGroup& group : poolGroupTable_) {
-        if(group.match(k)) {
-            group.all(pools);
-        }
-    }
-
-    return std::vector(pools.begin(), pools.end());
-}
-
-
-std::vector PoolManager::visitablePools(const std::set& keys) {
-
-    eckit::StringSet pools;
-
-    std::vector keystrings;
-    keystrings.reserve(keys.size());
-    for (const Key& k : keys) keystrings.emplace_back(k.valuesToString());
-
-    for (const PoolGroup& group : poolGroupTable_) {
-        for (const std::string& k : keystrings) {
-            if(group.match(k) || k.empty()) {
-                group.visitable(pools);
-                break;
-            }
-        }
-    }
-
-    LOG_DEBUG_LIB(LibFdb5) << "Visitable Pools " << pools << std::endl;
-
-    return std::vector(pools.begin(), pools.end());
-}
-
-std::vector PoolManager::visitablePools(const Key& key) {
-    return visitablePools(std::set { key });
-}
-
-std::vector PoolManager::visitablePools(const metkit::mars::MarsRequest& request) {
-    std::set keys;
-    config_.schema().matchFirstLevel(request, keys, "");
-    return visitablePools(keys);
-}
-
-std::vector PoolManager::writablePools(const Key& key) {
-
-    eckit::StringSet pools;
-
-    std::string k = key.valuesToString();
-
-    for (const PoolGroup& group : poolGroupTable_) {
-        if(group.match(k)) {
-            group.writable(pools);
-        }
-    }
-
-    return std::vector(pools.begin(), pools.end());
-}
-
-//----------------------------------------------------------------------------------------------------------------------
-
-}  // namespace fdb5
diff --git a/src/fdb5/pmem/PoolManager.h b/src/fdb5/pmem/PoolManager.h
deleted file mode 100644
index 2f3307341..000000000
--- a/src/fdb5/pmem/PoolManager.h
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Baudouin Raoult
-/// @author Tiago Quintino
-/// @author Simon Smart
-/// @date   Dec 2016
-
-#ifndef fdb5_PoolManager_H
-#define fdb5_PoolManager_H
-
-#include "eckit/utils/Regex.h"
-#include "eckit/filesystem/PathName.h"
-
-#include "fdb5/config/Config.h"
-#include "fdb5/pmem/PoolGroup.h"
-
-namespace metkit { class MarsRequest; }
-
-namespace fdb5 {
-
-class Key;
-
-//----------------------------------------------------------------------------------------------------------------------
-
-class PoolManager  {
-
-public: // methods
-
-    PoolManager(const Config& config=Config());
-
-    /// Uniquely selects a pool where the Key will be put or already exists
-    eckit::PathName pool(const Key& key);
-
-    /// Lists the roots that can be visited given a DB key
-    std::vector allPools(const Key& key);
-
-    /// Lists the roots that can be visited given a DB key
-    std::vector visitablePools(const Key& key);
-    std::vector visitablePools(const std::set& keys);
-    std::vector visitablePools(const metkit::mars::MarsRequest& request);
-
-    /// Lists the roots where a DB key would be able to be written
-    std::vector writablePools(const Key& key);
-
-private: // members
-
-    const std::vector poolGroupTable_;
-    Config config_;
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace fdb5
-
-#endif
diff --git a/src/fdb5/rules/Rule.cc b/src/fdb5/rules/Rule.cc
index 6fd9fe6f7..f872c38b0 100644
--- a/src/fdb5/rules/Rule.cc
+++ b/src/fdb5/rules/Rule.cc
@@ -50,7 +50,7 @@ Rule::Rule(eckit::Stream& s):
 }
 
 Rule::Rule(const Schema &schema, eckit::Stream& s):
-    schema_(schema), registry_(new TypesRegistry(s)) {
+    schema_(schema), registry_(s) {
 
     size_t numPredicates;
     size_t numRules;
@@ -68,7 +68,9 @@ Rule::Rule(const Schema &schema, eckit::Stream& s):
 }
 
 void Rule::encode(eckit::Stream& s) const {
-    registry_->encode(s);
+    
+    registry_.encode(s);
+
     s << line_;
     s << predicates_.size();
     for (const Predicate* predicate : predicates_) {
diff --git a/src/fdb5/rules/Schema.cc b/src/fdb5/rules/Schema.cc
index e89f4c7f9..a2dbe9a6f 100644
--- a/src/fdb5/rules/Schema.cc
+++ b/src/fdb5/rules/Schema.cc
@@ -39,7 +39,7 @@ Schema::Schema(std::istream& s) {
     load(s);
 }
 Schema::Schema(eckit::Stream& s) :
-    registry_(new TypesRegistry(s)) {
+    registry_(s) {
 
     size_t numRules;
     s >> path_;
@@ -52,7 +52,7 @@ Schema::Schema(eckit::Stream& s) :
 }
 
 void Schema::encode(eckit::Stream& s) const {
-    registry_->encode(s);
+    registry_.encode(s);
     s << path_;
     s << rules_.size();
     for (const Rule* rule : rules_) {
diff --git a/src/fdb5/toc/TocPurgeVisitor.cc b/src/fdb5/toc/TocPurgeVisitor.cc
index 34c73ea68..2015608b1 100644
--- a/src/fdb5/toc/TocPurgeVisitor.cc
+++ b/src/fdb5/toc/TocPurgeVisitor.cc
@@ -97,26 +97,6 @@ void TocPurgeVisitor::report(std::ostream& out) const {
         it.second.report(out, "          ");
     }
 
-    size_t indexToDelete = 0;
-    out << std::endl;
-    out << "Number of reachable fields per index file:" << std::endl;
-    for (const auto& it : indexUsage_) { // 
-        out << "    " << it.first << ": " << eckit::BigNum(it.second) << std::endl;
-        if (it.second == 0) {
-            indexToDelete++;
-        }
-    }
-
-    size_t dataToDelete = 0;
-    out << std::endl;
-    out << "Number of reachable fields per data file:" << std::endl;
-    for (const auto& it : dataUsage_) { // 
-        out << "    " << it.first << ": " << eckit::BigNum(it.second) << std::endl;
-        if (it.second == 0) {
-            dataToDelete++;
-        }
-    }
-
     out << std::endl;
     size_t cnt = 0;
     out << "Unreferenced owned data files:" << std::endl;
diff --git a/src/fdb5/types/TypesRegistry.cc b/src/fdb5/types/TypesRegistry.cc
index 6e054b425..5c65df01f 100644
--- a/src/fdb5/types/TypesRegistry.cc
+++ b/src/fdb5/types/TypesRegistry.cc
@@ -27,7 +27,7 @@ eckit::Reanimator TypesRegistry::reanimator_;
 
 TypesRegistry::TypesRegistry() = default;
 
-TypesRegistry::TypesRegistry(eckit::Stream& s) : {
+TypesRegistry::TypesRegistry(eckit::Stream& s) {
 
     size_t numTypes;
     std::string name;
diff --git a/tests/fdb/CMakeLists.txt b/tests/fdb/CMakeLists.txt
index c2353b3f8..d3a65dfad 100644
--- a/tests/fdb/CMakeLists.txt
+++ b/tests/fdb/CMakeLists.txt
@@ -7,10 +7,6 @@ configure_file( etc/fdb/schema      ${PROJECT_BINARY_DIR}/etc/fdb/schema      @O
 file( MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/tests/fdb/root ) # as defined in ${CMAKE_BINARY_DIR}/etc/fdb/roots file
 file( MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/tests/fdb/root2 ) # as defined in ${CMAKE_BINARY_DIR}/etc/fdb/roots file
 
-# for pmem based FDB
-configure_file( etc/fdb/pools      ${PROJECT_BINARY_DIR}/etc/fdb/pools       @ONLY )
-configure_file( etc/fdb/poolgroups ${PROJECT_BINARY_DIR}/etc/fdb/poolgroups  @ONLY )
-
 file( MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/tests/fdb/pool ) # as defined in ${CMAKE_BINARY_DIR}/etc/fdb/pools file
 
 ###############################################################################
@@ -67,10 +63,6 @@ foreach( _tst ${_test_data} )
 
 endforeach()
 
-#################################################################################
-# pmem tests make use of the test environment, so are added at the end
-
-add_subdirectory( pmem )
 add_subdirectory( api )
 add_subdirectory( database )
 add_subdirectory( tools )
diff --git a/tests/fdb/api/test_config.cc b/tests/fdb/api/test_config.cc
index 96a7c4403..5c9e5eb3e 100644
--- a/tests/fdb/api/test_config.cc
+++ b/tests/fdb/api/test_config.cc
@@ -38,7 +38,7 @@ CASE( "config_expands_from_environment_variable_json" ) {
     const std::string config_str(R"XX(
         {
             "type": "local",
-            "engine": "pmem",
+            "engine": "toc",
             "groups": [{
                 "pools": [{
                     "path": "/a/path/is/something"
@@ -52,7 +52,7 @@ CASE( "config_expands_from_environment_variable_json" ) {
     fdb5::Config expanded = fdb5::Config().expandConfig();
 
     EXPECT(expanded.getString("type") == "local");
-    EXPECT(expanded.getString("engine") == "pmem");
+    EXPECT(expanded.getString("engine") == "toc");
     EXPECT(expanded.getSubConfigurations("groups").size() == 1);
     EXPECT(expanded.getSubConfigurations("groups")[0].getSubConfigurations("pools").size() == 1);
     EXPECT(expanded.getSubConfigurations("groups")[0].getSubConfigurations("pools")[0].getString("path") == "/a/path/is/something");
diff --git a/tests/fdb/pmem/CMakeLists.txt b/tests/fdb/pmem/CMakeLists.txt
deleted file mode 100644
index 32408e5c2..000000000
--- a/tests/fdb/pmem/CMakeLists.txt
+++ /dev/null
@@ -1,27 +0,0 @@
-if( HAVE_PMEMFDB )
-
-    ecbuild_add_test( TARGET test_fdb5_pmem_pdatanode
-                      SOURCES test_pdatanode.cc
-                      INCLUDES ${PMEM_INCLUDE_DIRS}
-                      LIBS ${PMEM_LIBRARIES} fdb5
-                      ENVIRONMENT "${_test_environment}")
-
-    ecbuild_add_test( TARGET test_fdb5_pmem_pbranchingnode
-                      SOURCES test_pbranchingnode.cc
-                      INCLUDES ${PMEM_INCLUDE_DIRS}
-                      LIBS ${PMEM_LIBRARIES} fdb5
-                      ENVIRONMENT "${_test_environment}")
-
-    ecbuild_add_test( TARGET test_fdb5_pmem_pbasenode
-                      SOURCES test_pbasenode.cc
-                      INCLUDES ${PMEM_INCLUDE_DIRS}
-                      LIBS ${PMEM_LIBRARIES} fdb5
-                      ENVIRONMENT "${_test_environment}")
-
-    ecbuild_add_test( TARGET test_fdb5_pmem_pool_manager
-                      SOURCES test_pool_manager.cc
-                      INCLUDES ${PMEM_INCLUDE_DIRS}
-                      LIBS ${PMEM_LIBRARIES} fdb5
-                      ENVIRONMENT "${_test_environment}")
-
-endif()
diff --git a/tests/fdb/pmem/test_pbasenode.cc b/tests/fdb/pmem/test_pbasenode.cc
deleted file mode 100644
index b0814bc82..000000000
--- a/tests/fdb/pmem/test_pbasenode.cc
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-#include 
-
-
-#include "pmem/PersistentPtr.h"
-
-#include "fdb5/pmem/PBaseNode.h"
-#include "fdb5/pmem/PDataNode.h"
-#include "fdb5/pmem/PBranchingNode.h"
-
-#include "test_persistent_helpers.h"
-#include "eckit/testing/Test.h"
-
-using namespace eckit::testing;
-using namespace fdb5::pmem;
-using namespace pmem;
-
-namespace fdb {
-namespace test {
-
-//----------------------------------------------------------------------------------------------------------------------
-
-/// Provide a placeholder class to allow testing of the (protected) BaseNode functionality
-
-class ANode : public PBaseNode {
-
-public:  // types
-
-    enum NodeType {
-        NULL_NODE = PBaseNode::NULL_NODE,
-        DATA_NODE,
-        BRANCHING_NODE
-    };
-
-public:  // methods
-
-    ANode(ANode::NodeType type, const KeyType& key, const ValueType& value) :
-        PBaseNode(PBaseNode::NodeType(int(PBaseNode::NULL_NODE) + type - ANode::NULL_NODE), key, value) {}
-
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-/// Define a root type. Each test that does allocation should use a different element in the root object.
-
-const size_t root_elems = 4;
-
-class RootType {
-
-public:  // constructor
-
-    class Constructor : public AtomicConstructor {
-        virtual void make(RootType &object) const {
-            for (size_t i = 0; i < root_elems; i++) {
-                object.data_[i].nullify();
-            }
-        }
-    };
-
-public:  // members
-
-    PersistentPtr data_[root_elems];
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-// Only need type_id for RootType. Others defined in Pool.cc
-
-template<> uint64_t pmem::PersistentType::type_id = POBJ_ROOT_TYPE_NUM;
-
-// Create a global fixture, so that this pool is only created once, and destroyed once.
-
-PersistentPtr global_root;
-
-struct SuitePoolFixture {
-
-    SuitePoolFixture() : autoPool_(RootType::Constructor()) {
-        global_root = autoPool_.pool_.getRoot();
-    }
-    ~SuitePoolFixture() {
-        global_root.nullify();
-    }
-
-    AutoPool autoPool_;
-};
-
-SuitePoolFixture global_fixture;
-
-//----------------------------------------------------------------------------------------------------------------------
-
-CASE( "test_fdb5_pmem_pbasenode_construction" )
-{
-
-    class BNInspector : public PBaseNode {
-    public:
-        BNInspector() :
-            PBaseNode(PBaseNode::NULL_NODE, "key1", "value1") {}
-
-        bool check() const {
-
-            return type_ == PBaseNode::NULL_NODE &&
-                   idKey_ == std::string("key1") &&
-                   idValue_ == std::string("value1");
-        }
-    };
-
-    BNInspector bn;
-
-    EXPECT(bn.check());
-}
-
-CASE( "test_fdb5_pmem_pbasenode_types" )
-{
-    ANode n1(ANode::NULL_NODE, "key11", "value11");
-
-    EXPECT(n1.isNull());
-    EXPECT(!n1.isBranchingNode());
-    EXPECT(!n1.isDataNode());
-
-    ANode n2(ANode::DATA_NODE, "key11", "value11");
-
-    EXPECT(!n2.isNull());
-    EXPECT(!n2.isBranchingNode());
-    EXPECT(n2.isDataNode());
-
-    ANode n3(ANode::BRANCHING_NODE, "key11", "value11");
-
-    EXPECT(!n3.isNull());
-    EXPECT(n3.isBranchingNode());
-    EXPECT(!n3.isDataNode());
-}
-
-
-CASE( "test_fdb5_pmem_pbasenode_matches" )
-{
-    ANode n(ANode::NULL_NODE, "key11", "value11");
-
-    EXPECT(n.matches("key11", "value11"));
-    EXPECT(!n.matches("key11", "value12"));
-    EXPECT(!n.matches("key12", "value11"));
-    EXPECT(!n.matches("k", "v"));
-    EXPECT(!n.matches("key1111", "value1111"));
-}
-
-
-CASE( "test_fdb5_pmem_pbasenode_type_validator" )
-{
-    // As a base node, it is invalid to directly instantiate a PBaseNode. The type validation should
-    // fail for the base type, and pass for PDataNode and PBranchingNode
-
-    EXPECT(!::pmem::PersistentType::validate_type_id(
-                    ::pmem::PersistentType::type_id));
-    EXPECT(::pmem::PersistentType::validate_type_id(
-                    ::pmem::PersistentType::type_id));
-    EXPECT(::pmem::PersistentType::validate_type_id(
-                    ::pmem::PersistentType::type_id));
-}
-
-
-CASE( "test_fdb5_pmem_pbasenode_valid_allocated" )
-{
-    // When we allocate PDataNodes and PBranchingNodes into a PBasenode pointer, everything is just fine
-
-    std::string str_data("Testing testing");
-
-    global_root->data_[0].allocate_ctr(
-                PDataNode::BaseConstructor(
-                    PDataNode::Constructor("AKey", "AValue", str_data.data(), str_data.length())));
-
-    EXPECT(global_root->data_[0].valid());
-
-    global_root->data_[1].allocate_ctr(
-                PBranchingNode::BaseConstructor(
-                    AtomicConstructor2(
-                        std::string("key2"), std::string("value2"))));
-
-    EXPECT(global_root->data_[1].valid());
-
-    // However, you cannot allocate a base node. If we abuse the system and essentially force that
-    // it happens, things will fail on the valid() call.
-
-    class TweakConstructor : public AtomicConstructor {
-        virtual void make(PBaseNode& object) const {}
-    };
-
-    global_root->data_[2].allocate_ctr(TweakConstructor());
-
-    EXPECT(!global_root->data_[2].valid());
-}
-
-//-----------------------------------------------------------------------------
-
-}  // namespace test
-}  // namespace fdb
-
-int main(int argc, char **argv)
-{
-    return run_tests ( argc, argv );
-}
diff --git a/tests/fdb/pmem/test_pbranchingnode.cc b/tests/fdb/pmem/test_pbranchingnode.cc
deleted file mode 100644
index 481575ee5..000000000
--- a/tests/fdb/pmem/test_pbranchingnode.cc
+++ /dev/null
@@ -1,546 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-#include 
-
-#include "eckit/thread/Thread.h"
-#include "eckit/thread/ThreadControler.h"
-
-#include "pmem/PersistentPtr.h"
-
-#include "fdb5/pmem/PBranchingNode.h"
-#include "fdb5/pmem/PDataNode.h"
-#include "fdb5/pmem/DataPoolManager.h"
-#include "fdb5/database/Key.h"
-
-#include "test_persistent_helpers.h"
-#include "eckit/testing/Test.h"
-
-using namespace eckit::testing;
-using namespace fdb5::pmem;
-
-namespace fdb {
-namespace test {
-
-//----------------------------------------------------------------------------------------------------------------------
-
-class BrSpy : public PBranchingNode {
-public:
-    const pmem::PersistentVector& nodes() const { return nodes_; }
-    const pmem::PersistentPtr& axis() const { return axis_; }
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-/// Define a root type. Each test that does allocation should use a different element in the root object.
-
-const size_t root_elems = 6;
-
-class RootType {
-
-public: // constructor
-
-    class Constructor : public pmem::AtomicConstructor {
-        virtual void make(RootType &object) const {
-            for (size_t i = 0; i < root_elems; i++) {
-                object.data_[i].nullify();
-            }
-        }
-    };
-
-public: // members
-
-    pmem::PersistentPtr data_[root_elems];
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-// Only need type_id for RootType. Others defined in Pool.cc
-
-template<> uint64_t pmem::PersistentType::type_id = POBJ_ROOT_TYPE_NUM;
-
-// Create a global fixture, so that this pool is only created once, and destroyed once.
-
-pmem::PersistentPtr global_root;
-pmem::PersistentPool* global_pool;
-
-struct SuitePoolFixture {
-
-    SuitePoolFixture() : autoPool_(RootType::Constructor()) {
-        global_root = autoPool_.pool_.getRoot();
-        global_pool = &autoPool_.pool_;
-
-        for (size_t i = 0; i < root_elems; i++) {
-            global_root->data_[i].allocate("", "");
-        }
-    }
-    ~SuitePoolFixture() {
-        global_root.nullify();
-        global_pool = 0;
-    }
-
-    AutoPool autoPool_;
-};
-
-SuitePoolFixture global_fixture;
-
-//----------------------------------------------------------------------------------------------------------------------
-
-CASE( "test_fdb5_pmem_pbranchingnode_basenode_functionality" )
-{
-    std::vector elem(sizeof(PBranchingNode));
-
-    PBranchingNode& dn = *new (&elem[0]) PBranchingNode("key1", "value1");
-
-    EXPECT(!dn.isDataNode());
-    EXPECT(!dn.isNull());
-    EXPECT(dn.isBranchingNode());
-
-    EXPECT(dn.matches("key1", "value1"));
-    EXPECT(!dn.matches("key1", "value2"));
-    EXPECT(!dn.matches("key2", "value1"));
-}
-
-
-CASE( "test_fdb5_pmem_pbranchingnode_trivial_constructor" )
-{
-    std::vector elem(sizeof(PBranchingNode));
-
-    PBranchingNode& dn = *new (&elem[0]) PBranchingNode("key1", "value1");
-
-    EXPECT(static_cast(&dn)->nodes().size() == size_t(0));
-    EXPECT(static_cast(&dn)->axis().null());
-}
-
-
-CASE( "test_fdb5_pmem_pbranchingnode_insert_mismatched_key" )
-{
-    // If a data node is added whose key-value pair doesn't match that supplied to the
-    // branching node, then we should throw a wobbly.
-
-    PBranchingNode& root(*global_root->data_[0]);
-    DataPoolManager mgrMock("", *reinterpret_cast(0), global_root.uuid());
-
-    // Allocate a data node, and then insert it into the tree
-
-    fdb5::Key key;
-    key.push("key1", "value1");
-    key.push("key2", "value2");
-
-    std::string data1 = "This is some data. Wooooooooooooooooo!";
-
-    pmem::PersistentPtr ptr;
-    ptr.allocate_ctr(*global_pool, PDataNode::Constructor("keyz", "valuez", data1.data(), data1.size()));
-
-    EXPECT_THROWS_AS(root.insertDataNode(key, ptr, mgrMock), eckit::AssertionFailed);
-}
-
-
-CASE( "test_fdb5_pmem_pbranchingnode_insert_retrieve" )
-{
-    // Test that we can insert/retrieve with variously branching keys
-
-    // Global objects. The Manager won't do anything, as everything is within the pool pointed to
-    // by the global_root.uuid(), so the NULL reference will NEVER be used.
-    PBranchingNode& root(*global_root->data_[1]);
-    DataPoolManager mgrMock("", *reinterpret_cast(0), global_root.uuid());
-
-    // --
-
-    fdb5::Key key1;
-    key1.push("key1", "value1");
-    key1.push("key2", "value2");
-
-    std::string data1 = "This is some data. Wooooooooooooooooo!";
-
-    pmem::PersistentPtr ptr1;
-    ptr1.allocate_ctr(*global_pool, PDataNode::Constructor("key2", "value2", data1.data(), data1.size()));
-    root.insertDataNode(key1, ptr1, mgrMock);
-
-    // --
-
-    fdb5::Key key2;
-    key2.push("key1", "value1");
-    key2.push("key2a", "value2a");
-
-    std::string data2 = "Another bit of data";
-
-    pmem::PersistentPtr ptr2;
-    ptr2.allocate_ctr(*global_pool, PDataNode::Constructor("key2a", "value2a", data2.data(), data2.size()));
-    root.insertDataNode(key2, ptr2, mgrMock);
-
-
-    // --
-
-    fdb5::Key key3;
-    key3.push("key1a", "value1a");
-    key3.push("key2", "value2");
-
-    std::string data3 = "And again...";
-
-    pmem::PersistentPtr ptr3;
-    ptr3.allocate_ctr(*global_pool, PDataNode::Constructor("key2", "value2", data3.data(), data3.size()));
-    root.insertDataNode(key3, ptr3, mgrMock);
-
-    // --
-
-    pmem::PersistentPtr pdata1 = root.getDataNode(key1, mgrMock);
-    pmem::PersistentPtr pdata2 = root.getDataNode(key2, mgrMock);
-    pmem::PersistentPtr pdata3 = root.getDataNode(key3, mgrMock);
-
-    EXPECT(ptr1 == pdata1);
-    EXPECT(ptr2 == pdata2);
-    EXPECT(ptr3 == pdata3);
-
-    EXPECT(std::string(static_cast(pdata1->data()), pdata1->length()) == data1);
-    EXPECT(std::string(static_cast(pdata2->data()), pdata2->length()) == data2);
-    EXPECT(std::string(static_cast(pdata3->data()), pdata3->length()) == data3);
-}
-
-
-CASE( "test_fdb5_pmem_pbranchingnode_insert_masking" )
-{
-    // If data is repeatedly added with the same key, it masks existing data (which is not overwritten).
-
-    // Global objects. The Manager won't do anything, as everything is within the pool pointed to
-    // by the global_root.uuid(), so the NULL reference will NEVER be used.
-    PBranchingNode& root(*global_root->data_[2]);
-    DataPoolManager mgrMock("", *reinterpret_cast(0), global_root.uuid());
-
-    // Allocate a data node, and then insert it into the tree
-
-    fdb5::Key key;
-    key.push("key1", "value1");
-    key.push("key2", "value2");
-
-    std::string data1 = "This is some data. Wooooooooooooooooo!";
-
-    pmem::PersistentPtr ptr;
-    ptr.allocate_ctr(*global_pool, PDataNode::Constructor("key2", "value2", data1.data(), data1.size()));
-
-    root.insertDataNode(key, ptr, mgrMock);
-
-    // Add another data node with the same key
-
-    std::string data2 = "other data";
-
-    pmem::PersistentPtr ptr2;
-    ptr2.allocate_ctr(*global_pool, PDataNode::Constructor("key2", "value2", data2.data(), data2.size()));
-
-    root.insertDataNode(key, ptr2, mgrMock);
-
-    // Test that the second one is the one which is made accessible by the API
-
-    pmem::PersistentPtr pdata = root.getDataNode(key, mgrMock);
-    EXPECT(pdata == ptr2);
-
-    std::string check_str(static_cast(pdata->data()), pdata->length());
-    EXPECT(check_str == data2);
-
-    // Test that the original data is still there.
-
-    EXPECT(static_cast(&root)->nodes().size() == size_t(1));
-    EXPECT(static_cast(
-                                &static_cast(&root)->nodes()[0]->asBranchingNode()
-                              )->nodes().size() == size_t(2));
-    pmem::PersistentPtr pdata2 = static_cast(
-                                               &static_cast(&root)->nodes()[0]->asBranchingNode()
-                                            )->nodes()[0].as();
-    EXPECT(pdata2 == ptr);
-
-    std::string check_str2(static_cast(pdata2->data()), pdata2->length());
-    EXPECT(check_str2 == data1);
-}
-
-
-CASE( "test_fdb5_pmem_pbranchingnode_insert_collision" )
-{
-    // If we insert a data node that would mask a branching node, then something has gone
-    // wrong. We should hit an assertion.
-
-    PBranchingNode& root(*global_root->data_[3]);
-    DataPoolManager mgrMock("", *reinterpret_cast(0), global_root.uuid());
-
-    // --
-
-    fdb5::Key key1;
-    key1.push("key1", "value1");
-    key1.push("key2", "value2");
-    key1.push("key3", "value3");
-
-    std::string data1 = "This is some data. Wooooooooooooooooo!";
-
-    pmem::PersistentPtr ptr1;
-    ptr1.allocate_ctr(*global_pool, PDataNode::Constructor("key3", "value3", data1.data(), data1.size()));
-    root.insertDataNode(key1, ptr1, mgrMock);
-
-    // --
-
-    fdb5::Key key2;
-    key2.push("key1", "value1");
-    key2.push("key2", "value2");
-
-    std::string data2 = "Another bit of data";
-
-    pmem::PersistentPtr ptr2;
-    ptr2.allocate_ctr(*global_pool, PDataNode::Constructor("key2", "value2", data2.data(), data2.size()));
-
-    EXPECT_THROWS_AS(root.insertDataNode(key2, ptr2, mgrMock), eckit::AssertionFailed);
-}
-
-
-CASE( "test_fdb5_pmem_pbranchingnode_find_fail" )
-{
-    // If we attempt to find a non-existent data node (or worse, one that is present but not
-    // a data node) then fail gracefully.
-
-    // Global objects. The Manager won't do anything, as everything is within the pool pointed to
-    // by the global_root.uuid(), so the NULL reference will NEVER be used.
-    PBranchingNode& root(*global_root->data_[4]);
-    DataPoolManager mgrMock("", *reinterpret_cast(0), global_root.uuid());
-
-    // --
-
-    fdb5::Key key1;
-    key1.push("key1", "value1");
-    key1.push("key2", "value2");
-    key1.push("key3", "value3");
-
-    std::string data1 = "This is some data. Wooooooooooooooooo!";
-
-    pmem::PersistentPtr ptr1;
-    ptr1.allocate_ctr(*global_pool, PDataNode::Constructor("key3", "value3", data1.data(), data1.size()));
-    root.insertDataNode(key1, ptr1, mgrMock);
-
-    // -- Finding a non-existent node
-
-    fdb5::Key keyA;
-    keyA.push("a_key", "a_value");
-    keyA.push("anotherK", "anotherV");
-
-    pmem::PersistentPtr pdata1 = root.getDataNode(keyA, mgrMock);
-    EXPECT(pdata1.null());
-
-    // -- What if the searched for value isn't a DataNode. This is breaking the (implicit) schema
-    //    so it throws.
-
-    fdb5::Key keyB;
-    keyB.push("key1", "value1");
-    keyB.push("key2", "value2");
-
-    EXPECT_THROWS_AS(root.getDataNode(keyB, mgrMock), eckit::AssertionFailed);
-}
-
-
-CASE( "test_fdb5_pmem_pbranchingnode_thread_testing" )
-{
-    // Try and test the thread safety by throwing LOTS of data at the tree from multiple threads.
-
-    // Global objects. The Manager won't do anything, as everything is within the pool pointed to
-    // by the global_root.uuid(), so the NULL reference will NEVER be used.
-    PBranchingNode& root(*global_root->data_[5]);
-    DataPoolManager mgrMock("", *reinterpret_cast(0), global_root.uuid());
-
-    // --------------------
-
-    class WriterThread : public eckit::Thread {
-    public:
-        WriterThread(const std::vector >& keys, PBranchingNode& root) :
-            keys_(keys), root_(root),
-            mgrMock_("", *reinterpret_cast(0), global_root.uuid()) {}
-        virtual ~WriterThread() {}
-
-        virtual void run() {
-
-            // Iterate over the unique key descriptors passed in
-
-            for (std::vector >::const_iterator it = keys_.begin();
-                 it != keys_.end(); it++) {
-
-                std::string datakey = it->first.names()[it->first.names().size()-1];
-                std::string datavalue = it->first.value(datakey);
-                std::string data_str = it->second;
-
-                pmem::PersistentPtr pdata;
-                pdata.allocate_ctr(*global_pool, PDataNode::Constructor(datakey, datavalue, data_str.data(), data_str.length()));
-
-                // And insert into the tree
-
-                root_.insertDataNode(it->first, pdata, mgrMock_);
-            }
-        }
-
-    private:
-        const std::vector >& keys_;
-        PBranchingNode& root_;
-        DataPoolManager mgrMock_;
-    };
-
-    // --------------------
-
-    class Threads {
-
-    public: // methods
-
-        Threads() {}
-        ~Threads() {
-            for (std::vector::iterator it = threads_.begin(); it != threads_.end(); ++it)
-                delete *it;
-        }
-
-        void push(eckit::Thread* t) { threads_.push_back(new eckit::ThreadControler(t, false)); }
-
-        void go() {
-            for (std::vector::iterator it = threads_.begin(); it != threads_.end(); ++it)
-                (*it)->start();
-            for (std::vector::iterator it = threads_.begin(); it != threads_.end(); ++it)
-                (*it)->wait();
-        }
-
-    private: // members
-
-        std::vector threads_;
-    };
-
-    // --------------------
-
-    // Generate a list of sets of four unique indices. These will determine what each of the threads
-    // is going to add.
-
-    const size_t keys_per_thread = 10;
-    size_t thread_key_count = 0;
-    std::vector > > keys;
-    std::vector > part_key;
-
-    std::vector ks(4, 0);
-    std::vector k_max(4);
-    k_max[0] = 4;
-    k_max[1] = 5;
-    k_max[2] = 6;
-    k_max[3] = 7;
-
-    bool found;
-    do {
-        found = false;
-        for (size_t pos = 0; pos < 4; pos++) {
-            if (ks[pos] < k_max[pos]) {
-                ks[pos]++;
-                for (size_t pos2 = 0; pos2 < pos; pos2++)
-                    ks[pos2] = 0;
-
-                // Generate a unique key and data pair
-
-                fdb5::Key insert_key;
-                std::stringstream data_ss;
-                for (size_t i = 0; i < ks.size(); i++) {
-                    std::stringstream keybit, valuebit;
-                    keybit << "key" << i;
-                    valuebit << "value" << ks[i];
-                    data_ss << i << ks[i];
-                    insert_key.push(keybit.str(), valuebit.str());
-                }
-
-                // Add it to the list
-                part_key.push_back(std::make_pair(insert_key, data_ss.str()));
-
-                // Move onto the next one, as appropriate.
-                // (don't worry about the small overflow at the end, where things aren't nice
-                //  round multiples).
-
-                thread_key_count++;
-                if (thread_key_count >= keys_per_thread) {
-                    keys.push_back(part_key);
-                    part_key.clear();
-                    thread_key_count = 0;
-                }
-
-                found = true;
-                break;
-            }
-        }
-    } while(found);
-
-    // --------------------
-
-    Threads threads;
-    for (size_t i = 0; i < keys.size(); i++)
-        threads.push(new WriterThread(keys[i], root));
-    threads.go();
-
-    // --------------------
-
-    // And check that everything has been created correctly.
-
-    for (std::vector > >::const_iterator it = keys.begin();
-         it != keys.end(); ++it) {
-
-        for (std::vector >::const_iterator it2 = it->begin();
-             it2 != it->end(); ++it2) {
-
-            pmem::PersistentPtr pdata = root.getDataNode(it2->first, mgrMock);
-            EXPECT(!pdata.null());
-
-            std::string strnode(static_cast(pdata->data()), pdata->length());
-            EXPECT(strnode == it2->second);
-            eckit::Log::error() << "Check success" << std::endl;
-        }
-    }
-
-}
-
-
-CASE( "test_fdb5_pmem_pbranchingnode_datasize" )
-{
-    // Check that the objects haven't changed size unexpectedly
-    // (Important as data layout is persistent...)
-    // n.b. There is padding on the PBaseNode, which rounds it up in size...
-    EXPECT(sizeof(pmem::PersistentMutex) == size_t(64));
-    EXPECT(sizeof(PBranchingNode) == size_t(128));
-}
-
-
-CASE( "test_fdb5_pmem_branchingnode_atomicconstructor" )
-{
-    // Check that the AtomicConstructor correctly picks up the size
-
-    typedef pmem::AtomicConstructor2 BranchingConstructor2;
-
-    EXPECT(BranchingConstructor2("k1", "v1").size() == size_t(128));
-    EXPECT(BranchingConstructor2("k1", "v1").type_id() ==
-                            pmem::PersistentType::type_id);
-
-    // Check that the cast-to-PBaseNode constructor retains the type_id
-
-    EXPECT(PBranchingNode::BaseConstructor(
-                BranchingConstructor2("k1", "v1")).type_id()
-                == pmem::PersistentType::type_id);
-    EXPECT(PBranchingNode::BaseConstructor(
-                BranchingConstructor2("k1", "v1")).type_id()
-                != pmem::PersistentType::type_id);
-
-}
-
-
-//-----------------------------------------------------------------------------
-
-}  // namespace test
-}  // namespace fdb
-
-int main(int argc, char **argv)
-{
-    return run_tests ( argc, argv );
-}
diff --git a/tests/fdb/pmem/test_pdatanode.cc b/tests/fdb/pmem/test_pdatanode.cc
deleted file mode 100644
index 89a883156..000000000
--- a/tests/fdb/pmem/test_pdatanode.cc
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-#include 
-
-#include "pmem/PersistentPtr.h"
-
-#include "fdb5/pmem/PDataNode.h"
-
-#include 
-
-#include "eckit/testing/Test.h"
-
-using namespace eckit::testing;
-using namespace fdb5::pmem;
-
-namespace fdb {
-namespace test {
-
-//----------------------------------------------------------------------------------------------------------------------
-
-CASE( "test_fdb5_pmem_pdatanode_basenode_functionality" )
-{
-    std::string buf("Hello there, testing, testing");
-
-    std::vector elem(PDataNode::data_size(buf.length()));
-
-    PDataNode& dn = *new (&elem[0]) PDataNode("key1", "value1", buf.data(), buf.length());
-
-    EXPECT(dn.isDataNode());
-    EXPECT(!dn.isNull());
-    EXPECT(!dn.isBranchingNode());
-
-    EXPECT(dn.matches("key1", "value1"));
-    EXPECT(!dn.matches("key1", "value2"));
-    EXPECT(!dn.matches("key2", "value1"));
-}
-
-
-CASE( "test_fdb5_pmem_pdatanode_data" )
-{
-    std::string buf("Hello there, testing, testing");
-
-    std::vector elem(PDataNode::data_size(buf.length()));
-
-    PDataNode& dn = *new (&elem[0]) PDataNode("key1", "value1", buf.data(), buf.length());
-
-    EXPECT(dn.length() == eckit::Length(buf.length()));
-    EXPECT(dn.data() == (const void*)(&elem[40]));
-
-    std::string check_str(static_cast(dn.data()), dn.length());
-    EXPECT(buf == check_str);
-}
-
-
-CASE( "test_fdb5_pmem_pdatanode_datasize" )
-{
-    // Check that the objects haven't changed size unexpectedly
-    // (Important as data layout is persistent...)
-    // n.b. There is padding on the PBaseNode, which rounds it up in size...
-    EXPECT(sizeof(PDataNode) == size_t(48));
-
-    EXPECT(PDataNode::data_size(4) == size_t(44));
-    EXPECT(PDataNode::data_size(8) == size_t(48));
-    EXPECT(PDataNode::data_size(4096) == size_t(4136));
-}
-
-
-CASE( "test_fdb5_pmem_datanode_atomicconstructor" )
-{
-    // Check that the AtomicConstructor correctly picks up the size
-
-    EXPECT(PDataNode::Constructor("k1", "v1", NULL, 15).size() == size_t(55));
-    EXPECT(PDataNode::Constructor("k1", "v1", NULL, 15).type_id()
-                    == pmem::PersistentType::type_id);
-
-    // Check that the cast-to-PBaseNode constructor retains the type_id
-
-    EXPECT(PDataNode::BaseConstructor(
-                PDataNode::Constructor("k1", "v1", NULL, 15)).type_id()
-             == pmem::PersistentType::type_id);
-    EXPECT(PDataNode::BaseConstructor(
-                PDataNode::Constructor("k1", "v1", NULL, 15)).type_id()
-             != pmem::PersistentType::type_id);
-}
-
-//-----------------------------------------------------------------------------
-
-}  // namespace test
-}  // namespace fdb
-
-int main(int argc, char **argv)
-{
-    return run_tests ( argc, argv );
-}
diff --git a/tests/fdb/pmem/test_persistent_helpers.h b/tests/fdb/pmem/test_persistent_helpers.h
deleted file mode 100644
index cbfcd5ede..000000000
--- a/tests/fdb/pmem/test_persistent_helpers.h
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * (C) Copyright 1996-2015 ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-/// @author Simon Smart
-/// @date   April 2016
-
-#ifndef pmem_test_persistent_helpers_h
-#define pmem_test_persistent_helpers_h
-
-#include "eckit/filesystem/PathName.h"
-#include "eckit/memory/ScopedPtr.h"
-
-#include "pmem/AtomicConstructor.h"
-#include "pmem/PersistentPool.h"
-
-#include 
-
-//----------------------------------------------------------------------------------------------------------------------
-
-/// A fixture to obtain a unique name for this pool.
-
-struct UniquePool {
-
-    UniquePool() :
-        path_("") {
-
-        const char* tmpdir = ::getenv("TMPDIR");
-        if (!tmpdir)
-            tmpdir = ::getenv("SCRATCHDIR");
-        if (!tmpdir)
-            tmpdir = "/tmp";
-
-        eckit::PathName basepath(std::string(tmpdir) + "/pool");
-        path_ = eckit::PathName::unique(basepath);
-    }
-    ~UniquePool() {}
-
-    eckit::PathName path_;
-};
-
-
-/// Some standard constants to assist testing
-
-const size_t auto_pool_size = 1024 * 1024 * 20;
-const std::string auto_pool_name = "pool-name";
-
-/// A structure to automatically create and clean up a pool (used except in the tests where this
-/// functionality is being directly tested
-
-struct AutoPool {
-
-    AutoPool(const pmem::AtomicConstructorBase& constructor) :
-        path_(UniquePool().path_),
-        pool_(path_, auto_pool_size, auto_pool_name, constructor) {}
-    ~AutoPool() { pool_.remove(); }
-
-    eckit::PathName path_;
-    pmem::PersistentPool pool_;
-};
-
-
-
-/// A mock type, to call AtomicConstructors on appropriately sized regions of memory without going via the
-/// nvml library
-
-template 
-class PersistentMock {
-
-public: // methods
-
-    PersistentMock(const pmem::AtomicConstructor& ctr) :
-        data_(ctr.size()) {
-        ctr.make(object());
-    }
-
-    PersistentMock() :
-        ctr_(new pmem::AtomicConstructor0),
-        data_(ctr_->size()) {
-        ctr_->make(object());
-    }
-
-    template 
-    PersistentMock(const X1& x1) :
-        ctr_(new pmem::AtomicConstructor1(x1)),
-        data_(ctr_->size()) {
-        ctr_->make(object());
-    }
-
-    template 
-    PersistentMock(const X1& x1, const X2& x2) :
-        ctr_(new pmem::AtomicConstructor2(x1, x2)),
-        data_(ctr_->size()) {
-        ctr_->make(object());
-    }
-
-    T& object() {
-        return *reinterpret_cast(&data_[0]);
-    }
-
-private: // data
-
-    eckit::ScopedPtr > ctr_;
-    std::vector data_;
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-#endif // pmem_test_persistent_helpers_h
diff --git a/tests/fdb/pmem/test_pool_manager.cc b/tests/fdb/pmem/test_pool_manager.cc
deleted file mode 100644
index 4a6610343..000000000
--- a/tests/fdb/pmem/test_pool_manager.cc
+++ /dev/null
@@ -1,300 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/*
- * This software was developed as part of the EC H2020 funded project NextGenIO
- * (Project ID: 671951) www.nextgenio.eu
- */
-
-#include 
-#include 
-#include 
-
-#include "eckit/filesystem/PathName.h"
-#include "eckit/runtime/Main.h"
-#include "eckit/io/Buffer.h"
-
-#include "pmem/PersistentPtr.h"
-#include "pmem/AtomicConstructor.h"
-
-#include "fdb5/pmem/PRoot.h"
-#include "fdb5/pmem/PIndexRoot.h"
-#include "fdb5/pmem/DataPoolManager.h"
-
-#include "test_persistent_helpers.h"
-
-#include "eckit/testing/Test.h"
-
-using namespace fdb5::pmem;
-using namespace eckit;
-using namespace eckit::testing;
-using namespace pmem;
-
-namespace fdb {
-namespace test {
-
-//----------------------------------------------------------------------------------------------------------------------
-
-struct UniquePoolDir {
-    UniquePoolDir(const pmem::AtomicConstructorBase& ctr) :
-        path_(UniquePool().path_),
-        basePoolPath_((path_.mkdir(), path_ / "master")),
-        basePool_(basePoolPath_, 1025 * 1024 * 20, "pool-name", ctr) {
-
-        /// Use a non-standard size (which we can test...)
-        /// Minimum size is 8Mb. Use 11Mb
-        putenv(const_cast("fdbPMemDataPoolSize=11534336"));
-    }
-
-    ~UniquePoolDir() noexcept(false) {
-        // Clean up all the files
-
-        std::vector files;
-        std::vector dirs;
-        path_.children(files, dirs);
-
-        EXPECT(dirs.size() == 0);
-
-        for (std::vector::const_iterator fl = files.begin(); fl != files.end(); ++fl)
-            fl->unlink();
-
-        path_.rmdir();
-    }
-
-    eckit::PathName path_;
-    eckit::PathName basePoolPath_;
-    pmem::PersistentPool basePool_;
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-class MgrSpy : public DataPoolManager {
-public:
-    DataPool& currentWritePool() { return DataPoolManager::currentWritePool(); }
-    void invalidateCurrentPool(DataPool& p) { DataPoolManager::invalidateCurrentPool(p); }
-    const std::map& pools() const { return DataPoolManager::pools(); }
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-CASE( "test_fdb5_pmem_pool_manager_schema_create_pool" )
-{
-    UniquePoolDir up((PRoot::Constructor(PRoot::IndexClass)));
-    up.basePool_.getRoot()->buildRoot(fdb5::Key(), "");
-
-    PIndexRoot& ir(up.basePool_.getRoot()->indexRoot());
-
-    {
-        DataPoolManager mgr(up.path_, ir, 12345);
-
-        // Load up the schema from where it is installed in the test system, and then
-        // check that this equals whatever the DataPoolManager has stored.
-
-        EXPECT(ir.dataPoolUUIDs().size() == size_t(0));
-
-        DataPool& pool(static_cast(&mgr)->currentWritePool());
-        EXPECT(ir.dataPoolUUIDs().size() == size_t(1));
-
-        // Check that the size of the pool is correctly read
-        EXPECT(pool.size() == size_t(11534336));
-
-        EXPECT(pool.path().baseName() == "data-0");
-
-        // We can keep accessing the current write pool until it is invalidated (full)
-
-        DataPool& pool2(static_cast(&mgr)->currentWritePool());
-        EXPECT(&pool == &pool2);
-        EXPECT(pool.uuid() == pool2.uuid());
-        EXPECT(ir.dataPoolUUIDs().size() == size_t(1));
-
-        // But if the pool is invalidated, we get another one!
-
-        static_cast(&mgr)->invalidateCurrentPool(pool);
-       /// DataPool& pool3(static_cast(&mgr)->currentWritePool());
-       /// BOOST_CHECK_EQUAL(ir.dataPoolUUIDs().size(), size_t(2));
-       /// BOOST_CHECK(&pool != &pool3);
-       /// BOOST_CHECK(pool.uuid() != pool3.uuid());
-
-       /// BOOST_CHECK_EQUAL(pool3.path().baseName(), "data-1");
-    }
-}
-
-CASE( "test_fdb5_pmem_pool_manager_print" )
-{
-    UniquePoolDir up((PRoot::Constructor(PRoot::IndexClass)));
-    up.basePool_.getRoot()->buildRoot(fdb5::Key(), "");
-    PIndexRoot& ir(up.basePool_.getRoot()->indexRoot());
-
-    {
-        DataPoolManager mgr(up.path_, ir, 12345);
-
-        std::stringstream ss;
-        ss << mgr;
-
-        std::string correct = std::string("PMemDataPoolManager(") + std::string(up.path_) + ")";
-        EXPECT(ss.str() == correct);
-    }
-}
-
-CASE( "test_fdb5_pmem_pool_manager_pool_reopen" )
-{
-    UniquePoolDir up((PRoot::Constructor(PRoot::IndexClass)));
-    up.basePool_.getRoot()->buildRoot(fdb5::Key(), "");
-    PIndexRoot& ir(up.basePool_.getRoot()->indexRoot());
-    uint64_t uuid = 0;
-
-    {
-        DataPoolManager mgr(up.path_, ir, 12345);
-
-        DataPool& pool(static_cast(&mgr)->currentWritePool());
-        uuid = pool.uuid();
-
-        EXPECT(!pool.finalised());
-    }
-
-    // Re-open the same pool. Check that when we go to write, then we get back the same pool
-    // that we created before.
-
-    {
-        DataPoolManager mgr(up.path_, ir, 12345);
-
-        DataPool& pool(static_cast(&mgr)->currentWritePool());
-
-        EXPECT(uuid == pool.uuid());
-        EXPECT(!pool.finalised());
-
-        // Check that the size of the pool is correctly read
-        EXPECT(pool.size() == size_t(11534336));
-    }
-}
-
-
-CASE( "test_fdb5_pmem_pool_manager_pool_invalidation" )
-{
-    UniquePoolDir up((PRoot::Constructor(PRoot::IndexClass)));
-    up.basePool_.getRoot()->buildRoot(fdb5::Key(), "");
-    PIndexRoot& ir(up.basePool_.getRoot()->indexRoot());
-    uint64_t uuid = 0;
-
-    {
-        DataPoolManager mgr(up.path_, ir, 12345);
-
-        // n.b. Finilising a pool does NOT notify this to any other processes that have this pool
-        //      open for writing. They are free to (attempt) to write to the pool (and likely
-        //      fail and roll over).
-        //
-        //      All that marking this as finalised does is ensure that on re-opening a pool, we
-        //      don't attempt to re-open that one.
-
-        DataPool& pool(static_cast(&mgr)->currentWritePool());
-        uuid = pool.uuid();
-
-        static_cast(&mgr)->invalidateCurrentPool(pool);
-
-        EXPECT(pool.finalised());
-    }
-
-    // Re-open the same pool. Check that when we request the currentWritePool, it doesn't give us
-    // the finalised pool.
-
-    {
-        DataPoolManager mgr(up.path_, ir, 12345);
-
-        DataPool& pool(static_cast(&mgr)->currentWritePool());
-
-        EXPECT(uuid != pool.uuid());
-        EXPECT(!pool.finalised());
-    }
-}
-
-
-CASE( "test_fdb5_pmem_pool_manager_pool_load_pool" )
-{
-    UniquePoolDir up((PRoot::Constructor(PRoot::IndexClass)));
-    up.basePool_.getRoot()->buildRoot(fdb5::Key(), "");
-    PIndexRoot& ir(up.basePool_.getRoot()->indexRoot());
-    uint64_t uuid = 0;
-
-    {
-        DataPoolManager mgr(up.path_, ir, 12345);
-
-        // n.b. Finilising a pool does NOT notify this to any other processes that have this pool
-        //      open for writing. They are free to (attempt) to write to the pool (and likely
-        //      fail and roll over).
-        //
-        //      All that marking this as finalised does is ensure that on re-opening a pool, we
-        //      don't attempt to re-open that one.
-
-        DataPool& pool(static_cast(&mgr)->currentWritePool());
-        uuid = pool.uuid();
-    }
-
-    // Reopen the pool manager, and make it load the requested pool.
-
-    {
-        DataPoolManager mgr(up.path_, ir, 12345);
-
-        EXPECT(static_cast(&mgr)->pools().size() == size_t(0));
-
-        mgr.ensurePoolLoaded(uuid);
-
-        EXPECT(static_cast(&mgr)->pools().size() == size_t(1));
-        EXPECT(static_cast(&mgr)->pools().find(uuid) != static_cast(&mgr)->pools().end());
-    }
-}
-
-CASE( "test_fdb5_pmem_pool_manager_pool_allocate" )
-{
-    // Perform two allocations, one which fits and the other which will overflow the buffer.
-
-    UniquePoolDir up((PRoot::Constructor(PRoot::IndexClass)));
-    up.basePool_.getRoot()->buildRoot(fdb5::Key(), "");
-    PIndexRoot& ir(up.basePool_.getRoot()->indexRoot());
-
-    eckit::Buffer scratch_buf(6 * 1024 * 1024);
-
-    {
-        DataPoolManager mgr(up.path_, ir, 12345);
-
-        DataPool& pool1(static_cast(&mgr)->currentWritePool());
-        uint64_t uuid1 = pool1.uuid();
-
-        PersistentPtr tmp1;
-        mgr.allocate(tmp1, AtomicConstructor2(
-                         static_cast(scratch_buf), scratch_buf.size()));
-
-        DataPool& pool2(static_cast(&mgr)->currentWritePool());
-        uint64_t uuid2 = pool2.uuid();
-
-        EXPECT(uuid1 == uuid2);
-
-        PersistentPtr tmp2;
-        mgr.allocate(tmp2, AtomicConstructor2(
-                         static_cast(scratch_buf), scratch_buf.size()));
-
-        DataPool& pool3(static_cast(&mgr)->currentWritePool());
-        uint64_t uuid3 = pool3.uuid();
-
-        EXPECT(uuid1 != uuid3);
-    }
-}
-
-// TODO: Test allocate()
-
-//-----------------------------------------------------------------------------
-
-}  // namespace test
-}  // namespace fdb
-
-int main(int argc, char **argv)
-{
-    eckit::Main::initialise(argc, argv, "FDB_HOME");
-    return run_tests ( argc, argv, false );
-}

From 90ea48a575bc8293f10ce22410076266a2e62d18 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 24 Sep 2024 10:40:15 +0200
Subject: [PATCH 138/186] tag

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 5be94ede7..f7fe56fa5 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.13.100
\ No newline at end of file
+5.13.101

From 5a1865907a24bc2289b0f14a0f11fe1a2a53f5b3 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 30 Sep 2024 05:54:50 +0100
Subject: [PATCH 139/186] remote teardown on error

---
 src/fdb5/api/RemoteFDB.cc                     |  2 +-
 src/fdb5/api/RemoteFDB.h                      |  2 +-
 src/fdb5/remote/Connection.cc                 | 19 ++++++++++++
 src/fdb5/remote/Connection.h                  |  2 ++
 src/fdb5/remote/client/Client.cc              | 13 ++++++++
 src/fdb5/remote/client/Client.h               |  2 +-
 src/fdb5/remote/client/ClientConnection.cc    | 25 ++++------------
 src/fdb5/remote/client/ClientConnection.h     |  1 -
 .../remote/client/ClientConnectionRouter.cc   | 21 +++++++++++++
 .../remote/client/ClientConnectionRouter.h    |  2 ++
 src/fdb5/remote/client/RemoteCatalogue.cc     |  6 ++--
 src/fdb5/remote/client/RemoteCatalogue.h      |  2 +-
 src/fdb5/remote/client/RemoteStore.cc         |  6 ++--
 src/fdb5/remote/client/RemoteStore.h          |  2 +-
 src/fdb5/remote/server/ServerConnection.cc    | 30 +------------------
 15 files changed, 74 insertions(+), 61 deletions(-)

diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index 3fd798aaa..bd565e30b 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -308,7 +308,7 @@ bool RemoteFDB::handle(remote::Message message, bool control, uint32_t requestID
             return false;
     }
 }
-void RemoteFDB::handleException(std::exception_ptr e){NOTIMP;}
+// void RemoteFDB::handleException(std::exception_ptr e){NOTIMP;}
 
 static FDBBuilder builder("remote");
 
diff --git a/src/fdb5/api/RemoteFDB.h b/src/fdb5/api/RemoteFDB.h
index 76bd57984..7b9bad417 100644
--- a/src/fdb5/api/RemoteFDB.h
+++ b/src/fdb5/api/RemoteFDB.h
@@ -80,7 +80,7 @@ class RemoteFDB : public LocalFDB, public remote::Client {
 
     bool handle(remote::Message message, bool control, uint32_t requestID) override;
     bool handle(remote::Message message, bool control, uint32_t requestID, eckit::Buffer&& payload) override;
-    void handleException(std::exception_ptr e) override;
+    // void handleException(std::exception_ptr e) override;
 
 private: // members
 
diff --git a/src/fdb5/remote/Connection.cc b/src/fdb5/remote/Connection.cc
index 53d8b294e..97318b934 100644
--- a/src/fdb5/remote/Connection.cc
+++ b/src/fdb5/remote/Connection.cc
@@ -24,6 +24,25 @@ class TCPException : public eckit::Exception {
 
 }
 
+void Connection::teardown() {
+
+    if (!single_) {
+        // TODO make the data connection dying automatically, when there are no more async writes
+        try {
+            // all done - disconnecting
+            Connection::write(Message::Exit, false, 0, 0);
+        } catch(...) {
+            // if connection is already down, no need to escalate 
+        }
+    }
+    try {
+        // all done - disconnecting
+        Connection::write(Message::Exit, true, 0, 0);
+    } catch(...) {
+        // if connection is already down, no need to escalate 
+    }
+}
+
 //----------------------------------------------------------------------------------------------------------------------
 
 void Connection::writeUnsafe(bool control, const void* data, size_t length) {
diff --git a/src/fdb5/remote/Connection.h b/src/fdb5/remote/Connection.h
index 8f6ba872c..acd81e8aa 100644
--- a/src/fdb5/remote/Connection.h
+++ b/src/fdb5/remote/Connection.h
@@ -38,6 +38,8 @@ class Connection : eckit::NonCopyable {
     eckit::Buffer readControl(MessageHeader& hdr);
     eckit::Buffer readData(MessageHeader& hdr);
 
+    void teardown();
+
 private: // methods
 
     eckit::Buffer read(bool control, MessageHeader& hdr);
diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index 4436b0113..174b21f55 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -84,4 +84,17 @@ void Client::dataWrite(remote::Message msg, uint32_t requestID, std::vector> dataWriteQueue_;
     std::thread dataWriteThread_;
-    // std::future dataWriteFuture_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.cc b/src/fdb5/remote/client/ClientConnectionRouter.cc
index ea1e62c5c..3942b97b2 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.cc
+++ b/src/fdb5/remote/client/ClientConnectionRouter.cc
@@ -97,4 +97,25 @@ ClientConnectionRouter& ClientConnectionRouter::instance()
     return router;
 }
 
+void ClientConnectionRouter::teardown(std::exception_ptr e) {
+
+    try {
+        if (e) {
+            std::rethrow_exception(e);
+        }
+    }
+    catch(const std::exception& e) {
+        eckit::Log::error() << "error: " << e.what();
+    }
+
+    std::lock_guard lock(connectionMutex_);
+
+    for (const auto& [endp,conn] : connections_) {
+        if (conn) {
+            eckit::Log::warning() << "closing connection " << endp << std::endl;
+            conn->teardown();
+        }
+    }
+}
+
 }  // namespace fdb5::remote
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.h b/src/fdb5/remote/client/ClientConnectionRouter.h
index ea7e1755f..026511506 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.h
+++ b/src/fdb5/remote/client/ClientConnectionRouter.h
@@ -27,6 +27,8 @@ class ClientConnectionRouter : eckit::NonCopyable {
     ClientConnection& connection(const eckit::net::Endpoint& endpoint, const std::string& defaultEndpoint);
     ClientConnection& connection(const std::vector>& endpoints);
 
+    void teardown(std::exception_ptr e);
+
     void deregister(ClientConnection& connection);
 
 private:
diff --git a/src/fdb5/remote/client/RemoteCatalogue.cc b/src/fdb5/remote/client/RemoteCatalogue.cc
index 664c7ca6d..b184b3d1f 100644
--- a/src/fdb5/remote/client/RemoteCatalogue.cc
+++ b/src/fdb5/remote/client/RemoteCatalogue.cc
@@ -175,9 +175,9 @@ bool RemoteCatalogue::handle(Message message, bool control, uint32_t requestID,
     return false;
 }
 
-void RemoteCatalogue::handleException(std::exception_ptr e) {
-    NOTIMP;
-}
+// void RemoteCatalogue::handleException(std::exception_ptr e) {
+//     NOTIMP;
+// }
 
 void RemoteCatalogue::overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) {NOTIMP;}
 void RemoteCatalogue::index(const Key& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) {NOTIMP;}
diff --git a/src/fdb5/remote/client/RemoteCatalogue.h b/src/fdb5/remote/client/RemoteCatalogue.h
index 92b7acc0e..9d781ed12 100644
--- a/src/fdb5/remote/client/RemoteCatalogue.h
+++ b/src/fdb5/remote/client/RemoteCatalogue.h
@@ -73,7 +73,7 @@ class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public C
     // handlers for incoming messages - to be defined in the client class
     bool handle(Message message, bool control, uint32_t requestID) override;
     bool handle(Message message, bool control, uint32_t requestID, eckit::Buffer&& payload) override;
-    void handleException(std::exception_ptr e) override;
+    // void handleException(std::exception_ptr e) override;
 
 protected:
 
diff --git a/src/fdb5/remote/client/RemoteStore.cc b/src/fdb5/remote/client/RemoteStore.cc
index 8d068c5b3..03358de9f 100644
--- a/src/fdb5/remote/client/RemoteStore.cc
+++ b/src/fdb5/remote/client/RemoteStore.cc
@@ -403,9 +403,9 @@ bool RemoteStore::handle(Message message, bool control, uint32_t requestID, ecki
             return false;
     }
 }
-void RemoteStore::handleException(std::exception_ptr e) {
-    Log::error() << "RemoteStore::handleException " << std::endl;
-}
+// void RemoteStore::handleException(std::exception_ptr e) {
+//     Log::error() << "RemoteStore::handleException " << std::endl;
+// }
 
 eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation) {
     return dataHandle(fieldLocation, Key());
diff --git a/src/fdb5/remote/client/RemoteStore.h b/src/fdb5/remote/client/RemoteStore.h
index cedba9e50..22c426b62 100644
--- a/src/fdb5/remote/client/RemoteStore.h
+++ b/src/fdb5/remote/client/RemoteStore.h
@@ -155,7 +155,7 @@ class RemoteStore : public Store, public Client {
     // handlers for incoming messages - to be defined in the client class
     bool handle(Message message, bool control, uint32_t requestID) override;
     bool handle(Message message, bool control, uint32_t requestID, eckit::Buffer&& payload) override;
-    void handleException(std::exception_ptr e) override;
+    // void handleException(std::exception_ptr e) override;
 
 private: // members
 
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index 4a989048e..46825840b 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -443,21 +443,6 @@ void ServerConnection::handle() {
             if (hdr.message == Message::Exit) {
                 ASSERT(hdr.clientID() == 0);
 
-                // if (!single_) {
-                //     try {
-                //         // all done - disconnecting
-                //         Connection::write(Message::Exit, false, 0, 0);
-                //     } catch(...) {
-                //         // if connection is already down, no need to escalate 
-                //     }
-                // }
-                // try {
-                //     // all done - disconnecting
-                //     Connection::write(Message::Exit, true, 0, 0);
-                // } catch(...) {
-                //     // if connection is already down, no need to escalate 
-                // }
-
                 eckit::Log::status() << "Terminating CONTROL listener" << std::endl;
                 eckit::Log::info() << "Terminating CONTROL listener" << std::endl;
 
@@ -515,20 +500,7 @@ void ServerConnection::handle() {
     ASSERT(archiveQueue_.empty());
     archiveQueue_.close();
 
-    if (!single_) {
-        try {
-            // all done - disconnecting
-            Connection::write(Message::Exit, false, 0, 0);
-        } catch(...) {
-            // if connection is already down, no need to escalate 
-        }
-    }
-    try {
-        // all done - disconnecting
-        Connection::write(Message::Exit, true, 0, 0);
-    } catch(...) {
-        // if connection is already down, no need to escalate 
-    }
+    teardown();
 }
 
 void ServerConnection::handleException(std::exception_ptr e) {

From 6d99a27fd43f47de3603c3aaec1abf3fcdbac954 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 30 Sep 2024 09:10:18 +0100
Subject: [PATCH 140/186] fixed order of data connection init

---
 src/fdb5/remote/Connection.cc              |  1 +
 src/fdb5/remote/Connection.h               |  2 ++
 src/fdb5/remote/client/RemoteCatalogue.cc  |  2 +-
 src/fdb5/remote/server/ServerConnection.cc | 26 ++++++++++++++--------
 4 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/src/fdb5/remote/Connection.cc b/src/fdb5/remote/Connection.cc
index 97318b934..0eb998e7b 100644
--- a/src/fdb5/remote/Connection.cc
+++ b/src/fdb5/remote/Connection.cc
@@ -86,6 +86,7 @@ void Connection::readUnsafe(bool control, void* data, size_t length) {
 eckit::Buffer Connection::read(bool control, MessageHeader& hdr) {
     eckit::FixedString<4> tail;
 
+    std::lock_guard lock((control || single_) ? readControlMutex_ : readDataMutex_);
     readUnsafe(control, &hdr, sizeof(hdr));
 
     // std::cout << "READ [" <<  "endpoint=" <<  ((control || single_) ? controlSocket() : dataSocket()).remotePort() << ",message=" << hdr.message << ",clientID=" << hdr.clientID() << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
diff --git a/src/fdb5/remote/Connection.h b/src/fdb5/remote/Connection.h
index acd81e8aa..ec7900763 100644
--- a/src/fdb5/remote/Connection.h
+++ b/src/fdb5/remote/Connection.h
@@ -59,6 +59,8 @@ class Connection : eckit::NonCopyable {
 
     std::mutex controlMutex_;
     std::mutex dataMutex_;
+    std::mutex readControlMutex_;
+    std::mutex readDataMutex_;
 
 };
 
diff --git a/src/fdb5/remote/client/RemoteCatalogue.cc b/src/fdb5/remote/client/RemoteCatalogue.cc
index b184b3d1f..be1b95dd6 100644
--- a/src/fdb5/remote/client/RemoteCatalogue.cc
+++ b/src/fdb5/remote/client/RemoteCatalogue.cc
@@ -93,7 +93,7 @@ bool RemoteCatalogue::selectIndex(const Key& idxKey) {
 }
 
 const Index& RemoteCatalogue::currentIndex(){
-    return nullptr;
+    NOTIMP;
 }
 const Key RemoteCatalogue::currentIndexKey() {
     return currentIndexKey_;
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index 46825840b..c5020c3f8 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -240,6 +240,13 @@ void ServerConnection::initialiseConnections() {
         dataEndpoint = eckit::net::Endpoint{endpointFromClient.hostname(), dataSocket_->localPort()};
     }
 
+    std::future dataSocketInitFuture_;
+    if (!single_) {
+        dataSocketInitFuture_ = std::async(std::launch::async, [this] {
+            dataSocket_->accept();
+        });
+    }
+
     eckit::Log::info() << "Sending data endpoint to client: " << dataEndpoint << std::endl;
     {
         eckit::Buffer startupBuffer(1024);
@@ -255,14 +262,8 @@ void ServerConnection::initialiseConnections() {
         write(Message::Startup, true, 0, 0, std::vector>{{startupBuffer.data(), s.position()}});
     }
 
-
-    if (!errorMsg.empty()) {
-        error(errorMsg, hdr.clientID(), hdr.requestID);
-        return;
-    }
-
-    if (!single_) {
-        dataSocket_->accept();
+    if (!single_ && dataSocketInitFuture_.valid()) {
+        dataSocketInitFuture_.wait(); 
 
         // Check the response from the client.
         // Ensure that the hostname matches the original hostname, and that
@@ -290,8 +291,15 @@ void ServerConnection::initialiseConnections() {
             std::stringstream ss;
             ss << "Session IDs do not match: " << serverSession << " != " << sessionID_;
             throw eckit::BadValue(ss.str(), Here());
-        }
+        }       
     }
+
+    if (!errorMsg.empty()) {
+        error(errorMsg, hdr.clientID(), hdr.requestID);
+        return;
+    }
+
+    
 }
 
 int ServerConnection::selectDataPort() {

From 0b12259391d66ee5ce0b8bd0ffc8e1eccaeb6c71 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 30 Sep 2024 09:47:13 +0100
Subject: [PATCH 141/186] force single connection

---
 src/fdb5/remote/server/ServerConnection.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index c5020c3f8..88c36e08b 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -246,7 +246,7 @@ void ServerConnection::initialiseConnections() {
             dataSocket_->accept();
         });
     }
-
+    sleep(1);
     eckit::Log::info() << "Sending data endpoint to client: " << dataEndpoint << std::endl;
     {
         eckit::Buffer startupBuffer(1024);

From c21f5f32c8cd7bca2d9e640ccece457f1d34712c Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 30 Sep 2024 11:54:33 +0100
Subject: [PATCH 142/186] force single connection

---
 src/fdb5/remote/Connection.h               | 2 +-
 src/fdb5/remote/server/ServerConnection.cc | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/fdb5/remote/Connection.h b/src/fdb5/remote/Connection.h
index ec7900763..d8673333e 100644
--- a/src/fdb5/remote/Connection.h
+++ b/src/fdb5/remote/Connection.h
@@ -27,7 +27,7 @@ namespace fdb5::remote {
 class Connection : eckit::NonCopyable {
 
 public: // methods
-    Connection() : single_(false) {}
+    Connection() : single_(true) {}
     virtual ~Connection() {}
 
     void write(Message msg, bool control, uint32_t clientID, uint32_t requestID, const void* data, uint32_t length);
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index 88c36e08b..a9d329f1c 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -213,7 +213,7 @@ void ServerConnection::initialiseConnections() {
 
             LOG_DEBUG_LIB(LibFdb5) << "Protocol negotiation - NumberOfConnections " << ncSelected << std::endl;
             agreedConf_.set("NumberOfConnections", ncSelected);
-            single_ = (ncSelected == 1);
+            single_ = true; // (ncSelected == 1);
         }
         else {
             std::stringstream ss;
@@ -246,7 +246,6 @@ void ServerConnection::initialiseConnections() {
             dataSocket_->accept();
         });
     }
-    sleep(1);
     eckit::Log::info() << "Sending data endpoint to client: " << dataEndpoint << std::endl;
     {
         eckit::Buffer startupBuffer(1024);

From 07e4d3b8da76e2b5dda030d74f7d54408a70322f Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 30 Sep 2024 12:34:06 +0100
Subject: [PATCH 143/186] wip

---
 src/fdb5/remote/FdbServer.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/fdb5/remote/FdbServer.cc b/src/fdb5/remote/FdbServer.cc
index e9e9d53be..444ad9719 100644
--- a/src/fdb5/remote/FdbServer.cc
+++ b/src/fdb5/remote/FdbServer.cc
@@ -33,7 +33,10 @@ namespace fdb5::remote {
 FDBForker::FDBForker(net::TCPSocket &socket, const Config &config) :
     ProcessControler(true),
     socket_(socket),
-    config_(config) {}
+    config_(config) {
+
+        std::cout << socket << std::endl;
+    }
 
 FDBForker::~FDBForker() {}
 

From 28326383b8eb18e7fe25a47a0c87cf842e812ac3 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 30 Sep 2024 12:45:59 +0100
Subject: [PATCH 144/186] wip

---
 src/fdb5/remote/FdbServer.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/fdb5/remote/FdbServer.cc b/src/fdb5/remote/FdbServer.cc
index 444ad9719..50f02a33e 100644
--- a/src/fdb5/remote/FdbServer.cc
+++ b/src/fdb5/remote/FdbServer.cc
@@ -35,7 +35,7 @@ FDBForker::FDBForker(net::TCPSocket &socket, const Config &config) :
     socket_(socket),
     config_(config) {
 
-        std::cout << socket << std::endl;
+        std::cout << socket.localAddr() << "  " << socket.remoteAddr() << std::endl;
     }
 
 FDBForker::~FDBForker() {}

From 7f1fd533a0ea33e4f591e4a4d8d0f724d1b8cff4 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 30 Sep 2024 12:49:19 +0100
Subject: [PATCH 145/186] wip

---
 src/fdb5/remote/FdbServer.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/fdb5/remote/FdbServer.cc b/src/fdb5/remote/FdbServer.cc
index 50f02a33e..b1075bc87 100644
--- a/src/fdb5/remote/FdbServer.cc
+++ b/src/fdb5/remote/FdbServer.cc
@@ -35,7 +35,7 @@ FDBForker::FDBForker(net::TCPSocket &socket, const Config &config) :
     socket_(socket),
     config_(config) {
 
-        std::cout << socket.localAddr() << "  " << socket.remoteAddr() << std::endl;
+        std::cout << socket.localHost() << ":" << socket.localPort() << "  " << socket.remoteHost() << ":" << socket.remotePort() << std::endl;
     }
 
 FDBForker::~FDBForker() {}

From 4bbd91ceb40f3ecda65b7b08d379388561f89be8 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 30 Sep 2024 12:59:17 +0100
Subject: [PATCH 146/186] wip

---
 src/fdb5/remote/FdbServer.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/fdb5/remote/FdbServer.cc b/src/fdb5/remote/FdbServer.cc
index b1075bc87..b69ec7b29 100644
--- a/src/fdb5/remote/FdbServer.cc
+++ b/src/fdb5/remote/FdbServer.cc
@@ -34,8 +34,6 @@ FDBForker::FDBForker(net::TCPSocket &socket, const Config &config) :
     ProcessControler(true),
     socket_(socket),
     config_(config) {
-
-        std::cout << socket.localHost() << ":" << socket.localPort() << "  " << socket.remoteHost() << ":" << socket.remotePort() << std::endl;
     }
 
 FDBForker::~FDBForker() {}
@@ -50,6 +48,8 @@ void FDBForker::run() {
 
     eckit::Log::info() << "FDB forked pid " << ::getpid() << std::endl;
 
+    std::cout << socket.localHost() << ":" << socket.localPort() << "  " << socket.remoteHost() << ":" << socket.remotePort() << std::endl;
+
     if (config_.getString("type", "local") == "catalogue" || (::getenv("FDB_IS_CAT") && ::getenv("FDB_IS_CAT")[0] == '1')) {
         eckit::Log::info() << "FDB using Catalogue Handler" << std::endl;
         CatalogueHandler handler(socket_, config_);

From 9e80e07018eb6bb770a864b4a15eeffcbe8c630d Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 30 Sep 2024 13:00:54 +0100
Subject: [PATCH 147/186] wip

---
 src/fdb5/remote/FdbServer.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/fdb5/remote/FdbServer.cc b/src/fdb5/remote/FdbServer.cc
index b69ec7b29..cb2cf96f3 100644
--- a/src/fdb5/remote/FdbServer.cc
+++ b/src/fdb5/remote/FdbServer.cc
@@ -48,7 +48,7 @@ void FDBForker::run() {
 
     eckit::Log::info() << "FDB forked pid " << ::getpid() << std::endl;
 
-    std::cout << socket.localHost() << ":" << socket.localPort() << "  " << socket.remoteHost() << ":" << socket.remotePort() << std::endl;
+    std::cout << socket_.localHost() << ":" << socket_.localPort() << "  " << socket_.remoteHost() << ":" << socket_.remotePort() << std::endl;
 
     if (config_.getString("type", "local") == "catalogue" || (::getenv("FDB_IS_CAT") && ::getenv("FDB_IS_CAT")[0] == '1')) {
         eckit::Log::info() << "FDB using Catalogue Handler" << std::endl;

From 6263ddd26c73647ed8712f7454c76aa43a98ea71 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 30 Sep 2024 13:44:05 +0100
Subject: [PATCH 148/186] wip

---
 src/fdb5/remote/FdbServer.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/fdb5/remote/FdbServer.cc b/src/fdb5/remote/FdbServer.cc
index cb2cf96f3..0c80f2240 100644
--- a/src/fdb5/remote/FdbServer.cc
+++ b/src/fdb5/remote/FdbServer.cc
@@ -120,7 +120,7 @@ void FdbServerBase::doRun() {
     startPortReaperThread(config);
 
     int port = config.getInt("serverPort", 7654);
-    bool threaded = config.getBool("serverThreaded", false);
+    bool threaded = config.getBool("serverThreaded", true);
 
     net::TCPServer server(net::Port("fdb", port), net::SocketOptions::server().reusePort(true));
     server.closeExec(false);

From 3f57dd4b306941a888f2af8aabbf112911b344d0 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 30 Sep 2024 13:52:02 +0100
Subject: [PATCH 149/186] wip

---
 src/fdb5/remote/Connection.h               | 2 +-
 src/fdb5/remote/server/ServerConnection.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/fdb5/remote/Connection.h b/src/fdb5/remote/Connection.h
index d8673333e..ec7900763 100644
--- a/src/fdb5/remote/Connection.h
+++ b/src/fdb5/remote/Connection.h
@@ -27,7 +27,7 @@ namespace fdb5::remote {
 class Connection : eckit::NonCopyable {
 
 public: // methods
-    Connection() : single_(true) {}
+    Connection() : single_(false) {}
     virtual ~Connection() {}
 
     void write(Message msg, bool control, uint32_t clientID, uint32_t requestID, const void* data, uint32_t length);
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index a9d329f1c..f25d30a65 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -213,7 +213,7 @@ void ServerConnection::initialiseConnections() {
 
             LOG_DEBUG_LIB(LibFdb5) << "Protocol negotiation - NumberOfConnections " << ncSelected << std::endl;
             agreedConf_.set("NumberOfConnections", ncSelected);
-            single_ = true; // (ncSelected == 1);
+            single_ = (ncSelected == 1);
         }
         else {
             std::stringstream ss;

From 84cdfd4eafdaaddcec51fcace3f0b63500087dc5 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 30 Sep 2024 16:56:29 +0000
Subject: [PATCH 150/186] wip

---
 src/fdb5/remote/FdbServer.cc | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/fdb5/remote/FdbServer.cc b/src/fdb5/remote/FdbServer.cc
index 0c80f2240..e215b5d84 100644
--- a/src/fdb5/remote/FdbServer.cc
+++ b/src/fdb5/remote/FdbServer.cc
@@ -46,9 +46,7 @@ void FDBForker::run() {
     ::srand(::getpid() + ::time(nullptr));
     ::srandom(::getpid() + ::time(nullptr));
 
-    eckit::Log::info() << "FDB forked pid " << ::getpid() << std::endl;
-
-    std::cout << socket_.localHost() << ":" << socket_.localPort() << "  " << socket_.remoteHost() << ":" << socket_.remotePort() << std::endl;
+    eckit::Log::info() << "FDB forked pid " << ::getpid() << "  --  connection: " << socket_.localHost() << ":" << socket_.localPort() << "-->" << socket_.remoteHost() << ":" << socket_.remotePort() << std::endl;
 
     if (config_.getString("type", "local") == "catalogue" || (::getenv("FDB_IS_CAT") && ::getenv("FDB_IS_CAT")[0] == '1')) {
         eckit::Log::info() << "FDB using Catalogue Handler" << std::endl;

From 27ad20524536261600c38a25a0777ae9c9c994ff Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 30 Sep 2024 16:57:27 +0000
Subject: [PATCH 151/186] wip

---
 src/fdb5/remote/FdbServer.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/fdb5/remote/FdbServer.cc b/src/fdb5/remote/FdbServer.cc
index e215b5d84..6943fd121 100644
--- a/src/fdb5/remote/FdbServer.cc
+++ b/src/fdb5/remote/FdbServer.cc
@@ -118,7 +118,7 @@ void FdbServerBase::doRun() {
     startPortReaperThread(config);
 
     int port = config.getInt("serverPort", 7654);
-    bool threaded = config.getBool("serverThreaded", true);
+    bool threaded = config.getBool("serverThreaded", false);
 
     net::TCPServer server(net::Port("fdb", port), net::SocketOptions::server().reusePort(true));
     server.closeExec(false);

From 2d3ce4dcb0a4dc0bc24ae587a26d1eb5f151e49d Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 30 Sep 2024 17:12:04 +0000
Subject: [PATCH 152/186] wip

---
 src/fdb5/remote/server/ServerConnection.cc | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index f25d30a65..bacdd3c34 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -240,12 +240,6 @@ void ServerConnection::initialiseConnections() {
         dataEndpoint = eckit::net::Endpoint{endpointFromClient.hostname(), dataSocket_->localPort()};
     }
 
-    std::future dataSocketInitFuture_;
-    if (!single_) {
-        dataSocketInitFuture_ = std::async(std::launch::async, [this] {
-            dataSocket_->accept();
-        });
-    }
     eckit::Log::info() << "Sending data endpoint to client: " << dataEndpoint << std::endl;
     {
         eckit::Buffer startupBuffer(1024);
@@ -261,8 +255,8 @@ void ServerConnection::initialiseConnections() {
         write(Message::Startup, true, 0, 0, std::vector>{{startupBuffer.data(), s.position()}});
     }
 
-    if (!single_ && dataSocketInitFuture_.valid()) {
-        dataSocketInitFuture_.wait(); 
+    if (!single_) {
+        dataSocket_->accept();
 
         // Check the response from the client.
         // Ensure that the hostname matches the original hostname, and that

From 4c1ac8d33f832ffd10eec5e2561796d944cce029 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 1 Oct 2024 15:52:52 +0100
Subject: [PATCH 153/186] wip

---
 src/fdb5/remote/server/ServerConnection.cc | 3 +++
 src/fdb5/remote/server/ServerConnection.h  | 2 ++
 2 files changed, 5 insertions(+)

diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index bacdd3c34..c75d822ab 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -168,6 +168,8 @@ void ServerConnection::initialiseConnections() {
         errorMsg = "Error retrieving client protocol version";
     }
 
+    std::cout << "Connection from: " << endpointFromClient << "  sessionId" << clientSession << std::endl;
+
     if (errorMsg.empty() && !LibFdb5::instance().remoteProtocolVersion().check(remoteProtocolVersion, false)) {
         std::stringstream ss;
         ss << "FDB server version " << fdb5_version_str() << " - remote protocol version not supported:" << std::endl;
@@ -234,6 +236,7 @@ void ServerConnection::initialiseConnections() {
     if (single_) {
         dataEndpoint = endpointFromClient;
     } else {
+        std::lock_guard lock(dataPortMutex_);
         dataSocket_.reset(new eckit::net::EphemeralTCPServer(selectDataPort()));
         ASSERT(dataSocket_->socket() != -1);
 
diff --git a/src/fdb5/remote/server/ServerConnection.h b/src/fdb5/remote/server/ServerConnection.h
index 16f10b038..21257bc19 100644
--- a/src/fdb5/remote/server/ServerConnection.h
+++ b/src/fdb5/remote/server/ServerConnection.h
@@ -156,6 +156,8 @@ class ServerConnection : public Connection, public Handler {
 
 private:
 
+    std::mutex dataPortMutex_;
+
     // data connection
     std::unique_ptr dataSocket_;
     size_t dataListener_;

From 3825bb698a3e94b3a7a1f81cbd1191f06439c82b Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 3 Oct 2024 11:21:33 +0000
Subject: [PATCH 154/186] wip

---
 src/fdb5/remote/FdbServer.cc |  3 +++
 src/fdb5/tools/fdb-hammer.cc | 16 ++++++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/src/fdb5/remote/FdbServer.cc b/src/fdb5/remote/FdbServer.cc
index 6943fd121..066aca906 100644
--- a/src/fdb5/remote/FdbServer.cc
+++ b/src/fdb5/remote/FdbServer.cc
@@ -108,6 +108,8 @@ FdbServerBase::~FdbServerBase() {}
 
 void FdbServerBase::doRun() {
 
+    std::mutex forkMutex;
+
     hookUnique();
 
     Config config = LibFdb5::instance().defaultConfig();
@@ -132,6 +134,7 @@ void FdbServerBase::doRun() {
                 t.start();
             }
             else {
+                std::lock_guard lock(forkMutex);
                 FDBForker f(server.accept(), config);
                 f.start();
             }
diff --git a/src/fdb5/tools/fdb-hammer.cc b/src/fdb5/tools/fdb-hammer.cc
index d18400468..382cc302c 100644
--- a/src/fdb5/tools/fdb-hammer.cc
+++ b/src/fdb5/tools/fdb-hammer.cc
@@ -12,6 +12,9 @@
 #include 
 #include 
 #include 
+#include 
+#include 
+#include 
 
 #include "eccodes.h"
 
@@ -70,6 +73,7 @@ class FDBHammer : public fdb5::FDBTool {
         options_.push_back(new eckit::option::SimpleOption("nparams", "Number of parameters"));
         options_.push_back(new eckit::option::SimpleOption("verbose", "Print verbose output"));
         options_.push_back(new eckit::option::SimpleOption("disable-subtocs", "Disable use of subtocs"));
+        options_.push_back(new eckit::option::SimpleOption("delay", "Add random delay"));
     }
     ~FDBHammer() override {}
 
@@ -121,6 +125,8 @@ void FDBHammer::executeWrite(const eckit::option::CmdArgs &args) {
     size_t number = args.getLong("number", 1);
     size_t level = args.getLong("level", 1);
 
+    bool delay = args.getBool("delay", false);
+
 
     const char* buffer = nullptr;
     size_t size = 0;
@@ -128,6 +134,16 @@ void FDBHammer::executeWrite(const eckit::option::CmdArgs &args) {
     eckit::LocalConfiguration userConfig{};
     if (!args.has("disable-subtocs")) userConfig.set("useSubToc", true);
 
+    if (delay) {
+        std::random_device rd;
+        std::mt19937 mt(rd());
+        std::uniform_int_distribution dist(0, 10000);
+
+        int delayDuration = dist(mt);
+        std::cout << "delay duration: " << delayDuration << std::endl;
+        std::this_thread::sleep_for(std::chrono::milliseconds(delayDuration));
+    }
+
     fdb5::MessageArchiver archiver(fdb5::Key(), false, verbose_, config(args, userConfig));
 
     std::string expver = args.getString("expver");

From 7ec275b142839fc4430b2761127eba8d46ec9b63 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 7 Oct 2024 16:31:15 +0100
Subject: [PATCH 155/186] wip

---
 src/fdb5/api/DistFDB.h                     |  2 ++
 src/fdb5/api/FDBFactory.h                  |  2 +-
 src/fdb5/api/RemoteFDB.cc                  | 15 +++++++++++++++
 src/fdb5/api/RemoteFDB.h                   |  2 ++
 src/fdb5/database/Catalogue.h              |  2 +-
 src/fdb5/remote/Messages.cc                |  1 +
 src/fdb5/remote/Messages.h                 |  1 +
 src/fdb5/remote/client/RemoteCatalogue.h   |  1 -
 src/fdb5/remote/server/CatalogueHandler.cc | 19 +++++++++++++++++++
 src/fdb5/remote/server/CatalogueHandler.h  |  1 +
 tests/fdb/api/ApiSpy.h                     |  8 +++++++-
 11 files changed, 50 insertions(+), 4 deletions(-)

diff --git a/src/fdb5/api/DistFDB.h b/src/fdb5/api/DistFDB.h
index 97b81dc39..43e177f3f 100644
--- a/src/fdb5/api/DistFDB.h
+++ b/src/fdb5/api/DistFDB.h
@@ -47,6 +47,8 @@ class DistFDB : public FDBBase {
 
     ListIterator list(const FDBToolRequest& request) override;
 
+    AxesIterator axesIterator(const FDBToolRequest& request, int level=3) override { NOTIMP; }
+
     DumpIterator dump(const FDBToolRequest& request, bool simple) override;
 
     StatusIterator status(const FDBToolRequest& request) override;
diff --git a/src/fdb5/api/FDBFactory.h b/src/fdb5/api/FDBFactory.h
index 9ff9739ff..6661a475e 100644
--- a/src/fdb5/api/FDBFactory.h
+++ b/src/fdb5/api/FDBFactory.h
@@ -93,7 +93,7 @@ class FDBBase : private eckit::NonCopyable {
 
     virtual MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) = 0;
 
-    virtual AxesIterator axesIterator(const FDBToolRequest& request, int axes) { NOTIMP; }
+    virtual AxesIterator axesIterator(const FDBToolRequest& request, int axes) = 0;
 
     void registerArchiveCallback(ArchiveCallback callback) {callback_ = callback;}
 
diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index bd565e30b..ea2775bd4 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -56,6 +56,17 @@ struct ListHelper : BaseAPIHelper {
+    AxesHelper(int level) : level_(level) {}
+
+    void encodeExtra(eckit::Stream& s) const {
+        s << level_;
+    }
+private:
+    int level_;
+};
+
 struct InspectHelper : BaseAPIHelper {
 
     static fdb5::ListElement valueFromStream(eckit::Stream& s, fdb5::RemoteFDB* fdb) {
@@ -230,6 +241,10 @@ ListIterator RemoteFDB::list(const FDBToolRequest& request) {
     return forwardApiCall(ListHelper(), request);
 }
 
+AxesIterator RemoteFDB::axesIterator(const FDBToolRequest& request, int level) {
+    return forwardApiCall(AxesHelper(level), request);
+}
+
 ListIterator RemoteFDB::inspect(const metkit::mars::MarsRequest& request) {
     return forwardApiCall(InspectHelper(), request);
 }
diff --git a/src/fdb5/api/RemoteFDB.h b/src/fdb5/api/RemoteFDB.h
index 7b9bad417..4d911e39a 100644
--- a/src/fdb5/api/RemoteFDB.h
+++ b/src/fdb5/api/RemoteFDB.h
@@ -48,6 +48,8 @@ class RemoteFDB : public LocalFDB, public remote::Client {
 
     ListIterator list(const FDBToolRequest& request) override;
 
+    AxesIterator axesIterator(const FDBToolRequest& request, int level=3) override;
+
     DumpIterator dump(const FDBToolRequest& request, bool simple) override { NOTIMP; }
 
     StatusIterator status(const FDBToolRequest& request) override { NOTIMP; }
diff --git a/src/fdb5/database/Catalogue.h b/src/fdb5/database/Catalogue.h
index ae6b55bdd..e15a2c9cb 100644
--- a/src/fdb5/database/Catalogue.h
+++ b/src/fdb5/database/Catalogue.h
@@ -143,7 +143,7 @@ class CatalogueReader : virtual public Catalogue {
     virtual ~CatalogueReader() {}
 
     virtual DbStats stats() const = 0;
-    virtual bool axis(const std::string& keyword, eckit::StringSet& s) const = 0;
+    virtual bool axis(const std::string& keyword, eckit::StringSet& s) const { NOTIMP; }
     virtual bool retrieve(const Key& key, Field& field) const = 0;
 };
 
diff --git a/src/fdb5/remote/Messages.cc b/src/fdb5/remote/Messages.cc
index 6f9df650c..5e96fdab6 100644
--- a/src/fdb5/remote/Messages.cc
+++ b/src/fdb5/remote/Messages.cc
@@ -46,6 +46,7 @@ std::ostream& operator<<(std::ostream& s, const Message& m) {
         case Message::Read: s << "Read"; break;
         case Message::Move: s << "Move"; break;
         case Message::Store: s << "Store"; break;
+        case Message::Axes: s << "Axes"; break;
 
     // Responses
         case Message::Received: s << "Received"; break;
diff --git a/src/fdb5/remote/Messages.h b/src/fdb5/remote/Messages.h
index 00d14520d..76566a7a1 100644
--- a/src/fdb5/remote/Messages.h
+++ b/src/fdb5/remote/Messages.h
@@ -63,6 +63,7 @@ enum class Message : uint16_t {
     Read,
     Move,
     Store,
+    Axes,
 
     // Responses
     Received = 200,
diff --git a/src/fdb5/remote/client/RemoteCatalogue.h b/src/fdb5/remote/client/RemoteCatalogue.h
index 9d781ed12..1b94d888b 100644
--- a/src/fdb5/remote/client/RemoteCatalogue.h
+++ b/src/fdb5/remote/client/RemoteCatalogue.h
@@ -30,7 +30,6 @@ class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public C
 
     //From CatalogueReader
     DbStats stats() const override { return DbStats(); }
-    bool axis(const std::string& keyword, eckit::StringSet& s) const override { return false; }
     bool retrieve(const Key& key, Field& field) const override { return false; }
 
     // From Catalogue
diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc
index 91ab5d16a..e513dbd19 100644
--- a/src/fdb5/remote/server/CatalogueHandler.cc
+++ b/src/fdb5/remote/server/CatalogueHandler.cc
@@ -99,6 +99,10 @@ Handled CatalogueHandler::handleControl(Message message, uint32_t clientID, uint
                 list(clientID, requestID, std::move(payload));
                 return Handled::Yes;
 
+            case Message::Axes: // list request. Location are sent aynchronously over the data connection
+                axes(clientID, requestID, std::move(payload));
+                return Handled::Yes;
+
             case Message::Inspect: // inspect request. Location are sent aynchronously over the data connection
                 inspect(clientID, requestID, std::move(payload));
                 return Handled::Yes;
@@ -216,6 +220,17 @@ struct ListHelper : public BaseHelper {
     }
 };
 
+struct AxesHelper : public BaseHelper {
+    void extraDecode(eckit::Stream& s) {
+        s >> level_;
+    }
+    AxesIterator apiCall(FDB& fdb, const FDBToolRequest& request) const {
+        return fdb.axesIterator(request, level_);
+    }
+private:
+    int level_;
+};
+
 struct InspectHelper : public BaseHelper {
     ListIterator apiCall(FDB& fdb, const FDBToolRequest& request) const {
         return fdb.inspect(request.request());
@@ -283,6 +298,10 @@ void CatalogueHandler::list(uint32_t clientID, uint32_t requestID, eckit::Buffer
     forwardApiCall(clientID, requestID, std::move(payload));
 }
 
+void CatalogueHandler::axes(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) {
+    forwardApiCall(clientID, requestID, std::move(payload));
+}
+
 void CatalogueHandler::inspect(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) {
     forwardApiCall(clientID, requestID, std::move(payload));
 }
diff --git a/src/fdb5/remote/server/CatalogueHandler.h b/src/fdb5/remote/server/CatalogueHandler.h
index f60e151b2..dc92afd70 100644
--- a/src/fdb5/remote/server/CatalogueHandler.h
+++ b/src/fdb5/remote/server/CatalogueHandler.h
@@ -53,6 +53,7 @@ class CatalogueHandler : public ServerConnection {
 
     void flush(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload);
     void list(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload);
+    void axes(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload);
     void inspect(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload);
     void stats(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload);
     void schema(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload);
diff --git a/tests/fdb/api/ApiSpy.h b/tests/fdb/api/ApiSpy.h
index c25efeebf..40a160b62 100644
--- a/tests/fdb/api/ApiSpy.h
+++ b/tests/fdb/api/ApiSpy.h
@@ -39,11 +39,12 @@ class ApiSpy : public fdb5::FDBBase {
 
     struct Counts {
         Counts() :
-            archive(0), inspect(0), list(0), dump(0), status(0), wipe(0),
+            archive(0), inspect(0), list(0), axes(0), dump(0), status(0), wipe(0),
             purge(0), stats(0), flush(0), control(0), move(0) {}
         size_t archive;
         size_t inspect;
         size_t list;
+        size_t axes;
         size_t dump;
         size_t status;
         size_t wipe;
@@ -95,6 +96,11 @@ class ApiSpy : public fdb5::FDBBase {
         return fdb5::ListIterator(0);
     }
 
+    fdb5::AxesIterator axesIterator(const fdb5::FDBToolRequest& request, int level=3) override {
+        counts_.axes += 1;
+        return fdb5::AxesIterator(0);
+    }
+
     fdb5::DumpIterator dump(const fdb5::FDBToolRequest& request, bool simple) override {
         counts_.dump += 1;
         return fdb5::DumpIterator(0);

From 0d61134b7b577f7ad5bb03fb009397ff8dcae34b Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 31 Oct 2024 11:00:37 +0100
Subject: [PATCH 156/186] sleep before connection

---
 src/fdb5/remote/client/ClientConnectionRouter.cc | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/src/fdb5/remote/client/ClientConnectionRouter.cc b/src/fdb5/remote/client/ClientConnectionRouter.cc
index 3942b97b2..f847387cb 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.cc
+++ b/src/fdb5/remote/client/ClientConnectionRouter.cc
@@ -1,5 +1,9 @@
 #include "fdb5/remote/client/ClientConnectionRouter.h"
 
+#include 
+#include 
+#include 
+
 namespace{
     
 class ConnectionError : public eckit::Exception {
@@ -37,6 +41,17 @@ ClientConnection& ClientConnectionRouter::connection(const eckit::net::Endpoint&
         return *(it->second);
     } else {
         ClientConnection* clientConnection = new ClientConnection{endpoint, defaultEndpoint};
+        
+        static int fdbConnectMaxSleep = eckit::Resource("fdbConnectMaxSleep", 0);
+        
+        if (fdbConnectMaxSleep) {
+            std::random_device rd;
+            std::mt19937 mt(rd());
+            std::uniform_int_distribution dist(0, fdbConnectMaxSleep);
+
+            std::this_thread::sleep_for(std::chrono::milliseconds(dist(rd)));
+        }
+
         if (clientConnection->connect()) {
             auto it = (connections_.emplace(endpoint, std::unique_ptr(clientConnection))).first;
             return *(it->second);

From 52a87468f9513cd8c739283c0ec76f62e1196d2a Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 31 Oct 2024 11:07:50 +0100
Subject: [PATCH 157/186] sleep before connection

---
 src/fdb5/remote/client/ClientConnectionRouter.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/fdb5/remote/client/ClientConnectionRouter.cc b/src/fdb5/remote/client/ClientConnectionRouter.cc
index f847387cb..f2856174d 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.cc
+++ b/src/fdb5/remote/client/ClientConnectionRouter.cc
@@ -4,6 +4,8 @@
 #include 
 #include 
 
+#include "eckit/config/Resource.h"
+
 namespace{
     
 class ConnectionError : public eckit::Exception {

From ae94a85d96b4fdae266816fa6a269057686f2465 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 4 Nov 2024 07:41:15 +0100
Subject: [PATCH 158/186] data connection async accept

---
 .../remote/client/ClientConnectionRouter.cc   | 14 +++++------
 src/fdb5/remote/server/ServerConnection.cc    | 25 +++++++++++--------
 2 files changed, 21 insertions(+), 18 deletions(-)

diff --git a/src/fdb5/remote/client/ClientConnectionRouter.cc b/src/fdb5/remote/client/ClientConnectionRouter.cc
index f2856174d..69f741c64 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.cc
+++ b/src/fdb5/remote/client/ClientConnectionRouter.cc
@@ -44,15 +44,15 @@ ClientConnection& ClientConnectionRouter::connection(const eckit::net::Endpoint&
     } else {
         ClientConnection* clientConnection = new ClientConnection{endpoint, defaultEndpoint};
         
-        static int fdbConnectMaxSleep = eckit::Resource("fdbConnectMaxSleep", 0);
+        // static int fdbConnectMaxSleep = eckit::Resource("fdbConnectMaxSleep", 0);
         
-        if (fdbConnectMaxSleep) {
-            std::random_device rd;
-            std::mt19937 mt(rd());
-            std::uniform_int_distribution dist(0, fdbConnectMaxSleep);
+        // if (fdbConnectMaxSleep) {
+        //     std::random_device rd;
+        //     std::mt19937 mt(rd());
+        //     std::uniform_int_distribution dist(0, fdbConnectMaxSleep);
 
-            std::this_thread::sleep_for(std::chrono::milliseconds(dist(rd)));
-        }
+        //     std::this_thread::sleep_for(std::chrono::milliseconds(dist(rd)));
+        // }
 
         if (clientConnection->connect()) {
             auto it = (connections_.emplace(endpoint, std::unique_ptr(clientConnection))).first;
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index c75d822ab..71f683a3e 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -232,6 +232,7 @@ void ServerConnection::initialiseConnections() {
     //               server has multiple, then we use that on, whilst retaining
     //               the capacity in the protocol for the server to make a choice.
 
+    std::future dataSocketFuture;
     eckit::net::Endpoint dataEndpoint;
     if (single_) {
         dataEndpoint = endpointFromClient;
@@ -241,25 +242,27 @@ void ServerConnection::initialiseConnections() {
         ASSERT(dataSocket_->socket() != -1);
 
         dataEndpoint = eckit::net::Endpoint{endpointFromClient.hostname(), dataSocket_->localPort()};
+
+        dataSocketFuture = std::async(std::launch::async, [this] { dataSocket_->accept(); return true; });
     }
 
     eckit::Log::info() << "Sending data endpoint to client: " << dataEndpoint << std::endl;
-    {
-        eckit::Buffer startupBuffer(1024);
-        eckit::MemoryStream s(startupBuffer);
 
-        s << clientSession;
-        s << sessionID_;
-        s << dataEndpoint;
-        s << agreedConf_.get();
+    eckit::Buffer startupBuffer(1024);
+    eckit::MemoryStream s(startupBuffer);
 
-        LOG_DEBUG_LIB(LibFdb5) << "Protocol negotiation - configuration: " << agreedConf_ <>{{startupBuffer.data(), s.position()}});
-    }
+    LOG_DEBUG_LIB(LibFdb5) << "Protocol negotiation - configuration: " << agreedConf_ <>{{startupBuffer.data(), s.position()}});
 
     if (!single_) {
-        dataSocket_->accept();
+        ASSERT(dataSocketFuture.valid());
+        dataSocketFuture.wait();
 
         // Check the response from the client.
         // Ensure that the hostname matches the original hostname, and that

From a0087c2e7682f111969f63ee5b9ad440d0029c8d Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 4 Nov 2024 08:58:46 +0100
Subject: [PATCH 159/186] wip

---
 src/fdb5/remote/Connection.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/fdb5/remote/Connection.cc b/src/fdb5/remote/Connection.cc
index 0eb998e7b..2fe76c9c6 100644
--- a/src/fdb5/remote/Connection.cc
+++ b/src/fdb5/remote/Connection.cc
@@ -76,11 +76,14 @@ void Connection::readUnsafe(bool control, void* data, size_t length) {
         ss << "Read error. Expected " << length << ". Error = " << eckit::Log::syserr;
         throw TCPException(ss.str(), Here());
     }
-    if (length != read) {
+    if (length < read) {
         std::stringstream ss;
         ss << "Read error. Expected " << length << " bytes, read " << read;
         throw TCPException(ss.str(), Here());
     }
+    if (read == 0) {
+        readUnsafe(control,data,length);
+    }
 }
 
 eckit::Buffer Connection::read(bool control, MessageHeader& hdr) {

From 29ba56c026372f80ecaafb8445a32e0ee6d0c113 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 4 Nov 2024 09:11:41 +0100
Subject: [PATCH 160/186] wip

---
 src/fdb5/remote/Connection.cc              | 5 +----
 src/fdb5/remote/server/ServerConnection.cc | 8 ++++++++
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/fdb5/remote/Connection.cc b/src/fdb5/remote/Connection.cc
index 2fe76c9c6..0eb998e7b 100644
--- a/src/fdb5/remote/Connection.cc
+++ b/src/fdb5/remote/Connection.cc
@@ -76,14 +76,11 @@ void Connection::readUnsafe(bool control, void* data, size_t length) {
         ss << "Read error. Expected " << length << ". Error = " << eckit::Log::syserr;
         throw TCPException(ss.str(), Here());
     }
-    if (length < read) {
+    if (length != read) {
         std::stringstream ss;
         ss << "Read error. Expected " << length << " bytes, read " << read;
         throw TCPException(ss.str(), Here());
     }
-    if (read == 0) {
-        readUnsafe(control,data,length);
-    }
 }
 
 eckit::Buffer Connection::read(bool control, MessageHeader& hdr) {
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index 71f683a3e..7562e4ffb 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -14,6 +14,8 @@
  */
 
 #include 
+#include 
+#include 
 
 #include "eckit/config/Resource.h"
 #include "eckit/maths/Functions.h"
@@ -264,6 +266,12 @@ void ServerConnection::initialiseConnections() {
         ASSERT(dataSocketFuture.valid());
         dataSocketFuture.wait();
 
+        std::random_device rd;
+        std::mt19937 mt(rd());
+        std::uniform_int_distribution dist(0, 1000);
+        std::this_thread::sleep_for(std::chrono::milliseconds(dist(rd)));
+
+
         // Check the response from the client.
         // Ensure that the hostname matches the original hostname, and that
         // it returns the details we sent it

From a7dd7e54a67a4e0deb4721e2d0b18375350dd979 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 4 Nov 2024 09:37:39 +0100
Subject: [PATCH 161/186] wip

---
 src/fdb5/remote/client/ClientConnection.cc |  4 ++++
 src/fdb5/remote/server/ServerConnection.cc | 14 ++++++++------
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index a0f1a9a2d..4320367a7 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -301,6 +301,7 @@ void ClientConnection::writeControlStartupMessage() {
     //       essentially JSON) over the wire for flexibility.
     s << availableFunctionality().get();
 
+    eckit::Log::info() << "writeControlStartupMessage - Sending session " << sessionID_ << " to control " << controlEndpoint_ << std::endl;
     Connection::write(Message::Startup, true, 0, 0, payload, s.position());
 }
 
@@ -312,6 +313,7 @@ void ClientConnection::writeDataStartupMessage(const eckit::SessionID& serverSes
     s << sessionID_;
     s << serverSession;
 
+    eckit::Log::info() << "writeDataStartupMessage - Sending session " << sessionID_ << " to data " << dataEndpoint_ << std::endl;
     Connection::write(Message::Startup, false, 0, 0, payload, s.position());
 }
 
@@ -330,6 +332,8 @@ eckit::SessionID ClientConnection::verifyServerStartupResponse() {
 
     dataEndpoint_ = dataEndpoint;
 
+    eckit::Log::info() << "verifyServerStartupResponse - Received from server " << clientSession << " " << serverSession << " " << dataEndpoint << std::endl;
+
     if (dataEndpoint_.hostname() != controlEndpoint_.hostname()) {
         eckit::Log::warning() << "Data and control interface hostnames do not match. "
                        << dataEndpoint_.hostname() << " /= "
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index 7562e4ffb..bbf0996c8 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -170,8 +170,6 @@ void ServerConnection::initialiseConnections() {
         errorMsg = "Error retrieving client protocol version";
     }
 
-    std::cout << "Connection from: " << endpointFromClient << "  sessionId" << clientSession << std::endl;
-
     if (errorMsg.empty() && !LibFdb5::instance().remoteProtocolVersion().check(remoteProtocolVersion, false)) {
         std::stringstream ss;
         ss << "FDB server version " << fdb5_version_str() << " - remote protocol version not supported:" << std::endl;
@@ -238,12 +236,14 @@ void ServerConnection::initialiseConnections() {
     eckit::net::Endpoint dataEndpoint;
     if (single_) {
         dataEndpoint = endpointFromClient;
+        eckit::Log::info() << "initialiseConnections - received " << clientSession << " from " << endpointFromClient << " - single connection " << std::endl;
     } else {
         std::lock_guard lock(dataPortMutex_);
         dataSocket_.reset(new eckit::net::EphemeralTCPServer(selectDataPort()));
         ASSERT(dataSocket_->socket() != -1);
 
         dataEndpoint = eckit::net::Endpoint{endpointFromClient.hostname(), dataSocket_->localPort()};
+        eckit::Log::info() << "initialiseConnections - received " << clientSession << " from " << endpointFromClient << " - opening data connection " << dataEndpoint << std::endl;
 
         dataSocketFuture = std::async(std::launch::async, [this] { dataSocket_->accept(); return true; });
     }
@@ -266,10 +266,10 @@ void ServerConnection::initialiseConnections() {
         ASSERT(dataSocketFuture.valid());
         dataSocketFuture.wait();
 
-        std::random_device rd;
-        std::mt19937 mt(rd());
-        std::uniform_int_distribution dist(0, 1000);
-        std::this_thread::sleep_for(std::chrono::milliseconds(dist(rd)));
+        // std::random_device rd;
+        // std::mt19937 mt(rd());
+        // std::uniform_int_distribution dist(0, 1000);
+        // std::this_thread::sleep_for(std::chrono::milliseconds(dist(rd)));
 
 
         // Check the response from the client.
@@ -277,6 +277,8 @@ void ServerConnection::initialiseConnections() {
         // it returns the details we sent it
         // IE check that we are connected to the correct client!
 
+        eckit::Log::info() << "initialiseConnections - waiting for client message on data connection" << std::endl;
+
         MessageHeader dataHdr;
         eckit::Buffer payload2 = readData(dataHdr);
 

From e32c79da4e02a77f91af764e15e1088a42d0ec5f Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 6 Nov 2024 17:36:36 +0000
Subject: [PATCH 162/186] cleanup TCPException

---
 src/fdb5/remote/Connection.cc              | 16 ----------------
 src/fdb5/remote/Connection.h               | 14 ++++++++++++++
 src/fdb5/remote/FdbServer.cc               |  3 ---
 src/fdb5/remote/client/ClientConnection.cc | 17 -----------------
 src/fdb5/remote/server/ServerConnection.cc |  6 ------
 5 files changed, 14 insertions(+), 42 deletions(-)

diff --git a/src/fdb5/remote/Connection.cc b/src/fdb5/remote/Connection.cc
index 0eb998e7b..dec76567d 100644
--- a/src/fdb5/remote/Connection.cc
+++ b/src/fdb5/remote/Connection.cc
@@ -1,29 +1,13 @@
 #include "eckit/io/Buffer.h"
 #include "eckit/log/Log.h"
-#include "eckit/os/BackTrace.h"
 
 #include "fdb5/LibFdb5.h"
-// #include "fdb5/remote/Handler.h"
 #include "fdb5/remote/Connection.h"
 
 namespace fdb5::remote {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-namespace{
-
-class TCPException : public eckit::Exception {
-public:
-    TCPException(const std::string& msg, const eckit::CodeLocation& here) :
-        eckit::Exception(std::string("TCPException: ") + msg, here) {
-
-        eckit::Log::error() << "TCP Exception; backtrace(): " << std::endl;
-        eckit::Log::error() << eckit::BackTrace::dump() << std::endl;
-    }
-};
-
-}
-
 void Connection::teardown() {
 
     if (!single_) {
diff --git a/src/fdb5/remote/Connection.h b/src/fdb5/remote/Connection.h
index ec7900763..cbba43d51 100644
--- a/src/fdb5/remote/Connection.h
+++ b/src/fdb5/remote/Connection.h
@@ -10,7 +10,9 @@
 
 #pragma once
 
+#include "eckit/exception/Exceptions.h"
 #include "eckit/net/TCPSocket.h"
+#include "eckit/os/BackTrace.h"
 
 #include "fdb5/remote/Messages.h"
 
@@ -24,6 +26,18 @@ namespace fdb5::remote {
 
 //----------------------------------------------------------------------------------------------------------------------
 
+class TCPException : public eckit::Exception {
+public:
+    TCPException(const std::string& msg, const eckit::CodeLocation& here) :
+        eckit::Exception(std::string("TCPException: ") + msg, here) {
+
+        std::cerr << "TCP Exception; backtrace(): " << std::endl;
+        std::cerr << eckit::BackTrace::dump() << std::endl;
+    }
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
 class Connection : eckit::NonCopyable {
 
 public: // methods
diff --git a/src/fdb5/remote/FdbServer.cc b/src/fdb5/remote/FdbServer.cc
index 066aca906..6943fd121 100644
--- a/src/fdb5/remote/FdbServer.cc
+++ b/src/fdb5/remote/FdbServer.cc
@@ -108,8 +108,6 @@ FdbServerBase::~FdbServerBase() {}
 
 void FdbServerBase::doRun() {
 
-    std::mutex forkMutex;
-
     hookUnique();
 
     Config config = LibFdb5::instance().defaultConfig();
@@ -134,7 +132,6 @@ void FdbServerBase::doRun() {
                 t.start();
             }
             else {
-                std::lock_guard lock(forkMutex);
                 FDBForker f(server.accept(), config);
                 f.start();
             }
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index 4320367a7..cf6fb473b 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -9,7 +9,6 @@
 #include "eckit/log/Bytes.h"
 #include "eckit/log/Log.h"
 #include "eckit/message/Message.h"
-#include "eckit/os/BackTrace.h"
 #include "eckit/runtime/Main.h"
 #include "eckit/serialisation/MemoryStream.h"
 #include "eckit/utils/Translator.h"
@@ -24,22 +23,6 @@ namespace fdb5::remote {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-namespace{
-
-class TCPException : public eckit::Exception {
-public:
-    TCPException(const std::string& msg, const eckit::CodeLocation& here) :
-        eckit::Exception(std::string("TCPException: ") + msg, here) {
-
-        eckit::Log::error() << "TCP Exception; backtrace(): " << std::endl;
-        eckit::Log::error() << eckit::BackTrace::dump() << std::endl;
-    }
-};
-
-}
-
-//----------------------------------------------------------------------------------------------------------------------
-
 class DataWriteRequest {
 
 public:
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index bbf0996c8..135c46ce3 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -41,12 +41,6 @@ namespace fdb5::remote {
 // helpers
 namespace {
 
-class TCPException : public eckit::Exception {
-public:
-    TCPException(const std::string& msg, const eckit::CodeLocation& here) :
-        eckit::Exception(std::string("TCPException: ") + msg, here) {}
-};
-
 std::vector intersection(const eckit::LocalConfiguration& c1, const eckit::LocalConfiguration& c2, const std::string& field){
 
     std::vector v1 = c1.getIntVector(field);

From 5d83a55e1fd26e6fdc1e4f1bb6cebe228b25f1d9 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 6 Nov 2024 17:57:45 +0000
Subject: [PATCH 163/186] fix merge develop

---
 src/fdb5/tools/fdb-reconsolidate-toc.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/fdb5/tools/fdb-reconsolidate-toc.cc b/src/fdb5/tools/fdb-reconsolidate-toc.cc
index fa95726fd..5d5184dfe 100644
--- a/src/fdb5/tools/fdb-reconsolidate-toc.cc
+++ b/src/fdb5/tools/fdb-reconsolidate-toc.cc
@@ -9,7 +9,7 @@
  */
 
 #include "fdb5/tools/FDBTool.h"
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
 
 #include "eckit/option/CmdArgs.h"
 #include "eckit/config/LocalConfiguration.h"

From 2f048dfd50470c38ed1c2a4115ec74e235a1b0de Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 6 Nov 2024 18:10:10 +0000
Subject: [PATCH 164/186] cleanup

---
 src/fdb5/remote/Connection.cc                    |  7 -------
 src/fdb5/remote/client/ClientConnection.cc       |  1 -
 src/fdb5/remote/client/ClientConnectionRouter.cc | 16 ----------------
 src/fdb5/remote/client/RemoteCatalogue.cc        |  1 -
 src/fdb5/remote/server/CatalogueHandler.cc       |  8 --------
 src/fdb5/remote/server/ServerConnection.cc       | 14 --------------
 src/fdb5/toc/TocWipeVisitor.cc                   |  7 -------
 src/fdb5/tools/fdb-hammer.cc                     |  1 -
 tests/fdb/api/test_callback.cc                   |  4 ----
 tests/fdb/type/test_toKey.cc                     |  5 +----
 10 files changed, 1 insertion(+), 63 deletions(-)

diff --git a/src/fdb5/remote/Connection.cc b/src/fdb5/remote/Connection.cc
index dec76567d..d9b917787 100644
--- a/src/fdb5/remote/Connection.cc
+++ b/src/fdb5/remote/Connection.cc
@@ -73,8 +73,6 @@ eckit::Buffer Connection::read(bool control, MessageHeader& hdr) {
     std::lock_guard lock((control || single_) ? readControlMutex_ : readDataMutex_);
     readUnsafe(control, &hdr, sizeof(hdr));
 
-    // std::cout << "READ [" <<  "endpoint=" <<  ((control || single_) ? controlSocket() : dataSocket()).remotePort() << ",message=" << hdr.message << ",clientID=" << hdr.clientID() << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
-
     ASSERT(hdr.marker == StartMarker);
     ASSERT(hdr.version == CurrentVersion);
     ASSERT(single_ || hdr.control() == control);
@@ -88,14 +86,11 @@ eckit::Buffer Connection::read(bool control, MessageHeader& hdr) {
     ASSERT(tail == EndMarker);
 
     if (hdr.message == Message::Error) {
-        // std::cout << "ERROR while reading: ";
 
         char msg[hdr.payloadSize+1];
         if (hdr.payloadSize) {
             char msg[hdr.payloadSize+1];
-            // std::cout << static_cast(payload.data());
         }
-        // std::cout << std::endl;
     }
 
     return payload;
@@ -120,8 +115,6 @@ void Connection::write(remote::Message msg, bool control, uint32_t clientID, uin
 
     MessageHeader message{msg, control, clientID, requestID, payloadLength};
 
-    // std::cout << "WRITE [" << "endpoint=" << ((control || single_) ? controlSocket() : dataSocket()).remotePort() << ",message=" << message.message << ",clientID=" << message.clientID() << ",requestID=" << message.requestID << ",payload=" << message.payloadSize << "]" << std::endl;
-
     LOG_DEBUG_LIB(LibFdb5) << "Connection::write [message=" << msg << ",clientID=" << message.clientID() << ",control=" << control << ",requestID=" << requestID << ",data=" << data.size() << ",payload=" << payloadLength << "]" << std::endl;
 
     std::lock_guard lock((control || single_) ? controlMutex_ : dataMutex_);
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index cf6fb473b..91db976a5 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -191,7 +191,6 @@ std::future ClientConnection::controlWrite(Client& client, Messag
 }
 
 void ClientConnection::dataWrite(DataWriteRequest& r) {
-    // std::cout << "Write " << r.msg_ << ",clientID=" << r.client_->clientId() << ",requestID=" << r.id_ << std::endl;
     Connection::write(r.msg_, false, r.client_->clientId(), r.id_, r.data_.data(), r.data_.size());
 }
 
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.cc b/src/fdb5/remote/client/ClientConnectionRouter.cc
index 69f741c64..5a5242057 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.cc
+++ b/src/fdb5/remote/client/ClientConnectionRouter.cc
@@ -1,11 +1,5 @@
 #include "fdb5/remote/client/ClientConnectionRouter.h"
 
-#include 
-#include 
-#include 
-
-#include "eckit/config/Resource.h"
-
 namespace{
     
 class ConnectionError : public eckit::Exception {
@@ -43,16 +37,6 @@ ClientConnection& ClientConnectionRouter::connection(const eckit::net::Endpoint&
         return *(it->second);
     } else {
         ClientConnection* clientConnection = new ClientConnection{endpoint, defaultEndpoint};
-        
-        // static int fdbConnectMaxSleep = eckit::Resource("fdbConnectMaxSleep", 0);
-        
-        // if (fdbConnectMaxSleep) {
-        //     std::random_device rd;
-        //     std::mt19937 mt(rd());
-        //     std::uniform_int_distribution dist(0, fdbConnectMaxSleep);
-
-        //     std::this_thread::sleep_for(std::chrono::milliseconds(dist(rd)));
-        // }
 
         if (clientConnection->connect()) {
             auto it = (connections_.emplace(endpoint, std::unique_ptr(clientConnection))).first;
diff --git a/src/fdb5/remote/client/RemoteCatalogue.cc b/src/fdb5/remote/client/RemoteCatalogue.cc
index be1b95dd6..f43bedbd1 100644
--- a/src/fdb5/remote/client/RemoteCatalogue.cc
+++ b/src/fdb5/remote/client/RemoteCatalogue.cc
@@ -25,7 +25,6 @@ RemoteCatalogue::RemoteCatalogue(const Key& key, const Config& config):
     Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port")), ""),
     config_(config), schema_(nullptr), numLocations_(0) {
 
-    // std::cout << "Catalogue  " << clientId() << "  " << key << std::endl;
     loadSchema();
 }
 
diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc
index e513dbd19..a407882c0 100644
--- a/src/fdb5/remote/server/CatalogueHandler.cc
+++ b/src/fdb5/remote/server/CatalogueHandler.cc
@@ -416,19 +416,15 @@ void CatalogueHandler::flush(uint32_t clientID, uint32_t requestID, eckit::Buffe
     auto it = catalogues_.find(clientID);
     ASSERT(it != catalogues_.end());
 
-    // std::cout << "flush " << clientID << " archived " << it->second.locationsArchived << " expected " << it->second.locationsExpected << std::endl;
-
     {
         std::lock_guard lock(fieldLocationsMutex_);
         it->second.locationsExpected = numArchived;     // setting locationsExpected also means that a flush has been requested
         it->second.archivalCompleted = it->second.fieldLocationsReceived.get_future();
-        // std::cout << "flush post lock " << clientID << " archived " << it->second.locationsArchived << " expected " << it->second.locationsExpected << std::endl;
         if (it->second.locationsArchived == numArchived) {
             it->second.fieldLocationsReceived.set_value(numArchived);
         }
     }
 
-    // std::cout << "flush wait " << clientID << " archived " << it->second.locationsArchived << " expected " << it->second.locationsExpected << std::endl;
     it->second.archivalCompleted.wait();
     {
         std::lock_guard lock(fieldLocationsMutex_);
@@ -436,7 +432,6 @@ void CatalogueHandler::flush(uint32_t clientID, uint32_t requestID, eckit::Buffe
         it->second.locationsExpected = 0;
         it->second.locationsArchived = 0;
     }
-    // std::cout << "flush post wait " << clientID << " archived " << it->second.locationsArchived << " expected " << it->second.locationsExpected << std::endl;
 
     it->second.catalogue->flush(numArchived);
 
@@ -470,12 +465,9 @@ void CatalogueHandler::archiveBlob(const uint32_t clientID, const uint32_t reque
     it->second.catalogue->selectIndex(idxKey);
     it->second.catalogue->archive(idxKey, datumKey, std::move(location));
     {
-        // std::cout << "archiveBlob " << clientID << " archived " << it->second.locationsArchived << " expected " << it->second.locationsExpected << std::endl;
         std::lock_guard lock(fieldLocationsMutex_);
-        // std::cout << "archiveBlob post mutex " << std::endl;
         it->second.locationsArchived++;
         if (it->second.locationsExpected != 0 && it->second.archivalCompleted.valid() && it->second.locationsExpected == it->second.locationsArchived) {
-            // std::cout << "archiveBlob set_value " << std::endl;
             it->second.fieldLocationsReceived.set_value(it->second.locationsExpected);
         }
     }
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index 135c46ce3..4b7a04f1c 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -230,20 +230,16 @@ void ServerConnection::initialiseConnections() {
     eckit::net::Endpoint dataEndpoint;
     if (single_) {
         dataEndpoint = endpointFromClient;
-        eckit::Log::info() << "initialiseConnections - received " << clientSession << " from " << endpointFromClient << " - single connection " << std::endl;
     } else {
         std::lock_guard lock(dataPortMutex_);
         dataSocket_.reset(new eckit::net::EphemeralTCPServer(selectDataPort()));
         ASSERT(dataSocket_->socket() != -1);
 
         dataEndpoint = eckit::net::Endpoint{endpointFromClient.hostname(), dataSocket_->localPort()};
-        eckit::Log::info() << "initialiseConnections - received " << clientSession << " from " << endpointFromClient << " - opening data connection " << dataEndpoint << std::endl;
 
         dataSocketFuture = std::async(std::launch::async, [this] { dataSocket_->accept(); return true; });
     }
 
-    eckit::Log::info() << "Sending data endpoint to client: " << dataEndpoint << std::endl;
-
     eckit::Buffer startupBuffer(1024);
     eckit::MemoryStream s(startupBuffer);
 
@@ -260,19 +256,11 @@ void ServerConnection::initialiseConnections() {
         ASSERT(dataSocketFuture.valid());
         dataSocketFuture.wait();
 
-        // std::random_device rd;
-        // std::mt19937 mt(rd());
-        // std::uniform_int_distribution dist(0, 1000);
-        // std::this_thread::sleep_for(std::chrono::milliseconds(dist(rd)));
-
-
         // Check the response from the client.
         // Ensure that the hostname matches the original hostname, and that
         // it returns the details we sent it
         // IE check that we are connected to the correct client!
 
-        eckit::Log::info() << "initialiseConnections - waiting for client message on data connection" << std::endl;
-
         MessageHeader dataHdr;
         eckit::Buffer payload2 = readData(dataHdr);
 
@@ -301,8 +289,6 @@ void ServerConnection::initialiseConnections() {
         error(errorMsg, hdr.clientID(), hdr.requestID);
         return;
     }
-
-    
 }
 
 int ServerConnection::selectDataPort() {
diff --git a/src/fdb5/toc/TocWipeVisitor.cc b/src/fdb5/toc/TocWipeVisitor.cc
index 3c1fe9e3b..6d330e13f 100644
--- a/src/fdb5/toc/TocWipeVisitor.cc
+++ b/src/fdb5/toc/TocWipeVisitor.cc
@@ -164,15 +164,8 @@ bool TocWipeVisitor::visitIndex(const Index& index) {
 
     // Enumerate data files.
 
-    std::cout << index << std::endl;
     std::vector indexDataPaths(index.dataURIs());
-    store_.print(std::cout);
-    std::cout << std::endl;
     auto paths = store_.asCollocatedDataURIs(indexDataPaths);
-    std::cout << paths.size() << std::endl;
-    for (const auto& p : paths) {
-        std::cout << "  " << p << std::endl;
-    }
 
     for (const eckit::URI& uri : store_.asCollocatedDataURIs(indexDataPaths)) {
         if (include) {
diff --git a/src/fdb5/tools/fdb-hammer.cc b/src/fdb5/tools/fdb-hammer.cc
index 036e9dcfd..8257d61bc 100644
--- a/src/fdb5/tools/fdb-hammer.cc
+++ b/src/fdb5/tools/fdb-hammer.cc
@@ -139,7 +139,6 @@ void FDBHammer::executeWrite(const eckit::option::CmdArgs &args) {
         std::uniform_int_distribution dist(0, 10000);
 
         int delayDuration = dist(mt);
-        std::cout << "delay duration: " << delayDuration << std::endl;
         std::this_thread::sleep_for(std::chrono::milliseconds(delayDuration));
     }
 
diff --git a/tests/fdb/api/test_callback.cc b/tests/fdb/api/test_callback.cc
index d27d81d73..f0dccb057 100644
--- a/tests/fdb/api/test_callback.cc
+++ b/tests/fdb/api/test_callback.cc
@@ -53,10 +53,6 @@ CASE("Archive and flush callback") {
 
     EXPECT(map.size() == 3);
 
-    // for (const auto& [key, uri] : map) {
-    //     std::cout << key << " -> " << uri << std::endl;
-    // }
-
     for (const auto& [key, uri] : map) {
         bool found = false;
         for (const auto& originalKey : keys) {
diff --git a/tests/fdb/type/test_toKey.cc b/tests/fdb/type/test_toKey.cc
index b6b013f91..40dcc0254 100644
--- a/tests/fdb/type/test_toKey.cc
+++ b/tests/fdb/type/test_toKey.cc
@@ -271,10 +271,7 @@ CASE( "Date - string ctor - expansion" ) {
     eckit::Date now(-2);
     eckit::Translator t;
 
-    EXPECT(key.canonicalValue("date") == t(now.yyyymmdd()));
-    // std::cout << key.valuesToString() << std::endl;
-    // std::cout << ("od:0001:oper:ofb:"+t(now.yyyymmdd())+":0000:mhs:3001") << std::endl;
-    
+    EXPECT(key.canonicalValue("date") == t(now.yyyymmdd()));    
     EXPECT(key.valuesToString() == "od:0001:oper:ofb:"+t(now.yyyymmdd())+":0000:mhs:3001");
 
     fdb5::Archiver archiver;

From fdaa5d425289acd914e7cfc02c524f8baa272db0 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 8 Nov 2024 10:04:24 +0000
Subject: [PATCH 165/186] little cleanup

---
 src/fdb5/remote/client/ClientConnection.cc    | 31 +++++++++----------
 src/fdb5/remote/client/ClientConnection.h     |  2 +-
 .../remote/client/ClientConnectionRouter.cc   |  1 -
 src/fdb5/remote/server/ServerConnection.cc    |  1 -
 4 files changed, 15 insertions(+), 20 deletions(-)

diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index 91db976a5..ca8f1deeb 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -255,20 +255,19 @@ void ClientConnection::dataWriteThreadLoop() {
     // They will be released when flush() is called.
 }
 
-void ClientConnection::handleError(const MessageHeader& hdr, eckit::Buffer buffer) {
-    ASSERT(hdr.marker == StartMarker);
-    ASSERT(hdr.version == CurrentVersion);
+// void ClientConnection::handleError(const MessageHeader& hdr, eckit::Buffer buffer) {
+//     ASSERT(hdr.marker == StartMarker);
+//     ASSERT(hdr.version == CurrentVersion);
 
-    if (hdr.message == Message::Error) {
-        ASSERT(hdr.payloadSize > 9);
-        std::string what(buffer.size()+1, ' ');
-        buffer.copy(what.c_str(), buffer.size());
-        what[buffer.size()] = 0; // Just in case
-
-        throw RemoteFDBException(what, controlEndpoint_);
-    }
-}
+//     if (hdr.message == Message::Error) {
+//         ASSERT(hdr.payloadSize > 9);
+//         std::string what(buffer.size()+1, ' ');
+//         buffer.copy(what.c_str(), buffer.size());
+//         what[buffer.size()] = 0; // Just in case
 
+//         throw RemoteFDBException(what, controlEndpoint_);
+//     }
+// }
 
 void ClientConnection::writeControlStartupMessage() {
 
@@ -283,7 +282,7 @@ void ClientConnection::writeControlStartupMessage() {
     //       essentially JSON) over the wire for flexibility.
     s << availableFunctionality().get();
 
-    eckit::Log::info() << "writeControlStartupMessage - Sending session " << sessionID_ << " to control " << controlEndpoint_ << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "writeControlStartupMessage - Sending session " << sessionID_ << " to control " << controlEndpoint_ << std::endl;
     Connection::write(Message::Startup, true, 0, 0, payload, s.position());
 }
 
@@ -295,7 +294,7 @@ void ClientConnection::writeDataStartupMessage(const eckit::SessionID& serverSes
     s << sessionID_;
     s << serverSession;
 
-    eckit::Log::info() << "writeDataStartupMessage - Sending session " << sessionID_ << " to data " << dataEndpoint_ << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "writeDataStartupMessage - Sending session " << sessionID_ << " to data " << dataEndpoint_ << std::endl;
     Connection::write(Message::Startup, false, 0, 0, payload, s.position());
 }
 
@@ -314,8 +313,7 @@ eckit::SessionID ClientConnection::verifyServerStartupResponse() {
 
     dataEndpoint_ = dataEndpoint;
 
-    eckit::Log::info() << "verifyServerStartupResponse - Received from server " << clientSession << " " << serverSession << " " << dataEndpoint << std::endl;
-
+    LOG_DEBUG_LIB(LibFdb5) << "verifyServerStartupResponse - Received from server " << clientSession << " " << serverSession << " " << dataEndpoint << std::endl;
     if (dataEndpoint_.hostname() != controlEndpoint_.hostname()) {
         eckit::Log::warning() << "Data and control interface hostnames do not match. "
                        << dataEndpoint_.hostname() << " /= "
@@ -480,7 +478,6 @@ void ClientConnection::listeningDataThreadLoop() {
     } catch (...) {
        ClientConnectionRouter::instance().teardown(std::current_exception());
     }
-    // ClientConnectionRouter::instance().deregister(*this);
 }
 
 }  // namespace fdb5::remote
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index 52fe56b3c..790b4295f 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -67,7 +67,7 @@ class ClientConnection : protected Connection {
 
     eckit::SessionID verifyServerStartupResponse();
 
-    void handleError(const MessageHeader& hdr, eckit::Buffer buffer);
+    // void handleError(const MessageHeader& hdr, eckit::Buffer buffer);
 
     void listeningControlThreadLoop();
     void listeningDataThreadLoop();
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.cc b/src/fdb5/remote/client/ClientConnectionRouter.cc
index 5a5242057..3942b97b2 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.cc
+++ b/src/fdb5/remote/client/ClientConnectionRouter.cc
@@ -37,7 +37,6 @@ ClientConnection& ClientConnectionRouter::connection(const eckit::net::Endpoint&
         return *(it->second);
     } else {
         ClientConnection* clientConnection = new ClientConnection{endpoint, defaultEndpoint};
-
         if (clientConnection->connect()) {
             auto it = (connections_.emplace(endpoint, std::unique_ptr(clientConnection))).first;
             return *(it->second);
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index 4b7a04f1c..7879fe450 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -14,7 +14,6 @@
  */
 
 #include 
-#include 
 #include 
 
 #include "eckit/config/Resource.h"

From f2ed16656815d2180600a1ff6cf62ccdbcb6db1b Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 8 Nov 2024 10:06:55 +0000
Subject: [PATCH 166/186] version bump

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index ad59b0f9c..bae891de5 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.13.103
\ No newline at end of file
+5.13.104

From 8397826d0724262db64436bb3a64272a73392f98 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 11 Nov 2024 15:46:07 +0000
Subject: [PATCH 167/186] config remoteFDB connection(s)

---
 src/fdb5/remote/server/ServerConnection.cc | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index 7879fe450..770b7ba0c 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -135,9 +135,19 @@ Handled ServerConnection::handleData(Message message, uint32_t clientID, uint32_
 eckit::LocalConfiguration ServerConnection::availableFunctionality() const {
     eckit::LocalConfiguration conf;
 //    Add to the configuration all the components that require to be versioned, as in the following example, with a vector of supported version numbers
-    std::vector remoteFieldLocationVersions = {1};
+    static std::vector remoteFieldLocationVersions = {1};
+    static std::vector numberOfConnections = config_.getIntVector("supportedConnections", {1,2});
+
+    ASSERT(0 < numberOfConnections.size());
+    ASSERT(numberOfConnections[0] == 1 || numberOfConnections[0] == 2);
+
+    ASSERT(numberOfConnections.size() <= 2);
+    if (numberOfConnections.size() > 1) {
+        ASSERT(numberOfConnections[0] == 1);
+        ASSERT(numberOfConnections[1] == 2);
+    }
+
     conf.set("RemoteFieldLocation", remoteFieldLocationVersions);
-    std::vector numberOfConnections = {1,2};
     conf.set("NumberOfConnections", numberOfConnections);
     return conf;
 }

From 9057f3ef9547f2b8f4e13b33571bbabbeaafad0a Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 29 Nov 2024 14:16:51 +0000
Subject: [PATCH 168/186] wip

---
 src/fdb5/api/FDB.cc                       | 12 ++++----
 src/fdb5/api/FDB.h                        |  4 ++-
 src/fdb5/api/RemoteFDB.cc                 | 34 +++++++--------------
 src/fdb5/api/RemoteFDB.h                  |  8 ++---
 src/fdb5/remote/Connection.cc             |  3 ++
 src/fdb5/remote/Connection.h              |  4 +--
 src/fdb5/remote/client/Client.cc          | 13 --------
 src/fdb5/remote/client/Client.h           |  3 +-
 src/fdb5/remote/client/RemoteCatalogue.cc | 37 ++++-------------------
 src/fdb5/remote/client/RemoteCatalogue.h  |  5 +--
 src/fdb5/remote/client/RemoteStore.cc     |  3 --
 src/fdb5/remote/client/RemoteStore.h      | 10 ------
 src/fdb5/remote/server/CatalogueHandler.h |  5 +--
 src/fdb5/remote/server/ServerConnection.h |  2 +-
 src/fdb5/remote/server/StoreHandler.h     |  2 +-
 src/fdb5/types/TypesRegistry.h            |  4 +--
 16 files changed, 40 insertions(+), 109 deletions(-)

diff --git a/src/fdb5/api/FDB.cc b/src/fdb5/api/FDB.cc
index 77ff1ffcd..0fe967382 100644
--- a/src/fdb5/api/FDB.cc
+++ b/src/fdb5/api/FDB.cc
@@ -162,11 +162,7 @@ eckit::DataHandle* FDB::read(const std::vector& uris, bool sorted) {
     return result.dataHandle();
 }
 
-eckit::DataHandle* FDB::read(bool seekable, ListIterator& it, bool sorted) {
-
-    if (seekable) {
-        return new FieldHandle(it);
-    }
+eckit::DataHandle* FDB::read(ListIterator& it, bool sorted) {
 
     eckit::Timer timer;
     timer.start();
@@ -218,9 +214,13 @@ eckit::DataHandle* FDB::read(bool seekable, ListIterator& it, bool sorted) {
     return result.dataHandle();
 }
 
+eckit::DataHandle* FDB::readSeekable(ListIterator& it, bool sorted) {
+    return new FieldHandle(it);
+}
+
 eckit::DataHandle* FDB::retrieve(const metkit::mars::MarsRequest& request, bool seekable) {
     ListIterator it = inspect(request);
-    return read(seekable, it, sorted(request));
+    return (seekable ? readSeekable(it, sorted(request)) : read(it, sorted(request)));
 }
 
 ListIterator FDB::inspect(const metkit::mars::MarsRequest& request) {
diff --git a/src/fdb5/api/FDB.h b/src/fdb5/api/FDB.h
index d6ec236f3..a6dc3521a 100644
--- a/src/fdb5/api/FDB.h
+++ b/src/fdb5/api/FDB.h
@@ -89,7 +89,9 @@ class FDB {
 
     eckit::DataHandle* read(const std::vector& uris, bool sorted = false);
 
-    eckit::DataHandle* read(bool seekable, ListIterator& it, bool sorted = false);
+    eckit::DataHandle* read(ListIterator& it, bool sorted = false);
+
+    eckit::DataHandle* readSeekable(ListIterator& it, bool sorted = false);
 
     eckit::DataHandle* retrieve(const metkit::mars::MarsRequest& request, bool seekable = false);
 
diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index ea2775bd4..3e09fe6cc 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -56,7 +56,6 @@ struct ListHelper : BaseAPIHelper {
     AxesHelper(int level) : level_(level) {}
 
@@ -87,15 +86,6 @@ struct InspectHelper : BaseAPIHelper& queue) {
-            eckit::Buffer msg{0};
-                        while (true) {
-                            if (messageQueue->pop(msg) == -1) {
-                                break;
-                            } else {
-                                MemoryStream s(msg);
-                                queue.emplace(HelperClass::valueFromStream(s, remoteFDB));
-                            }
-                        }
-                        // messageQueue goes out of scope --> destructed
+                eckit::Buffer msg{0};
+                while (true) {
+                    if (messageQueue->pop(msg) == -1) {
+                        break;
+                    } else {
+                        MemoryStream s(msg);
+                        queue.emplace(HelperClass::valueFromStream(s, remoteFDB));
                     }
-                )
-           );
+                }
+                // messageQueue goes out of scope --> destructed
+            })
+        );
 }
 
 ListIterator RemoteFDB::list(const FDBToolRequest& request) {
@@ -323,7 +312,6 @@ bool RemoteFDB::handle(remote::Message message, bool control, uint32_t requestID
             return false;
     }
 }
-// void RemoteFDB::handleException(std::exception_ptr e){NOTIMP;}
 
 static FDBBuilder builder("remote");
 
diff --git a/src/fdb5/api/RemoteFDB.h b/src/fdb5/api/RemoteFDB.h
index 4d911e39a..2ada336ca 100644
--- a/src/fdb5/api/RemoteFDB.h
+++ b/src/fdb5/api/RemoteFDB.h
@@ -21,12 +21,11 @@
 #pragma once
 
 #include 
+#include 
 
 #include "fdb5/api/LocalFDB.h"
 #include "fdb5/remote/client/Client.h"
 
-#include 
-
 namespace fdb5 {
 
 //----------------------------------------------------------------------------------------------------------------------
@@ -36,13 +35,12 @@ class RemoteFDB : public LocalFDB, public remote::Client {
 
 public: // types
 
-//    using StoredMessage = std::pair;
     using MessageQueue = eckit::Queue;
 
 public: // method
 
     RemoteFDB(const eckit::Configuration& config, const std::string& name);
-    ~RemoteFDB() {}
+    ~RemoteFDB() override {}
 
     ListIterator inspect(const metkit::mars::MarsRequest& request) override;
 
@@ -79,10 +77,8 @@ class RemoteFDB : public LocalFDB, public remote::Client {
     FDBStats stats() const override { NOTIMP; }
 
     // Client
-
     bool handle(remote::Message message, bool control, uint32_t requestID) override;
     bool handle(remote::Message message, bool control, uint32_t requestID, eckit::Buffer&& payload) override;
-    // void handleException(std::exception_ptr e) override;
 
 private: // members
 
diff --git a/src/fdb5/remote/Connection.cc b/src/fdb5/remote/Connection.cc
index d9b917787..f714b5685 100644
--- a/src/fdb5/remote/Connection.cc
+++ b/src/fdb5/remote/Connection.cc
@@ -8,6 +8,9 @@ namespace fdb5::remote {
 
 //----------------------------------------------------------------------------------------------------------------------
 
+Connection::Connection() : single_(false) {}
+Connection::~Connection() {}
+
 void Connection::teardown() {
 
     if (!single_) {
diff --git a/src/fdb5/remote/Connection.h b/src/fdb5/remote/Connection.h
index cbba43d51..33131e7ee 100644
--- a/src/fdb5/remote/Connection.h
+++ b/src/fdb5/remote/Connection.h
@@ -41,8 +41,8 @@ class TCPException : public eckit::Exception {
 class Connection : eckit::NonCopyable {
 
 public: // methods
-    Connection() : single_(false) {}
-    virtual ~Connection() {}
+    Connection();
+    virtual ~Connection();
 
     void write(Message msg, bool control, uint32_t clientID, uint32_t requestID, const void* data, uint32_t length);
     void write(Message msg, bool control, uint32_t clientID, uint32_t requestID, std::vector> data = {});
diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index 174b21f55..4436b0113 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -84,17 +84,4 @@ void Client::dataWrite(remote::Message msg, uint32_t requestID, std::vector>& endpoints);
-    ~Client();
+    virtual ~Client();
 
     uint32_t clientId() const { return id_; }
     uint32_t id() const { return id_; }
@@ -52,7 +52,6 @@ class Client : eckit::NonCopyable {
     // handlers for incoming messages - to be defined in the client class
     virtual bool handle(Message message, bool control, uint32_t requestID) = 0;
     virtual bool handle(Message message, bool control, uint32_t requestID, eckit::Buffer&& payload) = 0;
-    // virtual void handleException(std::exception_ptr e) = 0;
 
 protected:
     
diff --git a/src/fdb5/remote/client/RemoteCatalogue.cc b/src/fdb5/remote/client/RemoteCatalogue.cc
index f43bedbd1..1af996920 100644
--- a/src/fdb5/remote/client/RemoteCatalogue.cc
+++ b/src/fdb5/remote/client/RemoteCatalogue.cc
@@ -28,36 +28,15 @@ RemoteCatalogue::RemoteCatalogue(const Key& key, const Config& config):
     loadSchema();
 }
 
+// Catalogue(URI, Config) is only used by the Visitors to traverse the catalogue. In the remote, we use the RemoteFDB for catalogue traversal
+// this ctor is here only to comply with the factory
 RemoteCatalogue::RemoteCatalogue(const eckit::URI& uri, const Config& config):
-    Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port")), ""), config_(config), schema_(nullptr), numLocations_(0)
-    {
-        NOTIMP;
-    }
-
-
-void RemoteCatalogue::sendArchiveData(uint32_t id, const Key& key, std::unique_ptr fieldLocation)
-{
-    ASSERT(!dbKey_.empty());
-    ASSERT(!currentIndexKey_.empty());
-    ASSERT(!key.empty());
-    ASSERT(fieldLocation);
-
-    Buffer keyBuffer(4096);
-    MemoryStream keyStream(keyBuffer);
-    keyStream << currentIndexKey_;
-    keyStream << key;
-
-    Buffer locBuffer(4096);
-    MemoryStream locStream(locBuffer);
-    locStream << *fieldLocation;
-
-    std::vector> payloads;
-    payloads.push_back(std::pair{keyBuffer, keyStream.position()});
-    payloads.push_back(std::pair{locBuffer, locStream.position()});
-
-    dataWrite(Message::Blob, id, payloads);
+    Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port")), ""), config_(config), schema_(nullptr), numLocations_(0) {
+    NOTIMP;
 }
 
+RemoteCatalogue::~RemoteCatalogue() {}
+
 void RemoteCatalogue::archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) {
 
     ASSERT(!datumKey.empty());
@@ -174,10 +153,6 @@ bool RemoteCatalogue::handle(Message message, bool control, uint32_t requestID,
     return false;
 }
 
-// void RemoteCatalogue::handleException(std::exception_ptr e) {
-//     NOTIMP;
-// }
-
 void RemoteCatalogue::overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) {NOTIMP;}
 void RemoteCatalogue::index(const Key& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) {NOTIMP;}
 void RemoteCatalogue::reconsolidate(){NOTIMP;}
diff --git a/src/fdb5/remote/client/RemoteCatalogue.h b/src/fdb5/remote/client/RemoteCatalogue.h
index 1b94d888b..07df9a781 100644
--- a/src/fdb5/remote/client/RemoteCatalogue.h
+++ b/src/fdb5/remote/client/RemoteCatalogue.h
@@ -19,7 +19,7 @@ class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public C
     RemoteCatalogue(const Key& key, const Config& config);
     RemoteCatalogue(const eckit::URI& uri, const Config& config);
 
-    ~RemoteCatalogue() {}
+    ~RemoteCatalogue() override;
 
     // From CatalogueWriter
     const Index& currentIndex() override;
@@ -59,8 +59,6 @@ class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public C
     void checkUID() const override;
     eckit::URI uri() const override;
 
-    void sendArchiveData(uint32_t id, const Key& key, std::unique_ptr fieldLocation);
-
 protected:
 
     // FDBStats archivalCompleted();
@@ -72,7 +70,6 @@ class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public C
     // handlers for incoming messages - to be defined in the client class
     bool handle(Message message, bool control, uint32_t requestID) override;
     bool handle(Message message, bool control, uint32_t requestID, eckit::Buffer&& payload) override;
-    // void handleException(std::exception_ptr e) override;
 
 protected:
 
diff --git a/src/fdb5/remote/client/RemoteStore.cc b/src/fdb5/remote/client/RemoteStore.cc
index 03358de9f..24a41febb 100644
--- a/src/fdb5/remote/client/RemoteStore.cc
+++ b/src/fdb5/remote/client/RemoteStore.cc
@@ -403,9 +403,6 @@ bool RemoteStore::handle(Message message, bool control, uint32_t requestID, ecki
             return false;
     }
 }
-// void RemoteStore::handleException(std::exception_ptr e) {
-//     Log::error() << "RemoteStore::handleException " << std::endl;
-// }
 
 eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation) {
     return dataHandle(fieldLocation, Key());
diff --git a/src/fdb5/remote/client/RemoteStore.h b/src/fdb5/remote/client/RemoteStore.h
index 22c426b62..16579d56e 100644
--- a/src/fdb5/remote/client/RemoteStore.h
+++ b/src/fdb5/remote/client/RemoteStore.h
@@ -155,7 +155,6 @@ class RemoteStore : public Store, public Client {
     // handlers for incoming messages - to be defined in the client class
     bool handle(Message message, bool control, uint32_t requestID) override;
     bool handle(Message message, bool control, uint32_t requestID, eckit::Buffer&& payload) override;
-    // void handleException(std::exception_ptr e) override;
 
 private: // members
 
@@ -163,25 +162,16 @@ class RemoteStore : public Store, public Client {
 
     const Config& config_;
 
-
     // @note This is a map of requestID:MessageQueue. At the point that a request is
     // complete, errored or otherwise killed, it needs to be removed from the map.
     // The shared_ptr allows this removal to be asynchronous with the actual task
     // cleaning up and returning to the client.
     std::map> messageQueues_;
     std::map> retrieveMessageQueues_;
-    // MessageQueue retrieveMessageQueue_;
 
     std::mutex retrieveMessageMutex_;
 
     Locations locations_;
-    // std::recursive_mutex locationMutex_;
-    // std::map fieldLocation)>> locations_;
-    // size_t fieldsArchived_;
-    // size_t locationsReceived_;
-    // std::promise promiseArchivalCompleted_;
-    // std::future archivalCompleted_;
-    // // std::mutex archiveMutex_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/remote/server/CatalogueHandler.h b/src/fdb5/remote/server/CatalogueHandler.h
index dc92afd70..e78a0b582 100644
--- a/src/fdb5/remote/server/CatalogueHandler.h
+++ b/src/fdb5/remote/server/CatalogueHandler.h
@@ -38,14 +38,12 @@ class CatalogueHandler : public ServerConnection {
 public:  // methods
 
     CatalogueHandler(eckit::net::TCPSocket& socket, const Config& config);
-    ~CatalogueHandler();
+    ~CatalogueHandler() override;
 
 private:  // methods
 
     Handled handleControl(Message message, uint32_t clientID, uint32_t requestID) override;
     Handled handleControl(Message message, uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) override;
-    // Handled handleData(Message message, uint32_t clientID, uint32_t requestID) override;
-    // Handled handleData(Message message, uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) override;
 
     // API functionality
     template 
@@ -64,7 +62,6 @@ class CatalogueHandler : public ServerConnection {
     bool remove(bool control, uint32_t clientID) override;
     // bool handlers() override;
 
-    // CatalogueWriter& catalogue(uint32_t catalogueID);
     CatalogueWriter& catalogue(uint32_t catalogueID, const Key& dbKey);
 
 private:  // member
diff --git a/src/fdb5/remote/server/ServerConnection.h b/src/fdb5/remote/server/ServerConnection.h
index 21257bc19..66ef81f56 100644
--- a/src/fdb5/remote/server/ServerConnection.h
+++ b/src/fdb5/remote/server/ServerConnection.h
@@ -87,7 +87,7 @@ struct ArchiveElem {
 class ServerConnection : public Connection, public Handler {
 public:  // methods
     ServerConnection(eckit::net::TCPSocket& socket, const Config& config);
-    ~ServerConnection();
+    ~ServerConnection() override;
 
     void initialiseConnections();
 
diff --git a/src/fdb5/remote/server/StoreHandler.h b/src/fdb5/remote/server/StoreHandler.h
index 74b2adec1..92abb9a36 100644
--- a/src/fdb5/remote/server/StoreHandler.h
+++ b/src/fdb5/remote/server/StoreHandler.h
@@ -33,7 +33,7 @@ class StoreHandler : public ServerConnection {
 public:  // methods
 
     StoreHandler(eckit::net::TCPSocket& socket, const Config& config);
-    ~StoreHandler();
+    ~StoreHandler() override;
 
 private:  // methods
 
diff --git a/src/fdb5/types/TypesRegistry.h b/src/fdb5/types/TypesRegistry.h
index b14e95101..057b9f7e9 100644
--- a/src/fdb5/types/TypesRegistry.h
+++ b/src/fdb5/types/TypesRegistry.h
@@ -39,9 +39,9 @@ class TypesRegistry : public eckit::Streamable  {
 public: // methods
 
     TypesRegistry();
-    TypesRegistry(eckit::Stream& s);
+    explicit TypesRegistry(eckit::Stream& s);
 
-    ~TypesRegistry();
+    ~TypesRegistry() override;
 
     const Type &lookupType(const std::string &keyword) const;
 

From db308cf9f7b655db663e84a1dfad77e00eef4429 Mon Sep 17 00:00:00 2001
From: Chris Bradley 
Date: Wed, 27 Nov 2024 13:44:33 +0000
Subject: [PATCH 169/186] Base tmpdir on cwd in auxiliary/callback tests

---
 tests/fdb/api/test_auxiliary.cc | 7 +++----
 tests/fdb/api/test_callback.cc  | 3 ++-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/fdb/api/test_auxiliary.cc b/tests/fdb/api/test_auxiliary.cc
index 67ccfbee9..1f88983d6 100644
--- a/tests/fdb/api/test_auxiliary.cc
+++ b/tests/fdb/api/test_auxiliary.cc
@@ -1,5 +1,6 @@
 #include "eckit/testing/Test.h"
 #include "eckit/filesystem/TmpDir.h"
+#include "eckit/filesystem/LocalPathName.h"
 #include "metkit/mars/MarsRequest.h"
 #include "fdb5/api/FDB.h"
 #include "fdb5/api/helpers/FDBToolRequest.h"
@@ -64,8 +65,7 @@ std::set setup(FDB& fdb) {
 //----------------------------------------------------------------------------------------------------------------------
 
 CASE("Wipe with extensions") {
-
-    eckit::TmpDir tmpdir;
+    eckit::TmpDir tmpdir(eckit::LocalPathName::cwd().c_str());
     eckit::testing::SetEnv env_config{"FDB_ROOT_DIRECTORY", tmpdir.asString().c_str()};
 
     eckit::testing::SetEnv env_extensions{"FDB_AUX_EXTENSIONS", "foo,bar"};
@@ -93,8 +93,7 @@ CASE("Wipe with extensions") {
 }
 
 CASE("Purge with extensions") {
-
-    eckit::TmpDir tmpdir;
+    eckit::TmpDir tmpdir(eckit::LocalPathName::cwd().c_str());
     eckit::testing::SetEnv env_config{"FDB_ROOT_DIRECTORY", tmpdir.asString().c_str()};
 
     eckit::testing::SetEnv env_extensions{"FDB_AUX_EXTENSIONS", "foo,bar"};
diff --git a/tests/fdb/api/test_callback.cc b/tests/fdb/api/test_callback.cc
index b68127c8e..2be8f9f0a 100644
--- a/tests/fdb/api/test_callback.cc
+++ b/tests/fdb/api/test_callback.cc
@@ -1,5 +1,6 @@
 #include "eckit/testing/Test.h"
 #include "eckit/filesystem/TmpDir.h"
+#include "eckit/filesystem/LocalPathName.h"
 #include "fdb5/api/FDB.h"
 
 namespace fdb5::test {
@@ -7,7 +8,7 @@ namespace fdb5::test {
 //----------------------------------------------------------------------------------------------------------------------
 CASE("Archive and flush callback") {
 
-    eckit::TmpDir tmpdir;
+    eckit::TmpDir tmpdir(eckit::LocalPathName::cwd().c_str());
     eckit::testing::SetEnv env_config{"FDB_ROOT_DIRECTORY", tmpdir.asString().c_str()};
 
     FDB fdb;

From 9d2bebda77acbf408531c9c14dd221e25a9373fd Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 27 Nov 2024 23:17:57 +0000
Subject: [PATCH 170/186] FDB-330 lustre FS identification

---
 src/fdb5/io/LustreFileHandle.h     |  9 +++++----
 src/fdb5/io/LustreSettings.cc      | 22 +++++++++++++++-------
 src/fdb5/io/LustreSettings.h       |  3 ++-
 src/fdb5/toc/TocCatalogueWriter.cc |  4 ++--
 src/fdb5/toc/TocHandler.cc         |  4 ++--
 5 files changed, 26 insertions(+), 16 deletions(-)

diff --git a/src/fdb5/io/LustreFileHandle.h b/src/fdb5/io/LustreFileHandle.h
index 1baa26cd4..7adcde756 100644
--- a/src/fdb5/io/LustreFileHandle.h
+++ b/src/fdb5/io/LustreFileHandle.h
@@ -56,18 +56,19 @@ class LustreFileHandle : public HANDLE {
 
     void openForAppend(const eckit::Length& len) override {
 
-        std::string path = HANDLE::path_;
+        std::string pathStr = HANDLE::path_;
+        eckit::PathName path{pathStr};
 
-        if(eckit::PathName(path).exists()) return; //< Lustre API outputs ioctl error messages when called on files exist
+        if(path.exists()) return; //< Lustre API outputs ioctl error messages when called on files exist
 
         /* From the docs: llapi_file_create closes the file descriptor. You must re-open the file afterwards */
 
-        LOG_DEBUG_LIB(LibFdb5) << "Creating Lustre file " << path
+        LOG_DEBUG_LIB(LibFdb5) << "Creating Lustre file " << pathStr
                                     << " with " << stripe_.count_ << " stripes "
                                     << "of " << eckit::Bytes(stripe_.size_)
                                     << std::endl;
 
-        int err = fdb5LustreapiFileCreate(path.c_str(), stripe_);
+        int err = fdb5LustreapiFileCreate(path, stripe_);
 
         if(err == EINVAL) {
 
diff --git a/src/fdb5/io/LustreSettings.cc b/src/fdb5/io/LustreSettings.cc
index 898306e68..7845113a6 100644
--- a/src/fdb5/io/LustreSettings.cc
+++ b/src/fdb5/io/LustreSettings.cc
@@ -15,8 +15,11 @@
 #include "fdb5/fdb5_config.h"
 #include "fdb5/LibFdb5.h"
 
+#define LL_SUPER_MAGIC  0x0BD00BD0
 
 #if defined(fdb5_HAVE_LUSTRE)
+#include 
+
 extern "C" {
 void fdb5_lustreapi_silence_msg();
 int  fdb5_lustreapi_file_create(const char* path, size_t stripesize, size_t stripecount);
@@ -35,19 +38,24 @@ bool fdb5LustreapiSupported() {
 #endif
 }
 
-int fdb5LustreapiFileCreate(const char* path, LustreStripe stripe) {
+int fdb5LustreapiFileCreate(eckit::PathName path, LustreStripe stripe) {
 
 #if defined(fdb5_HAVE_LUSTRE)
 
-    static bool lustreapi_silence = false;
+    struct statfs buf;
 
-    if(not lustreapi_silence) {
-        fdb5_lustreapi_silence_msg();
-        lustreapi_silence = true;
-    }
+    statfs(path.dirName().localPath(), &buf);
+    if (buf.type == LL_SUPER_MAGIC) {
 
-    return fdb5_lustreapi_file_create(path, stripe.size_, stripe.count_);
+        static bool lustreapi_silence = false;
 
+        if(not lustreapi_silence) {
+            fdb5_lustreapi_silence_msg();
+            lustreapi_silence = true;
+        }
+
+        return fdb5_lustreapi_file_create(path.localPath(), stripe.size_, stripe.count_);
+    }
 #endif
 
     /// @note since fdb5LustreapiSupported() should be guarding all calls to this function,
diff --git a/src/fdb5/io/LustreSettings.h b/src/fdb5/io/LustreSettings.h
index 40bb03262..464f4c673 100644
--- a/src/fdb5/io/LustreSettings.h
+++ b/src/fdb5/io/LustreSettings.h
@@ -16,6 +16,7 @@
 
 #include 
 
+#include "eckit/filesystem/PathName.h"
 namespace fdb5 {
 
 //----------------------------------------------------------------------------------------------------------------------
@@ -42,7 +43,7 @@ LustreStripe stripeDataLustreSettings();
 
 //----------------------------------------------------------------------------------------------------------------------
 
-int fdb5LustreapiFileCreate(const char* path, LustreStripe stripe);
+int fdb5LustreapiFileCreate(eckit::PathName path, LustreStripe stripe);
 
 bool fdb5LustreapiSupported();
 
diff --git a/src/fdb5/toc/TocCatalogueWriter.cc b/src/fdb5/toc/TocCatalogueWriter.cc
index 306a6bc3c..0099af0b2 100644
--- a/src/fdb5/toc/TocCatalogueWriter.cc
+++ b/src/fdb5/toc/TocCatalogueWriter.cc
@@ -63,7 +63,7 @@ bool TocCatalogueWriter::selectIndex(const Key& idxKey) {
 
         // Enforce lustre striping if requested
         if (stripeLustre()) {
-            fdb5LustreapiFileCreate(indexPath.localPath(), stripeIndexLustreSettings());
+            fdb5LustreapiFileCreate(indexPath, stripeIndexLustreSettings());
         }
 
         indexes_[idxKey] = Index(new TocIndex(idxKey, *this, indexPath, 0, TocIndex::WRITE));
@@ -85,7 +85,7 @@ bool TocCatalogueWriter::selectIndex(const Key& idxKey) {
 
             // Enforce lustre striping if requested
             if (stripeLustre()) {
-                fdb5LustreapiFileCreate(indexPath.localPath(), stripeIndexLustreSettings());
+                fdb5LustreapiFileCreate(indexPath, stripeIndexLustreSettings());
             }
 
             fullIndexes_[idxKey] = Index(new TocIndex(idxKey, *this, indexPath, 0, TocIndex::WRITE));
diff --git a/src/fdb5/toc/TocHandler.cc b/src/fdb5/toc/TocHandler.cc
index 34ab8dfe3..76f1cadc5 100644
--- a/src/fdb5/toc/TocHandler.cc
+++ b/src/fdb5/toc/TocHandler.cc
@@ -939,7 +939,7 @@ void TocHandler::writeInitRecord(const Key& key) {
 
     // enforce lustre striping if requested
     if (stripeLustre() && !tocPath_.exists()) {
-        fdb5LustreapiFileCreate(tocPath_.localPath(), stripeIndexLustreSettings());
+        fdb5LustreapiFileCreate(tocPath_, stripeIndexLustreSettings());
     }
 
     ASSERT(fd_ == -1);
@@ -977,7 +977,7 @@ void TocHandler::writeInitRecord(const Key& key) {
             // LustreFileHandle out(tmp, stripeIndexLustreSettings());
 
             if (stripeLustre()) {
-                fdb5LustreapiFileCreate(tmp.localPath(), stripeIndexLustreSettings());
+                fdb5LustreapiFileCreate(tmp, stripeIndexLustreSettings());
             }
             eckit::FileHandle out(tmp);
             in.copyTo(out);

From 99d130a01f0f396285a683c3028284ca1ce68f06 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 27 Nov 2024 23:42:47 +0000
Subject: [PATCH 171/186] FDB-330 lustre FS identification

---
 src/fdb5/io/LustreSettings.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/fdb5/io/LustreSettings.cc b/src/fdb5/io/LustreSettings.cc
index 7845113a6..4b3decb87 100644
--- a/src/fdb5/io/LustreSettings.cc
+++ b/src/fdb5/io/LustreSettings.cc
@@ -45,7 +45,7 @@ int fdb5LustreapiFileCreate(eckit::PathName path, LustreStripe stripe) {
     struct statfs buf;
 
     statfs(path.dirName().localPath(), &buf);
-    if (buf.type == LL_SUPER_MAGIC) {
+    if (buf.f_type == LL_SUPER_MAGIC) {
 
         static bool lustreapi_silence = false;
 

From e5f2ace8310cace6a3b2b901ad6d48fce62f337a Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 28 Nov 2024 11:42:32 +0000
Subject: [PATCH 172/186] addressed PR + fix toc async read

---
 src/fdb5/io/LustreSettings.cc | 2 +-
 src/fdb5/io/LustreSettings.h  | 2 +-
 src/fdb5/toc/TocHandler.cc    | 5 ++---
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/fdb5/io/LustreSettings.cc b/src/fdb5/io/LustreSettings.cc
index 4b3decb87..5af6ced71 100644
--- a/src/fdb5/io/LustreSettings.cc
+++ b/src/fdb5/io/LustreSettings.cc
@@ -38,7 +38,7 @@ bool fdb5LustreapiSupported() {
 #endif
 }
 
-int fdb5LustreapiFileCreate(eckit::PathName path, LustreStripe stripe) {
+int fdb5LustreapiFileCreate(const eckit::PathName& path, LustreStripe stripe) {
 
 #if defined(fdb5_HAVE_LUSTRE)
 
diff --git a/src/fdb5/io/LustreSettings.h b/src/fdb5/io/LustreSettings.h
index 464f4c673..c3c12d47e 100644
--- a/src/fdb5/io/LustreSettings.h
+++ b/src/fdb5/io/LustreSettings.h
@@ -43,7 +43,7 @@ LustreStripe stripeDataLustreSettings();
 
 //----------------------------------------------------------------------------------------------------------------------
 
-int fdb5LustreapiFileCreate(eckit::PathName path, LustreStripe stripe);
+int fdb5LustreapiFileCreate(const eckit::PathName& path, LustreStripe stripe);
 
 bool fdb5LustreapiSupported();
 
diff --git a/src/fdb5/toc/TocHandler.cc b/src/fdb5/toc/TocHandler.cc
index 76f1cadc5..ae978ec37 100644
--- a/src/fdb5/toc/TocHandler.cc
+++ b/src/fdb5/toc/TocHandler.cc
@@ -725,7 +725,7 @@ class SubtocPreloader {
                 int fd;
                 SYSCALL2((fd = ::open(path.localPath(), iomode)), path);
                 closers.emplace_back(AutoFDCloser{fd});
-                eckit::Length tocSize = 2*1024*1024;
+                eckit::Length tocSize = path.size();
 
                 aiocb& aio(aiocbs[i]);
                 zero(aio);
@@ -802,9 +802,8 @@ class SubtocPreloader {
             }
         }
 #else
-    NOTIMP
+    NOTIMP;
 #endif // eckit_HAVE_AIO
-
         return std::move(subTocReadCache_);
     }
 

From 49dcda0ce91d2ab3823bea856360fddbffbbd023 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 10 Dec 2024 10:11:42 +0000
Subject: [PATCH 173/186] retrieve seekable handle removed (and ported to
 feature/57-seekable branch)

---
 src/fdb5/CMakeLists.txt    |   2 -
 src/fdb5/api/FDB.cc        |   8 +-
 src/fdb5/api/FDB.h         |   4 +-
 src/fdb5/io/FieldHandle.cc | 291 -------------------------------------
 src/fdb5/io/FieldHandle.h  | 103 -------------
 5 files changed, 3 insertions(+), 405 deletions(-)
 delete mode 100644 src/fdb5/io/FieldHandle.cc
 delete mode 100644 src/fdb5/io/FieldHandle.h

diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt
index 2609214ad..732e8c99b 100644
--- a/src/fdb5/CMakeLists.txt
+++ b/src/fdb5/CMakeLists.txt
@@ -150,8 +150,6 @@ list( APPEND fdb5_srcs
     io/LustreFileHandle.h
     io/HandleGatherer.cc
     io/HandleGatherer.h
-    io/FieldHandle.cc
-    io/FieldHandle.h
     rules/MatchAlways.cc
     rules/MatchAlways.h
     rules/MatchAny.cc
diff --git a/src/fdb5/api/FDB.cc b/src/fdb5/api/FDB.cc
index 0fe967382..cb6a66161 100644
--- a/src/fdb5/api/FDB.cc
+++ b/src/fdb5/api/FDB.cc
@@ -214,13 +214,9 @@ eckit::DataHandle* FDB::read(ListIterator& it, bool sorted) {
     return result.dataHandle();
 }
 
-eckit::DataHandle* FDB::readSeekable(ListIterator& it, bool sorted) {
-    return new FieldHandle(it);
-}
-
-eckit::DataHandle* FDB::retrieve(const metkit::mars::MarsRequest& request, bool seekable) {
+eckit::DataHandle* FDB::retrieve(const metkit::mars::MarsRequest& request) {
     ListIterator it = inspect(request);
-    return (seekable ? readSeekable(it, sorted(request)) : read(it, sorted(request)));
+    return read(it, sorted(request));
 }
 
 ListIterator FDB::inspect(const metkit::mars::MarsRequest& request) {
diff --git a/src/fdb5/api/FDB.h b/src/fdb5/api/FDB.h
index a6dc3521a..28529795c 100644
--- a/src/fdb5/api/FDB.h
+++ b/src/fdb5/api/FDB.h
@@ -91,9 +91,7 @@ class FDB {
 
     eckit::DataHandle* read(ListIterator& it, bool sorted = false);
 
-    eckit::DataHandle* readSeekable(ListIterator& it, bool sorted = false);
-
-    eckit::DataHandle* retrieve(const metkit::mars::MarsRequest& request, bool seekable = false);
+    eckit::DataHandle* retrieve(const metkit::mars::MarsRequest& request);
 
     ListIterator inspect(const metkit::mars::MarsRequest& request);
 
diff --git a/src/fdb5/io/FieldHandle.cc b/src/fdb5/io/FieldHandle.cc
deleted file mode 100644
index f89ddefb4..000000000
--- a/src/fdb5/io/FieldHandle.cc
+++ /dev/null
@@ -1,291 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-#include 
-
-#include "eckit/io/MemoryHandle.h"
-#include "eckit/config/Resource.h"
-#include "eckit/log/Timer.h"
-#include "eckit/runtime/Metrics.h"
-#include "eckit/types/Types.h"
-
-#include "metkit/mars/MarsRequest.h"
-#include "metkit/hypercube/HyperCubePayloaded.h"
-
-#include "fdb5/LibFdb5.h"
-#include "fdb5/io/FieldHandle.h"
-
-namespace fdb5 {
-
-//----------------------------------------------------------------------------------------------------------------------
-
-FieldHandle::FieldHandle(ListIterator& it) :
-    datahandles_({}), totalSize_(0), currentIdx_(0), current_(nullptr), currentMemoryHandle_(false), buffer_(nullptr), sorted_(false), seekable_(true) {
-    ListElement el;
-    eckit::Length largest = 0;
-
-    static bool dedup = eckit::Resource("fdbDeduplicate;$FDB_DEDUPLICATE_FIELDS", false);
-    if (dedup) {
-        if (it.next(el)) {
-            // build the request representing the tensor-product of all retrieved fields
-            metkit::mars::MarsRequest cubeRequest = el.combinedKey().request();
-            std::vector elements{el};
-
-            while (it.next(el)) {
-                cubeRequest.merge(el.combinedKey().request());
-                elements.push_back(el);
-            }
-
-            // checking all retrieved fields against the hypercube, to remove duplicates
-            ListElementDeduplicator dedup;
-            metkit::hypercube::HyperCubePayloaded cube(cubeRequest, dedup);
-            for(auto el: elements) {
-                cube.add(el.combinedKey().request(), el);
-            }
-
-            if (cube.countVacant() > 0) {
-                std::stringstream ss;
-                ss << "No matching data for requests:" << std::endl;
-                for (auto req: cube.vacantRequests()) {
-                    ss << "    " << req << std::endl;
-                }
-                eckit::Log::warning() << ss.str() << std::endl;
-            }
-
-            for (size_t i=0; i< cube.size(); i++) {
-                ListElement element;
-                if (cube.find(i, element)) {
-                    eckit::Length len = element.location().length();
-                    eckit::DataHandle* dh = element.location().dataHandle();
-                    datahandles_.push_back(std::make_pair(len, dh));
-
-                    totalSize_ += len;
-                    bool canSeek = dh->canSeek();
-                    if (!canSeek) {
-                        largest = std::max(largest, len);
-                        seekable_ = false;
-                    }
-                }
-            }
-        }
-    }
-    else {
-        while (it.next(el)) {
-            eckit::Length len = el.location().length();
-            eckit::DataHandle* dh = el.location().dataHandle();
-            datahandles_.push_back(std::make_pair(len, dh));
-
-            totalSize_ += len;
-            bool canSeek = dh->canSeek();
-            if (!canSeek) {
-                largest = std::max(largest, len);
-                seekable_ = false;
-            }
-        }
-    }
-
-    if (!seekable_) {
-        // allocate a buffer that can fit the largest not seekable field (to avoid re-allocations)
-        buffer_ = new char[largest];
-    }
-}
-
-FieldHandle::~FieldHandle() {
-    for (size_t i = 0; i < datahandles_.size(); i++) {
-        delete datahandles_[i].second;
-    }
-    if (current_ && currentMemoryHandle_) {
-        delete current_;
-    }
-    if (buffer_) {
-        delete[] buffer_;
-    }
-}
-
-void FieldHandle::openCurrent() {
-
-    if (current_ && currentMemoryHandle_) {
-        delete current_;
-        currentMemoryHandle_ = false;
-    }
-
-    if (currentIdx_ < datahandles_.size()) {
-
-        eckit::Length currentSize = datahandles_[currentIdx_].first;
-        current_ = datahandles_[currentIdx_].second;
-        current_->openForRead();
-
-        if (!current_->canSeek()) {
-            long len = 0;
-            long toRead = currentSize;
-            long read = 0;
-            char *buf = buffer_;
-            while (toRead>0 && (len = current_->read(buf, toRead))>0) {
-                toRead -= len;
-                buf += len;
-                read += len;
-            }
-
-            if (read != currentSize) {
-                std::stringstream ss;
-                ss << "Error reading from " << *current_ << " - read " << read << ", expected " << currentSize;
-                throw eckit::ReadError(ss.str());
-            }
-            current_ = new eckit::MemoryHandle(buffer_, currentSize);
-            current_->openForRead();
-            currentMemoryHandle_ = true;
-        }
-    }
-}
-
-eckit::Length FieldHandle::openForRead() {
-    ASSERT(!current_);
-
-    currentIdx_=0;
-    openCurrent();
-
-    return totalSize_;
-}
-
-long FieldHandle::read1(char* buffer, long length) {
-    if (currentIdx_ >= datahandles_.size()) {
-        return 0;
-    }
-
-    long n = current_->read(buffer, length);
-    if (n <= 0) {
-        current_->close();
-        currentIdx_++;
-        openCurrent();
-        return read1(buffer, length);
-    }
-    return n;
-}
-
-long FieldHandle::read(void* buffer, long length) {
-    long requested = length;
-
-    char* p    = static_cast(buffer);
-    long n     = 0;
-    long total = 0;
-
-    while (length > 0 && (n = read1(p, length)) > 0) {
-        length -= n;
-        total += n;
-        p += n;
-    }
-
-    LOG_DEBUG_LIB(LibFdb5) << "FieldHandle::read - requested: " << requested << "  read: " << (total > 0 ? total : n) << std::endl;
-
-    return total > 0 ? total : n;
-}
-
-void FieldHandle::close() {
-    if (currentIdx_ < datahandles_.size()) {
-        current_->close();
-        if (currentMemoryHandle_) {
-            delete current_;
-        }
-        current_=nullptr;
-    }
-    currentIdx_ = datahandles_.size();
-}
-
-void FieldHandle::rewind() {
-    if (currentIdx_ == 0 || seekable_) {
-        if (current_ && currentIdx_ < datahandles_.size()) {
-            current_->close();
-        }
-        currentIdx_ = 0;
-        openCurrent();
-    } else {
-        throw eckit::ReadError("rewind not supported");
-    }
-}
-
-void FieldHandle::print(std::ostream& s) const {
-    if (eckit::format(s) == eckit::Log::compactFormat) {
-        s << "FieldHandle";
-    }
-    else {
-        s << "FieldHandle[";
-        for (size_t i = 0; i < datahandles_.size(); i++) {
-            if (i != 0) {
-                s << ",(";
-            }
-            datahandles_[i].second->print(s);
-            s << ")";
-        }
-        s << ']';
-    }
-}
-
-eckit::Length FieldHandle::size() {
-    return totalSize_;
-}
-
-eckit::Length FieldHandle::estimate() {
-    return totalSize_;
-}
-
-// only partial seek: within the current message and forward
-bool FieldHandle::canSeek() const {
-    return true;
-}
-
-eckit::Offset FieldHandle::position() {
-    long long accumulated = 0;
-    for (size_t idx = 0; idx < currentIdx_; idx++) {
-        accumulated += datahandles_[idx].first;
-    }
-    return accumulated + (currentIdx_ >= datahandles_.size() ? eckit::Offset(0) : current_->position());
-}
-
-eckit::Offset FieldHandle::seek(const eckit::Offset& offset) {
-    if (current_ && currentIdx_ < datahandles_.size()) {
-        current_->close();
-    }
-
-    const long long seekto = offset;
-    long long accumulated  = 0;
-
-    if (!seekable_) { // check that the offset is within the current message of later
-        for (size_t idx = 0; idx < currentIdx_; idx++) {
-            accumulated += datahandles_[idx].first;
-        }
-
-        if (seekto < accumulated) {
-            throw eckit::UserError("Cannot seek backward to previous data fields");
-        }
-    }
-
-    accumulated  = 0;
-    for (currentIdx_ = 0; currentIdx_ < datahandles_.size(); ++currentIdx_) {
-        long long size = datahandles_[currentIdx_].first;
-        if (accumulated <= seekto && seekto < accumulated + size) {
-            openCurrent();
-            current_->seek(seekto - accumulated);
-            return offset;
-        }
-        accumulated += size;
-    }
-    // check if we went beyond EOF which is POSIX compliant, but we ASSERT so we find possible bugs
-    eckit::Offset beyond = seekto - accumulated;
-    ASSERT(not beyond);
-    return offset;
-}
-
-bool FieldHandle::compress(bool) {
-    return false;
-}
-
-//----------------------------------------------------------------------------------------------------------------------
-
-}  // namespace eckit
diff --git a/src/fdb5/io/FieldHandle.h b/src/fdb5/io/FieldHandle.h
deleted file mode 100644
index 679279283..000000000
--- a/src/fdb5/io/FieldHandle.h
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * (C) Copyright 1996- ECMWF.
- *
- * This software is licensed under the terms of the Apache Licence Version 2.0
- * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
- * In applying this licence, ECMWF does not waive the privileges and immunities
- * granted to it by virtue of its status as an intergovernmental organisation nor
- * does it submit to any jurisdiction.
- */
-
-/// @author Emanuele Danovaro
-/// @date March 2024
-
-#pragma once
-
-#include "eckit/io/DataHandle.h"
-#include "eckit/io/Buffer.h"
-
-#include "fdb5/api/helpers/ListIterator.h"
-
-namespace fdb5 {
-
-//----------------------------------------------------------------------------------------------------------------------
-
-class ListElementDeduplicator : public metkit::hypercube::Deduplicator {
-public:
-    bool toReplace(const ListElement& existing, const ListElement& replacement) const override {
-        return existing.timestamp() < replacement.timestamp();
-    }
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-class FieldHandle : public eckit::DataHandle {
-public:
-
-    // -- Contructors
-
-    FieldHandle(ListIterator& it);
-
-    // -- Destructor
-
-    ~FieldHandle() override;
-
-    // -- Operators
-
-    // -- Overridden methods
-
-    // From DataHandle
-
-    eckit::Length openForRead() override;
-
-    long read(void*, long) override;
-    void close() override;
-    void rewind() override;
-    void print(std::ostream&) const override;
-
-    eckit::Offset position() override;
-    eckit::Offset seek(const eckit::Offset&) override;
-    bool canSeek() const override;
-
-    bool compress(bool = false) override;
-
-    eckit::Length size() override;
-    eckit::Length estimate() override;
-
-    // std::string title() const override;
-
-    // // From Streamable
-
-    // void encode(Stream&) const override;
-    // const ReanimatorBase& reanimator() const override { return reanimator_; }
-
-    // // -- Class methods
-
-    // static const ClassSpec& classSpec() { return classSpec_; }
-
-private:
-    // -- Methods
-
-    void openCurrent();
-    void open();
-    long read1(char*, long);
-
-private:
-    // -- Members
-
-    std::vector> datahandles_;
-    eckit::Length totalSize_;
-    
-    size_t currentIdx_;
-    DataHandle* current_;
-    bool currentMemoryHandle_;
-
-    char* buffer_{nullptr};
-    
-    bool sorted_;
-    bool seekable_;
-};
-
-//----------------------------------------------------------------------------------------------------------------------
-
-}  // namespace eckit

From b64b92439fbf98805fe1d9b968a939844b025fcf Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 10 Dec 2024 10:14:50 +0000
Subject: [PATCH 174/186] retrieve seekable handle removed (and ported to
 feature/57-seekable branch)

---
 src/fdb5/api/FDB.cc | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/fdb5/api/FDB.cc b/src/fdb5/api/FDB.cc
index cb6a66161..2720f46fa 100644
--- a/src/fdb5/api/FDB.cc
+++ b/src/fdb5/api/FDB.cc
@@ -28,7 +28,6 @@
 #include "fdb5/api/helpers/FDBToolRequest.h"
 #include "fdb5/database/Key.h"
 #include "fdb5/io/HandleGatherer.h"
-#include "fdb5/io/FieldHandle.h"
 #include "fdb5/message/MessageDecoder.h"
 #include "fdb5/types/Type.h"
 
@@ -146,6 +145,13 @@ bool FDB::sorted(const metkit::mars::MarsRequest &request) {
     return sorted;
 }
 
+class ListElementDeduplicator : public metkit::hypercube::Deduplicator {
+public:
+    bool toReplace(const ListElement& existing, const ListElement& replacement) const override {
+        return existing.timestamp() < replacement.timestamp();
+    }
+};
+
 eckit::DataHandle* FDB::read(const eckit::URI& uri) {
     auto location = std::unique_ptr(FieldLocationFactory::instance().build(uri.scheme(), uri));
     return location->dataHandle();
@@ -163,7 +169,6 @@ eckit::DataHandle* FDB::read(const std::vector& uris, bool sorted) {
 }
 
 eckit::DataHandle* FDB::read(ListIterator& it, bool sorted) {
-
     eckit::Timer timer;
     timer.start();
 

From 7e03919b5ad07402c079cd8c991771a1120c4f67 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Tue, 10 Dec 2024 10:28:19 +0000
Subject: [PATCH 175/186] retrieve seekable handle removed in the c API

---
 src/fdb5/api/fdb_c.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/fdb5/api/fdb_c.cc b/src/fdb5/api/fdb_c.cc
index 270a8c4f5..eb69bc9d5 100644
--- a/src/fdb5/api/fdb_c.cc
+++ b/src/fdb5/api/fdb_c.cc
@@ -386,7 +386,7 @@ int fdb_retrieve(fdb_handle_t* fdb, fdb_request_t* req, fdb_datareader_t* dr) {
         ASSERT(fdb);
         ASSERT(req);
         ASSERT(dr);
-        dr->set(fdb->retrieve(req->request(), true));
+        dr->set(fdb->retrieve(req->request()));
     });
 }
 int fdb_flush(fdb_handle_t* fdb) {

From 46508f358163a28f2b3b26f859aaa1b164cdb016 Mon Sep 17 00:00:00 2001
From: Kai Kratz 
Date: Wed, 11 Dec 2024 10:53:11 +0100
Subject: [PATCH 176/186] Remove commented code

---
 src/fdb5/CMakeLists.txt                       |  29 +-
 src/fdb5/api/RemoteFDB.cc                     |   1 -
 src/fdb5/api/local/AxesVisitor.h              |   2 -
 src/fdb5/api/local/ControlVisitor.cc          |   4 +-
 src/fdb5/api/local/ControlVisitor.h           |   1 -
 src/fdb5/api/local/ListVisitor.h              |   2 -
 src/fdb5/api/local/MoveVisitor.cc             |   4 -
 src/fdb5/api/local/MoveVisitor.h              |   1 -
 src/fdb5/api/local/PurgeVisitor.cc            |   6 -
 src/fdb5/api/local/PurgeVisitor.h             |   1 -
 src/fdb5/api/local/StatsVisitor.cc            |   1 -
 src/fdb5/api/local/StatsVisitor.h             |   1 -
 src/fdb5/api/local/WipeVisitor.cc             |   3 -
 src/fdb5/api/local/WipeVisitor.h              |   2 -
 src/fdb5/config/Config.cc                     |   1 -
 src/fdb5/daos/DaosCatalogue.h                 |   2 -
 src/fdb5/daos/DaosEngine.h                    |   2 -
 src/fdb5/database/Archiver.h                  |   2 -
 src/fdb5/database/BaseArchiveVisitor.cc       |   1 -
 src/fdb5/database/Catalogue.cc                |   2 +-
 src/fdb5/database/Catalogue.h                 |   5 -
 src/fdb5/database/Engine.h                    |   8 -
 src/fdb5/database/Index.cc                    |   2 +-
 src/fdb5/database/Manager.cc                  |  42 --
 src/fdb5/database/Manager.h                   |   9 -
 src/fdb5/database/ReadVisitor.cc              |  23 -
 src/fdb5/database/Store.cc                    |  20 -
 src/fdb5/database/Store.h                     |   4 -
 src/fdb5/rados/RadosFieldLocation.cc          |   5 -
 src/fdb5/rados/RadosFieldLocation.h           |  24 -
 src/fdb5/rados/RadosStore.cc                  |   3 -
 src/fdb5/remote/Connection.cc                 |  24 +-
 src/fdb5/remote/Messages.h                    |   2 -
 src/fdb5/remote/client/Client.cc              |   8 +-
 src/fdb5/remote/client/ClientConnection.cc    |  36 +-
 src/fdb5/remote/client/ClientConnection.h     |  12 +-
 .../remote/client/ClientConnectionRouter.cc   |   5 +-
 src/fdb5/remote/client/RemoteCatalogue.cc     |   8 +-
 src/fdb5/remote/client/RemoteCatalogue.h      |   4 +-
 src/fdb5/remote/client/RemoteStore.cc         |  35 +-
 src/fdb5/remote/client/RemoteStore.h          |   2 -
 src/fdb5/remote/server/CatalogueHandler.cc    |  62 +-
 src/fdb5/remote/server/CatalogueHandler.h     |   2 -
 src/fdb5/remote/server/ServerConnection.cc    |   3 -
 src/fdb5/rules/Matcher.cc                     |   3 -
 src/fdb5/rules/Matcher.h                      |   2 -
 src/fdb5/rules/Rule.cc                        |  29 -
 src/fdb5/toc/AdoptVisitor.cc                  |   2 -
 src/fdb5/toc/FileSpace.cc                     |   3 -
 src/fdb5/toc/TocCatalogue.cc                  |  26 -
 src/fdb5/toc/TocHandler.cc                    |   4 +-
 src/fdb5/toc/TocHandler.h                     |   3 -
 src/fdb5/toc/TocMoveVisitor.cc                |   2 -
 src/fdb5/toc/TocMoveVisitor.h                 |   2 -
 src/fdb5/toc/TocPurgeVisitor.cc               |   3 +-
 src/fdb5/toc/TocPurgeVisitor.h                |   1 -
 src/fdb5/toc/TocStats.cc                      |   1 -
 src/fdb5/toc/TocStats.h                       |   1 -
 src/fdb5/toc/TocWipeVisitor.cc                |   3 -
 src/fdb5/toc/TocWipeVisitor.h                 |   1 -
 tests/fdb/test_fdb5_service.cc                | 698 +++++++++---------
 61 files changed, 400 insertions(+), 800 deletions(-)

diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt
index 732e8c99b..062f210f5 100644
--- a/src/fdb5/CMakeLists.txt
+++ b/src/fdb5/CMakeLists.txt
@@ -213,7 +213,7 @@ list( APPEND fdb5_srcs
 )
 
 if(HAVE_FDB_REMOTE)
-    list( APPEND fdb5_srcs 
+    list( APPEND fdb5_srcs
         api/RemoteFDB.cc
         api/RemoteFDB.h
 
@@ -245,9 +245,6 @@ if(HAVE_FDB_REMOTE)
         remote/server/StoreHandler.cc
         remote/server/ServerConnection.h
         remote/server/ServerConnection.cc
-
-        # remote/RemoteConfiguration.h
-        # remote/RemoteConfiguration.cc
     )
 endif()
 
@@ -353,19 +350,19 @@ if( HAVE_DAOSFDB )
         daos/DaosArrayHandle.h
         daos/DaosKeyValueHandle.cc
         daos/DaosKeyValueHandle.h
-        daos/DaosStore.h 
-        daos/DaosStore.cc 
-        daos/DaosFieldLocation.h 
+        daos/DaosStore.h
+        daos/DaosStore.cc
+        daos/DaosFieldLocation.h
         daos/DaosFieldLocation.cc
         daos/DaosException.h
         daos/DaosException.cc
         daos/DaosEngine.h
-        daos/DaosEngine.cc 
-        daos/DaosCatalogue.h 
-        daos/DaosCatalogue.cc 
-        daos/DaosCatalogueWriter.h 
+        daos/DaosEngine.cc
+        daos/DaosCatalogue.h
+        daos/DaosCatalogue.cc
+        daos/DaosCatalogueWriter.h
         daos/DaosCatalogueWriter.cc
-        daos/DaosCatalogueReader.h 
+        daos/DaosCatalogueReader.h
         daos/DaosCatalogueReader.cc
         daos/DaosIndex.h
         daos/DaosIndex.cc
@@ -373,11 +370,11 @@ if( HAVE_DAOSFDB )
         daos/DaosIndexLocation.cc
         daos/DaosCommon.h
         daos/DaosCommon.cc
-        daos/DaosWipeVisitor.h 
+        daos/DaosWipeVisitor.h
         daos/DaosWipeVisitor.cc
-        daos/DaosArrayPartHandle.h 
+        daos/DaosArrayPartHandle.h
         daos/DaosArrayPartHandle.cc
-        daos/DaosLazyFieldLocation.h 
+        daos/DaosLazyFieldLocation.h
         daos/DaosLazyFieldLocation.cc
         daos/UUID.h
         daos/UUID.cc
@@ -392,7 +389,7 @@ endif()
 ecbuild_add_library(
 
     TARGET  fdb5
-    
+
     TYPE SHARED # Library needs to be shared for dynamic factory self registration
 
     SOURCES
diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index 3e09fe6cc..d52735005 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -32,7 +32,6 @@ struct BaseAPIHelper {
     static ValueType valueFromStream(eckit::Stream& s, fdb5::RemoteFDB* fdb) { return ValueType(s); }
 };
 
-// using ListHelper = BaseAPIHelper;
 using StatsHelper = BaseAPIHelper;
 
 struct ListHelper : BaseAPIHelper {
diff --git a/src/fdb5/api/local/AxesVisitor.h b/src/fdb5/api/local/AxesVisitor.h
index c4eaf5392..bbecb67d1 100644
--- a/src/fdb5/api/local/AxesVisitor.h
+++ b/src/fdb5/api/local/AxesVisitor.h
@@ -36,8 +36,6 @@ class AxesVisitor : public QueryVisitor {
     bool visitIndexes() override { return true; }
     bool visitEntries() override { return false; }
     void catalogueComplete(const fdb5::Catalogue& catalogue) override;
-
-//    bool preVisitDatabase(const eckit::URI& uri) override;
     bool visitDatabase(const Catalogue& catalogue) override;
     bool visitIndex(const Index&) override;
     void visitDatum(const Field&, const Key&) override { NOTIMP; }
diff --git a/src/fdb5/api/local/ControlVisitor.cc b/src/fdb5/api/local/ControlVisitor.cc
index 88b1d54e1..074feef3a 100644
--- a/src/fdb5/api/local/ControlVisitor.cc
+++ b/src/fdb5/api/local/ControlVisitor.cc
@@ -27,9 +27,9 @@ ControlVisitor::ControlVisitor(eckit::Queue& queue,
     identifiers_(identifiers) {}
 
 
-bool ControlVisitor::visitDatabase(const Catalogue& catalogue) { //, const Store& store) {
+bool ControlVisitor::visitDatabase(const Catalogue& catalogue) {
 
-    EntryVisitor::visitDatabase(catalogue); //, store);
+    EntryVisitor::visitDatabase(catalogue);
 
     // Only lock/unlock things that match exactly.
 
diff --git a/src/fdb5/api/local/ControlVisitor.h b/src/fdb5/api/local/ControlVisitor.h
index 2d5bd44df..5a98fe001 100644
--- a/src/fdb5/api/local/ControlVisitor.h
+++ b/src/fdb5/api/local/ControlVisitor.h
@@ -38,7 +38,6 @@ class ControlVisitor : public QueryVisitor {
     bool visitEntries() override { return false; }
 
     bool visitDatabase(const Catalogue& catalogue) override;
-//    bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
     bool visitIndex(const Index&) override { NOTIMP; }
     void visitDatum(const Field&, const Key&) override { NOTIMP; }
 
diff --git a/src/fdb5/api/local/ListVisitor.h b/src/fdb5/api/local/ListVisitor.h
index ec0a0e790..ac9c15ccf 100644
--- a/src/fdb5/api/local/ListVisitor.h
+++ b/src/fdb5/api/local/ListVisitor.h
@@ -43,7 +43,6 @@ struct ListVisitor : public QueryVisitor {
 
     /// Make a note of the current database. Subtract its key from the current
     /// request so we can test request is used in its entirety
-//    bool visitDatabase(const Catalogue& catalogue, const Store& store) override {
     bool visitDatabase(const Catalogue& catalogue) override {
 
         // If the DB is locked for listing, then it "doesn't exist"
@@ -51,7 +50,6 @@ struct ListVisitor : public QueryVisitor {
             return false;
         }
 
-//        bool ret = QueryVisitor::visitDatabase(catalogue, store);
         bool ret = QueryVisitor::visitDatabase(catalogue);
         ASSERT(catalogue.key().partialMatch(request_));
 
diff --git a/src/fdb5/api/local/MoveVisitor.cc b/src/fdb5/api/local/MoveVisitor.cc
index f3121ea8c..bf83fba9c 100644
--- a/src/fdb5/api/local/MoveVisitor.cc
+++ b/src/fdb5/api/local/MoveVisitor.cc
@@ -41,7 +41,6 @@ MoveVisitor::MoveVisitor(eckit::Queue& queue,
     QueryVisitor(queue, request),
     dest_(dest) {}
 
-//bool MoveVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
 bool MoveVisitor::visitDatabase(const Catalogue& catalogue) {
     if (catalogue.key().match(request_)) {
         catalogue.control(
@@ -54,11 +53,8 @@ bool MoveVisitor::visitDatabase(const Catalogue& catalogue) {
         ASSERT(!catalogue.enabled(ControlIdentifier::UniqueRoot));
 
         EntryVisitor::visitDatabase(catalogue);
-//        EntryVisitor::visitDatabase(catalogue, store);
 
         ASSERT(!internalVisitor_);
-        // internalVisitor_.reset(catalogue.moveVisitor(store, request_, dest_, queue_));
-        // internalVisitor_->visitDatabase(catalogue, store);
         internalVisitor_.reset(catalogue.moveVisitor(store(), request_, dest_, queue_));
         internalVisitor_->visitDatabase(catalogue);
 
diff --git a/src/fdb5/api/local/MoveVisitor.h b/src/fdb5/api/local/MoveVisitor.h
index 75ea7f854..2e43ac6aa 100644
--- a/src/fdb5/api/local/MoveVisitor.h
+++ b/src/fdb5/api/local/MoveVisitor.h
@@ -42,7 +42,6 @@ class MoveVisitor : public QueryVisitor {
     bool visitIndexes() override { return false; }
     bool visitEntries() override { return false; }
 
-//    bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
     bool visitDatabase(const Catalogue& catalogue) override;
     bool visitIndex(const Index&) override { NOTIMP; }
     void visitDatum(const Field&, const Key&) override { NOTIMP; }
diff --git a/src/fdb5/api/local/PurgeVisitor.cc b/src/fdb5/api/local/PurgeVisitor.cc
index c42290158..9cd9ab522 100644
--- a/src/fdb5/api/local/PurgeVisitor.cc
+++ b/src/fdb5/api/local/PurgeVisitor.cc
@@ -39,12 +39,6 @@ PurgeVisitor::PurgeVisitor(eckit::Queue& queue,
     doit_(doit),
     porcelain_(porcelain) {}
 
-/*
-bool PurgeVisitor::visitCatalogue(const Catalogue& catalogue) {
-    return false;
-}*/
-
-//bool PurgeVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
 bool PurgeVisitor::visitDatabase(const Catalogue& catalogue) {
 
     // If the DB is locked for wiping, then it "doesn't exist"
diff --git a/src/fdb5/api/local/PurgeVisitor.h b/src/fdb5/api/local/PurgeVisitor.h
index 0acdde7ff..985367b25 100644
--- a/src/fdb5/api/local/PurgeVisitor.h
+++ b/src/fdb5/api/local/PurgeVisitor.h
@@ -43,7 +43,6 @@ class PurgeVisitor : public QueryVisitor {
                  bool doit,
                  bool porcelain);
 
-//    bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
     bool visitDatabase(const Catalogue& catalogue) override;
     bool visitIndex(const Index& index) override;
     void catalogueComplete(const Catalogue& catalogue) override;
diff --git a/src/fdb5/api/local/StatsVisitor.cc b/src/fdb5/api/local/StatsVisitor.cc
index 080ec3654..25f669103 100644
--- a/src/fdb5/api/local/StatsVisitor.cc
+++ b/src/fdb5/api/local/StatsVisitor.cc
@@ -25,7 +25,6 @@ namespace local {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-// bool StatsVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
 bool StatsVisitor::visitDatabase(const Catalogue& catalogue) {
 
     EntryVisitor::visitDatabase(catalogue);
diff --git a/src/fdb5/api/local/StatsVisitor.h b/src/fdb5/api/local/StatsVisitor.h
index 6971238bb..e62dbee20 100644
--- a/src/fdb5/api/local/StatsVisitor.h
+++ b/src/fdb5/api/local/StatsVisitor.h
@@ -38,7 +38,6 @@ class StatsVisitor : public QueryVisitor {
 
     using QueryVisitor::QueryVisitor;
 
-//    bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
     bool visitDatabase(const Catalogue& catalogue) override;
     bool visitIndex(const Index& index) override;
     void catalogueComplete(const Catalogue& catalogue) override;
diff --git a/src/fdb5/api/local/WipeVisitor.cc b/src/fdb5/api/local/WipeVisitor.cc
index d290135e0..4e651e181 100644
--- a/src/fdb5/api/local/WipeVisitor.cc
+++ b/src/fdb5/api/local/WipeVisitor.cc
@@ -47,7 +47,6 @@ WipeVisitor::WipeVisitor(eckit::Queue& queue,
     unsafeWipeAll_(unsafeWipeAll) {}
 
 
-//bool WipeVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
 bool WipeVisitor::visitDatabase(const Catalogue& catalogue) {
 
     // If the DB is locked for wiping, then it "doesn't exist"
@@ -55,12 +54,10 @@ bool WipeVisitor::visitDatabase(const Catalogue& catalogue) {
         return false;
     }
 
-    // EntryVisitor::visitDatabase(catalogue, store);
     EntryVisitor::visitDatabase(catalogue);
 
     ASSERT(!internalVisitor_);
     internalVisitor_.reset(catalogue.wipeVisitor(store(), request_, out_, doit_, porcelain_, unsafeWipeAll_));
-    // internalVisitor_->visitDatabase(catalogue, store);
     internalVisitor_->visitDatabase(catalogue);
 
     return true; // Explore contained indexes
diff --git a/src/fdb5/api/local/WipeVisitor.h b/src/fdb5/api/local/WipeVisitor.h
index 6ac93dc06..7ef6f83c4 100644
--- a/src/fdb5/api/local/WipeVisitor.h
+++ b/src/fdb5/api/local/WipeVisitor.h
@@ -46,8 +46,6 @@ class WipeVisitor : public QueryVisitor {
 
     bool visitEntries() override { return false; }
     bool visitIndexes() override;
-
-//    bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
     bool visitDatabase(const Catalogue& catalogue) override;
     bool visitIndex(const Index& index) override;
     void catalogueComplete(const Catalogue& catalogue) override;
diff --git a/src/fdb5/config/Config.cc b/src/fdb5/config/Config.cc
index 42e198608..9644db66a 100644
--- a/src/fdb5/config/Config.cc
+++ b/src/fdb5/config/Config.cc
@@ -44,7 +44,6 @@ Config Config::make(const eckit::PathName& path, const eckit::Configuration& use
 
 Config::Config(const Configuration& config, const eckit::Configuration& userConfig) :
     LocalConfiguration(config), schemaPathInitialised_(false) {
-    // initializeSchemaPath();
     userConfig_ = std::make_shared(userConfig);
 }
 
diff --git a/src/fdb5/daos/DaosCatalogue.h b/src/fdb5/daos/DaosCatalogue.h
index b882fa350..cc5faec4a 100644
--- a/src/fdb5/daos/DaosCatalogue.h
+++ b/src/fdb5/daos/DaosCatalogue.h
@@ -32,8 +32,6 @@ class DaosCatalogue : public CatalogueImpl, public DaosCommon {
     DaosCatalogue(const Key& key, const fdb5::Config& config);
     DaosCatalogue(const eckit::URI& uri, const ControlIdentifiers& controlIdentifiers, const fdb5::Config& config);
 
-    // static const char* catalogueTypeName() { return fdb5::DaosEngine::typeName(); }
-    
     eckit::URI uri() const override;
     const Key& indexKey() const override { return currentIndexKey_; }
 
diff --git a/src/fdb5/daos/DaosEngine.h b/src/fdb5/daos/DaosEngine.h
index 09865fa12..98510aaab 100644
--- a/src/fdb5/daos/DaosEngine.h
+++ b/src/fdb5/daos/DaosEngine.h
@@ -44,8 +44,6 @@ class DaosEngine : public fdb5::Engine {
     std::vector visitableLocations(const Key& key, const Config& config) const override;
     std::vector visitableLocations(const metkit::mars::MarsRequest& rq, const Config& config) const override;
 
-    // std::vector writableLocations(const Key& key, const Config& config) const override { NOTIMP; };
-
     void print( std::ostream &out ) const override { NOTIMP; };
 
 private: // methods
diff --git a/src/fdb5/database/Archiver.h b/src/fdb5/database/Archiver.h
index b58c761a8..798986776 100644
--- a/src/fdb5/database/Archiver.h
+++ b/src/fdb5/database/Archiver.h
@@ -51,8 +51,6 @@ class Archiver : public eckit::NonCopyable {
 
     virtual ~Archiver();
 
-    // uint32_t id() const { return id_; }
-
     void archive(const Key& key, BaseArchiveVisitor& visitor);
     void archive(const Key& key, const void* data, size_t len);
 
diff --git a/src/fdb5/database/BaseArchiveVisitor.cc b/src/fdb5/database/BaseArchiveVisitor.cc
index b083c25ee..9be4c2096 100644
--- a/src/fdb5/database/BaseArchiveVisitor.cc
+++ b/src/fdb5/database/BaseArchiveVisitor.cc
@@ -34,7 +34,6 @@ bool BaseArchiveVisitor::selectDatabase(const Key& dbKey, const TypedKey& fullCo
 }
 
 bool BaseArchiveVisitor::selectIndex(const Key& idxKey, const TypedKey& fullComputedKey) {
-    // eckit::Log::info() << "selectIndex " << idxKey << std::endl;
     return catalogue()->selectIndex(idxKey);
 }
 
diff --git a/src/fdb5/database/Catalogue.cc b/src/fdb5/database/Catalogue.cc
index e51a2a6fb..573fe6c87 100644
--- a/src/fdb5/database/Catalogue.cc
+++ b/src/fdb5/database/Catalogue.cc
@@ -34,7 +34,7 @@ std::unique_ptr CatalogueImpl::buildStore() const {
     return StoreFactory::instance().build(key(), config_);
 }
 
-void Catalogue::visitEntries(EntryVisitor& visitor /*, const Store& store*/, bool sorted) {
+void Catalogue::visitEntries(EntryVisitor& visitor, bool sorted) {
 
     std::vector all = indexes(sorted);
 
diff --git a/src/fdb5/database/Catalogue.h b/src/fdb5/database/Catalogue.h
index e15a2c9cb..12ec5177a 100644
--- a/src/fdb5/database/Catalogue.h
+++ b/src/fdb5/database/Catalogue.h
@@ -136,8 +136,6 @@ class CatalogueReader : virtual public Catalogue {
 
 public:
 
-    // CatalogueReader(const Key& key, ControlIdentifiers controlIdentifiers, const fdb5::Config& config)
-    //     : Catalogue(key, controlIdentifiers, config) {}
     CatalogueReader() {}
     
     virtual ~CatalogueReader() {}
@@ -153,9 +151,6 @@ class CatalogueWriter : virtual public Catalogue  {
 public:
 
     CatalogueWriter() {}
-    // CatalogueWriter(const Key& key, ControlIdentifiers controlIdentifiers, const fdb5::Config& config)
-    //     : Catalogue(key, controlIdentifiers, config) {}
-    
     virtual ~CatalogueWriter() {}
 
     virtual const Index& currentIndex() = 0;
diff --git a/src/fdb5/database/Engine.h b/src/fdb5/database/Engine.h
index c7dc6dcf1..193b4a90b 100644
--- a/src/fdb5/database/Engine.h
+++ b/src/fdb5/database/Engine.h
@@ -25,8 +25,6 @@
 #include "eckit/memory/NonCopyable.h"
 #include "eckit/filesystem/URI.h"
 
-// #include "fdb5/database/DB.h"
-
 
 namespace fdb5 {
 
@@ -55,16 +53,10 @@ class Engine : private eckit::NonCopyable {
     /// Uniquely selects a location where the Key will be put or already exists
     virtual eckit::URI location(const Key& key, const Config& config) const = 0;
 
-    // /// Lists the roots that can be visited given a DB key
-    // virtual std::vector allLocations(const Key& key, const Config& config) const = 0;
-
     /// Lists the roots that can be visited given a DB key
     virtual std::vector visitableLocations(const Key& key, const Config& config) const = 0;
     virtual std::vector visitableLocations(const metkit::mars::MarsRequest& rq, const Config& config) const = 0;
 
-    /// Lists the roots where a DB key would be able to be written
-//    virtual std::vector writableLocations(const Key& key, const Config& config) const = 0;
-
     friend std::ostream &operator<<(std::ostream &s, const Engine& x);
 
 protected: // methods
diff --git a/src/fdb5/database/Index.cc b/src/fdb5/database/Index.cc
index 1219d384b..a08db81a6 100755
--- a/src/fdb5/database/Index.cc
+++ b/src/fdb5/database/Index.cc
@@ -122,7 +122,7 @@ void IndexBase::encodeLegacy(eckit::Stream& s, const int version) const {
 
     axes_.encode(s, version);
     s << key_;
-    s << ""; // key_.valuesToString();    we no longer write this field, required in the previous index format
+    s << ""; // we no longer write this field, required in the previous index format
     s << type_;
 }
 
diff --git a/src/fdb5/database/Manager.cc b/src/fdb5/database/Manager.cc
index 9f9d51090..3d888bf84 100644
--- a/src/fdb5/database/Manager.cc
+++ b/src/fdb5/database/Manager.cc
@@ -269,31 +269,6 @@ std::string Manager::engine(const URI& uri)
     throw eckit::BadParameter(oss.str(), Here());
 }
 
-// eckit::URI Manager::location(const Key& key) {
-
-//     const std::string& name = Manager::engine(key);
-
-//     return Engine::backend(name).location(key, config_);
-// }
-
-// std::vector Manager::allLocations(const Key& key)
-// {
-//     std::set engines = Manager::engines(key);
-
-//     LOG_DEBUG_LIB(LibFdb5) << "Matching engines for key " << key << " -> " << engines << std::endl;
-
-//     std::vector r; // union of all locations
-
-//     for(std::set::const_iterator i = engines.begin(); i != engines.end(); ++i) {
-//         LOG_DEBUG_LIB(LibFdb5) << "Selected FDB engine " << *i << std::endl;
-//         std::vector p = Engine::backend(*i).allLocations(key, config_);
-//         r.insert(r.end(), p.begin(), p.end());
-//     }
-
-//     return r;
-// }
-
-
 std::vector Manager::visitableLocations(const metkit::mars::MarsRequest& rq, bool all) {
 
     std::set engines = Manager::engines(rq, all);
@@ -318,23 +293,6 @@ std::vector Manager::visitableLocations(const metkit::mars::MarsRequ
 
 }
 
-// std::vector Manager::writableLocations(const Key& key) {
-
-//     std::set engines = Manager::engines(key);
-
-//     LOG_DEBUG_LIB(LibFdb5) << "Matching engines for key " << key << " -> " << engines << std::endl;
-
-//     std::vector r; // union of all locations
-
-//     for(std::set::const_iterator i = engines.begin(); i != engines.end(); ++i) {
-//         LOG_DEBUG_LIB(LibFdb5) << "Selected FDB engine " << *i << std::endl;
-//         std::vector p = Engine::backend(*i).writableLocations(key, config_);
-//         r.insert(r.end(), p.begin(), p.end());
-//     }
-
-//     return r;
-// }
-
 //----------------------------------------------------------------------------------------------------------------------
 
 }  // namespace fdb5
diff --git a/src/fdb5/database/Manager.h b/src/fdb5/database/Manager.h
index 40c392676..dc7d86c59 100644
--- a/src/fdb5/database/Manager.h
+++ b/src/fdb5/database/Manager.h
@@ -47,18 +47,9 @@ class Manager  {
     /// Uniquely selects the engine that will handle this URI by checking possible handlers
     std::string engine(const eckit::URI& uri);
 
-    /// Uniquely selects a location where the Key will be put or already exists
-    // eckit::URI location(const Key &key);
-
-    // /// Lists the roots that can be visited given a DB key
-    // std::vector allLocations(const Key& key);
-
     /// Lists the roots that can be visited given a DB key
     std::vector visitableLocations(const metkit::mars::MarsRequest& request, bool all);
 
-    // /// Lists the roots where a DB key would be able to be written
-    // std::vector writableLocations(const Key& key);
-
 private: // members
 
     eckit::PathName enginesFile_;
diff --git a/src/fdb5/database/ReadVisitor.cc b/src/fdb5/database/ReadVisitor.cc
index dfd4f4a4a..93281a7e6 100644
--- a/src/fdb5/database/ReadVisitor.cc
+++ b/src/fdb5/database/ReadVisitor.cc
@@ -7,27 +7,4 @@
  * granted to it by virtue of its status as an intergovernmental organisation nor
  * does it submit to any jurisdiction.
  */
-
-// #include "eckit/log/Log.h"
-
 #include "fdb5/database/ReadVisitor.h"
-#include "fdb5/database/Store.h"
-
-
-namespace fdb5 {
-
-//----------------------------------------------------------------------------------------------------------------------
-
-
-// CatalogueReader* ReadVisitor::reader() const {
-//     if (catalogue_) {
-//         CatalogueReader* catalogueReader = dynamic_cast(catalogue_);
-//         ASSERT(catalogueReader);
-//         return catalogueReader;
-//     }
-//     return nullptr;
-// }
-
-//----------------------------------------------------------------------------------------------------------------------
-
-} // namespace fdb5
diff --git a/src/fdb5/database/Store.cc b/src/fdb5/database/Store.cc
index 1b9615047..2c18203ef 100644
--- a/src/fdb5/database/Store.cc
+++ b/src/fdb5/database/Store.cc
@@ -100,26 +100,6 @@ std::unique_ptr StoreFactory::build(const Key& key, const Config& config)
     return (*j).second->make(key, config);
 }
 
-// std::unique_ptr StoreFactory::build(const eckit::URI& uri, const Config& config) {
-//     std::string name = uri.scheme();
-//     std::string nameLowercase = eckit::StringTools::lower(name);
-
-//     eckit::AutoLock lock(mutex_);
-//     auto j = builders_.find(nameLowercase);
-
-//     LOG_DEBUG_LIB(LibFdb5) << "Looking for StoreBuilder [" << nameLowercase << "]" << std::endl;
-
-//     if (j == builders_.end()) {
-//         eckit::Log::error() << "No StoreBuilder for [" << nameLowercase << "]" << std::endl;
-//         eckit::Log::error() << "StoreBuilders are:" << std::endl;
-//         for (j = builders_.begin(); j != builders_.end(); ++j)
-//             eckit::Log::error() << "   " << (*j).first << std::endl;
-//         throw eckit::SeriousBug(std::string("No StoreBuilder called ") + nameLowercase);
-//     }
-
-//     return (*j).second->make(uri, config);
-// }
-
 //----------------------------------------------------------------------------------------------------------------------
 
 StoreBuilderBase::StoreBuilderBase(const std::string& name) : name_(name) {
diff --git a/src/fdb5/database/Store.h b/src/fdb5/database/Store.h
index 9688703b8..2f7e7576f 100644
--- a/src/fdb5/database/Store.h
+++ b/src/fdb5/database/Store.h
@@ -66,10 +66,6 @@ class Store {
 
     virtual std::vector getAuxiliaryURIs(const eckit::URI&) const { NOTIMP; }
     virtual bool auxiliaryURIExists(const eckit::URI&) const { NOTIMP; }
-
-// TODO can we avoid using the schema here?
-// protected: // members
-//     const Schema& schema_;  //<< schema is owned by catalogue which always outlives the store
 };
 
 
diff --git a/src/fdb5/rados/RadosFieldLocation.cc b/src/fdb5/rados/RadosFieldLocation.cc
index 93bf21400..cf37be670 100644
--- a/src/fdb5/rados/RadosFieldLocation.cc
+++ b/src/fdb5/rados/RadosFieldLocation.cc
@@ -20,8 +20,6 @@ ::eckit::Reanimator RadosFieldLocation::reanimator_;
 
 //----------------------------------------------------------------------------------------------------------------------
 
-//RadosFieldLocation::RadosFieldLocation() {}
-
 RadosFieldLocation::RadosFieldLocation(const eckit::PathName path, eckit::Offset offset, eckit::Length length ) :
         FieldLocation(eckit::URI("rados", path), offset, length) {}
 
@@ -35,9 +33,6 @@ RadosFieldLocation::RadosFieldLocation(const eckit::URI &uri, eckit::Offset offs
 RadosFieldLocation::RadosFieldLocation(const RadosFieldLocation& rhs) :
     FieldLocation(rhs.uri_) {}
 
-// RadosFieldLocation::RadosFieldLocation(const FileStore &store, const FieldRef &ref) :
-//     FieldLocation(store.get(ref.pathId()), ref.offset(), ref.length()) {}
-
 RadosFieldLocation::RadosFieldLocation(eckit::Stream& s) :
     FieldLocation(s) {}
 
diff --git a/src/fdb5/rados/RadosFieldLocation.h b/src/fdb5/rados/RadosFieldLocation.h
index 77553dbbd..6d04ced4d 100644
--- a/src/fdb5/rados/RadosFieldLocation.h
+++ b/src/fdb5/rados/RadosFieldLocation.h
@@ -20,7 +20,6 @@
 
 #include "fdb5/database/FieldLocation.h"
 #include "fdb5/database/FileStore.h"
-//#include "fdb5/toc/FieldRef.h"
 
 namespace fdb5 {
 
@@ -29,24 +28,14 @@ namespace fdb5 {
 class RadosFieldLocation : public FieldLocation {
 public:
 
-    //RadosFieldLocation();
     RadosFieldLocation(const RadosFieldLocation& rhs);
     RadosFieldLocation(const eckit::PathName path, eckit::Offset offset, eckit::Length length);
     RadosFieldLocation(const eckit::URI &uri);
     RadosFieldLocation(const eckit::URI &uri, eckit::Offset offset, eckit::Length length);
-//    RadosFieldLocation(const FileStore& store, const FieldRef& ref);
     RadosFieldLocation(eckit::Stream&);
-
-//    const eckit::PathName path() const { return uri_.name(); }
-//    const eckit::Offset&   offset() const { return offset_; }
-
     eckit::DataHandle* dataHandle() const override;
     eckit::DataHandle* dataHandle(const Key& remapKey) const override;
-
-    // eckit::URI uri() const override;
-
     std::shared_ptr make_shared() const override;
-
     void visit(FieldLocationVisitor& visitor) const override;
 
 public: // For Streamable
@@ -56,25 +45,12 @@ class RadosFieldLocation : public FieldLocation {
 protected: // For Streamable
 
     const eckit::ReanimatorBase& reanimator() const override { return reanimator_; }
-    //void encode(eckit::Stream&) const override;
-
     static eckit::ClassSpec                    classSpec_;
     static eckit::Reanimator reanimator_;
 
 private: // methods
-
-//    void dump(std::ostream &out) const override;
-
     void print(std::ostream &out) const override;
-
     eckit::URI uri(const eckit::PathName &path);
-
-private: // members
-
-//    eckit::PathName path_;
-//    eckit::Offset offset_;
-
-    // For streamability
 };
 
 
diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc
index c4704e767..fe0ab6f2b 100644
--- a/src/fdb5/rados/RadosStore.cc
+++ b/src/fdb5/rados/RadosStore.cc
@@ -36,9 +36,6 @@ RadosStore(const Key& key, const Config& config, const eckit::net::Endpoint& con
     NOTIMP;
 }
 
-// RadosStore::RadosStore(const eckit::URI& uri, const Config& config) :
-//     Store(), directory_("mars:"+uri.path().dirName()), archivedFields_(0) {}
-
 eckit::URI RadosStore::uri() const {
     return URI("rados", directory_);
 }
diff --git a/src/fdb5/remote/Connection.cc b/src/fdb5/remote/Connection.cc
index f714b5685..f4468f85f 100644
--- a/src/fdb5/remote/Connection.cc
+++ b/src/fdb5/remote/Connection.cc
@@ -99,15 +99,10 @@ eckit::Buffer Connection::read(bool control, MessageHeader& hdr) {
     return payload;
 }
 
-// void Connection::write(remote::Message msg, bool control, const Handler& clientID, uint32_t requestID, const void* data, uint32_t length) {
-//     write(msg, control, clientID.clientId(), requestID, std::vector>{{data, length}});
-// }
-// void Connection::write(remote::Message msg, bool control, const Handler& clientID, uint32_t requestID, std::vector> data) {
-//     write(msg, control, clientID.clientId(), requestID, data);
-// }
 void Connection::write(remote::Message msg, bool control, uint32_t clientID, uint32_t requestID, const void* data, uint32_t length) {
     write(msg, control, clientID, requestID, std::vector>{{data, length}});
 }
+
 void Connection::write(remote::Message msg, bool control, uint32_t clientID, uint32_t requestID, std::vector> data) {
 
     uint32_t payloadLength = 0;
@@ -128,28 +123,15 @@ void Connection::write(remote::Message msg, bool control, uint32_t clientID, uin
     writeUnsafe(control, &EndMarker, sizeof(EndMarker));
 }
 
-// void Connection::writeControl(remote::Message msg, uint32_t clientID, uint32_t requestID, const void* data, uint32_t length) {
-//     write(msg, true, clientID, requestID, std::vector>{{data, length}});
-// }
-// void Connection::writeControl(remote::Message msg, uint32_t clientID, uint32_t requestID, std::vector> data) {
-//     write(msg, true, clientID, requestID, data);
-// }
-// void Connection::writeData(remote::Message msg, uint32_t clientID, uint32_t requestID, const void* data, uint32_t length) {
-//     write(msg, false, clientID, requestID, std::vector>{{data, length}});
-// }
-// void Connection::writeData(remote::Message msg, uint32_t clientID, uint32_t requestID, std::vector> data) {
-//     write(msg, false, clientID, requestID, data);
-// }
 void Connection::error(const std::string& msg, uint32_t clientID, uint32_t requestID) {
     eckit::Log::error() << "[clientID=" << clientID << ",requestID=" << requestID << "]  " << msg << std::endl;
     write(Message::Error, false, clientID, requestID, std::vector>{{msg.c_str(), msg.length()}});
 }
-// void Connection::error(const std::string& msg, const Handler& clientID, uint32_t requestID) {
-//     write(Message::Error, true, clientID.clientId(), requestID, std::vector>{{msg.c_str(), msg.length()}});
-// }
+
 eckit::Buffer Connection::readControl(MessageHeader& hdr) {
     return read(true, hdr);
 }
+
 eckit::Buffer Connection::readData(MessageHeader& hdr) {
     return read(false, hdr);
 }
diff --git a/src/fdb5/remote/Messages.h b/src/fdb5/remote/Messages.h
index 76566a7a1..ce203b39b 100644
--- a/src/fdb5/remote/Messages.h
+++ b/src/fdb5/remote/Messages.h
@@ -22,7 +22,6 @@
 #include 
 
 #include "eckit/types/FixedString.h"
-// #include "fdb5/remote/Handler.h"
 
 namespace eckit {
     class Stream;
@@ -89,7 +88,6 @@ class MessageHeader {
         requestID(0),
         payloadSize(0) {}
 
-    // MessageHeader(Message message, bool control, const Handler& clientID, uint32_t requestID, uint32_t payloadSize=0);
 
     MessageHeader(Message message, bool control, uint32_t clientID, uint32_t requestID, uint32_t payloadSize);
     
diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc
index 4436b0113..d443e37ea 100644
--- a/src/fdb5/remote/client/Client.cc
+++ b/src/fdb5/remote/client/Client.cc
@@ -13,7 +13,6 @@
 
 #include "fdb5/remote/client/Client.h"
 #include "fdb5/remote/client/ClientConnectionRouter.h"
-// #include 
 
 namespace fdb5::remote {
 
@@ -21,10 +20,7 @@ namespace fdb5::remote {
 
 void Client::setClientID() {
     static std::mutex idMutex_;
-    // static std::random_device rd;
-    // static std::mt19937 mt(rd());
-    // static std::uniform_int_distribution dist(0, 1000);
-    static uint32_t clientId_ = 0; // dist(mt)*1000;
+    static uint32_t clientId_ = 0;
 
     std::lock_guard lock(idMutex_);
     id_ = ++clientId_;
@@ -84,4 +80,4 @@ void Client::dataWrite(remote::Message msg, uint32_t requestID, std::vector lock(clientsMutex_);
-    // auto it = clients_.find(client.id());
-    // ASSERT(it == clients_.end());
-
     clients_[client.id()] = &client;
 }
 
@@ -71,7 +68,6 @@ bool ClientConnection::remove(uint32_t clientID) {
 
     if (clients_.empty()) {
         teardown();
-        // ClientConnectionRouter::instance().deregister(*this);
     }
 
     return clients_.empty();
@@ -94,17 +90,16 @@ uint32_t ClientConnection::generateRequestID() {
 
 bool ClientConnection::connect(bool singleAttempt) {
 
-//    std::lock_guard lock(requestMutex_);
     if (connected_) {
         eckit::Log::warning() << "ClientConnection::connect() called when already connected" << std::endl;
         return connected_;
     }
 
     int fdbMaxConnectRetries = (singleAttempt ? 1 : eckit::Resource("fdbMaxConnectRetries", 3));
-    int fdbConnectTimeout = eckit::Resource("fdbConnectTimeout", (singleAttempt ? 2 : 5)); // 0 = No timeout 
+    int fdbConnectTimeout = eckit::Resource("fdbConnectTimeout", (singleAttempt ? 2 : 5)); // 0 = No timeout
 
     try {
-        // Connect to server, and check that the server is happy on the response 
+        // Connect to server, and check that the server is happy on the response
         LOG_DEBUG_LIB(LibFdb5) << "Connecting to host: " << controlEndpoint_ << std::endl;
         controlClient_.connect(controlEndpoint_, fdbMaxConnectRetries, fdbConnectTimeout);
 
@@ -136,10 +131,6 @@ void ClientConnection::disconnect() {
     std::lock_guard lock(clientsMutex_);
     ASSERT(clients_.empty());
     if (connected_) {
-
-        // if (dataWriteFuture_.valid()) {
-        //     dataWriteFuture_.wait();
-        // }
         if (dataWriteThread_.joinable()) {
             dataWriteThread_.join();
         }
@@ -174,11 +165,6 @@ eckit::LocalConfiguration ClientConnection::availableFunctionality() const {
 // -----------------------------------------------------------------------------------------------------
 
 std::future ClientConnection::controlWrite(Client& client, Message msg, uint32_t requestID, bool dataListener, std::vector> data) {
-
-    // std::lock_guard lock(clientsMutex_);
-    // auto it = clients_.find(client.clientId());
-    // ASSERT(it != clients_.end());
-
     std::future f;
     {
         std::lock_guard lock(promisesMutex_);
@@ -201,7 +187,7 @@ void ClientConnection::dataWrite(Client& client, remote::Message msg, uint32_t r
         // retrieve or add client to the list
         std::lock_guard lock(clientsMutex_);
         auto it = clients_.find(client.clientId());
-        ASSERT(it != clients_.end());        
+        ASSERT(it != clients_.end());
     }
     {
         std::lock_guard lock(dataWriteMutex_);
@@ -211,7 +197,6 @@ void ClientConnection::dataWrite(Client& client, remote::Message msg, uint32_t r
 
             dataWriteQueue_.reset(new eckit::Queue{maxQueueLength});
             dataWriteThread_ = std::thread([this] { dataWriteThreadLoop(); });
-            // dataWriteFuture_ = std::async(std::launch::async, [this] { return dataWriteThreadLoop(); });
         }
     }
     uint32_t payloadLength = 0;
@@ -255,20 +240,6 @@ void ClientConnection::dataWriteThreadLoop() {
     // They will be released when flush() is called.
 }
 
-// void ClientConnection::handleError(const MessageHeader& hdr, eckit::Buffer buffer) {
-//     ASSERT(hdr.marker == StartMarker);
-//     ASSERT(hdr.version == CurrentVersion);
-
-//     if (hdr.message == Message::Error) {
-//         ASSERT(hdr.payloadSize > 9);
-//         std::string what(buffer.size()+1, ' ');
-//         buffer.copy(what.c_str(), buffer.size());
-//         what[buffer.size()] = 0; // Just in case
-
-//         throw RemoteFDBException(what, controlEndpoint_);
-//     }
-// }
-
 void ClientConnection::writeControlStartupMessage() {
 
     eckit::Buffer payload(4096);
@@ -413,7 +384,6 @@ void ClientConnection::listeningControlThreadLoop() {
     } catch (...) {
        ClientConnectionRouter::instance().teardown(std::current_exception());
     }
-    // ClientConnectionRouter::instance().deregister(*this);
 }
 
 void ClientConnection::listeningDataThreadLoop() {
diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h
index 790b4295f..f24cb8112 100644
--- a/src/fdb5/remote/client/ClientConnection.h
+++ b/src/fdb5/remote/client/ClientConnection.h
@@ -78,7 +78,7 @@ class ClientConnection : protected Connection {
 
 private: // members
 
-    eckit::SessionID sessionID_; 
+    eckit::SessionID sessionID_;
 
     eckit::net::Endpoint controlEndpoint_;
     eckit::net::Endpoint dataEndpoint_;
@@ -93,16 +93,16 @@ class ClientConnection : protected Connection {
 
     std::thread listeningControlThread_;
     std::thread listeningDataThread_;
-    
+
     std::mutex requestMutex_;
 
     // requestID
     std::mutex idMutex_;
     uint32_t id_;
 
-    bool connected_; 
-    bool controlStopping_; 
-    bool dataStopping_; 
+    bool connected_;
+    bool controlStopping_;
+    bool dataStopping_;
 
     std::mutex promisesMutex_;
     std::map> promises_;
@@ -114,4 +114,4 @@ class ClientConnection : protected Connection {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-}  // namespace fdb5::remote
\ No newline at end of file
+}  // namespace fdb5::remote
diff --git a/src/fdb5/remote/client/ClientConnectionRouter.cc b/src/fdb5/remote/client/ClientConnectionRouter.cc
index 3942b97b2..68d8c5a45 100644
--- a/src/fdb5/remote/client/ClientConnectionRouter.cc
+++ b/src/fdb5/remote/client/ClientConnectionRouter.cc
@@ -1,13 +1,13 @@
 #include "fdb5/remote/client/ClientConnectionRouter.h"
 
 namespace{
-    
+
 class ConnectionError : public eckit::Exception {
 public:
     ConnectionError();
     ConnectionError(const eckit::net::Endpoint&);
 
-    bool retryOnClient() const override { return true; } 
+    bool retryOnClient() const override { return true; }
 };
 
 ConnectionError::ConnectionError() {
@@ -84,7 +84,6 @@ ClientConnection& ClientConnectionRouter::connection(const std::vector lock(connectionMutex_);
     auto it = connections_.find(connection.controlEndpoint());
     if (it != connections_.end()) {
         connections_.erase(it);
diff --git a/src/fdb5/remote/client/RemoteCatalogue.cc b/src/fdb5/remote/client/RemoteCatalogue.cc
index 1af996920..a613b32bf 100644
--- a/src/fdb5/remote/client/RemoteCatalogue.cc
+++ b/src/fdb5/remote/client/RemoteCatalogue.cc
@@ -144,12 +144,6 @@ bool RemoteCatalogue::handle(Message message, bool control, uint32_t requestID)
 }
 bool RemoteCatalogue::handle(Message message, bool control, uint32_t requestID, eckit::Buffer&& payload) {
     LOG_DEBUG_LIB(LibFdb5) << *this << " - Received [message=" << ((uint) message) << ",requestID=" << requestID << ",payloadSize=" << payload.size() << "]" << std::endl;
-    // if (message == Message::Schema) {
-    //     LOG_DEBUG_LIB(LibFdb5) << "RemoteCatalogue::handle received payload size: " << payload.size() << std::endl;
-    //     MemoryStream s(payload);
-    //     schema_ = std::unique_ptr(eckit::Reanimator::reanimate(s));
-    //     return true;
-    // }
     return false;
 }
 
@@ -181,4 +175,4 @@ bool RemoteCatalogue::open() {
 
 static CatalogueReaderBuilder reader("remote");
 static CatalogueWriterBuilder writer("remote");
-} // namespace fdb5::remote
\ No newline at end of file
+} // namespace fdb5::remote
diff --git a/src/fdb5/remote/client/RemoteCatalogue.h b/src/fdb5/remote/client/RemoteCatalogue.h
index 07df9a781..d7a93db29 100644
--- a/src/fdb5/remote/client/RemoteCatalogue.h
+++ b/src/fdb5/remote/client/RemoteCatalogue.h
@@ -39,7 +39,7 @@ class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public C
     const Schema& schema() const override;
 
     std::vector metadataPaths() const override;
-    void visitEntries(EntryVisitor& visitor, /*const Store& store,*/ bool sorted = false) override;
+    void visitEntries(EntryVisitor& visitor, bool sorted = false) override;
     void dump(std::ostream& out, bool simple=false, const eckit::Configuration& conf = eckit::LocalConfiguration()) const override;
     StatsReportVisitor* statsReportVisitor() const override;
     PurgeVisitor* purgeVisitor(const Store& store) const override;
@@ -61,8 +61,6 @@ class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public C
 
 protected:
 
-    // FDBStats archivalCompleted();
-
     void loadSchema() override;
 
 private:
diff --git a/src/fdb5/remote/client/RemoteStore.cc b/src/fdb5/remote/client/RemoteStore.cc
index 24a41febb..2c0f4d285 100644
--- a/src/fdb5/remote/client/RemoteStore.cc
+++ b/src/fdb5/remote/client/RemoteStore.cc
@@ -47,7 +47,7 @@ namespace fdb5::remote {
 ///       in the stream
 ///
 /// --> Retrieve is a _streaming_ service.
-    
+
 
 class FDBRemoteDataHandle : public DataHandle {
 
@@ -120,7 +120,7 @@ class FDBRemoteDataHandle : public DataHandle {
 
             // Otherwise return the data!
             std::swap(currentBuffer_, msg.second);
-            
+
             n = bufferRead(p, sz);
             sz -= n;
             total += n;
@@ -197,15 +197,13 @@ std::vector> storeEndpoints(const C
 RemoteStore::RemoteStore(const Key& dbKey, const Config& config) :
     Client(storeEndpoints(config)),
     dbKey_(dbKey), config_(config)
-    // retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
     {}
 
 // this is used only in retrieval, with an URI already referring to an accessible Store
 RemoteStore::RemoteStore(const eckit::URI& uri, const Config& config) :
     Client(eckit::net::Endpoint(uri.hostport()), uri.hostport()),
     dbKey_(Key()), config_(config)
-    // retrieveMessageQueue_(eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200)),
-    { 
+    {
     // no need to set the local_ flag on the read path
     ASSERT(uri.scheme() == "fdb");
 }
@@ -215,8 +213,7 @@ RemoteStore::~RemoteStore() {
     // an error. n.b. if we don't do something, we will block in the destructor
     // of std::future.
     if (!locations_.complete()) {
-    // if (archivalCompleted_.valid() || !locations_.empty()) {
-        Log::error() << "Attempting to destruct RemoteStore with active archival" << std::endl; //  - location to receive: " << locations_.size() << " archivalCompleted_.valid() " << archivalCompleted_.valid() << std::endl;
+        Log::error() << "Attempting to destruct RemoteStore with active archival" << std::endl;
         eckit::Main::instance().terminate();
     }
 }
@@ -226,7 +223,7 @@ eckit::URI RemoteStore::uri() const {
 }
 
 bool RemoteStore::exists() const {
-    return true; // directory_.exists();
+    return true;
 }
 
 eckit::DataHandle* RemoteStore::retrieve(Field& field) const {
@@ -294,11 +291,10 @@ size_t RemoteStore::flush() {
 }
 
 void RemoteStore::close() {
-//    disconnect();
 }
 
 void RemoteStore::remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const {
-    NOTIMP;    
+    NOTIMP;
 }
 
 void RemoteStore::remove(const Key& key) const {
@@ -306,12 +302,12 @@ void RemoteStore::remove(const Key& key) const {
 }
 
 void RemoteStore::print(std::ostream &out) const {
-    out << "RemoteStore(host=" << controlEndpoint() /* << ", data=" << dataEndpoint() */ << ")";
+    out << "RemoteStore(host=" << controlEndpoint() << ")";
 }
 
 bool RemoteStore::handle(Message message, bool control, uint32_t requestID) {
 
-    switch (message) {  
+    switch (message) {
         case Message::Complete: {
             auto it = messageQueues_.find(requestID);
             if (it != messageQueues_.end()) {
@@ -353,7 +349,7 @@ bool RemoteStore::handle(Message message, bool control, uint32_t requestID) {
 bool RemoteStore::handle(Message message, bool control, uint32_t requestID, eckit::Buffer&& payload) {
 
     switch (message) {
-    
+
         case Message::Store: { // received a Field location from the remote store, can forward to the archiver for the indexing
             MemoryStream s(payload);
             std::unique_ptr location(eckit::Reanimator::reanimate(s));
@@ -389,13 +385,6 @@ bool RemoteStore::handle(Message message, bool control, uint32_t requestID, ecki
                 // goes out of scope in the worker thread).
                 messageQueues_.erase(it);
 
-            } else {
-//                 auto it = locations_.find(requestID);
-//                 if (it != locations_.end()) {
-// //                    archiver_->error(std::move(payload), controlEndpoint());
-//                 } else {
-//                     // retrieveMessageQueue_.emplace(message, std::move(payload));
-//                 }
             }
             return true;
         }
@@ -417,11 +406,7 @@ eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation, c
 
     uint32_t id = generateRequestID();
 
-    static size_t queueSize = 320; // eckit::Resource("fdbRemoteRetrieveQueueLength;$FDB_REMOTE_RETRIEVE_QUEUE_LENGTH", 200);
-                // auto id = retrieveMessageQueues_.find(requestID);
-                // ASSERT (it != retrieveMessageQueues_.end());
-                // it->second->emplace(std::make_pair(message, std::move(payload)));
-
+    static size_t queueSize = 320;
     std::shared_ptr queue = nullptr;
     {
         std::lock_guard lock(retrieveMessageMutex_);
diff --git a/src/fdb5/remote/client/RemoteStore.h b/src/fdb5/remote/client/RemoteStore.h
index 16579d56e..8bad69941 100644
--- a/src/fdb5/remote/client/RemoteStore.h
+++ b/src/fdb5/remote/client/RemoteStore.h
@@ -22,8 +22,6 @@
 
 namespace fdb5::remote {
 
-// class RemoteStoreArchiver;
-
 class Locations {
 
 public:
diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc
index a407882c0..79995845c 100644
--- a/src/fdb5/remote/server/CatalogueHandler.cc
+++ b/src/fdb5/remote/server/CatalogueHandler.cc
@@ -63,10 +63,6 @@ Handled CatalogueHandler::handleControl(Message message, uint32_t clientID, uint
                 archiver();
                 return Handled::YesAddArchiveListener;
 
-            // case Message::Flush: // notification that the client has sent all data locations for archival
-            //     flush(clientID, requestID, eckit::Buffer{0});
-            //     return Handled::Yes;
-
             default: {
                 std::stringstream ss;
                 ss << "ERROR: Unexpected message recieved (" << message << "). ABORTING";
@@ -134,57 +130,6 @@ Handled CatalogueHandler::handleControl(Message message, uint32_t clientID, uint
     return Handled::No;
 }
 
-// Handled CatalogueHandler::handleData(Message message, uint32_t clientID, uint32_t requestID) {
-//     try {
-//         switch (message) {
-//             case Message::Flush: // notification that the client has sent all data locations for archival
-//                 return Handled::YesRemoveArchiveListener; // ????
-//
-//             default: {
-//                 std::stringstream ss;
-//                 ss << "ERROR: Unexpected message recieved (" << message << "). ABORTING";
-//                 Log::status() << ss.str() << std::endl;
-//                 Log::error() << ss.str() << std::endl;
-//                 throw SeriousBug(ss.str(), Here());
-//             }
-//         }
-//     }
-//     catch (std::exception& e) {
-//         // n.b. more general than eckit::Exception
-//         error(e.what(), clientID, requestID);
-//     }
-//     catch (...) {
-//         error("Caught unexpected and unknown error", clientID, requestID);
-//     }
-//     return Handled::No;
-// }
-
-// Handled CatalogueHandler::handleData(Message message, uint32_t clientID, uint32_t requestID, /*eckit::net::Endpoint endpoint,*/ eckit::Buffer&& payload) {
-//     try {
-//         switch (message) {
-//             case Message::Blob:
-//             case Message::MultiBlob:
-//                 queue(message, clientID, requestID, std::move(payload));
-//                 return Handled::Yes;
-//
-//             default: {
-//                 std::stringstream ss;
-//                 ss << "ERROR: Unexpected message recieved (" << message << "). ABORTING";
-//                 Log::status() << ss.str() << std::endl;
-//                 Log::error() << ss.str() << std::endl;
-//                 throw SeriousBug(ss.str(), Here());
-//             }
-//         }
-//     }
-//     catch (std::exception& e) {
-//         // n.b. more general than eckit::Exception
-//         error(e.what(), clientID, requestID);
-//     }
-//     catch (...) {
-//         error("Caught unexpected and unknown error", clientID, requestID);
-//     }
-//     return Handled::No;
-// }
 
 // API forwarding logic, adapted from original remoteHandler
 // Used for Inspect and List
@@ -321,7 +266,7 @@ void CatalogueHandler::schema(uint32_t clientID, uint32_t requestID, eckit::Buff
         // 1. Read dbkey to select catalogue
         MemoryStream s(payload);
         Key dbKey(s);
-    
+
         // 2. Get catalogue
         Catalogue& cat = catalogue(clientID, dbKey);
         const Schema& schema = cat.schema();
@@ -402,7 +347,7 @@ void CatalogueHandler::stores(uint32_t clientID, uint32_t requestID) {
 
 
 void CatalogueHandler::flush(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) {
-        
+
     ASSERT(payload.size() > 0);
 
     size_t numArchived = 0;
@@ -474,7 +419,7 @@ void CatalogueHandler::archiveBlob(const uint32_t clientID, const uint32_t reque
 }
 
 bool CatalogueHandler::remove(bool control, uint32_t clientID) {
-    
+
     std::lock_guard lock(handlerMutex_);
 
     // is the client an FDB
@@ -513,7 +458,6 @@ CatalogueWriter& CatalogueHandler::catalogue(uint32_t id, const Key& dbKey) {
         numDataConnection_++;
     }
     return *((catalogues_.emplace(id, CatalogueArchiver(!single_, dbKey, config_)).first)->second.catalogue);
-    // return *((catalogues_[id] = std::make_pair(single_ ? 1 : 2, CatalogueArchiver(dbKey, config_))).second.catalogue);
 }
 
 }  // namespace fdb5::remote
diff --git a/src/fdb5/remote/server/CatalogueHandler.h b/src/fdb5/remote/server/CatalogueHandler.h
index e78a0b582..f63055b3a 100644
--- a/src/fdb5/remote/server/CatalogueHandler.h
+++ b/src/fdb5/remote/server/CatalogueHandler.h
@@ -60,7 +60,6 @@ class CatalogueHandler : public ServerConnection {
     void archiveBlob(const uint32_t clientID, const uint32_t requestID, const void* data, size_t length) override;
 
     bool remove(bool control, uint32_t clientID) override;
-    // bool handlers() override;
 
     CatalogueWriter& catalogue(uint32_t catalogueID, const Key& dbKey);
 
@@ -73,7 +72,6 @@ class CatalogueHandler : public ServerConnection {
     std::mutex fdbMutex_;
     std::mutex fieldLocationsMutex_;
 
-    // uint32_t fdbId_;
     bool fdbControlConnection_;
     bool fdbDataConnection_;
 };
diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc
index 770b7ba0c..8eaf0c5fc 100644
--- a/src/fdb5/remote/server/ServerConnection.cc
+++ b/src/fdb5/remote/server/ServerConnection.cc
@@ -404,7 +404,6 @@ void ServerConnection::listeningThreadLoopData() {
                         }
                     case Handled::Replied: // nothing to do
                     case Handled::Yes:
-        //                write(Message::Received, false, hdr.clientID(), hdr.requestID);
                         break;
                     case Handled::No:
                     default:
@@ -436,8 +435,6 @@ void ServerConnection::handle() {
 
     // listen loop
     while (true) {
-        //tidyWorkers();
-
         eckit::Buffer payload = readControl(hdr); // READ CONTROL
         LOG_DEBUG_LIB(LibFdb5) << "ServerConnection::handle - got [message=" << hdr.message << ",clientID="<< hdr.clientID() << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl;
 
diff --git a/src/fdb5/rules/Matcher.cc b/src/fdb5/rules/Matcher.cc
index 7ed3d492a..15a56ef76 100644
--- a/src/fdb5/rules/Matcher.cc
+++ b/src/fdb5/rules/Matcher.cc
@@ -21,9 +21,6 @@ namespace fdb5 {
 
 eckit::ClassSpec Matcher::classSpec_ = { &eckit::Streamable::classSpec(), "Matcher", };
 
-//eckit::Reanimator Matcher::reanimator_;
-
-
 Matcher::Matcher() {
 }
 
diff --git a/src/fdb5/rules/Matcher.h b/src/fdb5/rules/Matcher.h
index 3628df682..14b6f9627 100644
--- a/src/fdb5/rules/Matcher.h
+++ b/src/fdb5/rules/Matcher.h
@@ -19,7 +19,6 @@
 #include 
 #include 
 
-//#include "eckit/memory/NonCopyable.h"
 #include "eckit/serialisation/Streamable.h"
 
 class MarsTask;
@@ -60,7 +59,6 @@ class Matcher : public eckit::Streamable {
 
     friend std::ostream &operator<<(std::ostream &s, const Matcher &x);
 
-//	const eckit::ReanimatorBase& reanimator() const override { return reanimator_; }
 	static const eckit::ClassSpec&  classSpec() { return classSpec_; }
 
 private: // methods
diff --git a/src/fdb5/rules/Rule.cc b/src/fdb5/rules/Rule.cc
index 9fb2cd9ec..c87baaa2f 100644
--- a/src/fdb5/rules/Rule.cc
+++ b/src/fdb5/rules/Rule.cc
@@ -267,35 +267,6 @@ void Rule::expand(const Key& initialFieldKey, WriteVisitor &visitor, size_t dept
     expand(initialFieldKey, predicates_.begin(), depth, keys, fullComputedKey, visitor);
 }
 
-// void Rule::expandFirstLevel( const Key &dbKey, std::vector::const_iterator cur, Key &result, bool& found) const {
-
-//     if (cur == predicates_.end()) {
-//         found = true;
-//         result.rule(this);
-//         return;
-//     }
-
-//     std::vector::const_iterator next = cur;
-//     ++next;
-
-//     const std::string &keyword = (*cur)->keyword();
-//     const std::string &value = (*cur)->value(dbKey);
-
-//     result.push(keyword, value);
-
-//     if ((*cur)->match(result)) {
-//         expandFirstLevel(dbKey, next, result, found);
-//     }
-
-//     if (!found) {
-//         result.pop(keyword);
-//     }
-// }
-
-// void Rule::expandFirstLevel(const Key &dbKey,  Key &result, bool& found) const {
-//     expandFirstLevel(dbKey, predicates_.begin(), result, found);
-// }
-
 void Rule::expandFirstLevel(const metkit::mars::MarsRequest& rq, std::vector::const_iterator cur, TypedKey& result, bool& found) const {
 
     if (cur == predicates_.end()) {
diff --git a/src/fdb5/toc/AdoptVisitor.cc b/src/fdb5/toc/AdoptVisitor.cc
index 12526f235..386da2f19 100644
--- a/src/fdb5/toc/AdoptVisitor.cc
+++ b/src/fdb5/toc/AdoptVisitor.cc
@@ -31,8 +31,6 @@ AdoptVisitor::AdoptVisitor(Archiver& owner, const Key& initialFieldKey, const Pa
 }
 
 bool AdoptVisitor::selectDatum(const TypedKey& datumKey, const TypedKey& fullComputedKey) {
-
-    // Log::info() << "selectDatum " << key << ", " << full << " " << length_ << std::endl;
     checkMissingKeys(fullComputedKey);
 
     CatalogueWriter* cat = catalogue();
diff --git a/src/fdb5/toc/FileSpace.cc b/src/fdb5/toc/FileSpace.cc
index 5b0b0b2da..fce73de98 100644
--- a/src/fdb5/toc/FileSpace.cc
+++ b/src/fdb5/toc/FileSpace.cc
@@ -50,9 +50,7 @@ TocPath FileSpace::filesystem(const Key& key, const eckit::PathName& db) const {
     }
 
     LOG_DEBUG_LIB(LibFdb5) << "FDB for key " << key << " not found, selecting a root" << std::endl;
-    // LOG_DEBUG_LIB(LibFdb5) << eckit::BackTrace::dump() << std::endl;
     LOG_DEBUG_LIB(LibFdb5) << "FDB for key " << key << " not found, selecting a root" << std::endl;
-    // LOG_DEBUG_LIB(LibFdb5) << eckit::BackTrace::dump() << std::endl;
 
     return TocPath{FileSpaceHandler::lookup(handler_).selectFileSystem(key, *this) / db, ControlIdentifiers{}};
 }
@@ -127,7 +125,6 @@ bool FileSpace::existsDB(const Key& key, const eckit::PathName& db, TocPath& exi
     unsigned count = 0;
     bool found = false;
 
-//    std::vector visitables = visitable();
     std::string matchList;
     for (RootVec::const_iterator i = roots_.begin(); i != roots_.end(); ++i) {
         if (i->enabled(ControlIdentifier::List) && i->exists()) {
diff --git a/src/fdb5/toc/TocCatalogue.cc b/src/fdb5/toc/TocCatalogue.cc
index ea824eb2c..5fa12a5d0 100644
--- a/src/fdb5/toc/TocCatalogue.cc
+++ b/src/fdb5/toc/TocCatalogue.cc
@@ -84,35 +84,9 @@ std::vector TocCatalogue::metadataPaths() const {
     return paths;
 }
 
-// void TocCatalogue::visitEntries(EntryVisitor& visitor, /*const Store& store,*/ bool sorted) {
-
-//     std::vector all = indexes(sorted);
-
-//     // Allow the visitor to selectively reject this DB.
-//     if (visitor.visitDatabase(*this /*, store*/)) {
-//         if (visitor.visitIndexes()) {
-//             for (const Index& idx : all) {
-//                 if (visitor.visitEntries()) {
-//                     idx.entries(visitor); // contains visitIndex
-//                 } else {
-//                     visitor.visitIndex(idx);
-//                 }
-//             }
-//         }
-
-//         visitor.catalogueComplete(*this);
-//     }
-
-// }
-
 void TocCatalogue::loadSchema() {
     Timer timer("TocCatalogue::loadSchema()", Log::debug());
     schema_ = &SchemaRegistry::instance().get(schemaPath());
-    // if (schema_) {
-    //     schema_->load(schemaPath());
-    // } else {
-    //     schema_.reset(new Schema(schemaPath()));
-    // }
 }
 
 StatsReportVisitor* TocCatalogue::statsReportVisitor() const {
diff --git a/src/fdb5/toc/TocHandler.cc b/src/fdb5/toc/TocHandler.cc
index ae978ec37..90947c209 100644
--- a/src/fdb5/toc/TocHandler.cc
+++ b/src/fdb5/toc/TocHandler.cc
@@ -148,7 +148,7 @@ TocHandler::TocHandler(const eckit::PathName& directory, const Config& config) :
     const char* subTocOverride = ::getenv("FDB5_SUB_TOCS");
     if (subTocOverride) {
         useSubToc_ = true;
-    }    
+    }
 }
 
 TocHandler::TocHandler(const eckit::PathName& path, const Key& parentKey, MemoryHandle* cachedToc) :
@@ -1740,7 +1740,7 @@ size_t TocHandler::buildSubTocMaskRecord(TocRecord& r, const eckit::PathName& pa
 
 void TocHandler::control(const ControlAction& action, const ControlIdentifiers& identifiers) const {
 
-    
+
 
     for (ControlIdentifier identifier : identifiers) {
 
diff --git a/src/fdb5/toc/TocHandler.h b/src/fdb5/toc/TocHandler.h
index 9a8128b6e..a933ff39b 100644
--- a/src/fdb5/toc/TocHandler.h
+++ b/src/fdb5/toc/TocHandler.h
@@ -103,9 +103,6 @@ class TocHandler : public TocCommon, private eckit::NonCopyable {
     typedef std::vector< eckit::LocalPathName > TocPaths;
 
 public: // methods
-
-    // TocHandler( const Key& key, const Config& config);
-
     TocHandler( const eckit::PathName &dir, const Config& config);
 
     /// For initialising sub tocs or diagnostic interrogation.
diff --git a/src/fdb5/toc/TocMoveVisitor.cc b/src/fdb5/toc/TocMoveVisitor.cc
index e7e0fad4b..261bf1686 100644
--- a/src/fdb5/toc/TocMoveVisitor.cc
+++ b/src/fdb5/toc/TocMoveVisitor.cc
@@ -49,13 +49,11 @@ TocMoveVisitor::TocMoveVisitor(const TocCatalogue& catalogue,
 
 TocMoveVisitor::~TocMoveVisitor() {}
 
-//bool TocMoveVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
 bool TocMoveVisitor::visitDatabase(const Catalogue& catalogue) {
 
     // Overall checks
     ASSERT(&catalogue_ == &catalogue);
 
-//    MoveVisitor::visitDatabase(catalogue_, store);
     MoveVisitor::visitDatabase(catalogue_);
 
     // TOC specific checks: index files not locked
diff --git a/src/fdb5/toc/TocMoveVisitor.h b/src/fdb5/toc/TocMoveVisitor.h
index 3ff287c4b..ca18725a7 100644
--- a/src/fdb5/toc/TocMoveVisitor.h
+++ b/src/fdb5/toc/TocMoveVisitor.h
@@ -34,8 +34,6 @@ class TocMoveVisitor : public MoveVisitor {
     ~TocMoveVisitor() override;
 
 private: // methods
-
-    // bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
     bool visitDatabase(const Catalogue& catalogue) override;
 
     void move();
diff --git a/src/fdb5/toc/TocPurgeVisitor.cc b/src/fdb5/toc/TocPurgeVisitor.cc
index 2015608b1..8b49e4c9c 100644
--- a/src/fdb5/toc/TocPurgeVisitor.cc
+++ b/src/fdb5/toc/TocPurgeVisitor.cc
@@ -29,7 +29,6 @@ TocPurgeVisitor::TocPurgeVisitor(const TocCatalogue& catalogue, const Store& sto
 
 TocPurgeVisitor::~TocPurgeVisitor() {}
 
-//bool TocPurgeVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
 bool TocPurgeVisitor::visitDatabase(const Catalogue& catalogue) {
 
     std::set> metadata;
@@ -70,7 +69,7 @@ void TocPurgeVisitor::gatherAuxiliaryURIs() {
                 deletable = true;
             }
         }
-        
+
         // Add auxiliary files to the corresponding set
         eckit::URI uri(store_.type(), eckit::PathName(it.first));
         for (const auto& auxURI : store_.getAuxiliaryURIs(uri)) {
diff --git a/src/fdb5/toc/TocPurgeVisitor.h b/src/fdb5/toc/TocPurgeVisitor.h
index 3612f410d..96313be8a 100644
--- a/src/fdb5/toc/TocPurgeVisitor.h
+++ b/src/fdb5/toc/TocPurgeVisitor.h
@@ -32,7 +32,6 @@ class TocPurgeVisitor : public PurgeVisitor, public TocStatsReportVisitor {
     TocPurgeVisitor(const TocCatalogue& catalogue, const Store& store);
     ~TocPurgeVisitor() override;
 
-    // bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
     bool visitDatabase(const Catalogue& catalogue) override;
     void report(std::ostream& out) const override;
     void purge(std::ostream& out, bool porcelain, bool doit) const override;
diff --git a/src/fdb5/toc/TocStats.cc b/src/fdb5/toc/TocStats.cc
index eb3a3aedf..dde1f2692 100644
--- a/src/fdb5/toc/TocStats.cc
+++ b/src/fdb5/toc/TocStats.cc
@@ -218,7 +218,6 @@ TocStatsReportVisitor::TocStatsReportVisitor(const TocCatalogue& catalogue, bool
 
 TocStatsReportVisitor::~TocStatsReportVisitor() {}
 
-//bool TocStatsReportVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
 bool TocStatsReportVisitor::visitDatabase(const Catalogue& catalogue) {
     ASSERT(&catalogue == currentCatalogue_);
     return true;
diff --git a/src/fdb5/toc/TocStats.h b/src/fdb5/toc/TocStats.h
index 55809f059..80700fc61 100644
--- a/src/fdb5/toc/TocStats.h
+++ b/src/fdb5/toc/TocStats.h
@@ -159,7 +159,6 @@ class TocStatsReportVisitor : public virtual StatsReportVisitor {
 
 private: // methods
 
-    // bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
     bool visitDatabase(const Catalogue& catalogue) override;
     void visitDatum(const Field& field, const std::string& keyFingerprint) override;
     void visitDatum(const Field& field, const Key& datumKey) override { NOTIMP; }
diff --git a/src/fdb5/toc/TocWipeVisitor.cc b/src/fdb5/toc/TocWipeVisitor.cc
index 7cb422afb..bf2a13086 100644
--- a/src/fdb5/toc/TocWipeVisitor.cc
+++ b/src/fdb5/toc/TocWipeVisitor.cc
@@ -100,15 +100,12 @@ TocWipeVisitor::TocWipeVisitor(const TocCatalogue& catalogue,
 TocWipeVisitor::~TocWipeVisitor() {}
 
 
-// bool TocWipeVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
 bool TocWipeVisitor::visitDatabase(const Catalogue& catalogue) {
 
     // Overall checks
 
     ASSERT(&catalogue_ == &catalogue);
-//    ASSERT(&store_ == &store);
     ASSERT(catalogue.enabled(ControlIdentifier::Wipe));
-//    WipeVisitor::visitDatabase(catalogue, store);
     WipeVisitor::visitDatabase(catalogue);
 
     // Check that we are in a clean state (i.e. we only visit one DB).
diff --git a/src/fdb5/toc/TocWipeVisitor.h b/src/fdb5/toc/TocWipeVisitor.h
index f3fec1272..c8df9f5d2 100644
--- a/src/fdb5/toc/TocWipeVisitor.h
+++ b/src/fdb5/toc/TocWipeVisitor.h
@@ -37,7 +37,6 @@ class TocWipeVisitor : public WipeVisitor {
 
 private: // methods
 
-    // bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
     bool visitDatabase(const Catalogue& catalogue) override;
     bool visitIndex(const Index& index) override;
     void catalogueComplete(const Catalogue& catalogue) override;
diff --git a/tests/fdb/test_fdb5_service.cc b/tests/fdb/test_fdb5_service.cc
index 34dc335ee..6956d79a5 100644
--- a/tests/fdb/test_fdb5_service.cc
+++ b/tests/fdb/test_fdb5_service.cc
@@ -57,186 +57,186 @@ struct FixtureService {
     StringDict p;
     fdb5::Config config;
 
-	std::vector modelParams_;
+    std::vector modelParams_;
 
     FixtureService() : env("environ")
-	{
+    {
         p["class"]  = "rd";
-		p["stream"] = "oper";
-		p["domain"] = "g";
-		p["expver"] = "0001";
-		p["date"] = "20120911";
-		p["time"] = "0000";
+        p["stream"] = "oper";
+        p["domain"] = "g";
+        p["expver"] = "0001";
+        p["date"] = "20120911";
+        p["time"] = "0000";
         p["type"] = "fc";
 
         modelParams_.push_back( "130.128" );
         modelParams_.push_back( "138.128" );
-	}
+    }
 
     void write_cycle(fdb5::Archiver& fdb, StringDict& p )
-	{
-		Translator str;
-		std::vector::iterator param = modelParams_.begin();
-		for( ; param != modelParams_.end(); ++param )
-		{
-			p["param"] = *param;
-
-			p["levtype"] = "pl";
-
-			for( size_t step = 0; step < 12; ++step )
-			{
-				p["step"] = str(step*3);
-
-				for( size_t level = 0; level < 10; ++level )
-				{
-					p["levelist"] = str(level*100);
-
-					std::ostringstream data;
-					data << "Raining cats and dogs -- "
-						 << " param " << *param
-						 << " step "  << step
-						 << " level " << level
-						 << std::endl;
-					std::string data_str = data.str();
+    {
+        Translator str;
+        std::vector::iterator param = modelParams_.begin();
+        for( ; param != modelParams_.end(); ++param )
+        {
+            p["param"] = *param;
+
+            p["levtype"] = "pl";
+
+            for( size_t step = 0; step < 12; ++step )
+            {
+                p["step"] = str(step*3);
+
+                for( size_t level = 0; level < 10; ++level )
+                {
+                    p["levelist"] = str(level*100);
+
+                    std::ostringstream data;
+                    data << "Raining cats and dogs -- "
+                         << " param " << *param
+                         << " step "  << step
+                         << " level " << level
+                         << std::endl;
+                    std::string data_str = data.str();
 
                     fdb5::Key k{p};
                     ArchiveVisitor visitor(fdb, k, static_cast(data_str.c_str()), data_str.size());
                     fdb.archive(k, visitor);
-				}
-			}
-		}
-	}
+                }
+            }
+        }
+    }
 };
 
 //----------------------------------------------------------------------------------------------------------------------
 
 CASE ( "test_fdb_stepunit_archive" ) {
 
-	std::string data_str = "Raining cats and dogs";
-
-	fdb5::FDB fdb;
-
-	Key key;
-	key.set("class","od");
-	key.set("expver","0001");
-	key.set("type","fc");
-	key.set("stream","oper");
-	key.set("date","20101010");
-	key.set("time","0000");
-	key.set("domain","g");
-	key.set("levtype","sfc");
-	key.set("step","60");
-	key.set("param","130");
-	fdb.archive(key, static_cast(data_str.c_str()), data_str.size());
-	fdb.flush();
-
-	metkit::mars::MarsRequest req = key.request();
-
-	{
-		fdb5::FDBToolRequest r(req);
-		fdb5::ListIterator iter = fdb.list(r, true);
-		fdb5::ListElement el;
-		EXPECT(iter.next(el));
-		EXPECT(el.combinedKey().get("step") == "60");
-		EXPECT(!iter.next(el));
-	}
-
-	key.set("step","2");
-	key.set("stepunits","h");
-	fdb.archive(key, static_cast(data_str.c_str()), data_str.size());
-	fdb.flush();
-
-	req.setValue("step", "2");
-	{
-		fdb5::FDBToolRequest r(req);
-		fdb5::ListIterator iter = fdb.list(r, true);
-		fdb5::ListElement el;
-		EXPECT(iter.next(el));
-		EXPECT(el.combinedKey().get("step") == "2");
-		EXPECT(!iter.next(el));
-	}
-
-	req.setValuesTyped(new metkit::mars::TypeAny("step"), std::vector{"2","60"});
-	{
-		fdb5::FDBToolRequest r(req);
-		fdb5::ListIterator iter = fdb.list(r, true);
-		fdb5::ListElement el;
-		EXPECT(iter.next(el));
-		EXPECT(el.combinedKey().get("step") == "2");
-		EXPECT(iter.next(el));
-		EXPECT(el.combinedKey().get("step") == "60");
-		EXPECT(!iter.next(el));
-	}
-
-	key.set("step","30");
-	key.set("stepunits","m");
-	fdb.archive(key, static_cast(data_str.c_str()), data_str.size());
-	fdb.flush();
-
-	req.setValue("step", "30m");
-	{
-		fdb5::FDBToolRequest r(req);
-		fdb5::ListIterator iter = fdb.list(r, true);
-		fdb5::ListElement el;
-		EXPECT(iter.next(el));
-		EXPECT(el.combinedKey().get("step") == "30m");
-		EXPECT(!iter.next(el));
-	}
-
-	req.values("step", {"0","to","2","by","30m"});
-	req.unsetValues("param");
-	{
-		metkit::mars::MarsExpension expand{false};
-
-		metkit::mars::MarsRequest expandedRequests = expand.expand(req);
-		fdb5::FDBToolRequest r(expandedRequests);
-		fdb5::ListIterator iter = fdb.list(r, true);
-		fdb5::ListElement el;
-		EXPECT(iter.next(el));
-		EXPECT(el.combinedKey().get("step") == "30m");
-		EXPECT(iter.next(el));
-		EXPECT(el.combinedKey().get("step") == "2");
-		EXPECT(!iter.next(el));
-	}
+    std::string data_str = "Raining cats and dogs";
+
+    fdb5::FDB fdb;
+
+    Key key;
+    key.set("class","od");
+    key.set("expver","0001");
+    key.set("type","fc");
+    key.set("stream","oper");
+    key.set("date","20101010");
+    key.set("time","0000");
+    key.set("domain","g");
+    key.set("levtype","sfc");
+    key.set("step","60");
+    key.set("param","130");
+    fdb.archive(key, static_cast(data_str.c_str()), data_str.size());
+    fdb.flush();
+
+    metkit::mars::MarsRequest req = key.request();
+
+    {
+        fdb5::FDBToolRequest r(req);
+        fdb5::ListIterator iter = fdb.list(r, true);
+        fdb5::ListElement el;
+        EXPECT(iter.next(el));
+        EXPECT(el.combinedKey().get("step") == "60");
+        EXPECT(!iter.next(el));
+    }
+
+    key.set("step","2");
+    key.set("stepunits","h");
+    fdb.archive(key, static_cast(data_str.c_str()), data_str.size());
+    fdb.flush();
+
+    req.setValue("step", "2");
+    {
+        fdb5::FDBToolRequest r(req);
+        fdb5::ListIterator iter = fdb.list(r, true);
+        fdb5::ListElement el;
+        EXPECT(iter.next(el));
+        EXPECT(el.combinedKey().get("step") == "2");
+        EXPECT(!iter.next(el));
+    }
+
+    req.setValuesTyped(new metkit::mars::TypeAny("step"), std::vector{"2","60"});
+    {
+        fdb5::FDBToolRequest r(req);
+        fdb5::ListIterator iter = fdb.list(r, true);
+        fdb5::ListElement el;
+        EXPECT(iter.next(el));
+        EXPECT(el.combinedKey().get("step") == "2");
+        EXPECT(iter.next(el));
+        EXPECT(el.combinedKey().get("step") == "60");
+        EXPECT(!iter.next(el));
+    }
+
+    key.set("step","30");
+    key.set("stepunits","m");
+    fdb.archive(key, static_cast(data_str.c_str()), data_str.size());
+    fdb.flush();
+
+    req.setValue("step", "30m");
+    {
+        fdb5::FDBToolRequest r(req);
+        fdb5::ListIterator iter = fdb.list(r, true);
+        fdb5::ListElement el;
+        EXPECT(iter.next(el));
+        EXPECT(el.combinedKey().get("step") == "30m");
+        EXPECT(!iter.next(el));
+    }
+
+    req.values("step", {"0","to","2","by","30m"});
+    req.unsetValues("param");
+    {
+        metkit::mars::MarsExpension expand{false};
+
+        metkit::mars::MarsRequest expandedRequests = expand.expand(req);
+        fdb5::FDBToolRequest r(expandedRequests);
+        fdb5::ListIterator iter = fdb.list(r, true);
+        fdb5::ListElement el;
+        EXPECT(iter.next(el));
+        EXPECT(el.combinedKey().get("step") == "30m");
+        EXPECT(iter.next(el));
+        EXPECT(el.combinedKey().get("step") == "2");
+        EXPECT(!iter.next(el));
+    }
 }
 
 CASE ( "test_fdb_service" ) {
 
-	SETUP("Fixture") {
-		FixtureService f;
+    SETUP("Fixture") {
+        FixtureService f;
 
         SECTION( "test_fdb_service_write" )
-		{
-			fdb5::Archiver fdb;
+        {
+            fdb5::Archiver fdb;
 
             f.p["class"]  = "rd";
-			f.p["stream"] = "oper";
-			f.p["domain"] = "g";
-			f.p["expver"] = "0001";
+            f.p["stream"] = "oper";
+            f.p["domain"] = "g";
+            f.p["expver"] = "0001";
 
-			f.p["date"] = "20120911";
-			f.p["time"] = "0000";
-			f.p["type"] = "fc";
+            f.p["date"] = "20120911";
+            f.p["time"] = "0000";
+            f.p["type"] = "fc";
 
-			f.write_cycle(fdb, f.p);
+            f.write_cycle(fdb, f.p);
 
-			f.p["date"] = "20120911";
-			f.p["time"] = "0000";
-			f.p["type"] = "4v";
+            f.p["date"] = "20120911";
+            f.p["time"] = "0000";
+            f.p["type"] = "4v";
 
-			f.write_cycle(fdb, f.p);
+            f.write_cycle(fdb, f.p);
 
-			f.p["date"] = "20120912";
-			f.p["time"] = "0000";
-			f.p["type"] = "fc";
+            f.p["date"] = "20120912";
+            f.p["time"] = "0000";
+            f.p["type"] = "fc";
 
-			f.write_cycle(fdb, f.p);
+            f.write_cycle(fdb, f.p);
 
-			f.p["date"] = "20120912";
-			f.p["time"] = "0000";
-			f.p["type"] = "4v";
+            f.p["date"] = "20120912";
+            f.p["time"] = "0000";
+            f.p["type"] = "4v";
 
-			f.write_cycle(fdb, f.p);
+            f.write_cycle(fdb, f.p);
         }
 
 
@@ -244,24 +244,24 @@ CASE ( "test_fdb_service" ) {
         {
             fdb5::FDB retriever;
 
-			Buffer buffer(1024);
+            Buffer buffer(1024);
 
-			Translator str;
-			std::vector::iterator param = f.modelParams_.begin();
-			for( ; param != f.modelParams_.end(); ++param )
-			{
-				f.p["param"] = *param;
-				f.p["levtype"] = "pl";
+            Translator str;
+            std::vector::iterator param = f.modelParams_.begin();
+            for( ; param != f.modelParams_.end(); ++param )
+            {
+                f.p["param"] = *param;
+                f.p["levtype"] = "pl";
 
-				for( size_t step = 0; step < 2; ++step )
-				{
-					f.p["step"] = str(step*3);
+                for( size_t step = 0; step < 2; ++step )
+                {
+                    f.p["step"] = str(step*3);
 
-					for( size_t level = 0; level < 3; ++level )
-					{
-						f.p["levelist"] = str(level*100);
+                    for( size_t level = 0; level < 3; ++level )
+                    {
+                        f.p["levelist"] = str(level*100);
 
-						Log::info() << "Looking for: " << f.p << std::endl;
+                        Log::info() << "Looking for: " << f.p << std::endl;
 
                         metkit::mars::MarsRequest r("retrieve", f.p);
                         std::unique_ptr dh ( retriever.retrieve(r) );  AutoClose closer1(*dh);
@@ -281,9 +281,9 @@ CASE ( "test_fdb_service" ) {
                                 << std::endl;
 
                         EXPECT(::memcmp(buffer, data.str().c_str(), data.str().size()) == 0);
-					}
-				}
-			}
+                    }
+                }
+            }
         }
 
 
@@ -291,157 +291,157 @@ CASE ( "test_fdb_service" ) {
         {
             fdb5::FDB lister;
 
-			Translator str;
-			std::vector::iterator param = f.modelParams_.begin();
-			for( ; param != f.modelParams_.end(); ++param )
-			{
-				f.p["param"] = *param;
-				f.p["levtype"] = "pl";
-				f.p["step"] = str(0);
-				f.p["levelist"] = str(0);
-
-				Log::info() << "Looking for: " << f.p << std::endl;
-
-				metkit::mars::MarsRequest r("retrieve", f.p);
-				fdb5::FDBToolRequest req(r);
-				auto iter = lister.list(req);
-				fdb5::ListElement el;
-				EXPECT(iter.next(el));
-
-				eckit::PathName path = el.location().uri().path().dirName();
-
-				DIR* dirp = ::opendir(path.asString().c_str());
-            	struct dirent* dp;
-            	while ((dp = ::readdir(dirp)) != nullptr) {
-                	EXPECT_NOT(strstr( dp->d_name, "toc."));
-				}
-
-				// consuming the rest of the queue
-				while (iter.next(el));
-			}
+            Translator str;
+            std::vector::iterator param = f.modelParams_.begin();
+            for( ; param != f.modelParams_.end(); ++param )
+            {
+                f.p["param"] = *param;
+                f.p["levtype"] = "pl";
+                f.p["step"] = str(0);
+                f.p["levelist"] = str(0);
+
+                Log::info() << "Looking for: " << f.p << std::endl;
+
+                metkit::mars::MarsRequest r("retrieve", f.p);
+                fdb5::FDBToolRequest req(r);
+                auto iter = lister.list(req);
+                fdb5::ListElement el;
+                EXPECT(iter.next(el));
+
+                eckit::PathName path = el.location().uri().path().dirName();
+
+                DIR* dirp = ::opendir(path.asString().c_str());
+                struct dirent* dp;
+                while ((dp = ::readdir(dirp)) != nullptr) {
+                    EXPECT_NOT(strstr( dp->d_name, "toc."));
+                }
+
+                // consuming the rest of the queue
+                while (iter.next(el));
+            }
         }
 
         SECTION( "test_fdb_service_marsreques" )
         {
             std::vector steps;
-			steps.push_back( "15" );
-			steps.push_back( "18" );
-			steps.push_back( "24" );
+            steps.push_back( "15" );
+            steps.push_back( "18" );
+            steps.push_back( "24" );
 
-			std::vector levels;
-			levels.push_back( "100" );
-			levels.push_back( "500" );
+            std::vector levels;
+            levels.push_back( "100" );
+            levels.push_back( "500" );
 
-			std::vector params;
-			params.push_back( "130.128" );
-			params.push_back( "138.128" );
+            std::vector params;
+            params.push_back( "130.128" );
+            params.push_back( "138.128" );
 
-			std::vector dates;
-			dates.push_back( "20120911" );
-			dates.push_back( "20120912" );
+            std::vector dates;
+            dates.push_back( "20120911" );
+            dates.push_back( "20120912" );
 
 
             metkit::mars::MarsRequest r("retrieve");
 
             r.setValue("class","rd");
-			r.setValue("expver","0001");
-			r.setValue("type","fc");
-			r.setValue("stream","oper");
-			r.setValue("time","0000");
-			r.setValue("domain","g");
-			r.setValue("levtype","pl");
+            r.setValue("expver","0001");
+            r.setValue("type","fc");
+            r.setValue("stream","oper");
+            r.setValue("time","0000");
+            r.setValue("domain","g");
+            r.setValue("levtype","pl");
 
             r.setValuesTyped(new metkit::mars::TypeAny("param"), params);
             r.setValuesTyped(new metkit::mars::TypeAny("step"), steps);
             r.setValuesTyped(new metkit::mars::TypeAny("levelist"), levels);
             r.setValuesTyped(new metkit::mars::TypeAny("date"), dates);
 
-			Log::info() << r << std::endl;
+            Log::info() << r << std::endl;
 
             fdb5::FDB retriever;
 
             std::unique_ptr dh ( retriever.retrieve(r) );
 
-			PathName path ( "data_mars_request.data" );
-			path.unlink();
+            PathName path ( "data_mars_request.data" );
+            path.unlink();
 
-			dh->saveInto(path);
+            dh->saveInto(path);
         }
-	}
+    }
 }
 
 
 
 CASE ( "test_fdb_service_subtoc" ) {
 
-	SETUP("Fixture") {
-		FixtureService f;
+    SETUP("Fixture") {
+        FixtureService f;
 
-		eckit::LocalConfiguration userConf;
-    	userConf.set("useSubToc", true);
-		
-		fdb5::Config expanded = fdb5::Config().expandConfig();
+        eckit::LocalConfiguration userConf;
+        userConf.set("useSubToc", true);
 
-		fdb5::Config config(expanded, userConf);
+        fdb5::Config expanded = fdb5::Config().expandConfig();
+
+        fdb5::Config config(expanded, userConf);
 
         SECTION( "test_fdb_service_subtoc_write" )
-		{
-			fdb5::Archiver fdb(config);
+        {
+            fdb5::Archiver fdb(config);
 
             f.p["class"]  = "rd";
-			f.p["stream"] = "oper";
-			f.p["domain"] = "g";
-			f.p["expver"] = "0002";
+            f.p["stream"] = "oper";
+            f.p["domain"] = "g";
+            f.p["expver"] = "0002";
 
-			f.p["date"] = "20120911";
-			f.p["time"] = "0000";
-			f.p["type"] = "fc";
+            f.p["date"] = "20120911";
+            f.p["time"] = "0000";
+            f.p["type"] = "fc";
 
-			f.write_cycle(fdb, f.p);
+            f.write_cycle(fdb, f.p);
 
-			f.p["date"] = "20120911";
-			f.p["time"] = "0000";
-			f.p["type"] = "4v";
+            f.p["date"] = "20120911";
+            f.p["time"] = "0000";
+            f.p["type"] = "4v";
 
-			f.write_cycle(fdb, f.p);
+            f.write_cycle(fdb, f.p);
 
-			f.p["date"] = "20120912";
-			f.p["time"] = "0000";
-			f.p["type"] = "fc";
+            f.p["date"] = "20120912";
+            f.p["time"] = "0000";
+            f.p["type"] = "fc";
 
-			f.write_cycle(fdb, f.p);
+            f.write_cycle(fdb, f.p);
 
-			f.p["date"] = "20120912";
-			f.p["time"] = "0000";
-			f.p["type"] = "4v";
+            f.p["date"] = "20120912";
+            f.p["time"] = "0000";
+            f.p["type"] = "4v";
 
-			f.write_cycle(fdb, f.p);
+            f.write_cycle(fdb, f.p);
         }
 
         SECTION( "test_fdb_service_subtoc_readtobuffer" )
         {
             fdb5::FDB retriever;
 
-			Buffer buffer(1024);
+            Buffer buffer(1024);
 
-			f.p["expver"] = "0002";
+            f.p["expver"] = "0002";
 
-			Translator str;
-			std::vector::iterator param = f.modelParams_.begin();
-			for( ; param != f.modelParams_.end(); ++param )
-			{
-				f.p["param"] = *param;
-				f.p["levtype"] = "pl";
+            Translator str;
+            std::vector::iterator param = f.modelParams_.begin();
+            for( ; param != f.modelParams_.end(); ++param )
+            {
+                f.p["param"] = *param;
+                f.p["levtype"] = "pl";
 
-				for( size_t step = 0; step < 2; ++step )
-				{
-					f.p["step"] = str(step*3);
+                for( size_t step = 0; step < 2; ++step )
+                {
+                    f.p["step"] = str(step*3);
 
-					for( size_t level = 0; level < 3; ++level )
-					{
-						f.p["levelist"] = str(level*100);
+                    for( size_t level = 0; level < 3; ++level )
+                    {
+                        f.p["levelist"] = str(level*100);
 
-						Log::info() << "Looking for: " << f.p << std::endl;
+                        Log::info() << "Looking for: " << f.p << std::endl;
 
                         metkit::mars::MarsRequest r("retrieve", f.p);
                         std::unique_ptr dh ( retriever.retrieve(r) );  AutoClose closer1(*dh);
@@ -461,9 +461,9 @@ CASE ( "test_fdb_service_subtoc" ) {
                                 << std::endl;
 
                         EXPECT(::memcmp(buffer, data.str().c_str(), data.str().size()) == 0);
-					}
-				}
-			}
+                    }
+                }
+            }
         }
 
 
@@ -471,126 +471,126 @@ CASE ( "test_fdb_service_subtoc" ) {
         {
             fdb5::FDB lister;
 
-			f.p["expver"] = "0002";
-
-			Translator str;
-			std::vector::iterator param = f.modelParams_.begin();
-			for( ; param != f.modelParams_.end(); ++param )
-			{
-
-				f.p["param"] = *param;
-				f.p["levtype"] = "pl";
-				f.p["step"] = str(0);
-				f.p["levelist"] = str(0);
-
-				Log::info() << "Looking for: " << f.p << std::endl;
-
-				metkit::mars::MarsRequest r("retrieve", f.p);
-				fdb5::FDBToolRequest req(r);
-				auto iter = lister.list(req);
-				fdb5::ListElement el;
-				EXPECT(iter.next(el));
-
-				eckit::PathName path = el.location().uri().path().dirName();
-
-				DIR* dirp = ::opendir(path.asString().c_str());
-            	struct dirent* dp;
-				bool subtoc = false;
-            	while ((dp = ::readdir(dirp)) != nullptr) {
-                	if (strstr( dp->d_name, "toc.")) {
-						subtoc = true;
-					}
-				}
-				EXPECT(subtoc);
-
-				// consuming the rest of the queue
-				while (iter.next(el));
-			}
+            f.p["expver"] = "0002";
+
+            Translator str;
+            std::vector::iterator param = f.modelParams_.begin();
+            for( ; param != f.modelParams_.end(); ++param )
+            {
+
+                f.p["param"] = *param;
+                f.p["levtype"] = "pl";
+                f.p["step"] = str(0);
+                f.p["levelist"] = str(0);
+
+                Log::info() << "Looking for: " << f.p << std::endl;
+
+                metkit::mars::MarsRequest r("retrieve", f.p);
+                fdb5::FDBToolRequest req(r);
+                auto iter = lister.list(req);
+                fdb5::ListElement el;
+                EXPECT(iter.next(el));
+
+                eckit::PathName path = el.location().uri().path().dirName();
+
+                DIR* dirp = ::opendir(path.asString().c_str());
+                struct dirent* dp;
+                bool subtoc = false;
+                while ((dp = ::readdir(dirp)) != nullptr) {
+                    if (strstr( dp->d_name, "toc.")) {
+                        subtoc = true;
+                    }
+                }
+                EXPECT(subtoc);
+
+                // consuming the rest of the queue
+                while (iter.next(el));
+            }
         }
 
         SECTION( "test_fdb_service_subtoc_marsreques" )
         {
             std::vector steps;
-			steps.push_back( "15" );
-			steps.push_back( "18" );
-			steps.push_back( "24" );
+            steps.push_back( "15" );
+            steps.push_back( "18" );
+            steps.push_back( "24" );
 
-			std::vector levels;
-			levels.push_back( "100" );
-			levels.push_back( "500" );
+            std::vector levels;
+            levels.push_back( "100" );
+            levels.push_back( "500" );
 
-			std::vector params;
-			params.push_back( "130.128" );
-			params.push_back( "138.128" );
+            std::vector params;
+            params.push_back( "130.128" );
+            params.push_back( "138.128" );
 
-			std::vector dates;
-			dates.push_back( "20120911" );
-			dates.push_back( "20120912" );
+            std::vector dates;
+            dates.push_back( "20120911" );
+            dates.push_back( "20120912" );
 
 
             metkit::mars::MarsRequest r("retrieve");
 
             r.setValue("class","rd");
-			r.setValue("expver","0002");
-			r.setValue("type","fc");
-			r.setValue("stream","oper");
-			r.setValue("time","0000");
-			r.setValue("domain","g");
-			r.setValue("levtype","pl");
+            r.setValue("expver","0002");
+            r.setValue("type","fc");
+            r.setValue("stream","oper");
+            r.setValue("time","0000");
+            r.setValue("domain","g");
+            r.setValue("levtype","pl");
 
             r.setValuesTyped(new metkit::mars::TypeAny("param"), params);
             r.setValuesTyped(new metkit::mars::TypeAny("step"), steps);
             r.setValuesTyped(new metkit::mars::TypeAny("levelist"), levels);
             r.setValuesTyped(new metkit::mars::TypeAny("date"), dates);
 
-			Log::info() << r << std::endl;
+            Log::info() << r << std::endl;
 
             fdb5::FDB retriever;
 
             std::unique_ptr dh ( retriever.retrieve(r) );
 
-			PathName path ( "data_mars_request.data" );
-			path.unlink();
+            PathName path ( "data_mars_request.data" );
+            path.unlink();
 
-			dh->saveInto(path);
+            dh->saveInto(path);
         }
-	}
+    }
 }
 
 CASE( "schemaSerialisation" ) {
 
-	PathName filename    = PathName::unique("data");                                             
-	std::string filepath = filename.asString();
-
-	std::string original;
-	
-	{
-		eckit::FileStream sout(filepath.c_str(), "w");
-		auto c = eckit::closer(sout);
-
-		fdb5::Config config;
-		config = config.expandConfig();
-		
-		std::stringstream ss;
-		config.schema().dump(ss);
-		original = ss.str();
-
-		sout << config.schema();
-	}
-	{                                                                                           
-		eckit::FileStream sin(filepath.c_str(), "r");
-		auto c = eckit::closer(sin);
-
-		fdb5::Schema* clone = eckit::Reanimator::reanimate(sin);
-
-		std::stringstream ss;
-		clone->dump(ss);
-
-		EXPECT(original == ss.str());
-	}
-	if (filename.exists()) {
-		filename.unlink();
-	}
+    PathName filename    = PathName::unique("data");
+    std::string filepath = filename.asString();
+
+    std::string original;
+
+    {
+        eckit::FileStream sout(filepath.c_str(), "w");
+        auto c = eckit::closer(sout);
+
+        fdb5::Config config;
+        config = config.expandConfig();
+
+        std::stringstream ss;
+        config.schema().dump(ss);
+        original = ss.str();
+
+        sout << config.schema();
+    }
+    {
+        eckit::FileStream sin(filepath.c_str(), "r");
+        auto c = eckit::closer(sin);
+
+        fdb5::Schema* clone = eckit::Reanimator::reanimate(sin);
+
+        std::stringstream ss;
+        clone->dump(ss);
+
+        EXPECT(original == ss.str());
+    }
+    if (filename.exists()) {
+        filename.unlink();
+    }
 }
 
 //-----------------------------------------------------------------------------

From b70c8492c73c5f994e83b09c87f3bba3441b4514 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 11 Dec 2024 17:38:29 +0000
Subject: [PATCH 177/186] Fix PR

---
 src/fdb5/daos/DaosCatalogue.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/fdb5/daos/DaosCatalogue.cc b/src/fdb5/daos/DaosCatalogue.cc
index cd5597196..588479349 100644
--- a/src/fdb5/daos/DaosCatalogue.cc
+++ b/src/fdb5/daos/DaosCatalogue.cc
@@ -167,8 +167,7 @@ std::vector DaosCatalogue::indexes(bool) const {
 
 std::string DaosCatalogue::type() const {
 
-    // return DaosCatalogue::catalogueTypeName();
-    
+    return fdb5::DaosEngine::typeName();
 }
 
 void DaosCatalogue::remove(const fdb5::DaosNameBase& n, std::ostream& logAlways, std::ostream& logVerbose, bool doit) {

From 43ce247065b5424b52e8cd46b70ee86a500e9cb3 Mon Sep 17 00:00:00 2001
From: Kai Kratz 
Date: Wed, 11 Dec 2024 20:04:56 +0100
Subject: [PATCH 178/186] Remove commented out code & formatting

---
 src/fdb5/api/local/DumpVisitor.h           | 1 -
 src/fdb5/api/local/StatusVisitor.h         | 1 -
 src/fdb5/api/local/WipeVisitor.cc          | 2 +-
 src/fdb5/database/Catalogue.h              | 2 +-
 src/fdb5/database/EntryVisitMechanism.cc   | 3 +--
 src/fdb5/database/EntryVisitMechanism.h    | 6 ++++--
 src/fdb5/remote/Connection.h               | 3 +--
 src/fdb5/remote/client/RemoteCatalogue.cc  | 6 +++---
 src/fdb5/remote/server/CatalogueHandler.cc | 1 -
 src/fdb5/rules/MatchAlways.h               | 4 ++--
 src/fdb5/rules/MatchAny.h                  | 4 ++--
 src/fdb5/rules/MatchHidden.h               | 4 ++--
 src/fdb5/rules/MatchOptional.h             | 4 ++--
 src/fdb5/rules/MatchValue.h                | 4 ++--
 src/fdb5/rules/Rule.h                      | 4 ++--
 src/fdb5/rules/Schema.h                    | 4 ++--
 src/fdb5/toc/TocCatalogue.h                | 1 -
 17 files changed, 25 insertions(+), 29 deletions(-)

diff --git a/src/fdb5/api/local/DumpVisitor.h b/src/fdb5/api/local/DumpVisitor.h
index 417520ff3..8fe4785be 100644
--- a/src/fdb5/api/local/DumpVisitor.h
+++ b/src/fdb5/api/local/DumpVisitor.h
@@ -44,7 +44,6 @@ class DumpVisitor : public QueryVisitor {
     bool visitIndexes() override { return false; }
     bool visitEntries() override { return false; }
 
-//    bool visitDatabase(const Catalogue& catalogue, const Store& store) override {
     bool visitDatabase(const Catalogue& catalogue) override {
         catalogue.dump(out_, simple_);
         return true;
diff --git a/src/fdb5/api/local/StatusVisitor.h b/src/fdb5/api/local/StatusVisitor.h
index 646f840b6..87dd3dbfa 100644
--- a/src/fdb5/api/local/StatusVisitor.h
+++ b/src/fdb5/api/local/StatusVisitor.h
@@ -37,7 +37,6 @@ class StatusVisitor : public QueryVisitor {
     bool visitIndexes() override { return false; }
     bool visitEntries() override { return false; }
     bool visitDatabase(const Catalogue& catalogue) override  { queue_.emplace(catalogue); return true; }
-//    bool visitDatabase(const Catalogue& catalogue, const Store& store) override { queue_.emplace(catalogue); return true; }
     bool visitIndex(const Index&) override { NOTIMP; }
     void visitDatum(const Field&, const Key&) override { NOTIMP; }
 
diff --git a/src/fdb5/api/local/WipeVisitor.cc b/src/fdb5/api/local/WipeVisitor.cc
index 4e651e181..33170ef25 100644
--- a/src/fdb5/api/local/WipeVisitor.cc
+++ b/src/fdb5/api/local/WipeVisitor.cc
@@ -49,7 +49,7 @@ WipeVisitor::WipeVisitor(eckit::Queue& queue,
 
 bool WipeVisitor::visitDatabase(const Catalogue& catalogue) {
 
-    // If the DB is locked for wiping, then it "doesn't exist"
+    // If the Catalogue is locked for wiping, then it "doesn't exist"
     if (!catalogue.enabled(ControlIdentifier::Wipe)) {
         return false;
     }
diff --git a/src/fdb5/database/Catalogue.h b/src/fdb5/database/Catalogue.h
index 12ec5177a..9c798035e 100644
--- a/src/fdb5/database/Catalogue.h
+++ b/src/fdb5/database/Catalogue.h
@@ -58,7 +58,7 @@ class Catalogue {
 
     virtual std::vector metadataPaths() const = 0;
 
-    virtual void visitEntries(EntryVisitor& visitor, /*const Store& store,*/ bool sorted = false);
+    virtual void visitEntries(EntryVisitor& visitor, bool sorted = false);
 
     virtual void hideContents() = 0;
 
diff --git a/src/fdb5/database/EntryVisitMechanism.cc b/src/fdb5/database/EntryVisitMechanism.cc
index 2ebbad5c3..d174eff78 100644
--- a/src/fdb5/database/EntryVisitMechanism.cc
+++ b/src/fdb5/database/EntryVisitMechanism.cc
@@ -36,8 +36,7 @@ class FDBVisitException : public eckit::Exception {
 EntryVisitor::EntryVisitor() : currentCatalogue_(nullptr), currentStore_(nullptr), currentIndex_(nullptr) {}
 
 EntryVisitor::~EntryVisitor() {
-    if (currentStore_)
-        delete currentStore_;
+    delete currentStore_;
 }
 
 Store& EntryVisitor::store() const {
diff --git a/src/fdb5/database/EntryVisitMechanism.h b/src/fdb5/database/EntryVisitMechanism.h
index 1ff68bdd7..92139c8ca 100644
--- a/src/fdb5/database/EntryVisitMechanism.h
+++ b/src/fdb5/database/EntryVisitMechanism.h
@@ -59,11 +59,13 @@ class EntryVisitor : public eckit::NonCopyable {
     virtual void visitDatum(const Field& field, const Key& datumKey) = 0;
 
 protected:  // members
-
-    // n.b. non-owning
+    /// Non-owning
     const Catalogue* currentCatalogue_ = nullptr;
+    /// Owned store
     mutable Store* currentStore_ = nullptr;
+    /// Non-owning
     const Index* currentIndex_ = nullptr;
+    /// Non-owning
     const Rule* rule_ = nullptr;
 };
 
diff --git a/src/fdb5/remote/Connection.h b/src/fdb5/remote/Connection.h
index 33131e7ee..560f8b01c 100644
--- a/src/fdb5/remote/Connection.h
+++ b/src/fdb5/remote/Connection.h
@@ -67,7 +67,6 @@ class Connection : eckit::NonCopyable {
 protected: // members
 
     bool single_;
-//    bool exit_;
 
 private: // members
 
@@ -80,4 +79,4 @@ class Connection : eckit::NonCopyable {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-}  // namespace fdb5::remote
\ No newline at end of file
+}  // namespace fdb5::remote
diff --git a/src/fdb5/remote/client/RemoteCatalogue.cc b/src/fdb5/remote/client/RemoteCatalogue.cc
index a613b32bf..f7ac42924 100644
--- a/src/fdb5/remote/client/RemoteCatalogue.cc
+++ b/src/fdb5/remote/client/RemoteCatalogue.cc
@@ -120,7 +120,7 @@ eckit::URI RemoteCatalogue::uri() const {
 
 void RemoteCatalogue::loadSchema() {
     // NB we're at the db level, so get the db schema. We will want to get the master schema beforehand.
-    // (outside of the catalogue) 
+    // (outside of the catalogue)
 
     if (!schema_) {
         LOG_DEBUG_LIB(LibFdb5) << "RemoteCatalogue::loadSchema()" << std::endl;
@@ -129,7 +129,7 @@ void RemoteCatalogue::loadSchema() {
         eckit::Buffer keyBuffer(4096);
         eckit::MemoryStream keyStream(keyBuffer);
         keyStream << dbKey_;
-        
+
         eckit::Buffer buf = controlWriteReadResponse(Message::Schema, generateRequestID(), keyBuffer, keyStream.position());
 
         eckit::MemoryStream s(buf);
@@ -151,7 +151,7 @@ void RemoteCatalogue::overlayDB(const Catalogue& otherCatalogue, const std::set<
 void RemoteCatalogue::index(const Key& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) {NOTIMP;}
 void RemoteCatalogue::reconsolidate(){NOTIMP;}
 std::vector RemoteCatalogue::metadataPaths() const {NOTIMP;}
-void RemoteCatalogue::visitEntries(EntryVisitor& visitor, /*const Store& store,*/ bool sorted) {NOTIMP;}
+void RemoteCatalogue::visitEntries(EntryVisitor& visitor, bool sorted) {NOTIMP;}
 void RemoteCatalogue::dump(std::ostream& out, bool simple, const eckit::Configuration& conf) const {NOTIMP;}
 StatsReportVisitor* RemoteCatalogue::statsReportVisitor() const {NOTIMP;}
 PurgeVisitor* RemoteCatalogue::purgeVisitor(const Store& store) const {NOTIMP;}
diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc
index 79995845c..a6ed4e7d0 100644
--- a/src/fdb5/remote/server/CatalogueHandler.cc
+++ b/src/fdb5/remote/server/CatalogueHandler.cc
@@ -297,7 +297,6 @@ void CatalogueHandler::stores(uint32_t clientID, uint32_t requestID) {
 
     ASSERT(config_.has("stores"));
     std::map> stores;
-    // std::vector> stores;
     for (const auto& configStore: config_.getSubConfigurations("stores")) {
         ASSERT(configStore.has("default"));
         eckit::net::Endpoint fieldLocationEndpoint{configStore.getString("default")};
diff --git a/src/fdb5/rules/MatchAlways.h b/src/fdb5/rules/MatchAlways.h
index ebb80cef2..9d699c7b6 100644
--- a/src/fdb5/rules/MatchAlways.h
+++ b/src/fdb5/rules/MatchAlways.h
@@ -36,8 +36,8 @@ class MatchAlways : public Matcher {
 
     void dump(std::ostream& s, const std::string& keyword, const TypesRegistry& registry) const override;
 
-	const eckit::ReanimatorBase& reanimator() const override { return reanimator_; }
-	static const eckit::ClassSpec&  classSpec() { return classSpec_; }
+    const eckit::ReanimatorBase& reanimator() const override { return reanimator_; }
+    static const eckit::ClassSpec&  classSpec() { return classSpec_; }
 
 private: // methods
 
diff --git a/src/fdb5/rules/MatchAny.h b/src/fdb5/rules/MatchAny.h
index b95473902..37294c9cb 100644
--- a/src/fdb5/rules/MatchAny.h
+++ b/src/fdb5/rules/MatchAny.h
@@ -37,8 +37,8 @@ class MatchAny : public Matcher{
 
     void dump(std::ostream& s, const std::string& keyword, const TypesRegistry& registry) const override;
 
-	const eckit::ReanimatorBase& reanimator() const override { return reanimator_; }
-	static const eckit::ClassSpec&  classSpec() { return classSpec_; }
+    const eckit::ReanimatorBase& reanimator() const override { return reanimator_; }
+    static const eckit::ClassSpec&  classSpec() { return classSpec_; }
 
 private: // methods
 
diff --git a/src/fdb5/rules/MatchHidden.h b/src/fdb5/rules/MatchHidden.h
index c0d7a0356..115dc916c 100644
--- a/src/fdb5/rules/MatchHidden.h
+++ b/src/fdb5/rules/MatchHidden.h
@@ -38,8 +38,8 @@ class MatchHidden : public Matcher{
 
     void dump(std::ostream& s, const std::string& keyword, const TypesRegistry& registry) const override;
 
-	const eckit::ReanimatorBase& reanimator() const override { return reanimator_; }
-	static const eckit::ClassSpec&  classSpec() { return classSpec_; }
+    const eckit::ReanimatorBase& reanimator() const override { return reanimator_; }
+    static const eckit::ClassSpec&  classSpec() { return classSpec_; }
 
 private: // methods
 
diff --git a/src/fdb5/rules/MatchOptional.h b/src/fdb5/rules/MatchOptional.h
index 1d7a6a884..837d88e00 100644
--- a/src/fdb5/rules/MatchOptional.h
+++ b/src/fdb5/rules/MatchOptional.h
@@ -38,8 +38,8 @@ class MatchOptional : public Matcher{
 
     void dump(std::ostream& s, const std::string& keyword, const TypesRegistry& registry) const override;
 
-	const eckit::ReanimatorBase& reanimator() const override { return reanimator_; }
-	static const eckit::ClassSpec&  classSpec() { return classSpec_; }
+    const eckit::ReanimatorBase& reanimator() const override { return reanimator_; }
+    static const eckit::ClassSpec&  classSpec() { return classSpec_; }
 
 private: // methods
 
diff --git a/src/fdb5/rules/MatchValue.h b/src/fdb5/rules/MatchValue.h
index bf26dbc5c..b8663f918 100644
--- a/src/fdb5/rules/MatchValue.h
+++ b/src/fdb5/rules/MatchValue.h
@@ -38,8 +38,8 @@ class MatchValue : public Matcher{
 
     void dump(std::ostream &s, const std::string &keyword, const TypesRegistry ®istry) const override;
 
-	const eckit::ReanimatorBase& reanimator() const override { return reanimator_; }
-	static const eckit::ClassSpec&  classSpec() { return classSpec_; }
+    const eckit::ReanimatorBase& reanimator() const override { return reanimator_; }
+    static const eckit::ClassSpec&  classSpec() { return classSpec_; }
 
 private: // methods
 
diff --git a/src/fdb5/rules/Rule.h b/src/fdb5/rules/Rule.h
index 300f9bd47..fc22c53cd 100644
--- a/src/fdb5/rules/Rule.h
+++ b/src/fdb5/rules/Rule.h
@@ -89,8 +89,8 @@ class Rule : public eckit::Streamable {
     const Schema &schema() const;
     const TypesRegistry& registry() const;
 
-	const eckit::ReanimatorBase& reanimator() const override { return reanimator_; }
-	static const eckit::ClassSpec&  classSpec() { return classSpec_; }
+    const eckit::ReanimatorBase& reanimator() const override { return reanimator_; }
+    static const eckit::ClassSpec&  classSpec() { return classSpec_; }
 
     void check(const Key& key) const;
 
diff --git a/src/fdb5/rules/Schema.h b/src/fdb5/rules/Schema.h
index 7b51c04d4..db56cc946 100644
--- a/src/fdb5/rules/Schema.h
+++ b/src/fdb5/rules/Schema.h
@@ -84,8 +84,8 @@ class Schema : public eckit::Streamable {
 
     const TypesRegistry& registry() const;
     
-	const eckit::ReanimatorBase& reanimator() const override { return reanimator_; }
-	static const eckit::ClassSpec&  classSpec() { return classSpec_; }
+    const eckit::ReanimatorBase& reanimator() const override { return reanimator_; }
+    static const eckit::ClassSpec&  classSpec() { return classSpec_; }
 
 private: // methods
 
diff --git a/src/fdb5/toc/TocCatalogue.h b/src/fdb5/toc/TocCatalogue.h
index ca89323c3..9088853bf 100644
--- a/src/fdb5/toc/TocCatalogue.h
+++ b/src/fdb5/toc/TocCatalogue.h
@@ -61,7 +61,6 @@ class TocCatalogue : public CatalogueImpl, public TocHandler {
 
     void checkUID() const override;
     bool exists() const override;
-    // void visitEntries(EntryVisitor& visitor, /*const Store& store,*/ bool sorted) override;
     void dump(std::ostream& out, bool simple, const eckit::Configuration& conf) const override;
     std::vector metadataPaths() const override;
     const Schema& schema() const override;

From 5974d926e682624302eee86bac8a0ef0b1a8d9e3 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 18 Dec 2024 09:41:22 +0000
Subject: [PATCH 179/186] addressed PR comments (mutex on flush while evicting
 a catalogue from cache + Client handle simplified)

---
 src/fdb5/api/RemoteFDB.cc                  |  4 +-
 src/fdb5/api/RemoteFDB.h                   |  4 +-
 src/fdb5/database/Archiver.cc              | 58 ++++++++++++----------
 src/fdb5/database/Archiver.h               |  1 +
 src/fdb5/database/EntryVisitMechanism.cc   | 13 ++---
 src/fdb5/database/EntryVisitMechanism.h    |  2 +-
 src/fdb5/remote/client/Client.h            |  4 +-
 src/fdb5/remote/client/ClientConnection.cc |  8 +--
 src/fdb5/remote/client/RemoteCatalogue.cc  |  7 +--
 src/fdb5/remote/client/RemoteCatalogue.h   |  6 +--
 src/fdb5/remote/client/RemoteStore.cc      |  4 +-
 src/fdb5/remote/client/RemoteStore.h       |  4 +-
 12 files changed, 58 insertions(+), 57 deletions(-)

diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
index d52735005..c2401657f 100644
--- a/src/fdb5/api/RemoteFDB.cc
+++ b/src/fdb5/api/RemoteFDB.cc
@@ -246,7 +246,7 @@ void RemoteFDB::print(std::ostream& s) const {
 }
 
 // Client
-bool RemoteFDB::handle(remote::Message message, bool control, uint32_t requestID) {
+bool RemoteFDB::handle(remote::Message message, uint32_t requestID) {
     
     switch (message) {
         case fdb5::remote::Message::Complete: {
@@ -279,7 +279,7 @@ bool RemoteFDB::handle(remote::Message message, bool control, uint32_t requestID
             return false;
     }
 }
-bool RemoteFDB::handle(remote::Message message, bool control, uint32_t requestID, eckit::Buffer&& payload) {
+bool RemoteFDB::handle(remote::Message message, uint32_t requestID, eckit::Buffer&& payload) {
 
     switch (message) {
         case fdb5::remote::Message::Blob: {
diff --git a/src/fdb5/api/RemoteFDB.h b/src/fdb5/api/RemoteFDB.h
index 2ada336ca..7524896bc 100644
--- a/src/fdb5/api/RemoteFDB.h
+++ b/src/fdb5/api/RemoteFDB.h
@@ -77,8 +77,8 @@ class RemoteFDB : public LocalFDB, public remote::Client {
     FDBStats stats() const override { NOTIMP; }
 
     // Client
-    bool handle(remote::Message message, bool control, uint32_t requestID) override;
-    bool handle(remote::Message message, bool control, uint32_t requestID, eckit::Buffer&& payload) override;
+    bool handle(remote::Message message, uint32_t requestID) override;
+    bool handle(remote::Message message, uint32_t requestID, eckit::Buffer&& payload) override;
 
 private: // members
 
diff --git a/src/fdb5/database/Archiver.cc b/src/fdb5/database/Archiver.cc
index e77411b38..7b49b9cb4 100644
--- a/src/fdb5/database/Archiver.cc
+++ b/src/fdb5/database/Archiver.cc
@@ -77,37 +77,43 @@ void Archiver::selectDatabase(const Key& dbKey) {
 
     static size_t fdbMaxNbDBsOpen = eckit::Resource("fdbMaxNbDBsOpen", 64);
 
-    if (databases_.size() >= fdbMaxNbDBsOpen) {
-        bool found = false;
-        time_t oldest = ::time(0) + 24 * 60 * 60;
-        Key oldK;
-        for (auto i = databases_.begin(); i != databases_.end(); ++i) {
-            if (i->second.time_ <= oldest) {
-                found = true;
-                oldK = i->first;
-                oldest = i->second.time_;
+    {
+        std::lock_guard cacheLock(cacheMutex_);
+        if (databases_.size() >= fdbMaxNbDBsOpen) {
+            bool found = false;
+            time_t oldest = ::time(0) + 24 * 60 * 60;
+            Key oldK;
+            for (auto i = databases_.begin(); i != databases_.end(); ++i) {
+                if (i->second.time_ <= oldest) {
+                    found = true;
+                    oldK = i->first;
+                    oldest = i->second.time_;
+                }
+            }
+            if (found) {
+                // flushing before evicting from cache
+                std::lock_guard lock(flushMutex_);
+
+                databases_[oldK].catalogue_->flush(databases_[oldK].store_->flush());
+                
+                eckit::Log::info() << "Closing database " << *databases_[oldK].catalogue_ << std::endl;
+                databases_.erase(oldK);
             }
         }
-        if (found) {
-            databases_[oldK].catalogue_->flush(databases_[oldK].store_->flush());
-            
-            eckit::Log::info() << "Closing database " << *databases_[oldK].catalogue_ << std::endl;
-            databases_.erase(oldK);
-        }
-    }
 
-    std::unique_ptr cat = CatalogueWriterFactory::instance().build(dbKey, dbConfig_);
-    ASSERT(cat);
+        std::unique_ptr cat = CatalogueWriterFactory::instance().build(dbKey, dbConfig_);
+        ASSERT(cat);
 
-    // If this database is locked for writing then this is an error
-    if (!cat->enabled(ControlIdentifier::Archive)) {
-        std::ostringstream ss;
-        ss << "Database " << *cat << " matched for archived is LOCKED against archiving";
-        throw eckit::UserError(ss.str(), Here());
-    }
+        // If this database is locked for writing then this is an error
+        if (!cat->enabled(ControlIdentifier::Archive)) {
+            std::ostringstream ss;
+            ss << "Database " << *cat << " matched for archived is LOCKED against archiving";
+            throw eckit::UserError(ss.str(), Here());
+        }
 
-    std::unique_ptr str = cat->buildStore();
-    db_ = &(databases_[dbKey] = Database{::time(0), std::move(cat), std::move(str)});
+        std::unique_ptr str = cat->buildStore();
+        db_ = &(databases_[dbKey] = Database{::time(0), std::move(cat), std::move(str)});
+    }
 }
 
 void Archiver::print(std::ostream& out) const {
diff --git a/src/fdb5/database/Archiver.h b/src/fdb5/database/Archiver.h
index 798986776..e2e525ac6 100644
--- a/src/fdb5/database/Archiver.h
+++ b/src/fdb5/database/Archiver.h
@@ -81,6 +81,7 @@ class Archiver : public eckit::NonCopyable {
     Database* db_;
 
     std::mutex flushMutex_;
+    std::mutex cacheMutex_;
     const ArchiveCallback& callback_;
 };
 
diff --git a/src/fdb5/database/EntryVisitMechanism.cc b/src/fdb5/database/EntryVisitMechanism.cc
index d174eff78..d022dd509 100644
--- a/src/fdb5/database/EntryVisitMechanism.cc
+++ b/src/fdb5/database/EntryVisitMechanism.cc
@@ -33,16 +33,14 @@ class FDBVisitException : public eckit::Exception {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-EntryVisitor::EntryVisitor() : currentCatalogue_(nullptr), currentStore_(nullptr), currentIndex_(nullptr) {}
+EntryVisitor::~EntryVisitor() {}
 
-EntryVisitor::~EntryVisitor() {
-    delete currentStore_;
-}
+EntryVisitor::EntryVisitor() : currentCatalogue_(nullptr), currentStore_(nullptr), currentIndex_(nullptr) {}
 
 Store& EntryVisitor::store() const {
     if (!currentStore_) {
         ASSERT(currentCatalogue_);
-        currentStore_ = currentCatalogue_->buildStore().release();
+        currentStore_ = currentCatalogue_->buildStore();
         ASSERT(currentStore_);
     }
     return *currentStore_;
@@ -50,7 +48,7 @@ Store& EntryVisitor::store() const {
 
 bool EntryVisitor::visitDatabase(const Catalogue& catalogue) {
     currentCatalogue_ = &catalogue;
-    currentStore_ = nullptr;
+    currentStore_.reset();
     currentIndex_ = nullptr;
     rule_ = nullptr;
     return true;
@@ -61,8 +59,7 @@ void EntryVisitor::catalogueComplete(const Catalogue& catalogue) {
         ASSERT(currentCatalogue_ == &catalogue);
     }
     currentCatalogue_ = nullptr;
-    delete currentStore_;
-    currentStore_ = nullptr;
+    currentStore_.reset();
     currentIndex_ = nullptr;
     rule_ = nullptr;
 }
diff --git a/src/fdb5/database/EntryVisitMechanism.h b/src/fdb5/database/EntryVisitMechanism.h
index 92139c8ca..512a255c4 100644
--- a/src/fdb5/database/EntryVisitMechanism.h
+++ b/src/fdb5/database/EntryVisitMechanism.h
@@ -62,7 +62,7 @@ class EntryVisitor : public eckit::NonCopyable {
     /// Non-owning
     const Catalogue* currentCatalogue_ = nullptr;
     /// Owned store
-    mutable Store* currentStore_ = nullptr;
+    mutable std::unique_ptr currentStore_ = nullptr;
     /// Non-owning
     const Index* currentIndex_ = nullptr;
     /// Non-owning
diff --git a/src/fdb5/remote/client/Client.h b/src/fdb5/remote/client/Client.h
index 9e58e5de3..689e42607 100644
--- a/src/fdb5/remote/client/Client.h
+++ b/src/fdb5/remote/client/Client.h
@@ -50,8 +50,8 @@ class Client : eckit::NonCopyable {
     void dataWrite(remote::Message msg, uint32_t requestID, std::vector> data={});
     
     // handlers for incoming messages - to be defined in the client class
-    virtual bool handle(Message message, bool control, uint32_t requestID) = 0;
-    virtual bool handle(Message message, bool control, uint32_t requestID, eckit::Buffer&& payload) = 0;
+    virtual bool handle(Message message, uint32_t requestID) = 0;
+    virtual bool handle(Message message, uint32_t requestID, eckit::Buffer&& payload) = 0;
 
 protected:
     
diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc
index 7fbbd4c20..f85707156 100644
--- a/src/fdb5/remote/client/ClientConnection.cc
+++ b/src/fdb5/remote/client/ClientConnection.cc
@@ -360,10 +360,10 @@ void ClientConnection::listeningControlThreadLoop() {
                         }
 
                         if (hdr.payloadSize == 0) {
-                            handled = client->handle(hdr.message, hdr.control(), hdr.requestID);
+                            handled = client->handle(hdr.message, hdr.requestID);
                         }
                         else {
-                            handled = client->handle(hdr.message, hdr.control(), hdr.requestID, std::move(payload));
+                            handled = client->handle(hdr.message, hdr.requestID, std::move(payload));
                         }
                     }
 
@@ -425,10 +425,10 @@ void ClientConnection::listeningDataThreadLoop() {
                     ASSERT(client);
                     ASSERT(!hdr.control());
                     if (hdr.payloadSize == 0) {
-                        handled = client->handle(hdr.message, hdr.control(), hdr.requestID);
+                        handled = client->handle(hdr.message, hdr.requestID);
                     }
                     else {
-                        handled = client->handle(hdr.message, hdr.control(), hdr.requestID, std::move(payload));
+                        handled = client->handle(hdr.message, hdr.requestID, std::move(payload));
                     }
 
                     if (!handled) {
diff --git a/src/fdb5/remote/client/RemoteCatalogue.cc b/src/fdb5/remote/client/RemoteCatalogue.cc
index f7ac42924..e0a93d854 100644
--- a/src/fdb5/remote/client/RemoteCatalogue.cc
+++ b/src/fdb5/remote/client/RemoteCatalogue.cc
@@ -35,8 +35,6 @@ RemoteCatalogue::RemoteCatalogue(const eckit::URI& uri, const Config& config):
     NOTIMP;
 }
 
-RemoteCatalogue::~RemoteCatalogue() {}
-
 void RemoteCatalogue::archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) {
 
     ASSERT(!datumKey.empty());
@@ -137,12 +135,11 @@ void RemoteCatalogue::loadSchema() {
     }
 }
 
-bool RemoteCatalogue::handle(Message message, bool control, uint32_t requestID) {
+bool RemoteCatalogue::handle(Message message, uint32_t requestID) {
     Log::warning() << *this << " - Received [message=" << ((uint) message) << ",requestID=" << requestID << "]" << std::endl;
-    NOTIMP;
     return false;
 }
-bool RemoteCatalogue::handle(Message message, bool control, uint32_t requestID, eckit::Buffer&& payload) {
+bool RemoteCatalogue::handle(Message message, uint32_t requestID, eckit::Buffer&& payload) {
     LOG_DEBUG_LIB(LibFdb5) << *this << " - Received [message=" << ((uint) message) << ",requestID=" << requestID << ",payloadSize=" << payload.size() << "]" << std::endl;
     return false;
 }
diff --git a/src/fdb5/remote/client/RemoteCatalogue.h b/src/fdb5/remote/client/RemoteCatalogue.h
index d7a93db29..1d10b9716 100644
--- a/src/fdb5/remote/client/RemoteCatalogue.h
+++ b/src/fdb5/remote/client/RemoteCatalogue.h
@@ -19,7 +19,7 @@ class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public C
     RemoteCatalogue(const Key& key, const Config& config);
     RemoteCatalogue(const eckit::URI& uri, const Config& config);
 
-    ~RemoteCatalogue() override;
+    ~RemoteCatalogue() override = default;
 
     // From CatalogueWriter
     const Index& currentIndex() override;
@@ -66,8 +66,8 @@ class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public C
 private:
     // From Client
     // handlers for incoming messages - to be defined in the client class
-    bool handle(Message message, bool control, uint32_t requestID) override;
-    bool handle(Message message, bool control, uint32_t requestID, eckit::Buffer&& payload) override;
+    bool handle(Message message, uint32_t requestID) override;
+    bool handle(Message message, uint32_t requestID, eckit::Buffer&& payload) override;
 
 protected:
 
diff --git a/src/fdb5/remote/client/RemoteStore.cc b/src/fdb5/remote/client/RemoteStore.cc
index 2c0f4d285..d1951f77b 100644
--- a/src/fdb5/remote/client/RemoteStore.cc
+++ b/src/fdb5/remote/client/RemoteStore.cc
@@ -305,7 +305,7 @@ void RemoteStore::print(std::ostream &out) const {
     out << "RemoteStore(host=" << controlEndpoint() << ")";
 }
 
-bool RemoteStore::handle(Message message, bool control, uint32_t requestID) {
+bool RemoteStore::handle(Message message, uint32_t requestID) {
 
     switch (message) {
         case Message::Complete: {
@@ -346,7 +346,7 @@ bool RemoteStore::handle(Message message, bool control, uint32_t requestID) {
             return false;
     }
 }
-bool RemoteStore::handle(Message message, bool control, uint32_t requestID, eckit::Buffer&& payload) {
+bool RemoteStore::handle(Message message, uint32_t requestID, eckit::Buffer&& payload) {
 
     switch (message) {
 
diff --git a/src/fdb5/remote/client/RemoteStore.h b/src/fdb5/remote/client/RemoteStore.h
index 8bad69941..3446d318e 100644
--- a/src/fdb5/remote/client/RemoteStore.h
+++ b/src/fdb5/remote/client/RemoteStore.h
@@ -151,8 +151,8 @@ class RemoteStore : public Store, public Client {
 private: // methods
 
     // handlers for incoming messages - to be defined in the client class
-    bool handle(Message message, bool control, uint32_t requestID) override;
-    bool handle(Message message, bool control, uint32_t requestID, eckit::Buffer&& payload) override;
+    bool handle(Message message, uint32_t requestID) override;
+    bool handle(Message message, uint32_t requestID, eckit::Buffer&& payload) override;
 
 private: // members
 

From 598a1d7f0d094317c9de510d6ca06c3bfa66c935 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Thu, 19 Dec 2024 08:29:50 +0000
Subject: [PATCH 180/186] compute Axes encoding size (to allocate the correct
 buffer size)

---
 src/fdb5/api/helpers/AxesIterator.cc       |  4 ++++
 src/fdb5/api/helpers/AxesIterator.h        |  1 +
 src/fdb5/database/IndexAxis.cc             | 17 +++++++++++++++++
 src/fdb5/database/IndexAxis.h              |  1 +
 src/fdb5/database/Key.cc                   | 16 ++++++++++++++++
 src/fdb5/database/Key.h                    |  2 ++
 src/fdb5/remote/server/CatalogueHandler.cc |  2 ++
 7 files changed, 43 insertions(+)

diff --git a/src/fdb5/api/helpers/AxesIterator.cc b/src/fdb5/api/helpers/AxesIterator.cc
index 99cdfe89a..2d4ada8e5 100644
--- a/src/fdb5/api/helpers/AxesIterator.cc
+++ b/src/fdb5/api/helpers/AxesIterator.cc
@@ -31,6 +31,10 @@ void AxesElement::print(std::ostream& out) const {
     out << "Axes(db=" << dbKey_ << ", axes=" << axes_ << ")";
 }
 
+size_t AxesElement::encodeSize() const {
+    return dbKey_.encodeSize() + axes_.encodeSize(IndexAxis::currentVersion());
+}
+
 void AxesElement::encode(eckit::Stream& s) const {
     s << dbKey_;
     axes_.encode(s, IndexAxis::currentVersion());
diff --git a/src/fdb5/api/helpers/AxesIterator.h b/src/fdb5/api/helpers/AxesIterator.h
index 9881358dd..3da56a2cb 100644
--- a/src/fdb5/api/helpers/AxesIterator.h
+++ b/src/fdb5/api/helpers/AxesIterator.h
@@ -40,6 +40,7 @@ class AxesElement {
     const IndexAxis& axes() const { return axes_; }
 
     void print(std::ostream& out) const;
+    size_t encodeSize() const;
 
 private: // methods
 
diff --git a/src/fdb5/database/IndexAxis.cc b/src/fdb5/database/IndexAxis.cc
index 18e7b9b3f..6fba17385 100755
--- a/src/fdb5/database/IndexAxis.cc
+++ b/src/fdb5/database/IndexAxis.cc
@@ -74,6 +74,23 @@ bool IndexAxis::operator!=(const IndexAxis& rhs) const {
     return !(*this == rhs);
 }
 
+size_t encodeString(size_t len) {
+    return (5 + len);
+}
+
+size_t IndexAxis::encodeSize(const int version) const {
+    size_t size=2;
+    size += encodeString(4) + 5;
+    size += encodeString(4);
+    for (const auto& [key, vals] : axis_) {
+        size += encodeString(key.length()) + 5;
+        for (const auto& v : *vals) {
+            size += encodeString(v.length());
+        }
+    }
+    return size;
+}
+
 void IndexAxis::encode(eckit::Stream &s, const int version) const {
     if (version >= 3) {
         encodeCurrent(s, version);
diff --git a/src/fdb5/database/IndexAxis.h b/src/fdb5/database/IndexAxis.h
index 392e96716..d57c3b51c 100644
--- a/src/fdb5/database/IndexAxis.h
+++ b/src/fdb5/database/IndexAxis.h
@@ -58,6 +58,7 @@ class IndexAxis : private eckit::NonCopyable {
     void insert(const Key& key);
     /// @note: the values are required to be cannonicalised
     void insert(const std::string& axis, const std::vector& values);
+    size_t encodeSize(const int version) const;
     void encode(eckit::Stream &s, const int version) const;
     static int currentVersion() { return 3; }
 
diff --git a/src/fdb5/database/Key.cc b/src/fdb5/database/Key.cc
index 0d6d548b2..bd1abee5c 100644
--- a/src/fdb5/database/Key.cc
+++ b/src/fdb5/database/Key.cc
@@ -61,6 +61,22 @@ void BaseKey::decode(eckit::Stream& s) {
     }
 }
 
+size_t encodeString(const std::string& str) {
+    return (5 + str.length());
+}
+
+size_t BaseKey::encodeSize() const {
+    size_t size = 5;
+    for (const auto& [key_name, key_value] : keys_) {
+        size += encodeString(key_name) + encodeString(canonicalise(key_name, key_value));
+    }
+    size += 5;
+    for (const auto& name : names_) {
+        size += encodeString(name) + encodeString(type(name));
+    }
+    return size;
+}
+
 void BaseKey::encode(eckit::Stream& s) const {
 
     s << keys_.size();
diff --git a/src/fdb5/database/Key.h b/src/fdb5/database/Key.h
index 058e7c0b6..523a7b015 100644
--- a/src/fdb5/database/Key.h
+++ b/src/fdb5/database/Key.h
@@ -138,6 +138,8 @@ class BaseKey {
 
     virtual operator eckit::StringDict() const;
 
+    size_t encodeSize() const;
+
 protected: // members
 
     //TODO add unit test for each type
diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc
index a6ed4e7d0..22cccf76b 100644
--- a/src/fdb5/remote/server/CatalogueHandler.cc
+++ b/src/fdb5/remote/server/CatalogueHandler.cc
@@ -166,6 +166,8 @@ struct ListHelper : public BaseHelper {
 };
 
 struct AxesHelper : public BaseHelper {
+    virtual size_t encodeBufferSize(const AxesElement& el) const { return el.encodeSize(); }
+
     void extraDecode(eckit::Stream& s) {
         s >> level_;
     }

From 10d10b088a77e6f70be9c508fe7bd9fd0beda519 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 20 Dec 2024 14:57:52 +0000
Subject: [PATCH 181/186] added Schema dependency

---
 src/fdb5/api/local/QueryVisitor.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/fdb5/api/local/QueryVisitor.h b/src/fdb5/api/local/QueryVisitor.h
index 7c99054f3..6e35f56e8 100644
--- a/src/fdb5/api/local/QueryVisitor.h
+++ b/src/fdb5/api/local/QueryVisitor.h
@@ -19,6 +19,7 @@
 #ifndef fdb5_api_local_QueryVisitor_H
 #define fdb5_api_local_QueryVisitor_H
 
+#include "fdb5/database/Store.h"
 #include "fdb5/database/EntryVisitMechanism.h"
 
 #include "eckit/container/Queue.h"

From 5a6a08f40ce34e5c50671de476c3303a3e23c25c Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 20 Dec 2024 15:34:27 +0000
Subject: [PATCH 182/186] wip

---
 src/fdb5/api/LocalFDB.cc                 | 1 +
 src/fdb5/api/local/QueryVisitor.h        | 2 +-
 src/fdb5/database/EntryVisitMechanism.cc | 2 +-
 src/fdb5/toc/TocCatalogueWriter.cc       | 1 +
 4 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/fdb5/api/LocalFDB.cc b/src/fdb5/api/LocalFDB.cc
index 8c2d680a7..2996fcaa8 100644
--- a/src/fdb5/api/LocalFDB.cc
+++ b/src/fdb5/api/LocalFDB.cc
@@ -22,6 +22,7 @@
 #include "fdb5/api/LocalFDB.h"
 #include "fdb5/database/Archiver.h"
 #include "fdb5/database/Catalogue.h"
+#include "fdb5/database/Store.h"
 #include "fdb5/database/EntryVisitMechanism.h"
 #include "fdb5/database/Index.h"
 #include "fdb5/database/Inspector.h"
diff --git a/src/fdb5/api/local/QueryVisitor.h b/src/fdb5/api/local/QueryVisitor.h
index 6e35f56e8..6828504ed 100644
--- a/src/fdb5/api/local/QueryVisitor.h
+++ b/src/fdb5/api/local/QueryVisitor.h
@@ -19,8 +19,8 @@
 #ifndef fdb5_api_local_QueryVisitor_H
 #define fdb5_api_local_QueryVisitor_H
 
-#include "fdb5/database/Store.h"
 #include "fdb5/database/EntryVisitMechanism.h"
+#include "fdb5/database/Store.h"
 
 #include "eckit/container/Queue.h"
 
diff --git a/src/fdb5/database/EntryVisitMechanism.cc b/src/fdb5/database/EntryVisitMechanism.cc
index d022dd509..34f372e66 100644
--- a/src/fdb5/database/EntryVisitMechanism.cc
+++ b/src/fdb5/database/EntryVisitMechanism.cc
@@ -15,9 +15,9 @@
 #include "fdb5/api/helpers/FDBToolRequest.h"
 #include "fdb5/database/Manager.h"
 #include "fdb5/database/Key.h"
+#include "fdb5/database/Store.h"
 #include "fdb5/LibFdb5.h"
 #include "fdb5/rules/Schema.h"
-#include "fdb5/database/Store.h"
 
 using namespace eckit;
 
diff --git a/src/fdb5/toc/TocCatalogueWriter.cc b/src/fdb5/toc/TocCatalogueWriter.cc
index 0099af0b2..6ad90f009 100644
--- a/src/fdb5/toc/TocCatalogueWriter.cc
+++ b/src/fdb5/toc/TocCatalogueWriter.cc
@@ -15,6 +15,7 @@
 #include "eckit/log/Bytes.h"
 #include "eckit/io/EmptyHandle.h"
 
+#include "fdb5/database/Store.h"
 #include "fdb5/database/EntryVisitMechanism.h"
 #include "fdb5/io/FDBFileHandle.h"
 #include "fdb5/LibFdb5.h"

From 4ea6a5b80c3345c0fbed16a38471bcfd18b7bfa9 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 20 Dec 2024 15:37:17 +0000
Subject: [PATCH 183/186] Revert "wip"

This reverts commit 5a6a08f40ce34e5c50671de476c3303a3e23c25c.
---
 src/fdb5/api/LocalFDB.cc                 | 1 -
 src/fdb5/api/local/QueryVisitor.h        | 2 +-
 src/fdb5/database/EntryVisitMechanism.cc | 2 +-
 src/fdb5/toc/TocCatalogueWriter.cc       | 1 -
 4 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/fdb5/api/LocalFDB.cc b/src/fdb5/api/LocalFDB.cc
index 2996fcaa8..8c2d680a7 100644
--- a/src/fdb5/api/LocalFDB.cc
+++ b/src/fdb5/api/LocalFDB.cc
@@ -22,7 +22,6 @@
 #include "fdb5/api/LocalFDB.h"
 #include "fdb5/database/Archiver.h"
 #include "fdb5/database/Catalogue.h"
-#include "fdb5/database/Store.h"
 #include "fdb5/database/EntryVisitMechanism.h"
 #include "fdb5/database/Index.h"
 #include "fdb5/database/Inspector.h"
diff --git a/src/fdb5/api/local/QueryVisitor.h b/src/fdb5/api/local/QueryVisitor.h
index 6828504ed..6e35f56e8 100644
--- a/src/fdb5/api/local/QueryVisitor.h
+++ b/src/fdb5/api/local/QueryVisitor.h
@@ -19,8 +19,8 @@
 #ifndef fdb5_api_local_QueryVisitor_H
 #define fdb5_api_local_QueryVisitor_H
 
-#include "fdb5/database/EntryVisitMechanism.h"
 #include "fdb5/database/Store.h"
+#include "fdb5/database/EntryVisitMechanism.h"
 
 #include "eckit/container/Queue.h"
 
diff --git a/src/fdb5/database/EntryVisitMechanism.cc b/src/fdb5/database/EntryVisitMechanism.cc
index 34f372e66..d022dd509 100644
--- a/src/fdb5/database/EntryVisitMechanism.cc
+++ b/src/fdb5/database/EntryVisitMechanism.cc
@@ -15,9 +15,9 @@
 #include "fdb5/api/helpers/FDBToolRequest.h"
 #include "fdb5/database/Manager.h"
 #include "fdb5/database/Key.h"
-#include "fdb5/database/Store.h"
 #include "fdb5/LibFdb5.h"
 #include "fdb5/rules/Schema.h"
+#include "fdb5/database/Store.h"
 
 using namespace eckit;
 
diff --git a/src/fdb5/toc/TocCatalogueWriter.cc b/src/fdb5/toc/TocCatalogueWriter.cc
index 6ad90f009..0099af0b2 100644
--- a/src/fdb5/toc/TocCatalogueWriter.cc
+++ b/src/fdb5/toc/TocCatalogueWriter.cc
@@ -15,7 +15,6 @@
 #include "eckit/log/Bytes.h"
 #include "eckit/io/EmptyHandle.h"
 
-#include "fdb5/database/Store.h"
 #include "fdb5/database/EntryVisitMechanism.h"
 #include "fdb5/io/FDBFileHandle.h"
 #include "fdb5/LibFdb5.h"

From b07fa9f982d42cca03eb49d8493f5d44f80f5867 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 20 Dec 2024 15:46:31 +0000
Subject: [PATCH 184/186] reverted to Store* in EntryVisitor

---
 src/fdb5/api/local/QueryVisitor.h        |  1 -
 src/fdb5/database/EntryVisitMechanism.cc | 13 +++++++++----
 src/fdb5/database/EntryVisitMechanism.h  |  2 +-
 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/src/fdb5/api/local/QueryVisitor.h b/src/fdb5/api/local/QueryVisitor.h
index 6e35f56e8..7c99054f3 100644
--- a/src/fdb5/api/local/QueryVisitor.h
+++ b/src/fdb5/api/local/QueryVisitor.h
@@ -19,7 +19,6 @@
 #ifndef fdb5_api_local_QueryVisitor_H
 #define fdb5_api_local_QueryVisitor_H
 
-#include "fdb5/database/Store.h"
 #include "fdb5/database/EntryVisitMechanism.h"
 
 #include "eckit/container/Queue.h"
diff --git a/src/fdb5/database/EntryVisitMechanism.cc b/src/fdb5/database/EntryVisitMechanism.cc
index d022dd509..bc2b8d328 100644
--- a/src/fdb5/database/EntryVisitMechanism.cc
+++ b/src/fdb5/database/EntryVisitMechanism.cc
@@ -33,14 +33,18 @@ class FDBVisitException : public eckit::Exception {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-EntryVisitor::~EntryVisitor() {}
+EntryVisitor::~EntryVisitor() {
+    if (currentStore_) {
+        delete currentStore_;
+    }
+}
 
 EntryVisitor::EntryVisitor() : currentCatalogue_(nullptr), currentStore_(nullptr), currentIndex_(nullptr) {}
 
 Store& EntryVisitor::store() const {
     if (!currentStore_) {
         ASSERT(currentCatalogue_);
-        currentStore_ = currentCatalogue_->buildStore();
+        currentStore_ = currentCatalogue_->buildStore().release();
         ASSERT(currentStore_);
     }
     return *currentStore_;
@@ -48,7 +52,7 @@ Store& EntryVisitor::store() const {
 
 bool EntryVisitor::visitDatabase(const Catalogue& catalogue) {
     currentCatalogue_ = &catalogue;
-    currentStore_.reset();
+    currentStore_ = nullptr;
     currentIndex_ = nullptr;
     rule_ = nullptr;
     return true;
@@ -59,7 +63,8 @@ void EntryVisitor::catalogueComplete(const Catalogue& catalogue) {
         ASSERT(currentCatalogue_ == &catalogue);
     }
     currentCatalogue_ = nullptr;
-    currentStore_.reset();
+    delete currentStore_;
+    currentStore_ = nullptr;
     currentIndex_ = nullptr;
     rule_ = nullptr;
 }
diff --git a/src/fdb5/database/EntryVisitMechanism.h b/src/fdb5/database/EntryVisitMechanism.h
index 512a255c4..92139c8ca 100644
--- a/src/fdb5/database/EntryVisitMechanism.h
+++ b/src/fdb5/database/EntryVisitMechanism.h
@@ -62,7 +62,7 @@ class EntryVisitor : public eckit::NonCopyable {
     /// Non-owning
     const Catalogue* currentCatalogue_ = nullptr;
     /// Owned store
-    mutable std::unique_ptr currentStore_ = nullptr;
+    mutable Store* currentStore_ = nullptr;
     /// Non-owning
     const Index* currentIndex_ = nullptr;
     /// Non-owning

From 92e37c7b67d7842874a8972261a9e634440c9ca3 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 20 Dec 2024 16:03:25 +0000
Subject: [PATCH 185/186] version bump

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index e09e09fa3..377371fa1 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.13.105
+5.13.106
\ No newline at end of file

From 8b3afcb34f76a38cbb342e9b00fe86fe5740432b Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 1 Jan 2025 16:19:14 +0100
Subject: [PATCH 186/186] Archiver::flushMutex_ as recursive_mutex to protect
 catalogue eviction from cache

---
 src/fdb5/database/Archiver.cc | 6 +++---
 src/fdb5/database/Archiver.h  | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/fdb5/database/Archiver.cc b/src/fdb5/database/Archiver.cc
index 7b49b9cb4..3b1840546 100644
--- a/src/fdb5/database/Archiver.cc
+++ b/src/fdb5/database/Archiver.cc
@@ -42,7 +42,7 @@ void Archiver::archive(const Key& key, const void* data, size_t len) {
 
 void Archiver::archive(const Key& key, BaseArchiveVisitor& visitor) {
 
-    std::lock_guard lock(flushMutex_);
+    std::lock_guard lock(flushMutex_);
     visitor.rule(nullptr);
     
     dbConfig_.schema().expand(key, visitor);
@@ -58,7 +58,7 @@ void Archiver::archive(const Key& key, BaseArchiveVisitor& visitor) {
 }
 
 void Archiver::flush() {
-    std::lock_guard lock(flushMutex_);
+    std::lock_guard lock(flushMutex_);
     for (auto i = databases_.begin(); i != databases_.end(); ++i) {
         // flush the store, pass the number of flushed fields to the catalogue
         i->second.catalogue_->flush(i->second.store_->flush());
@@ -92,7 +92,7 @@ void Archiver::selectDatabase(const Key& dbKey) {
             }
             if (found) {
                 // flushing before evicting from cache
-                std::lock_guard lock(flushMutex_);
+                std::lock_guard lock(flushMutex_);
 
                 databases_[oldK].catalogue_->flush(databases_[oldK].store_->flush());
                 
diff --git a/src/fdb5/database/Archiver.h b/src/fdb5/database/Archiver.h
index e2e525ac6..8d4b623b2 100644
--- a/src/fdb5/database/Archiver.h
+++ b/src/fdb5/database/Archiver.h
@@ -80,7 +80,7 @@ class Archiver : public eckit::NonCopyable {
 
     Database* db_;
 
-    std::mutex flushMutex_;
+    std::recursive_mutex flushMutex_;
     std::mutex cacheMutex_;
     const ArchiveCallback& callback_;
 };