Merge pull request #31100 from vespa-engine/toregge/expose-imported-a…

…ttributes-in-state-explorer

Expose imported attributes in state explorer.

searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp

@@ -2,11 +2,19 @@
 
 #include <vespa/searchcore/proton/attribute/attribute_manager_explorer.h>
 #include <vespa/searchcore/proton/attribute/attributemanager.h>
+#include <vespa/searchcore/proton/attribute/imported_attributes_repo.h>
+#include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h>
+#include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h>
 #include <vespa/searchcore/proton/test/attribute_utils.h>
 #include <vespa/searchcore/proton/test/attribute_vectors.h>
+#include <vespa/searchlib/attribute/attributefactory.h>
+#include <vespa/searchlib/attribute/imported_attribute_vector.h>
+#include <vespa/searchlib/attribute/imported_attribute_vector_factory.h>
 #include <vespa/searchlib/attribute/interlock.h>
+#include <vespa/searchlib/attribute/reference_attribute.h>
 #include <vespa/searchlib/index/dummyfileheadercontext.h>
 #include <vespa/searchlib/test/directory_handler.h>
+#include <vespa/searchlib/test/mock_gid_to_lid_mapping.h>
 #include <vespa/searchcommon/attribute/config.h>
 #include <vespa/vespalib/data/slime/slime.h>
 #include <vespa/vespalib/gtest/gtest.h>
@@ -21,6 +29,12 @@ using namespace proton;
 using namespace proton::test;
 using search::AttributeVector;
 using search::DictionaryConfig;
+using search::attribute::BasicType;
+using search::attribute::Config;
+using search::attribute::ImportedAttributeVector;
+using search::attribute::ImportedAttributeVectorFactory;
+using search::attribute::ReferenceAttribute;
+using search::attribute::test::MockGidToLidMapperFactory;
 using vespalib::ForegroundThreadExecutor;
 using vespalib::ISequencedTaskExecutor;
 using vespalib::SequencedTaskExecutor;
@@ -32,6 +46,10 @@ using vespalib::HwInfo;
 
 const vespalib::string TEST_DIR = "test_output";
 
+const vespalib::string ref_name("ref");
+const vespalib::string target_name("f3");
+const vespalib::string imported_name("my_f3");
+
 namespace {
 VESPA_THREAD_STACK_TAG(test_executor)
 }
@@ -43,28 +61,13 @@ struct AttributesStateExplorerTest : public ::testing::Test
     std::unique_ptr<ISequencedTaskExecutor> _attribute_field_writer;
     ForegroundThreadExecutor _shared;
     HwInfo                 _hwInfo;
+    std::shared_ptr<const IDocumentMetaStoreContext> _parent_dms;
+    std::shared_ptr<IDocumentMetaStoreContext> _dms;
+    std::shared_ptr<AttributeManager> _parent_mgr;
     AttributeManager::SP _mgr;
     AttributeManagerExplorer _explorer;
-    AttributesStateExplorerTest()
-        : _dirHandler(TEST_DIR),
-          _fileHeaderContext(),
-          _attribute_field_writer(SequencedTaskExecutor::create(test_executor, 1)),
-          _shared(),
-          _hwInfo(),
-          _mgr(new AttributeManager(TEST_DIR, "test.subdb", TuneFileAttributes(),
-                                    _fileHeaderContext,
-                                    std::make_shared<search::attribute::Interlock>(),
-                                    *_attribute_field_writer,
-                                    _shared,
-                                    _hwInfo)),
-          _explorer(_mgr)
-    {
-        addAttribute("regular");
-        addExtraAttribute("extra");
-        add_fast_search_attribute("btree", DictionaryConfig::Type::BTREE);
-        add_fast_search_attribute("hybrid", DictionaryConfig::Type::BTREE_AND_HASH);
-        add_fast_search_attribute("hash", DictionaryConfig::Type::HASH);
-    }
+    AttributesStateExplorerTest() noexcept;
+    ~AttributesStateExplorerTest() override;
     void addAttribute(const vespalib::string &name) {
         _mgr->addAttribute({name, AttributeUtils::getInt32Config()}, 1);
     }
@@ -84,16 +87,71 @@ struct AttributesStateExplorerTest : public ::testing::Test
         _explorer.get_child(name)->get_state(inserter, true);
         return result;
     }
-
+    void add_reference_attribute() {
+        search::attribute::Config cfg(BasicType::REFERENCE);
+        _mgr->addAttribute({ ref_name, cfg }, 1);
+        auto& ref_attr = dynamic_cast<ReferenceAttribute&>(**_mgr->getAttribute(ref_name));
+        ref_attr.setGidToLidMapperFactory(std::make_shared<MockGidToLidMapperFactory>());
+    }
+    std::shared_ptr<ReferenceAttribute> get_reference_attribute() {
+        return std::dynamic_pointer_cast<ReferenceAttribute>(_mgr->getAttribute(ref_name)->getSP());
+    }
+    void add_imported_attributes() {
+        auto repo = std::make_unique<ImportedAttributesRepo>();
+        auto attr = ImportedAttributeVectorFactory::create(imported_name,
+                                                           get_reference_attribute(),
+                                                           _dms,
+                                                           _parent_mgr->getAttribute(target_name)->getSP(),
+                                                           _parent_dms,
+                                                           false);
+        repo->add(imported_name, attr);
+        _mgr->setImportedAttributes(std::move(repo));
+    }
 };
 
+AttributesStateExplorerTest::AttributesStateExplorerTest() noexcept
+    : _dirHandler(TEST_DIR),
+      _fileHeaderContext(),
+      _attribute_field_writer(SequencedTaskExecutor::create(test_executor, 1)),
+      _shared(),
+      _hwInfo(),
+      _parent_dms(std::make_shared<const DocumentMetaStoreContext>(std::make_shared<bucketdb::BucketDBOwner>())),
+      _dms(),
+      _parent_mgr(std::make_shared<AttributeManager>
+                  (TEST_DIR, "test.parent.subdb", TuneFileAttributes(),
+                   _fileHeaderContext,
+                   std::make_shared<search::attribute::Interlock>(),
+                   *_attribute_field_writer,
+                   _shared,
+                   _hwInfo)),
+      _mgr(new AttributeManager(TEST_DIR, "test.subdb", TuneFileAttributes(),
+                                _fileHeaderContext,
+                                std::make_shared<search::attribute::Interlock>(),
+                                *_attribute_field_writer,
+                                _shared,
+                                _hwInfo)),
+      _explorer(_mgr)
+{
+    _parent_mgr->addAttribute({target_name, AttributeUtils::getInt32Config()}, 1);
+    addAttribute("regular");
+    addExtraAttribute("extra");
+    add_fast_search_attribute("btree", DictionaryConfig::Type::BTREE);
+    add_fast_search_attribute("hybrid", DictionaryConfig::Type::BTREE_AND_HASH);
+    add_fast_search_attribute("hash", DictionaryConfig::Type::HASH);
+    add_reference_attribute();
+    add_imported_attributes();
+}
+
+AttributesStateExplorerTest::~AttributesStateExplorerTest() = default;
+
+
 using StringVector = std::vector<vespalib::string>;
 
 TEST_F(AttributesStateExplorerTest, require_that_attributes_are_exposed_as_children_names)
 {
     StringVector children = _explorer.get_children_names();
     std::sort(children.begin(), children.end());
-    EXPECT_EQ(StringVector({"btree", "hash", "hybrid", "regular"}), children);
+    EXPECT_EQ(StringVector({"btree", "hash", "hybrid", "my_f3", "ref", "regular"}), children);
 }
 
 TEST_F(AttributesStateExplorerTest, require_that_attributes_are_explorable)
@@ -125,4 +183,12 @@ TEST_F(AttributesStateExplorerTest, require_that_dictionary_memory_usage_is_repo
     }
 }
 
+TEST_F(AttributesStateExplorerTest, require_that_imported_attribute_shows_memory_usage)
+{
+    vespalib::string cache_memory_usage("cacheMemoryUsage");
+    auto slime = explore_attribute(imported_name);
+    EXPECT_LT(0, slime[cache_memory_usage]["allocated"].asLong());
+    EXPECT_LT(0, slime[cache_memory_usage]["used"].asLong());
+}
+
 GTEST_MAIN_RUN_ALL_TESTS()

searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt

@@ -34,6 +34,7 @@ vespa_add_library(searchcore_attribute STATIC
     document_field_retriever.cpp
     filter_attribute_manager.cpp
     flushableattribute.cpp
+    imported_attribute_vector_explorer.cpp
     imported_attributes_context.cpp
     imported_attributes_repo.cpp
     initialized_attributes_result.cpp

searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_explorer.cpp

@@ -3,9 +3,13 @@
 #include "attribute_manager_explorer.h"
 #include "attribute_executor.h"
 #include "attribute_vector_explorer.h"
+#include "imported_attribute_vector_explorer.h"
+#include "imported_attributes_repo.h"
 #include <vespa/searchlib/attribute/attributevector.h>
+#include <vespa/searchlib/attribute/imported_attribute_vector.h>
 
 using search::AttributeVector;
+using search::attribute::ImportedAttributeVector;
 using vespalib::slime::Inserter;
 
 namespace proton {
@@ -32,13 +36,30 @@ AttributeManagerExplorer::get_children_names() const
     for (const auto &attr : attributes) {
         names.push_back(attr->getName());
     }
+    auto imported = _mgr->getImportedAttributes();
+    if (imported != nullptr) {
+        std::vector<std::shared_ptr<ImportedAttributeVector>> i_list;
+        imported->getAll(i_list);
+        for (const auto& attr : i_list) {
+            names.push_back(attr->getName());
+        }
+    }
     return names;
 }
 
 std::unique_ptr<vespalib::StateExplorer>
 AttributeManagerExplorer::get_child(vespalib::stringref name) const
 {
     auto guard = _mgr->getAttribute(name);
+    if (!guard || !guard->getSP()) {
+        auto imported = _mgr->getImportedAttributes();
+        if (imported != nullptr) {
+            auto& imported_attr = imported->get(name);
+            if (imported_attr) {
+                return std::make_unique<ImportedAttributeVectorExplorer>(imported_attr);
+            }
+        }
+    }
     auto attr = guard ? guard->getSP() : std::shared_ptr<AttributeVector>();
     if (attr && _mgr->getWritableAttribute(name) != nullptr) {
         auto executor = std::make_unique<AttributeExecutor>(_mgr, std::move(attr));

searchcore/src/vespa/searchcore/proton/attribute/imported_attribute_vector_explorer.cpp

@@ -0,0 +1,28 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "imported_attribute_vector_explorer.h"
+#include <vespa/searchlib/attribute/imported_attribute_vector.h>
+#include <vespa/searchlib/util/state_explorer_utils.h>
+#include <vespa/vespalib/data/slime/cursor.h>
+#include <vespa/vespalib/util/memoryusage.h>
+
+using search::StateExplorerUtils;
+using search::attribute::ImportedAttributeVector;
+using namespace vespalib::slime;
+
+namespace proton {
+
+ImportedAttributeVectorExplorer::ImportedAttributeVectorExplorer(std::shared_ptr<ImportedAttributeVector> attr)
+    : _attr(std::move(attr))
+{
+}
+
+void
+ImportedAttributeVectorExplorer::get_state(const vespalib::slime::Inserter &inserter, bool) const
+{
+    Cursor &object = inserter.insertObject();
+    auto memory_usage = _attr->get_memory_usage();
+    StateExplorerUtils::memory_usage_to_slime(memory_usage, object.setObject("cacheMemoryUsage"));
+}
+
+}

searchcore/src/vespa/searchcore/proton/attribute/imported_attribute_vector_explorer.h

@@ -0,0 +1,26 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/net/http/state_explorer.h>
+
+namespace search::attribute { class ImportedAttributeVector; }
+
+namespace proton {
+
+/**
+ * Class used to explore the state of an imported attribute vector.
+ */
+class ImportedAttributeVectorExplorer : public vespalib::StateExplorer
+{
+private:
+    std::shared_ptr<search::attribute::ImportedAttributeVector> _attr;
+
+public:
+    ImportedAttributeVectorExplorer(std::shared_ptr<search::attribute::ImportedAttributeVector> attr);
+
+    // Implements vespalib::StateExplorer
+    void get_state(const vespalib::slime::Inserter &inserter, bool full) const override;
+};
+
+}