rework checkpoint mem estimation (#4055)

add debug prefix to enable_multi_writes

src/include/main/db_config.h

@@ -16,7 +16,7 @@ class ClientContext;
 struct SystemConfig;
 
 typedef void (*set_context)(ClientContext* context, const common::Value& parameter);
-typedef common::Value (*get_setting)(ClientContext* context);
+typedef common::Value (*get_setting)(const ClientContext* context);
 
 enum class OptionType : uint8_t { CONFIGURATION = 0, EXTENSION = 1 };
 

src/include/main/settings.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "common/exception/not_implemented.h"
 #include "common/types/value/value.h"
 #include "main/client_context.h"
 #include "main/db_config.h"
@@ -8,202 +9,204 @@ namespace kuzu {
 namespace main {
 
 struct ThreadsSetting {
-    static constexpr const char* name = "threads";
-    static constexpr const common::LogicalTypeID inputType = common::LogicalTypeID::INT64;
+    static constexpr auto name = "threads";
+    static constexpr auto inputType = common::LogicalTypeID::INT64;
     static void setContext(ClientContext* context, const common::Value& parameter) {
         parameter.validateType(inputType);
         context->getClientConfigUnsafe()->numThreads = parameter.getValue<int64_t>();
     }
-    static common::Value getSetting(ClientContext* context) {
+    static common::Value getSetting(const ClientContext* context) {
         return common::Value(context->getClientConfig()->numThreads);
     }
 };
 
 struct TimeoutSetting {
-    static constexpr const char* name = "timeout";
-    static constexpr const common::LogicalTypeID inputType = common::LogicalTypeID::INT64;
+    static constexpr auto name = "timeout";
+    static constexpr auto inputType = common::LogicalTypeID::INT64;
     static void setContext(ClientContext* context, const common::Value& parameter) {
         parameter.validateType(inputType);
         context->getClientConfigUnsafe()->timeoutInMS = parameter.getValue<int64_t>();
     }
-    static common::Value getSetting(ClientContext* context) {
+    static common::Value getSetting(const ClientContext* context) {
         return common::Value(context->getClientConfig()->timeoutInMS);
     }
 };
 
 struct ProgressBarSetting {
-    static constexpr const char* name = "progress_bar";
-    static constexpr const common::LogicalTypeID inputType = common::LogicalTypeID::BOOL;
+    static constexpr auto name = "progress_bar";
+    static constexpr auto inputType = common::LogicalTypeID::BOOL;
     static void setContext(ClientContext* context, const common::Value& parameter) {
         parameter.validateType(inputType);
         context->getClientConfigUnsafe()->enableProgressBar = parameter.getValue<bool>();
         context->getProgressBar()->toggleProgressBarPrinting(parameter.getValue<bool>());
     }
-    static common::Value getSetting(ClientContext* context) {
+    static common::Value getSetting(const ClientContext* context) {
         return common::Value(context->getClientConfig()->enableProgressBar);
     }
 };
 
 struct ProgressBarTimerSetting {
-    static constexpr const char* name = "progress_bar_time";
-    static constexpr const common::LogicalTypeID inputType = common::LogicalTypeID::INT64;
+    static constexpr auto name = "progress_bar_time";
+    static constexpr auto inputType = common::LogicalTypeID::INT64;
     static void setContext(ClientContext* context, const common::Value& parameter) {
         parameter.validateType(inputType);
         context->getClientConfigUnsafe()->showProgressAfter = parameter.getValue<int64_t>();
         context->getProgressBar()->setShowProgressAfter(parameter.getValue<int64_t>());
     }
-    static common::Value getSetting(ClientContext* context) {
+    static common::Value getSetting(const ClientContext* context) {
         return common::Value(context->getClientConfig()->showProgressAfter);
     }
 };
 
 struct VarLengthExtendMaxDepthSetting {
-    static constexpr const char* name = "var_length_extend_max_depth";
-    static constexpr const common::LogicalTypeID inputType = common::LogicalTypeID::INT64;
+    static constexpr auto name = "var_length_extend_max_depth";
+    static constexpr auto inputType = common::LogicalTypeID::INT64;
     static void setContext(ClientContext* context, const common::Value& parameter) {
         parameter.validateType(inputType);
         context->getClientConfigUnsafe()->varLengthMaxDepth = parameter.getValue<int64_t>();
     }
-    static common::Value getSetting(ClientContext* context) {
+    static common::Value getSetting(const ClientContext* context) {
         return common::Value(context->getClientConfig()->varLengthMaxDepth);
     }
 };
 
 struct EnableSemiMaskSetting {
-    static constexpr const char* name = "enable_semi_mask";
-    static constexpr const common::LogicalTypeID inputType = common::LogicalTypeID::BOOL;
+    static constexpr auto name = "enable_semi_mask";
+    static constexpr auto inputType = common::LogicalTypeID::BOOL;
     static void setContext(ClientContext* context, const common::Value& parameter) {
         parameter.validateType(inputType);
         context->getClientConfigUnsafe()->enableSemiMask = parameter.getValue<bool>();
     }
-    static common::Value getSetting(ClientContext* context) {
+    static common::Value getSetting(const ClientContext* context) {
         return common::Value(context->getClientConfig()->enableSemiMask);
     }
 };
 
 struct DisableMapKeyCheck {
-    static constexpr const char* name = "disable_map_key_check";
-    static constexpr const common::LogicalTypeID inputType = common::LogicalTypeID::BOOL;
+    static constexpr auto name = "disable_map_key_check";
+    static constexpr auto inputType = common::LogicalTypeID::BOOL;
     static void setContext(ClientContext* context, const common::Value& parameter) {
         parameter.validateType(inputType);
         context->getClientConfigUnsafe()->disableMapKeyCheck = parameter.getValue<bool>();
     }
-    static common::Value getSetting(ClientContext* context) {
+    static common::Value getSetting(const ClientContext* context) {
         return common::Value(context->getClientConfig()->disableMapKeyCheck);
     }
 };
 
 struct EnableZoneMapSetting {
-    static constexpr const char* name = "enable_zone_map";
-    static constexpr const common::LogicalTypeID inputType = common::LogicalTypeID::BOOL;
+    static constexpr auto name = "enable_zone_map";
+    static constexpr auto inputType = common::LogicalTypeID::BOOL;
     static void setContext(ClientContext* /*context*/, const common::Value& parameter) {
         parameter.validateType(inputType);
+        // TODO(Guodong/Xiyang/Ben): Turn me on when zone map is ready.
+        throw common::NotImplementedException("Zone map is not yet ready to be turned on.");
         // context->getClientConfigUnsafe()->enableZoneMap = parameter.getValue<bool>();
     }
-    static common::Value getSetting(ClientContext* context) {
+    static common::Value getSetting(const ClientContext* context) {
         return common::Value(context->getClientConfig()->enableZoneMap);
     }
 };
 
 struct HomeDirectorySetting {
-    static constexpr const char* name = "home_directory";
-    static constexpr const common::LogicalTypeID inputType = common::LogicalTypeID::STRING;
+    static constexpr auto name = "home_directory";
+    static constexpr auto inputType = common::LogicalTypeID::STRING;
     static void setContext(ClientContext* context, const common::Value& parameter) {
         parameter.validateType(inputType);
         context->getClientConfigUnsafe()->homeDirectory = parameter.getValue<std::string>();
     }
-    static common::Value getSetting(ClientContext* context) {
+    static common::Value getSetting(const ClientContext* context) {
         return common::Value::createValue(context->getClientConfig()->homeDirectory);
     }
 };
 
 struct FileSearchPathSetting {
-    static constexpr const char* name = "file_search_path";
-    static constexpr const common::LogicalTypeID inputType = common::LogicalTypeID::STRING;
+    static constexpr auto name = "file_search_path";
+    static constexpr auto inputType = common::LogicalTypeID::STRING;
     static void setContext(ClientContext* context, const common::Value& parameter) {
         parameter.validateType(inputType);
         context->getClientConfigUnsafe()->fileSearchPath = parameter.getValue<std::string>();
     }
-    static common::Value getSetting(ClientContext* context) {
+    static common::Value getSetting(const ClientContext* context) {
         return common::Value::createValue(context->getClientConfig()->fileSearchPath);
     }
 };
 
 struct RecursivePatternSemanticSetting {
-    static constexpr const char* name = "recursive_pattern_semantic";
-    static constexpr const common::LogicalTypeID inputType = common::LogicalTypeID::STRING;
+    static constexpr auto name = "recursive_pattern_semantic";
+    static constexpr auto inputType = common::LogicalTypeID::STRING;
     static void setContext(ClientContext* context, const common::Value& parameter) {
         parameter.validateType(inputType);
-        auto input = parameter.getValue<std::string>();
+        const auto input = parameter.getValue<std::string>();
         context->getClientConfigUnsafe()->recursivePatternSemantic =
             common::PathSemanticUtils::fromString(input);
     }
-    static common::Value getSetting(ClientContext* context) {
-        auto result = common::PathSemanticUtils::toString(
+    static common::Value getSetting(const ClientContext* context) {
+        const auto result = common::PathSemanticUtils::toString(
             context->getClientConfig()->recursivePatternSemantic);
         return common::Value::createValue(result);
     }
 };
 
 struct RecursivePatternFactorSetting {
-    static constexpr const char* name = "recursive_pattern_factor";
-    static constexpr const common::LogicalTypeID inputType = common::LogicalTypeID::INT64;
+    static constexpr auto name = "recursive_pattern_factor";
+    static constexpr auto inputType = common::LogicalTypeID::INT64;
     static void setContext(ClientContext* context, const common::Value& parameter) {
         parameter.validateType(inputType);
         context->getClientConfigUnsafe()->recursivePatternCardinalityScaleFactor =
             parameter.getValue<std::int64_t>();
     }
-    static common::Value getSetting(ClientContext* context) {
+    static common::Value getSetting(const ClientContext* context) {
         return common::Value::createValue(
             context->getClientConfig()->recursivePatternCardinalityScaleFactor);
     }
 };
 
 struct EnableMVCCSetting {
-    static constexpr const char* name = "enable_multi_writes";
-    static constexpr const common::LogicalTypeID inputType = common::LogicalTypeID::BOOL;
+    static constexpr auto name = "debug_enable_multi_writes";
+    static constexpr auto inputType = common::LogicalTypeID::BOOL;
     static void setContext(ClientContext* context, const common::Value& parameter) {
         KU_ASSERT(parameter.getDataType().getLogicalTypeID() == common::LogicalTypeID::BOOL);
         // TODO: This is a temporary solution to make tests of multiple write transactions easier.
         context->getDBConfigUnsafe()->enableMultiWrites = parameter.getValue<bool>();
     }
-    static common::Value getSetting(ClientContext* context) {
+    static common::Value getSetting(const ClientContext* context) {
         return common::Value(context->getDBConfig()->enableMultiWrites);
     }
 };
 
 struct CheckpointThresholdSetting {
-    static constexpr const char* name = "checkpoint_threshold";
-    static constexpr common::LogicalTypeID inputType = common::LogicalTypeID::INT64;
+    static constexpr auto name = "checkpoint_threshold";
+    static constexpr auto inputType = common::LogicalTypeID::INT64;
     static void setContext(ClientContext* context, const common::Value& parameter) {
         parameter.validateType(inputType);
         context->getDBConfigUnsafe()->checkpointThreshold = parameter.getValue<int64_t>();
     }
-    static common::Value getSetting(ClientContext* context) {
+    static common::Value getSetting(const ClientContext* context) {
         return common::Value(context->getDBConfig()->checkpointThreshold);
     }
 };
 
 struct AutoCheckpointSetting {
-    static constexpr const char* name = "auto_checkpoint";
-    static constexpr common::LogicalTypeID inputType = common::LogicalTypeID::BOOL;
+    static constexpr auto name = "auto_checkpoint";
+    static constexpr auto inputType = common::LogicalTypeID::BOOL;
     static void setContext(ClientContext* context, const common::Value& parameter) {
         parameter.validateType(inputType);
         context->getDBConfigUnsafe()->autoCheckpoint = parameter.getValue<bool>();
     }
-    static common::Value getSetting(ClientContext* context) {
+    static common::Value getSetting(const ClientContext* context) {
         return common::Value(context->getDBConfig()->autoCheckpoint);
     }
 };
 
 struct ForceCheckpointClosingDBSetting {
-    static constexpr const char* name = "force_checkpoint_on_close";
-    static constexpr common::LogicalTypeID inputType = common::LogicalTypeID::BOOL;
+    static constexpr auto name = "force_checkpoint_on_close";
+    static constexpr auto inputType = common::LogicalTypeID::BOOL;
     static void setContext(ClientContext* context, const common::Value& parameter) {
         parameter.validateType(inputType);
         context->getDBConfigUnsafe()->forceCheckpointOnClose = parameter.getValue<bool>();
     }
-    static common::Value getSetting(ClientContext* context) {
+    static common::Value getSetting(const ClientContext* context) {
         return common::Value(context->getDBConfig()->forceCheckpointOnClose);
     }
 };

src/include/storage/index/in_mem_hash_index.h

@@ -145,6 +145,7 @@ class InMemHashIndex final {
 
     uint64_t numPrimarySlots() const { return pSlots->size(); }
     uint64_t numOverflowSlots() const { return oSlots->size(); }
+    uint64_t getEstimatedMemUsage() const { return pSlots->getMemUsage() + oSlots->getMemUsage(); }
 
     const HashIndexHeader& getIndexHeader() const { return indexHeader; }
 

src/include/storage/local_storage/local_hash_index.h

@@ -12,6 +12,7 @@ namespace storage {
 class BaseHashIndexLocalStorage {
 public:
     virtual ~BaseHashIndexLocalStorage() = default;
+    virtual uint64_t getEstimatedMemUsage() = 0;
 };
 
 enum class HashIndexLocalLookupState : uint8_t { KEY_FOUND, KEY_DELETED, KEY_NOT_EXIST };
@@ -78,6 +79,11 @@ class HashIndexLocalStorage final : public BaseHashIndexLocalStorage {
 
     const InMemHashIndex<T>& getInsertions() { return localInsertions; }
 
+    uint64_t getEstimatedMemUsage() override {
+        return localInsertions.getEstimatedMemUsage() +
+               localDeletions.size() * sizeof(OwnedKeyType);
+    }
+
 private:
     // When the storage type is string, allow the key type to be string_view with a custom hash
     // function
@@ -176,6 +182,8 @@ class LocalHashIndex {
             ->deleteKey(key);
     }
 
+    uint64_t getEstimatedMemUsage() { return localIndex->getEstimatedMemUsage(); }
+
 private:
     common::PhysicalTypeID keyDataTypeID;
     std::unique_ptr<BaseHashIndexLocalStorage> localIndex;

src/include/storage/local_storage/local_node_table.h

@@ -22,6 +22,7 @@ class LocalNodeTable final : public LocalTable {
     bool delete_(transaction::Transaction* transaction, TableDeleteState& deleteState) override;
     bool addColumn(transaction::Transaction* transaction,
         TableAddColumnState& addColumnState) override;
+    uint64_t getEstimatedMemUsage() override;
 
     common::offset_t validateUniquenessConstraint(const transaction::Transaction* transaction,
         const common::ValueVector& pkVector);

src/include/storage/local_storage/local_rel_table.h

@@ -26,6 +26,7 @@ class LocalRelTable final : public LocalTable {
     bool delete_(transaction::Transaction* transaction, TableDeleteState& state) override;
     bool addColumn(transaction::Transaction* transaction,
         TableAddColumnState& addColumnState) override;
+    uint64_t getEstimatedMemUsage() override;
 
     void checkIfNodeHasRels(common::ValueVector* srcNodeIDVector) const;
 

src/include/storage/local_storage/local_storage.h

@@ -29,6 +29,8 @@ class LocalStorage {
     void commit();
     void rollback();
 
+    uint64_t getEstimatedMemUsage() const;
+
 private:
     main::ClientContext& clientContext;
     std::unordered_map<common::table_id_t, std::unique_ptr<LocalTable>> tables;

src/include/storage/local_storage/local_table.h

@@ -33,6 +33,7 @@ class LocalTable {
         TableAddColumnState& addColumnState) = 0;
     virtual void clear() = 0;
     virtual common::TableType getTableType() const = 0;
+    virtual uint64_t getEstimatedMemUsage() = 0;
 
     template<class TARGET>
     const TARGET& constCast() {

src/include/storage/storage_manager.h

@@ -45,8 +45,6 @@ class StorageManager {
     bool isReadOnly() const { return readOnly; }
     bool compressionEnabled() const { return enableCompression; }
 
-    uint64_t getEstimatedMemoryUsage();
-
 private:
     BMFileHandle* initFileHandle(const std::string& fileName, common::VirtualFileSystem* vfs,
         main::ClientContext* context) const;

src/include/storage/storage_structure/disk_array.h

@@ -357,6 +357,10 @@ class BlockVectorInternal {
     // memory and not on disk (nor on the wal).
     uint8_t* operator[](uint64_t idx);
 
+    uint64_t getMemUsage() const {
+        return inMemArrayPages.size() * common::BufferPoolConstants::PAGE_4KB_SIZE;
+    }
+
 protected:
     inline uint64_t addInMemoryArrayPage(bool setToZero) {
         inMemArrayPages.emplace_back(
@@ -399,6 +403,8 @@ class BlockVector {
         return DiskArray<U>::getAlignedElementSize();
     }
 
+    uint64_t getMemUsage() const { return vector.getMemUsage(); }
+
 private:
     BlockVectorInternal vector;
 };

src/include/storage/store/node_table.h

@@ -153,8 +153,6 @@ class NodeTable final : public Table {
     void rollback(LocalTable* localTable) override;
     void checkpoint(common::Serializer& ser) override;
 
-    uint64_t getEstimatedMemoryUsage() const override;
-
     common::node_group_idx_t getNumCommittedNodeGroups() const {
         return nodeGroups->getNumNodeGroups();
     }

src/include/storage/store/rel_table.h

@@ -167,8 +167,6 @@ class RelTable final : public Table {
     void rollback(LocalTable* localTable) override;
     void checkpoint(common::Serializer& ser) override;
 
-    uint64_t getEstimatedMemoryUsage() const override { return 0; }
-
     common::row_idx_t getNumRows() override { return nextRelOffset; }
 
     RelTableData* getDirectedTableData(common::RelDataDirection direction) const {

src/include/storage/store/table.h

@@ -167,8 +167,6 @@ class Table {
     virtual void rollback(LocalTable* localTable) = 0;
     virtual void checkpoint(common::Serializer& ser) = 0;
 
-    virtual uint64_t getEstimatedMemoryUsage() const = 0;
-
     virtual common::row_idx_t getNumRows() = 0;
 
     void setHasChanges() { hasChanges = true; }