From ac111e7a2939d861a80fbe25413d781bf114b9b1 Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte Date: Tue, 26 Nov 2024 16:08:51 +0100 Subject: [PATCH] MGM/INSPECTOR: Honor user-defined interval option. Fixes EOS-6248 --- common/json/Json.hh | 15 +++ mgm/inspector/FileInspector.cc | 114 ++++++++++-------- mgm/inspector/FileInspector.hh | 14 +-- mgm/inspector/FileInspectorStats.cc | 176 ---------------------------- mgm/inspector/FileInspectorStats.hh | 70 +---------- 5 files changed, 93 insertions(+), 296 deletions(-) diff --git a/common/json/Json.hh b/common/json/Json.hh index 1ac8fac24..f4fa6be18 100644 --- a/common/json/Json.hh +++ b/common/json/Json.hh @@ -284,6 +284,14 @@ void ConvertFromJson(const Json::Value& val, std::unordered_map& out) } } +/** + * @brief Serializes an object into a JSON string. + * + * @tparam T The type of the object to serialize. + * @param input The object to be serialized. + * @param indentation (Optional) Indentation string for pretty-printing. Defaults to no indentation. + * @return A string containing the JSON representation of the object. + */ template std::string Marshal(const T& input, const char* indentation = "") { @@ -293,6 +301,13 @@ std::string Marshal(const T& input, const char* indentation = "") return Json::writeString(writer, root); } +/** + * @brief Deserializes a JSON string into an object. + * + * @tparam T The type of the object to deserialize into. + * @param input The JSON string to deserialize. + * @param out The object where the deserialized data will be stored. + */ template void Unmarshal(const std::string input, T& out) { diff --git a/mgm/inspector/FileInspector.cc b/mgm/inspector/FileInspector.cc index 7ebe11d1f..f5f372946 100644 --- a/mgm/inspector/FileInspector.cc +++ b/mgm/inspector/FileInspector.cc @@ -26,6 +26,7 @@ #include "common/IntervalStopwatch.hh" #include "common/LayoutId.hh" #include "common/Timing.hh" +#include "common/json/Json.hh" #include "common/ParseUtils.hh" #include "mgm/FsView.hh" #include "mgm/inspector/FileInspector.hh" @@ -137,7 +138,7 @@ FileInspector::Options FileInspector::getOptions(const LockFsView lockfsview) if (opts.enabled) { enable(); - eos_static_debug("file inspector is enabled - interval = %ld seconds", + eos_static_debug("msg=\"file inspector is enabled\" interval=%lds", opts.interval.count()); } else { disable(); @@ -148,7 +149,7 @@ FileInspector::Options FileInspector::getOptions(const LockFsView lockfsview) //------------------------------------------------------------------------------ -// Background Thread cleaning up left-over atomic uploads +// Background Thread to compute the stats //------------------------------------------------------------------------------ void FileInspector::backgroundThread(ThreadAssistant& assistant) noexcept @@ -164,7 +165,7 @@ FileInspector::backgroundThread(ThreadAssistant& assistant) noexcept } assistant.wait_for(std::chrono::seconds(10)); - eos_static_info("msg=\"async thread started\""); + eos_static_info("%s", "msg=\"async thread started\""); if (mQdbHelper.HasStats()) { mQdbHelper.Load(mLastStats); @@ -180,13 +181,14 @@ FileInspector::backgroundThread(ThreadAssistant& assistant) noexcept disable(); } - common::IntervalStopwatch stopwatch(std::chrono::seconds(60)); + common::IntervalStopwatch stopwatch(std::chrono::seconds( + opts.interval.count())); if (opts.enabled && gOFS->mMaster->IsMaster()) { - eos_static_info("msg=\"scan started!\""); + eos_static_info("%s", "msg=\"scan started\""); mCurrentStats.TimeScan = time(NULL); performCycleQDB(assistant); - eos_static_info("msg=\"scan finished!\""); + eos_static_info("%s", "msg=\"scan finished\""); } assistant.wait_for(stopwatch.timeRemainingInCycle()); @@ -198,7 +200,7 @@ FileInspector::backgroundThread(ThreadAssistant& assistant) noexcept //------------------------------------------------------------------------------ void FileInspector::performCycleQDB(ThreadAssistant& assistant) noexcept { - eos_static_info("msg=\"start FileInspector scan on QDB\""); + eos_static_info("%s", "msg=\"start FileInspector scan on QDB\""); // Initialize qclient.. if (!mQcl) { @@ -271,7 +273,7 @@ void FileInspector::performCycleQDB(ThreadAssistant& assistant) noexcept } if (scanner.hasError(err)) { - eos_static_err("msg=\"QDB scanner error - interrupting scan\" error=\"%s\"", + eos_static_err("msg=\"QDB scanner error, interrupting scan\" error=\"%s\"", err.c_str()); break; } @@ -1341,48 +1343,68 @@ FileInspector::Dump(std::string& out, std::string_view options, void FileInspector::QdbHelper::Store(const FileInspectorStats& stats) { - FileInspectorStatsSerializer s(stats); - mQHashStats.hset(SCAN_STATS_KEY, s.SerializeScanStats()); - mQHashStats.hset(FAULTY_FILES_KEY, s.SerializeFaultyFiles()); - mQHashStats.hset(ACCESS_TIME_FILES_KEY, s.SerializeAccessTimeFiles()); - mQHashStats.hset(ACCESS_TIME_VOLUME_KEY, s.SerializeAccessTimeVolume()); - mQHashStats.hset(BIRTH_TIME_FILES_KEY, s.SerializeBirthTimeFiles()); - mQHashStats.hset(BIRTH_TIME_VOLUME_KEY, s.SerializeBirthTimeVolume()); - mQHashStats.hset(BIRTH_VS_ACCESS_TIME_FILES_KEY, - s.SerializeBirthVsAccessTimeFiles()); - mQHashStats.hset(BIRTH_VS_ACCESS_TIME_VOLUME_KEY, - s.SerializeBirthVsAccessTimeVolume()); - mQHashStats.hset(USER_COSTS_KEY, s.SerializeUserCosts()); - mQHashStats.hset(GROUP_COSTS_KEY, s.SerializeGroupCosts()); - mQHashStats.hset(TOTAL_COSTS_KEY, s.SerializeTotalCosts()); - mQHashStats.hset(USER_BYTES_KEY, s.SerializeUserBytes()); - mQHashStats.hset(GROUP_BYTES_KEY, s.SerializeGroupBytes()); - mQHashStats.hset(TOTAL_BYTES_KEY, s.SerializeTotalBytes()); - mQHashStats.hset(NUM_FAULTY_FILES_KEY, s.SerializeNumFaultyFiles()); - mQHashStats.hset(TIME_SCAN_KEY, s.SerializeTimeScan()); + mQHashStats.hmset({ + SCAN_STATS_KEY, Marshal(stats.FaultyFiles), + FAULTY_FILES_KEY, Marshal(stats.FaultyFiles), + ACCESS_TIME_FILES_KEY, Marshal(stats.AccessTimeFiles), + ACCESS_TIME_VOLUME_KEY, Marshal(stats.AccessTimeVolume), + BIRTH_TIME_FILES_KEY, Marshal(stats.BirthTimeFiles), + BIRTH_TIME_VOLUME_KEY, Marshal(stats.BirthTimeVolume), + BIRTH_VS_ACCESS_TIME_FILES_KEY, Marshal(stats.BirthVsAccessTimeFiles), + BIRTH_VS_ACCESS_TIME_VOLUME_KEY, Marshal(stats.BirthVsAccessTimeVolume), + USER_COSTS_KEY, Marshal(stats.UserCosts), + GROUP_COSTS_KEY, Marshal(stats.GroupCosts), + TOTAL_COSTS_KEY, Marshal(stats.TotalCosts), + USER_BYTES_KEY, Marshal(stats.UserBytes), + GROUP_BYTES_KEY, Marshal(stats.GroupBytes), + TOTAL_BYTES_KEY, Marshal(stats.TotalBytes), + NUM_FAULTY_FILES_KEY, Marshal(stats.NumFaultyFiles), + TIME_SCAN_KEY, Marshal(stats.TimeScan) + }); } void FileInspector::QdbHelper::Load(FileInspectorStats& stats) { - FileInspectorStatsDeserializer s; - s.DeserializeScanStats(mQHashStats.hget(SCAN_STATS_KEY), stats); - s.DeserializeFaultyFiles(mQHashStats.hget(FAULTY_FILES_KEY), stats); - s.DeserializeAccessTimeFiles(mQHashStats.hget(ACCESS_TIME_FILES_KEY), stats); - s.DeserializeAccessTimeVolume(mQHashStats.hget(ACCESS_TIME_VOLUME_KEY), stats); - s.DeserializeBirthTimeFiles(mQHashStats.hget(BIRTH_TIME_FILES_KEY), stats); - s.DeserializeBirthTimeVolume(mQHashStats.hget(BIRTH_TIME_VOLUME_KEY), stats); - s.DeserializeBirthVsAccessTimeFiles(mQHashStats.hget( - BIRTH_VS_ACCESS_TIME_FILES_KEY), stats); - s.DeserializeBirthVsAccessTimeVolume(mQHashStats.hget( - BIRTH_VS_ACCESS_TIME_VOLUME_KEY), stats); - s.DeserializeUserCosts(mQHashStats.hget(USER_COSTS_KEY), stats); - s.DeserializeGroupCosts(mQHashStats.hget(GROUP_COSTS_KEY), stats); - s.DeserializeTotalCosts(mQHashStats.hget(TOTAL_COSTS_KEY), stats); - s.DeserializeUserBytes(mQHashStats.hget(USER_BYTES_KEY), stats); - s.DeserializeGroupBytes(mQHashStats.hget(GROUP_BYTES_KEY), stats); - s.DeserializeTotalBytes(mQHashStats.hget(TOTAL_BYTES_KEY), stats); - s.DeserializeNumFaultyFiles(mQHashStats.hget(NUM_FAULTY_FILES_KEY), stats); - s.DeserializeTimeScan(mQHashStats.hget(TIME_SCAN_KEY), stats); + std::vector members = mQHashStats.hgetall(); + + for (int i = 0; i < members.size() - 1; i += 2) { + std::string key = members[i]; + std::string value = members[i + 1]; + + if (key == SCAN_STATS_KEY) { + Unmarshal(value, stats.ScanStats); + } else if (key == FAULTY_FILES_KEY) { + Unmarshal(value, stats.FaultyFiles); + } else if (key == ACCESS_TIME_FILES_KEY) { + Unmarshal(value, stats.AccessTimeFiles); + } else if (key == ACCESS_TIME_VOLUME_KEY) { + Unmarshal(value, stats.AccessTimeVolume); + } else if (key == BIRTH_TIME_FILES_KEY) { + Unmarshal(value, stats.BirthTimeFiles); + } else if (key == BIRTH_TIME_VOLUME_KEY) { + Unmarshal(value, stats.BirthTimeVolume); + } else if (key == BIRTH_VS_ACCESS_TIME_FILES_KEY) { + Unmarshal(value, stats.BirthVsAccessTimeFiles); + } else if (key == BIRTH_VS_ACCESS_TIME_VOLUME_KEY) { + Unmarshal(value, stats.BirthVsAccessTimeVolume); + } else if (key == USER_COSTS_KEY) { + Unmarshal(value, stats.UserCosts); + } else if (key == GROUP_COSTS_KEY) { + Unmarshal(value, stats.GroupCosts); + } else if (key == TOTAL_COSTS_KEY) { + Unmarshal(value, stats.TotalCosts); + } else if (key == USER_BYTES_KEY) { + Unmarshal(value, stats.UserBytes); + } else if (key == GROUP_BYTES_KEY) { + Unmarshal(value, stats.GroupBytes); + } else if (key == TOTAL_BYTES_KEY) { + Unmarshal(value, stats.TotalBytes); + } else if (key == NUM_FAULTY_FILES_KEY) { + Unmarshal(value, stats.NumFaultyFiles); + } else if (key == TIME_SCAN_KEY) { + Unmarshal(value, stats.TimeScan); + } + } } diff --git a/mgm/inspector/FileInspector.hh b/mgm/inspector/FileInspector.hh index d6c272d93..66f096818 100644 --- a/mgm/inspector/FileInspector.hh +++ b/mgm/inspector/FileInspector.hh @@ -23,20 +23,19 @@ #pragma once +#include "mgm/Namespace.hh" #include "common/VirtualIdentity.hh" #include "common/AssistedThread.hh" #include "mgm/inspector/FileInspectorStats.hh" -#include -#include "mgm/Namespace.hh" +#include "namespace/ns_quarkdb/QdbContactDetails.hh" +#include "namespace/ns_quarkdb/qclient/include/qclient/QClient.hh" +#include "namespace/ns_quarkdb/qclient/include/qclient/structures/QHash.hh" #include "namespace/interface/IFileMD.hh" +#include #include #include #include #include -#include "namespace/ns_quarkdb/QdbContactDetails.hh" -#include "namespace/ns_quarkdb/qclient/include/qclient/QClient.hh" -#include "namespace/ns_quarkdb/qclient/include/qclient/structures/QHash.hh" - EOSMGMNAMESPACE_BEGIN @@ -62,12 +61,13 @@ public: //! Constructor //! //! @param space_name corresponding space name + //! @param qdb_details for connecting to QuarkDB //---------------------------------------------------------------------------- FileInspector(std::string_view space_name, const eos::QdbContactDetails& qdb_details); //---------------------------------------------------------------------------- - //! Constructor + //! Destructor //---------------------------------------------------------------------------- virtual ~FileInspector(); diff --git a/mgm/inspector/FileInspectorStats.cc b/mgm/inspector/FileInspectorStats.cc index c095bf91c..644690113 100644 --- a/mgm/inspector/FileInspectorStats.cc +++ b/mgm/inspector/FileInspectorStats.cc @@ -104,180 +104,4 @@ noexcept return *this; } -std::string FileInspectorStatsSerializer::SerializeScanStats() -{ - return Marshal(mFileInspectorStats.ScanStats); -} - -std::string FileInspectorStatsSerializer::SerializeFaultyFiles() -{ - return Marshal(mFileInspectorStats.FaultyFiles); -} - -std::string FileInspectorStatsSerializer::SerializeAccessTimeFiles() -{ - return Marshal(mFileInspectorStats.AccessTimeFiles); -} - -std::string FileInspectorStatsSerializer::SerializeAccessTimeVolume() -{ - return Marshal(mFileInspectorStats.AccessTimeVolume); -} - -std::string FileInspectorStatsSerializer::SerializeBirthTimeFiles() -{ - return Marshal(mFileInspectorStats.BirthTimeFiles); -} - -std::string FileInspectorStatsSerializer::SerializeBirthTimeVolume() -{ - return Marshal(mFileInspectorStats.BirthTimeVolume); -} - -std::string FileInspectorStatsSerializer::SerializeBirthVsAccessTimeFiles() -{ - return Marshal(mFileInspectorStats.BirthVsAccessTimeFiles); -} - -std::string FileInspectorStatsSerializer::SerializeBirthVsAccessTimeVolume() -{ - return Marshal(mFileInspectorStats.BirthVsAccessTimeVolume); -} - -std::string FileInspectorStatsSerializer::SerializeUserCosts() -{ - return Marshal(mFileInspectorStats.UserCosts); -} - -std::string FileInspectorStatsSerializer::SerializeGroupCosts() -{ - return Marshal(mFileInspectorStats.GroupCosts); -} - -std::string FileInspectorStatsSerializer::SerializeTotalCosts() -{ - return Marshal(mFileInspectorStats.TotalCosts); -} - -std::string FileInspectorStatsSerializer::SerializeUserBytes() -{ - return Marshal(mFileInspectorStats.UserBytes); -} - -std::string FileInspectorStatsSerializer::SerializeGroupBytes() -{ - return Marshal(mFileInspectorStats.GroupBytes); -} - -std::string FileInspectorStatsSerializer::SerializeTotalBytes() -{ - return Marshal(mFileInspectorStats.TotalBytes); -} - -std::string FileInspectorStatsSerializer::SerializeNumFaultyFiles() -{ - return Marshal(mFileInspectorStats.NumFaultyFiles); -} - -std::string FileInspectorStatsSerializer::SerializeTimeScan() -{ - return Marshal(mFileInspectorStats.TimeScan); -} - -void FileInspectorStatsDeserializer::DeserializeScanStats( - const std::string& out, FileInspectorStats& stats) const -{ - Unmarshal(out, stats.ScanStats); -} - -void FileInspectorStatsDeserializer::DeserializeFaultyFiles( - const std::string& out, FileInspectorStats& stats) const -{ - Unmarshal(out, stats.FaultyFiles); -} - -void FileInspectorStatsDeserializer::DeserializeAccessTimeFiles( - const std::string& out, FileInspectorStats& stats) const -{ - Unmarshal(out, stats.AccessTimeFiles); -} - -void FileInspectorStatsDeserializer::DeserializeAccessTimeVolume( - const std::string& out, FileInspectorStats& stats) const -{ - Unmarshal(out, stats.AccessTimeVolume); -} - -void FileInspectorStatsDeserializer::DeserializeBirthTimeFiles( - const std::string& out, FileInspectorStats& stats) const -{ - Unmarshal(out, stats.BirthTimeFiles); -} - -void FileInspectorStatsDeserializer::DeserializeBirthTimeVolume( - const std::string& out, FileInspectorStats& stats) const -{ - Unmarshal(out, stats.BirthTimeVolume); -} - -void FileInspectorStatsDeserializer::DeserializeBirthVsAccessTimeFiles( - const std::string& out, FileInspectorStats& stats) const -{ - Unmarshal(out, stats.BirthVsAccessTimeFiles); -} - -void FileInspectorStatsDeserializer::DeserializeBirthVsAccessTimeVolume( - const std::string& out, FileInspectorStats& stats) const -{ - Unmarshal(out, stats.BirthVsAccessTimeVolume); -} - -void FileInspectorStatsDeserializer::DeserializeUserCosts( - const std::string& out, FileInspectorStats& stats) const -{ - Unmarshal(out, stats.UserCosts); -} - -void FileInspectorStatsDeserializer::DeserializeGroupCosts( - const std::string& out, FileInspectorStats& stats) const -{ - Unmarshal(out, stats.GroupCosts); -} - -void FileInspectorStatsDeserializer::DeserializeTotalCosts( - const std::string& out, FileInspectorStats& stats) const -{ - Unmarshal(out, stats.TotalCosts); -} - -void FileInspectorStatsDeserializer::DeserializeUserBytes( - const std::string& out, FileInspectorStats& stats) const -{ - Unmarshal(out, stats.UserBytes); -} - -void FileInspectorStatsDeserializer::DeserializeGroupBytes( - const std::string& out, FileInspectorStats& stats) const -{ - Unmarshal(out, stats.GroupBytes); -} - -void FileInspectorStatsDeserializer::DeserializeTotalBytes( - const std::string& out, FileInspectorStats& stats) const -{ - Unmarshal(out, stats.TotalBytes); -} - -void FileInspectorStatsDeserializer::DeserializeNumFaultyFiles( - const std::string& out, FileInspectorStats& stats) const -{ - Unmarshal(out, stats.NumFaultyFiles); -} - -void FileInspectorStatsDeserializer::DeserializeTimeScan(const std::string& out, - FileInspectorStats& stats) const -{ - Unmarshal(out, stats.TimeScan); -} - EOSMGMNAMESPACE_END \ No newline at end of file diff --git a/mgm/inspector/FileInspectorStats.hh b/mgm/inspector/FileInspectorStats.hh index 9c0ac87ff..645049642 100644 --- a/mgm/inspector/FileInspectorStats.hh +++ b/mgm/inspector/FileInspectorStats.hh @@ -51,7 +51,9 @@ EOSMGMNAMESPACE_BEGIN #define NUM_FAULTY_FILES_KEY "num-faulty-files" #define TIME_SCAN_KEY "time-scan" - +//------------------------------------------------------------------------------ +//! Class holding the stats information +//------------------------------------------------------------------------------ struct FileInspectorStats { FileInspectorStats() : TimeScan(0) {} @@ -113,71 +115,5 @@ private: const std::string kFileInspectorStats = "eos-file-inspector-stats"; }; -struct FileInspectorStatsSerializer { - friend class FileInspectorStats; - - FileInspectorStatsSerializer(const FileInspectorStats& stats) : - mFileInspectorStats(stats) {} - - std::string SerializeScanStats(); - std::string SerializeFaultyFiles(); - std::string SerializeAccessTimeFiles(); - std::string SerializeAccessTimeVolume(); - std::string SerializeBirthTimeFiles(); - std::string SerializeBirthTimeVolume(); - std::string SerializeBirthVsAccessTimeFiles(); - std::string SerializeBirthVsAccessTimeVolume(); - std::string SerializeUserCosts(); - std::string SerializeGroupCosts(); - std::string SerializeTotalCosts(); - std::string SerializeUserBytes(); - std::string SerializeGroupBytes(); - std::string SerializeTotalBytes(); - std::string SerializeNumFaultyFiles(); - std::string SerializeTimeScan(); - -private: - const FileInspectorStats& mFileInspectorStats; -}; - -struct FileInspectorStatsDeserializer { - friend class FileInspectorStats; - - FileInspectorStatsDeserializer() {} - - void DeserializeScanStats(const std::string& out, - FileInspectorStats& stats) const; - void DeserializeFaultyFiles(const std::string& out, - FileInspectorStats& stats) const; - void DeserializeAccessTimeFiles(const std::string& out, - FileInspectorStats& stats) const; - void DeserializeAccessTimeVolume(const std::string& out, - FileInspectorStats& stats) const; - void DeserializeBirthTimeFiles(const std::string& out, - FileInspectorStats& stats) const; - void DeserializeBirthTimeVolume(const std::string& out, - FileInspectorStats& stats) const; - void DeserializeBirthVsAccessTimeFiles(const std::string& out, - FileInspectorStats& stats) const; - void DeserializeBirthVsAccessTimeVolume(const std::string& out, - FileInspectorStats& stats) const; - void DeserializeUserCosts(const std::string& out, - FileInspectorStats& stats) const; - void DeserializeGroupCosts(const std::string& out, - FileInspectorStats& stats) const; - void DeserializeTotalCosts(const std::string& out, - FileInspectorStats& stats) const; - void DeserializeUserBytes(const std::string& out, - FileInspectorStats& stats) const; - void DeserializeGroupBytes(const std::string& out, - FileInspectorStats& stats) const; - void DeserializeTotalBytes(const std::string& out, - FileInspectorStats& stats) const; - void DeserializeNumFaultyFiles(const std::string& out, - FileInspectorStats& stats) const; - void DeserializeTimeScan(const std::string& out, - FileInspectorStats& stats) const; -}; - EOSMGMNAMESPACE_END \ No newline at end of file