Skip to content

Commit

Permalink
Add LinuxMemoryChecker checks/warnings to ensure system-memory-gb < s…
Browse files Browse the repository at this point in the history 
…ystem-mem-limit-gb < memory limit for process.
  • Loading branch information
@minhancao
minhancao committed Feb 6, 2025
1 parent d827df7 commit 80ca9b5
Show file tree
Hide file tree
Showing 5 changed files with 386 additions and 113 deletions.
132 changes: 121 additions & 11 deletions presto-native-execution/presto_cpp/main/LinuxMemoryChecker.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#include <sys/stat.h>
#include "presto_cpp/main/PeriodicMemoryChecker.h"
#include "presto_cpp/main/common/Configs.h"
#include "presto_cpp/main/common/Utils.h"

namespace facebook::presto {

Expand All @@ -30,12 +31,35 @@ class LinuxMemoryChecker : public PeriodicMemoryChecker {
struct stat buffer;
if ((stat(kCgroupV1Path, &buffer) == 0)) {
statFile_ = kCgroupV1Path;
PRESTO_STARTUP_LOG(INFO)
<< fmt::format("Using cgroup v1 memory stat file: {}", statFile_);
} else if ((stat(kCgroupV2Path, &buffer) == 0)) {
statFile_ = kCgroupV2Path;
PRESTO_STARTUP_LOG(INFO)
<< fmt::format("Using cgroup v2 memory stat file: {}", statFile_);
} else {
statFile_ = "None";
PRESTO_STARTUP_LOG(INFO)
<< fmt::format("Could not find memory stat file: {}", statFile_);
}

// Set memMaxFile_ to:
// "/sys/fs/cgroup/memory/memory.limit_in_bytes" for cgroup v1
// or
// "/sys/fs/cgroup/memory.max" for cgroup v2.
if ((stat(kCgroupV1MaxMemFilePath, &buffer) == 0)) {
memMaxFile_ = kCgroupV1MaxMemFilePath;
PRESTO_STARTUP_LOG(INFO)
<< fmt::format("Using cgroup v1 memory max file {}", memMaxFile_);
} else if ((stat(kCgroupV2MaxMemFilePath, &buffer) == 0)) {
memMaxFile_ = kCgroupV2MaxMemFilePath;
PRESTO_STARTUP_LOG(INFO)
<< fmt::format("Using cgroup v2 memory max file {}", memMaxFile_);
} else {
memMaxFile_ = "None";
PRESTO_STARTUP_LOG(INFO)
<< fmt::format("Could not find memory max file: {}", memMaxFile_);
}
LOG(INFO) << fmt::format("Using memory stat file {}", statFile_);
}

~LinuxMemoryChecker() override {}
Expand All @@ -49,6 +73,87 @@ class LinuxMemoryChecker : public PeriodicMemoryChecker {
LOG(INFO) << fmt::format("Changed to using memory stat file {}", statFile_);
}

void setMemMaxFile(std::string memMaxFile) {
memMaxFile_ = memMaxFile;
LOG(INFO) << fmt::format(
"Changed to using memory max file {}", memMaxFile_);
}

void setMemInfo(std::string memInfoFile) {
memInfoFile_ = memInfoFile;
LOG(INFO) << fmt::format("Changed to using meminfo file {}", memInfoFile_);
}

void start() override {
// Check system-memory-gb < system-mem-limit-gb < memory limit for process.
auto* systemConfig = SystemConfig::instance();
int64_t systemMemoryInBytes = systemConfig->systemMemoryGb() << 30;
PRESTO_STARTUP_LOG(INFO)
<< fmt::format("System memory in bytes: {}", systemMemoryInBytes);

PRESTO_STARTUP_LOG(INFO) << fmt::format(
"System memory limit in bytes: {}", config_.systemMemLimitBytes);

int64_t memoryLimitForProcess = getMemoryLimitForProcess();
PRESTO_STARTUP_LOG(INFO) << fmt::format(
"Memory limit for process in bytes: {}", memoryLimitForProcess);

VELOX_CHECK_LE(
config_.systemMemLimitBytes,
memoryLimitForProcess,
"system memory limit = {} bytes is higher than the memory limit for process = {} bytes.",
config_.systemMemLimitBytes,
memoryLimitForProcess);

if (config_.systemMemLimitBytes < systemMemoryInBytes) {
LOG(WARNING) << "system-mem-limit-gb is smaller than system-memory-gb. "
<< "Expected: system-mem-limit-gb >= system-memory-gb.";
}

PeriodicMemoryChecker::start();
}

int64_t getMemoryLimitForProcess() {
// Set the memory limit for process to be the smaller number between
// /proc/meminfo and memMaxFile_.
int64_t memoryLimitForProcess = 0;
// meminfo's units is in kB.
folly::gen::byLine(memInfoFile_.c_str()) |
[&](const folly::StringPiece& line) -> void {
if (memoryLimitForProcess != 0) {
return;
}
memoryLimitForProcess = static_cast<int64_t>(
extractNumericConfigValueWithRegex(line, kMemTotalRegex) * 1024);
};

// For cgroup v1, memory.limit_in_bytes can default to a really big numeric
// value in bytes like 9223372036854771712 to represent that
// memory.limit_in_bytes is not set to a value. The default value here is
// set to PAGE_COUNTER_MAX, which is LONG_MAX/PAGE_SIZE on the 64-bit
// platform. The default value can vary based upon the platform's PAGE_SIZE.
// If memory.limit_in_bytes contains a really big numeric value, then we
// will use MemTotal from /proc/meminfo.

// For cgroup v2, memory.max can contain a numeric value in bytes or string
// "max" which represents no value has been set. If memory.max contains
// "max", then we will use MemTotal from /proc/meminfo.
if (memMaxFile_ != "None") {
folly::gen::byLine(memMaxFile_.c_str()) |
[&](const folly::StringPiece& line) -> void {
if (line == "max") {
return;
}
memoryLimitForProcess =
std::min(memoryLimitForProcess, folly::to<int64_t>(line));
return;
};
}

// Unit is in bytes.
return memoryLimitForProcess;
}

protected:
// Current memory calculation used is inactive_anon + active_anon
// Our first attempt was using memInfo memTotal - memAvailable.
Expand Down Expand Up @@ -83,6 +188,10 @@ class LinuxMemoryChecker : public PeriodicMemoryChecker {
if (statFile_ != "None") {
folly::gen::byLine(statFile_.c_str()) |
[&](const folly::StringPiece& line) -> void {
if (activeAnon != 0 && inactiveAnon != 0) {
return;
}

if (inactiveAnon == 0) {
inactiveAnon =
extractNumericConfigValueWithRegex(line, kInactiveAnonRegex);
Expand All @@ -92,19 +201,19 @@ class LinuxMemoryChecker : public PeriodicMemoryChecker {
activeAnon =
extractNumericConfigValueWithRegex(line, kActiveAnonRegex);
}

if (activeAnon != 0 && inactiveAnon != 0) {
return;
}
};

// Unit is in bytes.
return inactiveAnon + activeAnon;
}

// Last resort use host machine info.
folly::gen::byLine("/proc/meminfo") |
folly::gen::byLine(memInfoFile_.c_str()) |
[&](const folly::StringPiece& line) -> void {
if (memAvailable != 0 && memTotal != 0) {
return;
}

if (memAvailable == 0) {
memAvailable =
extractNumericConfigValueWithRegex(line, kMemAvailableRegex) * 1024;
Expand All @@ -114,10 +223,6 @@ class LinuxMemoryChecker : public PeriodicMemoryChecker {
memTotal =
extractNumericConfigValueWithRegex(line, kMemTotalRegex) * 1024;
}

if (memAvailable != 0 && memTotal != 0) {
return;
}
};
// Unit is in bytes.
return (memAvailable && memTotal) ? memTotal - memAvailable : 0;
Expand All @@ -143,10 +248,15 @@ class LinuxMemoryChecker : public PeriodicMemoryChecker {
const boost::regex kInactiveAnonRegex{R"!(inactive_anon\s*(\d+)\s*)!"};
const boost::regex kActiveAnonRegex{R"!(active_anon\s*(\d+)\s*)!"};
const boost::regex kMemAvailableRegex{R"!(MemAvailable:\s*(\d+)\s*kB)!"};
const boost::regex kMemTotalRegex{R"!(MemTotal:\s*(\d+)\s*kB)!"};
const boost::regex kMemTotalRegex{R"!(MemTotal:\s*(\d+)\s+kB)!"};
const char* kCgroupV1Path = "/sys/fs/cgroup/memory/memory.stat";
const char* kCgroupV2Path = "/sys/fs/cgroup/memory.stat";
const char* kCgroupV1MaxMemFilePath =
"/sys/fs/cgroup/memory/memory.limit_in_bytes";
const char* kCgroupV2MaxMemFilePath = "/sys/fs/cgroup/memory.max";
std::string statFile_;
std::string memInfoFile_ = "/proc/meminfo";
std::string memMaxFile_;

size_t extractNumericConfigValueWithRegex(
const folly::StringPiece& line,
Expand Down
4 changes: 2 additions & 2 deletions presto-native-execution/presto_cpp/main/PeriodicMemoryChecker.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,10 +71,10 @@ class PeriodicMemoryChecker {

/// Starts the 'PeriodicMemoryChecker'. A background scheduler will be
/// launched to perform the checks. This should only be called once.
void start();
virtual void start();

/// Stops the 'PeriodicMemoryChecker'.
void stop();
virtual void stop();

protected:
/// Returns current system memory usage. The returned value is used to compare
Expand Down
Loading

0 comments on commit 80ca9b5

Please sign in to comment.