From 95a8caf1a204e9ec34b3c76b2f3676e89727c304 Mon Sep 17 00:00:00 2001 From: Vlad Volodkin Date: Thu, 9 Jan 2025 14:14:33 +0000 Subject: [PATCH 1/4] Emit malloc metrics Signed-off-by: Vlad Volodkin --- mountpoint-s3/src/cli.rs | 13 +++++++++++++ mountpoint-s3/src/metrics.rs | 20 ++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/mountpoint-s3/src/cli.rs b/mountpoint-s3/src/cli.rs index 9f0686989..b5538e32b 100644 --- a/mountpoint-s3/src/cli.rs +++ b/mountpoint-s3/src/cli.rs @@ -581,12 +581,25 @@ impl CliArgs { } } +fn configure_malloc() { + unsafe { + println!( + "glibc version: {:?}", + std::ffi::CStr::from_ptr(libc::gnu_get_libc_version()) + ); + // NOTE: M_MMAP_MAX is 65536. With 8MiB allocations it allows up to 512GiB of simultaneously allocated memory via mmap, which should fit for most of the cases. For more info: + // https://github.com/bminor/glibc/blob/release/2.26/master/malloc/malloc.c#L983 + libc::mallopt(libc::M_MMAP_THRESHOLD, 131072); // disable dynamic MMAP_THRESHOLD: https://github.com/bminor/glibc/blob/release/2.26/master/malloc/malloc.c#L1739 + } +} + pub fn main(client_builder: ClientBuilder) -> anyhow::Result<()> where ClientBuilder: FnOnce(&CliArgs) -> anyhow::Result<(Client, Runtime, S3Personality)>, Client: ObjectClient + Clone + Send + Sync + 'static, Runtime: Spawn + Clone + Send + Sync + 'static, { + configure_malloc(); let args = CliArgs::parse(); let successful_mount_msg = format!( "{} is mounted at {}", diff --git a/mountpoint-s3/src/metrics.rs b/mountpoint-s3/src/metrics.rs index 4cc693fb9..ce4301a3e 100644 --- a/mountpoint-s3/src/metrics.rs +++ b/mountpoint-s3/src/metrics.rs @@ -44,6 +44,7 @@ pub fn install() -> MetricsSinkHandle { Ok(()) | Err(RecvTimeoutError::Disconnected) => break, Err(RecvTimeoutError::Timeout) => { poll_process_metrics(&mut sys); + poll_malloc_metrics(); inner.publish() } } @@ -52,6 +53,7 @@ pub fn install() -> MetricsSinkHandle { // any new metrics data after the sink shuts down, but we assume a clean shutdown // stops generating new metrics before shutting down the sink. poll_process_metrics(&mut sys); + poll_malloc_metrics(); inner.publish(); }) }; @@ -87,6 +89,24 @@ fn poll_process_metrics(sys: &mut System) { } } +fn poll_malloc_metrics() { + unsafe { + let mallinfo = libc::mallinfo(); + // Space wasted due to fragmentation: `process.mallinfo.arena - process.mallinfo.fordblks` + metrics::gauge!("process.mallinfo.arena").set(mallinfo.arena as f64); /* non-mmapped space allocated from system */ + metrics::gauge!("process.mallinfo.ordblks").set(mallinfo.ordblks as f64); /* number of free chunks */ + metrics::gauge!("process.mallinfo.smblks").set(mallinfo.smblks as f64); /* number of fastbin blocks */ + metrics::gauge!("process.mallinfo.hblks").set(mallinfo.hblks as f64); /* number of mmapped regions */ + metrics::gauge!("process.mallinfo.hblkhd").set(mallinfo.hblkhd as f64); /* space in mmapped regions */ + metrics::gauge!("process.mallinfo.usmblks").set(mallinfo.usmblks as f64); /* always 0, preserved for backwards compatibility */ + metrics::gauge!("process.mallinfo.fsmblks").set(mallinfo.fsmblks as f64); /* space available in freed fastbin blocks */ + metrics::gauge!("process.mallinfo.uordblks").set(mallinfo.uordblks as f64); /* total allocated space */ + metrics::gauge!("process.mallinfo.fordblks").set(mallinfo.fordblks as f64); /* total free space */ + /* top-most, releasable (via malloc_trim) space */ + metrics::gauge!("process.mallinfo.keepcost").set(mallinfo.keepcost as f64); + } +} + #[derive(Debug)] struct MetricsSink { metrics: DashMap, From 165b5c615429367ce5a3c185aa59ea66953ef3b0 Mon Sep 17 00:00:00 2001 From: Vlad Volodkin Date: Thu, 9 Jan 2025 14:22:18 +0000 Subject: [PATCH 2/4] Fixes Signed-off-by: Vlad Volodkin --- mountpoint-s3/src/cli.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mountpoint-s3/src/cli.rs b/mountpoint-s3/src/cli.rs index b5538e32b..1af836064 100644 --- a/mountpoint-s3/src/cli.rs +++ b/mountpoint-s3/src/cli.rs @@ -581,12 +581,9 @@ impl CliArgs { } } +#[cfg(target_os = "linux")] fn configure_malloc() { unsafe { - println!( - "glibc version: {:?}", - std::ffi::CStr::from_ptr(libc::gnu_get_libc_version()) - ); // NOTE: M_MMAP_MAX is 65536. With 8MiB allocations it allows up to 512GiB of simultaneously allocated memory via mmap, which should fit for most of the cases. For more info: // https://github.com/bminor/glibc/blob/release/2.26/master/malloc/malloc.c#L983 libc::mallopt(libc::M_MMAP_THRESHOLD, 131072); // disable dynamic MMAP_THRESHOLD: https://github.com/bminor/glibc/blob/release/2.26/master/malloc/malloc.c#L1739 @@ -599,6 +596,7 @@ where Client: ObjectClient + Clone + Send + Sync + 'static, Runtime: Spawn + Clone + Send + Sync + 'static, { + #[cfg(target_os = "linux")] configure_malloc(); let args = CliArgs::parse(); let successful_mount_msg = format!( From 1f3b04a2107e49dc18a8739672d21851a9b4fc29 Mon Sep 17 00:00:00 2001 From: Vlad Volodkin Date: Thu, 9 Jan 2025 15:00:51 +0000 Subject: [PATCH 3/4] Fix on macOS Signed-off-by: Vlad Volodkin --- mountpoint-s3/src/metrics.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mountpoint-s3/src/metrics.rs b/mountpoint-s3/src/metrics.rs index ce4301a3e..4ca11abf9 100644 --- a/mountpoint-s3/src/metrics.rs +++ b/mountpoint-s3/src/metrics.rs @@ -44,6 +44,7 @@ pub fn install() -> MetricsSinkHandle { Ok(()) | Err(RecvTimeoutError::Disconnected) => break, Err(RecvTimeoutError::Timeout) => { poll_process_metrics(&mut sys); + #[cfg(target_os = "linux")] poll_malloc_metrics(); inner.publish() } @@ -53,6 +54,7 @@ pub fn install() -> MetricsSinkHandle { // any new metrics data after the sink shuts down, but we assume a clean shutdown // stops generating new metrics before shutting down the sink. poll_process_metrics(&mut sys); + #[cfg(target_os = "linux")] poll_malloc_metrics(); inner.publish(); }) @@ -89,6 +91,7 @@ fn poll_process_metrics(sys: &mut System) { } } +#[cfg(target_os = "linux")] fn poll_malloc_metrics() { unsafe { let mallinfo = libc::mallinfo(); From e59b2aa623e7e95f12a1cf3cd8299cc3802f0b31 Mon Sep 17 00:00:00 2001 From: Vlad Volodkin Date: Thu, 9 Jan 2025 18:52:29 +0000 Subject: [PATCH 4/4] Set MMAP_THRESHOLD to 4MiB Signed-off-by: Vlad Volodkin --- mountpoint-s3/src/cli.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mountpoint-s3/src/cli.rs b/mountpoint-s3/src/cli.rs index 1af836064..e036beec1 100644 --- a/mountpoint-s3/src/cli.rs +++ b/mountpoint-s3/src/cli.rs @@ -584,9 +584,11 @@ impl CliArgs { #[cfg(target_os = "linux")] fn configure_malloc() { unsafe { + // Disable dynamic MMAP_THRESHOLD and set it to 4MiB: https://github.com/bminor/glibc/blob/release/2.26/master/malloc/malloc.c#L1739 + // NOTE: M_MMAP_MAX is 65536. With 8MiB allocations it allows up to 512GiB of simultaneously allocated memory via mmap, which should fit for most of the cases. For more info: // https://github.com/bminor/glibc/blob/release/2.26/master/malloc/malloc.c#L983 - libc::mallopt(libc::M_MMAP_THRESHOLD, 131072); // disable dynamic MMAP_THRESHOLD: https://github.com/bminor/glibc/blob/release/2.26/master/malloc/malloc.c#L1739 + libc::mallopt(libc::M_MMAP_THRESHOLD, 4194304); } }