From e49cf4bdc1c557552360eadd319e4af2aa9879b7 Mon Sep 17 00:00:00 2001 From: Lipeng Zhu Date: Fri, 14 Jun 2024 07:48:45 -0400 Subject: [PATCH 1/3] Introduce Thread-local storage variable to to reduce atomic contention when update used_memory metrics. Signed-off-by: Lipeng Zhu --- src/evict.c | 4 ++-- src/zmalloc.c | 34 +++++++++++++++++++++++++++++----- src/zmalloc.h | 1 + 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/evict.c b/src/evict.c index fb04616871..8519ab6ceb 100644 --- a/src/evict.c +++ b/src/evict.c @@ -676,12 +676,12 @@ int performEvictions(void) { * AOF and Output buffer memory will be freed eventually so * we only care about memory used by the key space. */ enterExecutionUnit(1, 0); - delta = (long long)zmalloc_used_memory(); + delta = (long long)zmalloc_used_memory_with_thread_delta(); latencyStartMonitor(eviction_latency); dbGenericDelete(db, keyobj, server.lazyfree_lazy_eviction, DB_FLAG_KEY_EVICTED); latencyEndMonitor(eviction_latency); latencyAddSampleIfNeeded("eviction-del", eviction_latency); - delta -= (long long)zmalloc_used_memory(); + delta -= (long long)zmalloc_used_memory_with_thread_delta(); mem_freed += delta; server.stat_evictedkeys++; signalModifiedKey(NULL, db, keyobj); diff --git a/src/zmalloc.c b/src/zmalloc.c index 0117d8d91a..97e0431b1b 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -87,10 +87,26 @@ void zlibc_free(void *ptr) { #define dallocx(ptr, flags) je_dallocx(ptr, flags) #endif -#define update_zmalloc_stat_alloc(__n) atomic_fetch_add_explicit(&used_memory, (__n), memory_order_relaxed) -#define update_zmalloc_stat_free(__n) atomic_fetch_sub_explicit(&used_memory, (__n), memory_order_relaxed) +static _Atomic int64_t used_memory = 0; +static __thread int64_t thread_used_memory_delta = 0; -static _Atomic size_t used_memory = 0; +#define THREAD_MEM_MAX_DELTA (1024 * 1024) + +static inline void update_zmalloc_stat_alloc(size_t size) { + thread_used_memory_delta += size; + if (thread_used_memory_delta >= THREAD_MEM_MAX_DELTA) { + atomic_fetch_add_explicit(&used_memory, thread_used_memory_delta, memory_order_relaxed); + thread_used_memory_delta = 0; + } +} + +static inline void update_zmalloc_stat_free(size_t size) { + thread_used_memory_delta -= size; + if (thread_used_memory_delta <= THREAD_MEM_MAX_DELTA) { + atomic_fetch_add_explicit(&used_memory, thread_used_memory_delta, memory_order_relaxed); + thread_used_memory_delta = 0; + } +} static void zmalloc_default_oom(size_t size) { fprintf(stderr, "zmalloc: Out of memory trying to allocate %zu bytes\n", size); @@ -388,8 +404,16 @@ char *zstrdup(const char *s) { } size_t zmalloc_used_memory(void) { - size_t um = atomic_load_explicit(&used_memory, memory_order_relaxed); - return um; + /* Maybe become negative */ + int64_t um = atomic_load_explicit(&used_memory, memory_order_relaxed); + return um < 0 ? 0 : um; +} + +/* Flush the thread_used_memory_delta to global counter. */ +size_t zmalloc_used_memory_with_thread_delta(void) { + atomic_fetch_add_explicit(&used_memory, thread_used_memory_delta, memory_order_relaxed); + thread_used_memory_delta = 0; + return zmalloc_used_memory(); } void zmalloc_set_oom_handler(void (*oom_handler)(size_t)) { diff --git a/src/zmalloc.h b/src/zmalloc.h index a909366c13..ff4f4ee452 100644 --- a/src/zmalloc.h +++ b/src/zmalloc.h @@ -126,6 +126,7 @@ void *ztrycalloc_usable(size_t size, size_t *usable); void *ztryrealloc_usable(void *ptr, size_t size, size_t *usable); __attribute__((malloc)) char *zstrdup(const char *s); size_t zmalloc_used_memory(void); +size_t zmalloc_used_memory_with_thread_delta(void); void zmalloc_set_oom_handler(void (*oom_handler)(size_t)); size_t zmalloc_get_rss(void); int zmalloc_get_allocator_info(size_t *allocated, From a860c21a3dbfb57e78721eff2c7796ed266e6fbc Mon Sep 17 00:00:00 2001 From: Lipeng Zhu Date: Fri, 14 Jun 2024 09:35:35 -0400 Subject: [PATCH 2/3] use standard thread_local. Signed-off-by: Lipeng Zhu --- src/zmalloc.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/zmalloc.c b/src/zmalloc.c index 97e0431b1b..772cfb7902 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -52,6 +52,7 @@ void zlibc_free(void *ptr) { #include #include "zmalloc.h" #include +#include #define UNUSED(x) ((void)(x)) @@ -88,7 +89,7 @@ void zlibc_free(void *ptr) { #endif static _Atomic int64_t used_memory = 0; -static __thread int64_t thread_used_memory_delta = 0; +static thread_local int64_t thread_used_memory_delta = 0; #define THREAD_MEM_MAX_DELTA (1024 * 1024) From aeb8159d5b9d2bba41ede137e793d351e600632c Mon Sep 17 00:00:00 2001 From: Lipeng Zhu Date: Fri, 14 Jun 2024 10:06:03 -0400 Subject: [PATCH 3/3] fix failure for macos. Signed-off-by: Lipeng Zhu --- src/zmalloc.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/zmalloc.c b/src/zmalloc.c index 772cfb7902..a514075cbc 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -52,7 +52,6 @@ void zlibc_free(void *ptr) { #include #include "zmalloc.h" #include -#include #define UNUSED(x) ((void)(x)) @@ -88,6 +87,12 @@ void zlibc_free(void *ptr) { #define dallocx(ptr, flags) je_dallocx(ptr, flags) #endif +#if __STDC_NO_THREADS__ +#define thread_local __thread +#else +#include +#endif + static _Atomic int64_t used_memory = 0; static thread_local int64_t thread_used_memory_delta = 0;