diff --git a/include/mimalloc.h b/include/mimalloc.h index fe5aa8f34..5d2ef8886 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -311,6 +311,9 @@ typedef enum mi_option_e { mi_option_page_reset, mi_option_abandoned_page_reset, mi_option_segment_reset, +#if defined(__linux__) + mi_option_prefault, +#endif mi_option_eager_commit_delay, mi_option_reset_delay, mi_option_use_numa_nodes, diff --git a/readme.md b/readme.md index cdb1b82aa..1ef7fd7d6 100644 --- a/readme.md +++ b/readme.md @@ -302,6 +302,10 @@ or via environment variables: `MIMALLOC_EAGER_COMMIT_DELAY=N` (`N` is 1 by default) to delay the initial `N` segments (of 4MiB) of a thread to not allocate in the huge OS pages; this prevents threads that are short lived and allocate just a little to take up space in the huge OS page area (which cannot be reset). +- `MIMALLOC_PREFAULT=1`: (Linux only) enable memory prefaulting when available. This option instructs the kernel to synchronously + load the entire mapped region into active memory by specifying `MAP_POPULATE` in `mmap`. It will cause read-ahead on that memory, + and then the subsequent accesses to the memory can proceed without page faults, improving some performance, but might also reduce + some available memory. In `mimalloc_test_stress`, it is known to reduce about 95% page-faults. Use caution when using `fork` in combination with either large or huge OS pages: on a fork, the OS uses copy-on-write for all pages in the original process including the huge OS pages. When any memory is now written in that area, the diff --git a/src/options.c b/src/options.c index 30025db22..758357f6b 100644 --- a/src/options.c +++ b/src/options.c @@ -81,6 +81,9 @@ static mi_option_desc_t options[_mi_option_last] = { 1, UNINIT, MI_OPTION(page_reset) }, // reset page memory on free { 0, UNINIT, MI_OPTION(abandoned_page_reset) },// reset free page memory when a thread terminates { 0, UNINIT, MI_OPTION(segment_reset) }, // reset segment memory on free (needs eager commit) +#if defined(__linux__) + { 0, UNINIT, MI_OPTION(prefault) }, +#endif #if defined(__NetBSD__) { 0, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed #else diff --git a/src/os.c b/src/os.c index 85415232d..676c08d3d 100644 --- a/src/os.c +++ b/src/os.c @@ -362,6 +362,10 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro #define MAP_NORESERVE 0 #endif int flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE; + #if defined(__linux__) + if (mi_option_get(mi_option_prefault)) + flags = flags | MAP_POPULATE; + #endif int fd = -1; #if defined(MAP_ALIGNED) // BSD if (try_alignment > 0) { @@ -392,6 +396,9 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro } else { int lflags = flags & ~MAP_NORESERVE; // using NORESERVE on huge pages seems to fail on Linux + #if defined(__linux__) + lflags = lflags & ~MAP_POPULATE; // don't use MAP_POPULATE on huge pages + #endif int lfd = fd; #ifdef MAP_ALIGNED_SUPER lflags |= MAP_ALIGNED_SUPER;