From c13d8bbba13f8866b556b567f24409e82e43d72e Mon Sep 17 00:00:00 2001 From: Janick Martinez Esturo Date: Tue, 21 Jan 2025 17:58:32 +0100 Subject: [PATCH 1/4] Increase jemalloc aarch64 page size limit (#5244) Pass JEMALLOC_SYS_WITH_LG_PAGE=16 to aarch64 cross-compilation to support systems with up to 64-KiB page sizes. --- Cross.toml | 11 +++++++++++ Makefile | 5 ++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/Cross.toml b/Cross.toml index 871391253d3..8181967f327 100644 --- a/Cross.toml +++ b/Cross.toml @@ -3,3 +3,14 @@ pre-build = ["apt-get install -y cmake clang-5.0"] [target.aarch64-unknown-linux-gnu] pre-build = ["apt-get install -y cmake clang-5.0"] + +# Allow setting page size limits for jemalloc at build time: +# For certain architectures (like aarch64), we must compile +# jemalloc with support for large page sizes, otherwise the host's +# system page size will be used, which may not work on the target systems. +# JEMALLOC_SYS_WITH_LG_PAGE=16 tells jemalloc to support up to 64-KiB +# pages. See: https://github.com/sigp/lighthouse/issues/5244 +[build.env] +passthrough = [ + "JEMALLOC_SYS_WITH_LG_PAGE", +] diff --git a/Makefile b/Makefile index 4d95f50c5ce..f3233ef25a7 100644 --- a/Makefile +++ b/Makefile @@ -63,7 +63,10 @@ install-lcli: build-x86_64: cross build --bin lighthouse --target x86_64-unknown-linux-gnu --features "portable,$(CROSS_FEATURES)" --profile "$(CROSS_PROFILE)" --locked build-aarch64: - cross build --bin lighthouse --target aarch64-unknown-linux-gnu --features "portable,$(CROSS_FEATURES)" --profile "$(CROSS_PROFILE)" --locked + # JEMALLOC_SYS_WITH_LG_PAGE=16 tells jemalloc to support up to 64-KiB + # pages, which are commonly used by aarch64 systems. + # See: https://github.com/sigp/lighthouse/issues/5244 + JEMALLOC_SYS_WITH_LG_PAGE=16 cross build --bin lighthouse --target aarch64-unknown-linux-gnu --features "portable,$(CROSS_FEATURES)" --profile "$(CROSS_PROFILE)" --locked build-lcli-x86_64: cross build --bin lcli --target x86_64-unknown-linux-gnu --features "portable" --profile "$(CROSS_PROFILE)" --locked From 3206e29a20776860bb009667825263b3b673b735 Mon Sep 17 00:00:00 2001 From: Janick Martinez Esturo Date: Thu, 23 Jan 2025 07:58:46 +0100 Subject: [PATCH 2/4] - review comment also add JEMALLOC_SYS_WITH_LG_PAGE to `build-lcli-aarch64` target --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f3233ef25a7..2ce278a4f2f 100644 --- a/Makefile +++ b/Makefile @@ -71,7 +71,10 @@ build-aarch64: build-lcli-x86_64: cross build --bin lcli --target x86_64-unknown-linux-gnu --features "portable" --profile "$(CROSS_PROFILE)" --locked build-lcli-aarch64: - cross build --bin lcli --target aarch64-unknown-linux-gnu --features "portable" --profile "$(CROSS_PROFILE)" --locked + # JEMALLOC_SYS_WITH_LG_PAGE=16 tells jemalloc to support up to 64-KiB + # pages, which are commonly used by aarch64 systems. + # See: https://github.com/sigp/lighthouse/issues/5244 + JEMALLOC_SYS_WITH_LG_PAGE=16 cross build --bin lcli --target aarch64-unknown-linux-gnu --features "portable" --profile "$(CROSS_PROFILE)" --locked # Create a `.tar.gz` containing a binary for a specific target. define tarball_release_binary From 1425ace0a939c199a0115f421ea7fd0d3e2e2793 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 30 Jan 2025 11:34:44 +1100 Subject: [PATCH 3/4] Spaces to tabs --- Makefile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 2ce278a4f2f..3cc9299ecfe 100644 --- a/Makefile +++ b/Makefile @@ -63,17 +63,17 @@ install-lcli: build-x86_64: cross build --bin lighthouse --target x86_64-unknown-linux-gnu --features "portable,$(CROSS_FEATURES)" --profile "$(CROSS_PROFILE)" --locked build-aarch64: - # JEMALLOC_SYS_WITH_LG_PAGE=16 tells jemalloc to support up to 64-KiB - # pages, which are commonly used by aarch64 systems. - # See: https://github.com/sigp/lighthouse/issues/5244 + # JEMALLOC_SYS_WITH_LG_PAGE=16 tells jemalloc to support up to 64-KiB + # pages, which are commonly used by aarch64 systems. + # See: https://github.com/sigp/lighthouse/issues/5244 JEMALLOC_SYS_WITH_LG_PAGE=16 cross build --bin lighthouse --target aarch64-unknown-linux-gnu --features "portable,$(CROSS_FEATURES)" --profile "$(CROSS_PROFILE)" --locked build-lcli-x86_64: cross build --bin lcli --target x86_64-unknown-linux-gnu --features "portable" --profile "$(CROSS_PROFILE)" --locked build-lcli-aarch64: - # JEMALLOC_SYS_WITH_LG_PAGE=16 tells jemalloc to support up to 64-KiB - # pages, which are commonly used by aarch64 systems. - # See: https://github.com/sigp/lighthouse/issues/5244 + # JEMALLOC_SYS_WITH_LG_PAGE=16 tells jemalloc to support up to 64-KiB + # pages, which are commonly used by aarch64 systems. + # See: https://github.com/sigp/lighthouse/issues/5244 JEMALLOC_SYS_WITH_LG_PAGE=16 cross build --bin lcli --target aarch64-unknown-linux-gnu --features "portable" --profile "$(CROSS_PROFILE)" --locked # Create a `.tar.gz` containing a binary for a specific target. From 13dac8b5ebc657d131efa5aa2e012e8cca9bbdc4 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 30 Jan 2025 13:57:42 +1100 Subject: [PATCH 4/4] Show page size in lighthouse --version --- .cargo/config.toml | 1 - common/malloc_utils/src/jemalloc.rs | 17 ++++++++++++++++- common/malloc_utils/src/lib.rs | 4 ++-- lighthouse/src/main.rs | 14 +++++++++----- 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index dac01630032..a408305c4d1 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,4 +1,3 @@ [env] # Set the number of arenas to 16 when using jemalloc. JEMALLOC_SYS_WITH_MALLOC_CONF = "abort_conf:true,narenas:16" - diff --git a/common/malloc_utils/src/jemalloc.rs b/common/malloc_utils/src/jemalloc.rs index 0e2e00cb0ef..f3a35fc41c0 100644 --- a/common/malloc_utils/src/jemalloc.rs +++ b/common/malloc_utils/src/jemalloc.rs @@ -9,7 +9,7 @@ //! B) `_RJEM_MALLOC_CONF` at runtime. use metrics::{set_gauge, try_create_int_gauge, IntGauge}; use std::sync::LazyLock; -use tikv_jemalloc_ctl::{arenas, epoch, stats, Error}; +use tikv_jemalloc_ctl::{arenas, epoch, stats, Access, AsName, Error}; #[global_allocator] static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; @@ -52,3 +52,18 @@ pub fn scrape_jemalloc_metrics_fallible() -> Result<(), Error> { Ok(()) } + +pub fn page_size() -> Result { + // Full list of keys: https://jemalloc.net/jemalloc.3.html + "arenas.page\0".name().read() +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn page_size_ok() { + assert!(page_size().is_ok()); + } +} diff --git a/common/malloc_utils/src/lib.rs b/common/malloc_utils/src/lib.rs index 3bb242369f7..50d2785a744 100644 --- a/common/malloc_utils/src/lib.rs +++ b/common/malloc_utils/src/lib.rs @@ -29,10 +29,10 @@ not(target_env = "musl"), not(feature = "jemalloc") ))] -mod glibc; +pub mod glibc; #[cfg(feature = "jemalloc")] -mod jemalloc; +pub mod jemalloc; pub use interface::*; diff --git a/lighthouse/src/main.rs b/lighthouse/src/main.rs index 43c5e1107ca..dd7401d49e6 100644 --- a/lighthouse/src/main.rs +++ b/lighthouse/src/main.rs @@ -66,11 +66,15 @@ fn bls_hardware_acceleration() -> bool { return std::arch::is_aarch64_feature_detected!("neon"); } -fn allocator_name() -> &'static str { - if cfg!(target_os = "windows") { - "system" - } else { - "jemalloc" +fn allocator_name() -> String { + #[cfg(target_os = "windows")] + { + "system".to_string() + } + #[cfg(not(target_os = "windows"))] + match malloc_utils::jemalloc::page_size() { + Ok(page_size) => format!("jemalloc ({}K)", page_size / 1024), + Err(e) => format!("jemalloc (error: {e:?})"), } }