From fd978acac001708043aabbd173c263a1fd677e91 Mon Sep 17 00:00:00 2001 From: Michal Rostecki Date: Fri, 15 Oct 2021 17:38:54 +0200 Subject: [PATCH] bpf: Adjust PID-related BPF map sizes to the current pid_max Before this change, we assumed the worst case scenario about the number of processes which can be schedules - the upper configurable limit for x86_64 - which in theory is 4194304. Allocating such a map size resulted in huge memory usage by lockc BPF programs, over 500 MB, which is unacceptable for nodes with <=2 GB RAM (which is usual both in IoT/ARM world and public clouds). This change reads the currently configured limit by reading the kernel.pid_max sysctl. On the most of systems it's configured to 32768. That's way less than the maximum worst case scenario. Memory usage before this change: lockc-control-plane-0:~ # free -m total used free shared buff/cache available Mem: 1968 811 689 8 467 1011 Swap: 0 0 0 lockc-control-plane-0:~ # systemctl start lockcd lockc-control-plane-0:~ # free -m total used free shared buff/cache available Mem: 1968 1420 154 8 393 339 Swap: 0 0 0 After this change: lockc-control-plane-0:~ # free -m total used free shared buff/cache available Mem: 1968 793 220 3 955 1037 Swap: 0 0 0 lockc-control-plane-0:~ # systemctl start lockcd lockc-control-plane-0:~ # free -m total used free shared buff/cache available Mem: 1968 803 210 3 955 1028 Swap: 0 0 Fixes: #82 Signed-off-by: Michal Rostecki --- docs/src/SUMMARY.md | 1 + docs/src/tuning/README.md | 37 +++++++++++++++++++++++++ lockc/Cargo.toml | 1 + lockc/src/lib.rs | 57 ++++++++++++++++++++++++++++----------- 4 files changed, 80 insertions(+), 16 deletions(-) create mode 100644 docs/src/tuning/README.md diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 061ff06..f0c55d4 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -17,5 +17,6 @@ - [File access](policies/file-access.md) - [Mount](policies/mount.md) - [Syslog](policies/syslog.md) +- [Tuning](tuning/README.md) - [Demos](demos/README.md) - [Mount](demos/mount.md) diff --git a/docs/src/tuning/README.md b/docs/src/tuning/README.md new file mode 100644 index 0000000..261e668 --- /dev/null +++ b/docs/src/tuning/README.md @@ -0,0 +1,37 @@ +## Tuning + +This guide shows options and tricks to gain an optimal performance and resouce +usage. + +### Memory usage + +Memory usage by lockc depends mostly on BPF maps size. BPF maps are stored in +memory and the biggest BPF maps are the ones related to tracking processes and +containers. Size of those maps depends on the limit of processes (in separate +memory spaces) in the system. That limit is determined by the `kernel.pid_max` +sysctl. By default the limit is 32768. With such limit, memory usage by lockc +should be aproximately 10-20 MB. + +If you observe too much memory being used after installing lockc, try to check +the value of `kernel.pid_max`, which can be done with: + +```bash +sudo sysctl kernel.pid_max +``` + +Change of that value (i.e. to 10000) can be done with: + +```bash +sudo sysctl kernel.pid_max=10000 +``` + +But that change will be not persistent after reboot. Changing it persistently +requires adding a configuration to `/etc/sysctl.d`. I.e. we could create the +file `/etc/sysctl.d/05-lockc.conf` with the following content: + +``` +kernel.pid_max = 10000 +``` + +After creating that file, the lower limit is going to be persistent after +reboot. diff --git a/lockc/Cargo.toml b/lockc/Cargo.toml index 2aa87df..4cb041b 100644 --- a/lockc/Cargo.toml +++ b/lockc/Cargo.toml @@ -37,6 +37,7 @@ procfs = "0.11" regex = { version = "1.5", default-features = false, features = ["perf"] } serde = "1.0" serde_json = "1.0" +sysctl = "0.4" thiserror = "1.0" tokio = { version = "1.7", features = ["macros", "process", "rt-multi-thread"] } uuid = { version = "0.8", default-features = false, features = ["v4"] } diff --git a/lockc/src/lib.rs b/lockc/src/lib.rs index 4b1fc18..2459db6 100644 --- a/lockc/src/lib.rs +++ b/lockc/src/lib.rs @@ -5,9 +5,13 @@ #[macro_use] extern crate lazy_static; -use bpfstructs::BpfStruct; +use std::{convert::TryInto, fs, io, io::prelude::*, num, path}; + use byteorder::{NativeEndian, WriteBytesExt}; -use std::{convert::TryInto, fs, io, io::prelude::*, path}; +use sysctl::Sysctl; + +use bpfstructs::BpfStruct; + #[rustfmt::skip] mod bpf; @@ -135,28 +139,39 @@ pub fn init_allowed_paths(mut maps: LockcMapsMut) -> Result<(), InitAllowedPaths } #[derive(thiserror::Error, Debug)] -pub enum LoadProgramError { - #[error("hash error")] - HashError(#[from] HashError), +pub enum GetPidMaxError { + #[error(transparent)] + ParseInt(#[from] num::ParseIntError), - #[error("init runtimes error")] - InitRuntimesError(#[from] InitRuntimesError), + #[error(transparent)] + Sysctl(#[from] sysctl::SysctlError), +} - #[error("init allowed paths error")] - InitAllowedPathsError(#[from] InitAllowedPathsError), +/// Gets the max PID number configured in the system. +fn get_pid_max() -> Result { + let pid_max_s = sysctl::Ctl::new("kernel.pid_max")?.value_string()?; + let pid_max = pid_max_s.parse::()?; + Ok(pid_max) +} - #[error("could not convert the hash to a byte array")] - ByteWriteError(#[from] io::Error), +#[derive(thiserror::Error, Debug)] +pub enum LoadProgramError { + #[error(transparent)] + Libbpf(#[from] libbpf_rs::Error), - #[error("libbpf error")] - LibbpfError(#[from] libbpf_rs::Error), + #[error(transparent)] + GetPidMax(#[from] GetPidMaxError), + + #[error(transparent)] + InitAllowedPaths(#[from] InitAllowedPathsError), - #[error("could not align the byte data")] - ByteAlignError, + #[error(transparent)] + InitRuntimes(#[from] InitRuntimesError), } /// Performs the following BPF-related operations: /// - loading BPF programs +/// - resizing PID-related BPF maps /// - pinning BPF maps in BPFFS /// - pinning BPF programs in BPFFS /// - attaching BPF programs, creating links @@ -174,7 +189,12 @@ pub enum LoadProgramError { pub fn load_programs>(path_base_ts_r: P) -> Result<(), LoadProgramError> { let path_base_ts = path_base_ts_r.as_ref(); let skel_builder = LockcSkelBuilder::default(); - let open_skel = skel_builder.open()?; + let mut open_skel = skel_builder.open()?; + + let pid_max = get_pid_max()?; + open_skel.maps_mut().containers().set_max_entries(pid_max)?; + open_skel.maps_mut().processes().set_max_entries(pid_max)?; + let mut skel = open_skel.load()?; let mut path_map_runtimes = path_base_ts.join("map_runtimes"); @@ -488,6 +508,11 @@ mod tests { assert_eq!(returned_hash, correct_hash); } + #[test] + fn get_pid_max_when_correct() { + assert!(get_pid_max().is_ok()); + } + // It doesn't work on Github actions, see // https://github.com/rancher-sandbox/lockc/issues/65 #[test]