Merge pull request #84 from vadorovsky/fix-memory-alloc

bpf: Adjust PID-related BPF map sizes to the current pid_max

docs/src/SUMMARY.md

@@ -17,5 +17,6 @@
     - [File access](policies/file-access.md)
     - [Mount](policies/mount.md)
     - [Syslog](policies/syslog.md)
+- [Tuning](tuning/README.md)
 - [Demos](demos/README.md)
     - [Mount](demos/mount.md)

docs/src/tuning/README.md

@@ -0,0 +1,37 @@
+## Tuning
+
+This guide shows options and tricks to gain an optimal performance and resouce
+usage.
+
+### Memory usage
+
+Memory usage by lockc depends mostly on BPF maps size. BPF maps are stored in
+memory and the biggest BPF maps are the ones related to tracking processes and
+containers. Size of those maps depends on the limit of processes (in separate
+memory spaces) in the system. That limit is determined by the `kernel.pid_max`
+sysctl. By default the limit is 32768. With such limit, memory usage by lockc
+should be aproximately 10-20 MB.
+
+If you observe too much memory being used after installing lockc, try to check
+the value of `kernel.pid_max`, which can be done with:
+
+```bash
+sudo sysctl kernel.pid_max
+```
+
+Change of that value (i.e. to 10000) can be done with:
+
+```bash
+sudo sysctl kernel.pid_max=10000
+```
+
+But that change will be not persistent after reboot. Changing it persistently
+requires adding a configuration to `/etc/sysctl.d`. I.e. we could create the
+file `/etc/sysctl.d/05-lockc.conf` with the following content:
+
+```
+kernel.pid_max = 10000
+```
+
+After creating that file, the lower limit is going to be persistent after
+reboot.

lockc/Cargo.toml

@@ -37,6 +37,7 @@ procfs = "0.11"
 regex = { version = "1.5", default-features = false, features = ["perf"] }
 serde = "1.0"
 serde_json = "1.0"
+sysctl = "0.4"
 thiserror = "1.0"
 tokio = { version = "1.7", features = ["macros", "process", "rt-multi-thread"] }
 uuid = { version = "0.8", default-features = false, features = ["v4"] }

lockc/src/lib.rs

@@ -5,9 +5,13 @@
 #[macro_use]
 extern crate lazy_static;
 
-use bpfstructs::BpfStruct;
+use std::{convert::TryInto, fs, io, io::prelude::*, num, path};
+
 use byteorder::{NativeEndian, WriteBytesExt};
-use std::{convert::TryInto, fs, io, io::prelude::*, path};
+use sysctl::Sysctl;
+
+use bpfstructs::BpfStruct;
+
 
 #[rustfmt::skip]
 mod bpf;
@@ -135,28 +139,39 @@ pub fn init_allowed_paths(mut maps: LockcMapsMut) -> Result<(), InitAllowedPaths
 }
 
 #[derive(thiserror::Error, Debug)]
-pub enum LoadProgramError {
-    #[error("hash error")]
-    HashError(#[from] HashError),
+pub enum GetPidMaxError {
+    #[error(transparent)]
+    ParseInt(#[from] num::ParseIntError),
 
-    #[error("init runtimes error")]
-    InitRuntimesError(#[from] InitRuntimesError),
+    #[error(transparent)]
+    Sysctl(#[from] sysctl::SysctlError),
+}
 
-    #[error("init allowed paths error")]
-    InitAllowedPathsError(#[from] InitAllowedPathsError),
+/// Gets the max PID number configured in the system.
+fn get_pid_max() -> Result<u32, GetPidMaxError> {
+    let pid_max_s = sysctl::Ctl::new("kernel.pid_max")?.value_string()?;
+    let pid_max = pid_max_s.parse::<u32>()?;
+    Ok(pid_max)
+}
 
-    #[error("could not convert the hash to a byte array")]
-    ByteWriteError(#[from] io::Error),
+#[derive(thiserror::Error, Debug)]
+pub enum LoadProgramError {
+    #[error(transparent)]
+    Libbpf(#[from] libbpf_rs::Error),
 
-    #[error("libbpf error")]
-    LibbpfError(#[from] libbpf_rs::Error),
+    #[error(transparent)]
+    GetPidMax(#[from] GetPidMaxError),
+
+    #[error(transparent)]
+    InitAllowedPaths(#[from] InitAllowedPathsError),
 
-    #[error("could not align the byte data")]
-    ByteAlignError,
+    #[error(transparent)]
+    InitRuntimes(#[from] InitRuntimesError),
 }
 
 /// Performs the following BPF-related operations:
 /// - loading BPF programs
+/// - resizing PID-related BPF maps
 /// - pinning BPF maps in BPFFS
 /// - pinning BPF programs in BPFFS
 /// - attaching BPF programs, creating links
@@ -174,7 +189,12 @@ pub enum LoadProgramError {
 pub fn load_programs<P: AsRef<path::Path>>(path_base_ts_r: P) -> Result<(), LoadProgramError> {
     let path_base_ts = path_base_ts_r.as_ref();
     let skel_builder = LockcSkelBuilder::default();
-    let open_skel = skel_builder.open()?;
+    let mut open_skel = skel_builder.open()?;
+
+    let pid_max = get_pid_max()?;
+    open_skel.maps_mut().containers().set_max_entries(pid_max)?;
+    open_skel.maps_mut().processes().set_max_entries(pid_max)?;
+
     let mut skel = open_skel.load()?;
 
     let mut path_map_runtimes = path_base_ts.join("map_runtimes");
@@ -488,6 +508,11 @@ mod tests {
         assert_eq!(returned_hash, correct_hash);
     }
 
+    #[test]
+    fn get_pid_max_when_correct() {
+        assert!(get_pid_max().is_ok());
+    }
+
     // It doesn't work on Github actions, see
     // https://github.com/rancher-sandbox/lockc/issues/65
     #[test]