bpf: Fix problems with container starts

This change consists of several fixes which were causing random
problems with containers getting Ready.

* Allowing access to /run directory inside container filesystem. It's
  used both by popular applications (like nginx) and by kubelet to store
  the network namespace (/run/netns).
* Using uprobes for registering containers and processes in BPF maps.
  Doing BPF operations from the wrapper had the weird issue - by
  registering the wrapper process, we were in fact preventing the same
  lockc-runc-wrapper process from doing any more BPF map modifications.
  Using uprobes has also an another advantage - it will allow rootless
  containers to work.
* Proper cleanup of old containers and processes.

Majority of uprobes code on the userspace side comes from
bpfcontain-rs.

Fixes: #52
Fixes: #92
Signed-off-by: Michal Rostecki <mrostecki@opensuse.org>
Signed-off-by: William Findlay <william@williamfindlay.com>

Cargo.toml

@@ -1,5 +1,6 @@
 [workspace]
 members = [
     "lockc",
+    "lockc-uprobes",
     "xtask",
 ]

contrib/etc/lockc/lockc.toml

@@ -225,6 +225,7 @@ allowed_paths_access_restricted = [
     "/home",
     "/lib",
     "/proc",
+    "/run",
     "/sys/fs/cgroup",
     "/tmp",
     "/usr",
@@ -244,6 +245,7 @@ allowed_paths_access_baseline = [
     "/home",
     "/lib",
     "/proc",
+    "/run",
     "/sys/fs/cgroup",
     "/tmp",
     "/usr",
@@ -252,9 +254,9 @@ allowed_paths_access_baseline = [
 
 denied_paths_access_restricted = [
     "/proc/acpi",
+    "/proc/sys",
 ]
 
 denied_paths_access_baseline = [
     "/proc/acpi",
-    "/proc/sys",
 ]

contrib/systemd/lockcd.service.in

@@ -3,7 +3,9 @@ Description=lockc daemon
 After=network-online.target
 
 [Service]
-Type=oneshot
+Type=simple
+Restart=always
+RestartSec=1
 ExecStart={{ bindir }}/lockcd
 StandardOutput=journal
 

lockc-uprobes/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "lockc-uprobes"
+version = "0.1.0"
+edition = "2021"
+
+license = "Apache-2.0"
+
+[dependencies]
+libc = "0.2"

lockc-uprobes/src/lib.rs

@@ -0,0 +1,13 @@
+use libc::pid_t;
+
+#[no_mangle]
+#[inline(never)]
+pub extern "C" fn add_container(_retp: *mut i32, _container_id: u32, _pid: pid_t, _policy: i32) {}
+
+#[no_mangle]
+#[inline(never)]
+pub extern "C" fn delete_container(_retp: *mut i32, _container_id: u32) {}
+
+#[no_mangle]
+#[inline(never)]
+pub extern "C" fn add_process(_retp: *mut i32, _container_id: u32, _pid: pid_t) {}

lockc/Cargo.toml

@@ -22,13 +22,16 @@ bindgen = "0.59"
 byteorder = "1.4"
 chrono = { version = "0.4", default-features = false, features = ["clock"] }
 config = { version = "0.11", default-features = false, features = ["toml"] }
+ctrlc = "3.2"
 dirs = "4.0"
 futures = "0.3"
+goblin = "0.4"
 kube = "0.63"
 k8s-openapi = { version = "0.13", default-features = false, features = ["v1_21"] }
 lazy_static = "1.4"
 libc = { version = "0.2", features = [ "extra_traits" ] }
-libbpf-rs = "0.13"
+libbpf-rs = { git = "https://github.com/libbpf/libbpf-rs", rev = "ebc55ba" }
+lockc-uprobes = { path = "../lockc-uprobes" }
 log = "0.4"
 log4rs = "1.0"
 nix = "0.23"
@@ -41,6 +44,7 @@ sysctl = "0.4"
 thiserror = "1.0"
 tokio = { version = "1.7", features = ["macros", "process", "rt-multi-thread"] }
 uuid = { version = "0.8", default-features = false, features = ["v4"] }
+which = "4.2"
 
 [build-dependencies]
 anyhow = "1.0"

lockc/src/bin/lockc-runc-wrapper.rs

@@ -6,6 +6,8 @@ use log4rs::append::file::FileAppender;
 use log4rs::config::{runtime::ConfigErrors, Appender, Config, Root};
 use uuid::Uuid;
 
+use lockc_uprobes::{add_container, add_process, delete_container};
+
 // TODO: To be used for cri-o.
 // static ANNOTATION_K8S_LABELS: &str = "io.kubernetes.cri-o.Labels";
 
@@ -140,13 +142,13 @@ struct Mount {
     destination: String,
     r#type: String,
     source: String,
-    options: Vec<String>
+    options: Vec<String>,
 }
 
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 struct Mounts {
-    mounts: Vec<Mount>
+    mounts: Vec<Mount>,
 }
 
 fn docker_config<P: AsRef<std::path::Path>>(
@@ -161,9 +163,9 @@ fn docker_config<P: AsRef<std::path::Path>>(
 
     for test in m.mounts {
         let source: Vec<&str> = test.source.split('/').collect();
-        if source.len() > 1 && source[ source.len() - 1 ] == "hostname" {
-                let config_v2= str::replace(&test.source, "hostname", "config.v2.json");
-                return Ok(std::path::PathBuf::from(config_v2));
+        if source.len() > 1 && source[source.len() - 1] == "hostname" {
+            let config_v2 = str::replace(&test.source, "hostname", "config.v2.json");
+            return Ok(std::path::PathBuf::from(config_v2));
         }
     }
 
@@ -185,15 +187,11 @@ fn docker_label<P: AsRef<std::path::Path>>(
 
     match x {
         Some(x) => match x {
-            "restricted" => {
-                Ok(lockc::bpfstructs::container_policy_level_POLICY_LEVEL_RESTRICTED)
-            }
+            "restricted" => Ok(lockc::bpfstructs::container_policy_level_POLICY_LEVEL_RESTRICTED),
             "baseline" => Ok(lockc::bpfstructs::container_policy_level_POLICY_LEVEL_BASELINE),
-            "privileged" => {
-                Ok(lockc::bpfstructs::container_policy_level_POLICY_LEVEL_PRIVILEGED)
-            }
-            _ => Ok(lockc::bpfstructs::container_policy_level_POLICY_LEVEL_BASELINE)
-        }
+            "privileged" => Ok(lockc::bpfstructs::container_policy_level_POLICY_LEVEL_PRIVILEGED),
+            _ => Ok(lockc::bpfstructs::container_policy_level_POLICY_LEVEL_BASELINE),
+        },
         None => Ok(lockc::bpfstructs::container_policy_level_POLICY_LEVEL_BASELINE),
     }
 }
@@ -265,12 +263,33 @@ fn setup_logging() -> Result<(), SetupLoggingError> {
     Ok(())
 }
 
+#[derive(thiserror::Error, Debug)]
+enum UprobeError {
+    #[error("failed to call into uprobe, BPF programs are most likely not running")]
+    Call,
+
+    #[error("BPF program error")]
+    BPF,
+
+    #[error("unknown error")]
+    Unknown,
+}
+
+fn check_uprobe_ret(ret: i32) -> Result<(), UprobeError> {
+    match ret {
+        0 => Ok(()),
+        n if n == -libc::EAGAIN => Err(UprobeError::Call),
+        n if n == -libc::EINVAL => Err(UprobeError::BPF),
+        _ => Err(UprobeError::Unknown),
+    }
+}
+
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
     setup_logging()?;
 
     let pid = nix::unistd::getpid();
-    let pid_u = u32::try_from(libc::pid_t::from(pid))?;
+    let pid_u = libc::pid_t::from(pid);
 
     let mut opt_parsing_action = OptParsingAction::NoPositional;
     let mut arg_parsing_action = ArgParsingAction::None;
@@ -353,29 +372,34 @@ async fn main() -> anyhow::Result<()> {
 
     match container_action {
         ContainerAction::Other => {
+            info!("other action");
             match container_id_o {
                 Some(v) => {
                     let container_key = lockc::hash(&v)?;
-                    lockc::add_process(container_key, pid_u)?;
+
+                    let mut ret: i32 = -libc::EAGAIN;
+                    add_process(&mut ret as *mut i32, container_key, pid_u);
+                    check_uprobe_ret(ret)?;
+
                     cmd.status().await?;
-                    lockc::delete_process(pid_u)?;
                 }
                 None => {
                     // The purpose of this fake "container" is only to allow the runc
                     // subcommand to be detected as wrapped and thus allowed by
                     // the LSM program to execute. It's only to handle subcommands
                     // like `init`, `list` or `spec`, so we make it restricted.
-                    lockc::add_container(
-                        0,
-                        pid_u,
-                        lockc::bpfstructs::container_policy_level_POLICY_LEVEL_RESTRICTED,
-                    )?;
+                    // lockc::add_container(
+                    //     0,
+                    //     pid_u,
+                    //     lockc::bpfstructs::container_policy_level_POLICY_LEVEL_RESTRICTED,
+                    // )?;
                     cmd.status().await?;
-                    lockc::delete_container(0)?;
+                    // lockc::delete_container(0)?;
                 }
             }
         }
         ContainerAction::Create => {
+            info!("create action");
             let container_key = lockc::hash(&container_id_o.unwrap())?;
             let container_bundle = match container_bundle_o {
                 Some(v) => std::path::PathBuf::from(v),
@@ -392,15 +416,29 @@ async fn main() -> anyhow::Result<()> {
                     policy = docker_label(docker_conf)?;
                 }
             };
-            lockc::add_container(container_key, pid_u, policy)?;
+
+            info!("before uprobe");
+            let mut ret: i32 = -libc::EAGAIN;
+            add_container(&mut ret as *mut i32, container_key, pid_u, policy);
+            info!("after uprobe, ret: {}", ret);
+            check_uprobe_ret(ret)?;
+            info!("uprobe checked");
+
             cmd.status().await?;
+            info!("runc executed");
         }
         ContainerAction::Delete => {
+            info!("delete action");
             let container_key = lockc::hash(&container_id_o.unwrap())?;
-            lockc::delete_container(container_key)?;
+
             cmd.status().await?;
+
+            let mut ret: i32 = -libc::EAGAIN;
+            delete_container(&mut ret as *mut i32, container_key);
+            check_uprobe_ret(ret)?;
         }
         ContainerAction::Start => {
+            info!("start action");
             cmd.status().await?;
         }
     }

lockc/src/bin/lockcd.rs

@@ -20,8 +20,11 @@ fn main() -> anyhow::Result<()> {
 
     let path_base_ts = path_base.join(&dirname);
 
-    lockc::load_programs(path_base_ts)?;
+    // lockc::load_programs(path_base_ts)?;
+    let skel = lockc::BpfContext::new(path_base_ts)?;
     lockc::cleanup(path_base, &dirname)?;
 
+    skel.work_loop()?;
+
     Ok(())
 }