//! Host checks for the builtin executor, behind `forge-runner check`. //! //! The check runs one trivial job through the real sandbox, with a rootfs of host binaries, and //! reports each requirement as ok, failed, or skipped. It also holds [`build_rootfs`], which //! builds such a rootfs for the tests. use std::fs; use std::io; use std::path::{Path, PathBuf}; use std::process::{ExitStatus, Stdio}; use tokio::io::AsyncWriteExt; use tokio::process::Command; use crate::config::{Limits, UsernsMode}; use crate::sandbox::SandboxRequest; use crate::sandbox::idmap::IdMap; /// The probe job. It prints the seccomp mode of the job, then succeeds. const PROBE_SCRIPT: &str = "grep '^Seccomp:' /proc/self/status; true"; /// The probe job in subuid mode. It also chowns a file in `/tmp` to 1000:1000 and prints its /// owner, which needs the subordinate range. const PROBE_SUBUID_SCRIPT: &str = "grep '^Seccomp:' /proc/self/status; \ touch /tmp/forge-probe && chown 1000:1000 /tmp/forge-probe \ && stat -c 'owner %u %g' /tmp/forge-probe; true"; /// The utilities the probe needs in its rootfs. const PROBE_BINARIES: &[&str] = &["sh", "grep", "touch", "chown", "stat"]; /// The controllers the job cgroup needs for `[limits]`. const LIMIT_CONTROLLERS: [&str; 3] = ["cpu", "memory", "pids"]; /// Copies `binaries` (looked up in `/usr/bin` and `/bin`) and their shared libraries into `root`, /// at the same absolute paths. It also creates the empty mount points the sandbox needs. pub fn build_rootfs(root: &Path, binaries: &[&str]) -> io::Result<()> { for dir in [ "bin", "proc", "dev", "sys", "tmp", "workspace", "etc", "root", ] { fs::create_dir_all(root.join(dir))?; } for name in ["null", "zero", "full", "random", "urandom"] { fs::write(root.join("dev").join(name), b"")?; } for name in ["etc/hosts", "etc/hostname", "etc/resolv.conf"] { fs::write(root.join(name), b"")?; } let mut libs: Vec = Vec::new(); for name in binaries { let host = ["/usr/bin", "/bin"] .iter() .map(|dir| Path::new(dir).join(name)) .find(|p| p.exists()) .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, format!("{name} not found")))?; let real = fs::canonicalize(&host)?; fs::copy(&real, root.join("bin").join(name))?; libs.extend(shared_libs(&real)); } libs.sort(); libs.dedup(); for lib in &libs { copy_into(root, lib)?; } Ok(()) } /// The shared libraries `ldd` reports for `binary`, as absolute host paths. fn shared_libs(binary: &Path) -> Vec { let Ok(out) = std::process::Command::new("ldd").arg(binary).output() else { return Vec::new(); }; String::from_utf8_lossy(&out.stdout) .lines() .filter_map(|line| { let line = line.trim(); let path = match line.split_once(" => ") { Some((_, rest)) => rest.split(" (").next()?, None => line.split(" (").next()?, }; path.starts_with('/').then(|| PathBuf::from(path)) }) .collect() } fn copy_into(root: &Path, host: &Path) -> io::Result<()> { let real = fs::canonicalize(host)?; let dest = root.join(host.strip_prefix("/").unwrap_or(host)); if let Some(parent) = dest.parent() { fs::create_dir_all(parent)?; } fs::copy(&real, &dest)?; Ok(()) } /// How one requirement came out. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Status { Ok, Fail, Skip, } /// One line of the report. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Line { pub status: Status, pub name: &'static str, pub detail: String, } impl Line { fn ok(name: &'static str, detail: impl Into) -> Self { Self { status: Status::Ok, name, detail: detail.into(), } } fn fail(name: &'static str, detail: impl Into) -> Self { Self { status: Status::Fail, name, detail: detail.into(), } } fn skip(name: &'static str, detail: impl Into) -> Self { Self { status: Status::Skip, name, detail: detail.into(), } } } /// The helper's output and exit status for the probe job. struct Probe { status: ExitStatus, stdout: String, stderr: String, } /// Runs the probe job through the sandbox helper at `helper`, under `map`. `work_dir` must be /// absolute. async fn run_probe(helper: &Path, work_dir: &Path, map: &IdMap) -> Result { let root = work_dir.join(format!("check-{}", std::process::id())); let rootfs = root.join("rootfs"); let build_into = rootfs.clone(); let built = tokio::task::spawn_blocking(move || build_rootfs(&build_into, PROBE_BINARIES)) .await .map_err(|e| format!("cannot build the test rootfs: {e}"))?; built.map_err(|e| format!("cannot build the test rootfs: {e}"))?; let workspace = root.join("workspace"); fs::create_dir_all(&workspace) .map_err(|e| format!("cannot create {}: {e}", workspace.display()))?; let script = if map.is_subuid() { PROBE_SUBUID_SCRIPT } else { PROBE_SCRIPT }; let request = SandboxRequest { job_id: "forge-check".to_owned(), rootfs, job_dir: root.join("job"), workspace, image_env: Vec::new(), env: Vec::new(), run: script.to_owned(), network: false, limits: Limits::default(), idmap: map.clone(), }; let json = serde_json::to_vec(&request).map_err(|e| e.to_string())?; let output = async { let mut child = Command::new(helper) .arg("__sandbox") .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .kill_on_drop(true) .spawn()?; if let Some(mut stdin) = child.stdin.take() { stdin.write_all(&json).await?; } child.wait_with_output().await } .await .map_err(|e| format!("cannot start the sandbox helper: {e}")); let _ = fs::remove_dir_all(&root); let output = output?; Ok(Probe { status: output.status, stdout: String::from_utf8_lossy(&output.stdout).into_owned(), stderr: String::from_utf8_lossy(&output.stderr).into_owned(), }) } /// The helper's `forge-sandbox: ` lines, without the prefix. fn sandbox_lines(stderr: &str) -> Vec<&str> { stderr .lines() .filter_map(|line| line.strip_prefix("forge-sandbox: ")) .collect() } /// The cgroup v2 controllers enabled for the job cgroup, which is a child of the runner's parent /// slice (see the helper). Returns the enabled list, or why it could not be read. fn delegated_controllers() -> Result, String> { let text = fs::read_to_string("/proc/self/cgroup") .map_err(|e| format!("cannot read /proc/self/cgroup: {e}"))?; let own = text .lines() .find_map(|line| line.strip_prefix("0::")) .ok_or_else(|| "cgroup v2 is not in use by this process".to_owned())?; let own = PathBuf::from(own.trim()); let base = own.parent().unwrap_or(&own); let file = Path::new("/sys/fs/cgroup") .join(base.strip_prefix("/").unwrap_or(base)) .join("cgroup.subtree_control"); let enabled = fs::read_to_string(&file).map_err(|e| format!("cannot read {}: {e}", file.display()))?; Ok(enabled.split_whitespace().map(str::to_owned).collect()) } /// Builds the report from the probe result. Pure, so the rules can be tested. fn report(probe: Result<&Probe, &str>, controllers: Result<&[String], &str>) -> Vec { let mut lines = Vec::new(); match controllers { Ok(enabled) => { let missing: Vec<&str> = LIMIT_CONTROLLERS .iter() .copied() .filter(|c| !enabled.iter().any(|e| e == c)) .collect(); if missing.is_empty() { lines.push(Line::ok( "cgroup v2 delegation", format!("controllers enabled: {}", enabled.join(" ")), )); } else { lines.push(Line::fail( "cgroup v2 delegation", format!( "missing {}; [limits] cannot be applied (enabled: {})", missing.join(" "), enabled.join(" ") ), )); } } Err(reason) => lines.push(Line::fail("cgroup v2 delegation", reason)), } let probe = match probe { Ok(probe) => probe, Err(reason) => { lines.push(Line::fail("sandbox run", reason)); for name in [ "user namespaces", "overlayfs in a user namespace", "resource limits", "seccomp filter", ] { lines.push(Line::skip(name, "not tested: the probe job did not run")); } return lines; } }; let setup_errors = sandbox_lines(&probe.stderr); let userns_error = setup_errors .iter() .find(|l| l.contains("user namespaces unavailable")); let started = probe.stdout.lines().any(|l| l.starts_with("Seccomp:")); if let Some(line) = userns_error { lines.push(Line::fail("user namespaces", *line)); } else if started { lines.push(Line::ok( "user namespaces", "unprivileged user namespaces work", )); } else { lines.push(Line::skip( "user namespaces", "not tested: the probe job did not start", )); } if !started { let reason = setup_errors .first() .map(|l| (*l).to_owned()) .unwrap_or_else(|| format!("the probe exited with {}", probe.status)); lines.push(Line::fail("sandbox run", reason)); lines.push(Line::skip( "overlayfs in a user namespace", "not tested: the probe job did not start", )); lines.push(Line::skip( "resource limits", "not tested: the probe job did not start", )); lines.push(Line::skip( "seccomp filter", "not tested: the probe job did not start", )); return lines; } lines.push(Line::ok( "sandbox run", format!("probe job exited with {}", probe.status), )); match setup_errors .iter() .find(|l| l.starts_with("warning: overlay mount failed")) { Some(line) => lines.push(Line::fail( "overlayfs in a user namespace", format!("{line}; jobs copy the rootfs instead (slower, uses disk)"), )), None => lines.push(Line::ok( "overlayfs in a user namespace", "overlay mounts work", )), } match setup_errors .iter() .find(|l| l.starts_with("warning: resource limits unavailable")) { Some(line) => lines.push(Line::fail("resource limits", *line)), None => lines.push(Line::ok( "resource limits", "cpu, memory and pids limits apply", )), } let mode = probe .stdout .lines() .find_map(|l| l.strip_prefix("Seccomp:")) .map(str::trim) .unwrap_or_default(); if mode == "2" { lines.push(Line::ok( "seccomp filter", "the job runs with a seccomp filter", )); } else { lines.push(Line::fail( "seccomp filter", format!("the job's seccomp mode is {mode:?}, expected 2"), )); } lines } /// The subordinate id check, in subuid mode. The probe must have started, and must have chowned /// its file in `/tmp` to 1000:1000, which is container id 1000 and needs the subordinate range. fn ownership_line(probe: &Probe) -> Line { const NAME: &str = "subordinate ids"; if !probe.stdout.lines().any(|l| l.starts_with("Seccomp:")) { return Line::skip(NAME, "not tested: the probe job did not start"); } if probe.stdout.lines().any(|l| l.trim() == "owner 1000 1000") { return Line::ok(NAME, "a job can chown files to 1000:1000"); } let reason = probe .stderr .lines() .find(|l| !l.starts_with("forge-sandbox: ") && !l.trim().is_empty()) .unwrap_or("no output"); Line::fail( NAME, format!("a job cannot chown 1000:1000 in /tmp: {reason}"), ) } /// Runs every check and prints one line per requirement. Returns `true` when all pass. /// /// `helper` is the `forge-runner` binary, and `work_dir` is where the probe runs. The probe needs /// the same filesystem that jobs will use. `mode` is the `[sandbox] userns` setting. pub async fn run(helper: &Path, work_dir: &Path, mode: UsernsMode) -> bool { let work_dir = match std::path::absolute(work_dir) { Ok(path) => path, Err(e) => { println!("FAIL work dir: {}: {e}", work_dir.display()); return false; } }; let map = match IdMap::resolve(mode) { Ok(map) => map, Err(e) => { println!("FAIL user ids: {e}"); return false; } }; let probe = run_probe(helper, &work_dir, &map).await; let controllers = delegated_controllers(); let mut lines = vec![Line::ok("user ids", map.describe())]; lines.extend(report( probe.as_ref().map_err(String::as_str), controllers.as_deref().map_err(String::as_str), )); if map.is_subuid() { lines.push(match &probe { Ok(probe) => ownership_line(probe), Err(_) => Line::skip("subordinate ids", "not tested: the probe job did not run"), }); } let mut all_ok = true; for line in &lines { let tag = match line.status { Status::Ok => "ok ", Status::Fail => { all_ok = false; "FAIL" } Status::Skip => { all_ok = false; "skip" } }; println!("{tag} {}: {}", line.name, line.detail); } all_ok } #[cfg(test)] mod tests { use super::*; use std::os::unix::process::ExitStatusExt; fn probe(stdout: &str, stderr: &str) -> Probe { Probe { status: ExitStatus::from_raw(0), stdout: stdout.to_owned(), stderr: stderr.to_owned(), } } fn all() -> Vec { LIMIT_CONTROLLERS.iter().map(|c| (*c).to_owned()).collect() } #[test] fn a_healthy_host_passes_every_check() { let p = probe("Seccomp:\t2\n", ""); let lines = report(Ok(&p), Ok(&all())); assert!(lines.iter().all(|l| l.status == Status::Ok), "{lines:?}"); } #[test] fn missing_userns_is_reported_with_its_reason() { let p = probe("", "forge-sandbox: user namespaces unavailable: EPERM\n"); let lines = report(Ok(&p), Ok(&all())); let userns = lines .iter() .find(|l| l.name == "user namespaces") .expect("line"); assert_eq!(userns.status, Status::Fail); assert!(userns.detail.contains("EPERM")); assert!( lines .iter() .any(|l| l.name == "sandbox run" && l.status == Status::Fail) ); } #[test] fn overlay_and_limit_warnings_are_failures() { let p = probe( "Seccomp:\t2\n", "forge-sandbox: warning: overlay mount failed (EPERM), copying the rootfs instead\n\ forge-sandbox: warning: resource limits unavailable: cgroup\n", ); let lines = report(Ok(&p), Ok(&all())); let overlay = lines .iter() .find(|l| l.name.starts_with("overlayfs")) .expect("line"); assert_eq!(overlay.status, Status::Fail); let limits = lines .iter() .find(|l| l.name == "resource limits") .expect("line"); assert_eq!(limits.status, Status::Fail); } #[test] fn missing_controllers_fail_the_delegation_check() { let p = probe("Seccomp:\t2\n", ""); let enabled = vec!["memory".to_owned()]; let lines = report(Ok(&p), Ok(&enabled)); let line = &lines[0]; assert_eq!(line.status, Status::Fail); assert!(line.detail.contains("cpu pids")); } #[test] fn a_probe_that_never_ran_skips_the_rest() { let lines = report(Err("cannot build the test rootfs"), Ok(&all())); assert_eq!(lines[1].status, Status::Fail); assert!(lines.iter().skip(2).all(|l| l.status == Status::Skip)); } }