//! Real sandbox tests. They run `forge-runner __sandbox` with a rootfs built here from the host's //! shell and a few utilities plus their shared libraries. Nothing is pulled from a registry, and //! there is no network access. The tests skip, with a message, when unprivileged user //! namespaces are unavailable. use std::fs; use std::io::Write; use std::path::{Path, PathBuf}; use std::process::{Command, Output, Stdio}; use std::time::{Duration, Instant}; use std::os::unix::fs::MetadataExt; use forge_runner::config::Limits; use forge_runner::sandbox::SandboxRequest; use forge_runner::sandbox::idmap::{IdMap, UsernsMode}; /// Utilities copied into the test rootfs. `sh` is the job shell, and the rest are used by the /// test scripts. const BINARIES: &[&str] = &[ "sh", "id", "cat", "grep", "touch", "sleep", "unshare", "chown", "stat", "mkdir", ]; fn userns_available() -> bool { let max = fs::read_to_string("/proc/sys/user/max_user_namespaces") .ok() .and_then(|s| s.trim().parse::().ok()) .unwrap_or(0); if max == 0 { return false; } Command::new("unshare") .args(["--user", "--map-root-user", "true"]) .status() .is_ok_and(|s| s.success()) } /// Skips the test when user namespaces are unavailable. Returns `true` when the test should run. fn can_run() -> bool { if userns_available() { true } else { eprintln!("skipping: unprivileged user namespaces are not available on this host"); false } } /// The shared libraries `ldd` reports for `binary`, as absolute host paths. fn shared_libs(binary: &Path) -> Vec { let Ok(out) = Command::new("ldd").arg(binary).output() else { return Vec::new(); }; String::from_utf8_lossy(&out.stdout) .lines() .filter_map(|line| { let line = line.trim(); let path = match line.split_once(" => ") { Some((_, rest)) => rest.split(" (").next()?, None => line.split(" (").next()?, }; path.starts_with('/').then(|| PathBuf::from(path)) }) .collect() } fn copy_into(root: &Path, host: &Path) { let real = fs::canonicalize(host).expect("canonicalize host file"); let dest = root.join(host.strip_prefix("/").unwrap_or(host)); if let Some(parent) = dest.parent() { fs::create_dir_all(parent).expect("create dest dir"); } fs::copy(&real, &dest).expect("copy file into rootfs"); } /// Builds a tiny rootfs in `root`: the binaries in `/bin`, their shared libraries at the same /// absolute paths, and the empty mount points the spec needs. fn build_rootfs(root: &Path) { for dir in [ "bin", "proc", "dev", "sys", "tmp", "workspace", "etc", "root", ] { fs::create_dir_all(root.join(dir)).expect("create rootfs dir"); } for name in ["null", "zero", "full", "random", "urandom"] { fs::write(root.join("dev").join(name), b"").expect("placeholder device"); } fs::write(root.join("etc/hosts"), b"").expect("placeholder hosts"); fs::write(root.join("etc/hostname"), b"").expect("placeholder hostname"); fs::write(root.join("etc/resolv.conf"), b"").expect("placeholder resolv.conf"); let mut libs: Vec = Vec::new(); for name in BINARIES { let host = ["/usr/bin", "/bin"] .iter() .map(|dir| Path::new(dir).join(name)) .find(|p| p.exists()) .unwrap_or_else(|| panic!("{name} not found on this host")); let real = fs::canonicalize(&host).expect("canonicalize binary"); fs::copy(&real, root.join("bin").join(name)).expect("copy binary"); libs.extend(shared_libs(&real)); } libs.sort(); libs.dedup(); for lib in &libs { copy_into(root, lib); } } fn request(dir: &Path, rootfs: &Path, run: &str, network: bool) -> SandboxRequest { let workspace = dir.join("workspace"); fs::create_dir_all(&workspace).expect("create workspace"); SandboxRequest { job_id: "job_01".to_owned(), rootfs: rootfs.to_owned(), job_dir: dir.join("jobs/job_01"), workspace, image_env: Vec::new(), env: vec![("FORGE_TEST".to_owned(), "yes".to_owned())], run: run.to_owned(), network, limits: Limits::default(), idmap: IdMap::single(), } } /// The subuid map, or `None` with a message when the host can't provide one. fn subuid_map() -> Option { match IdMap::resolve(UsernsMode::Subuid) { Ok(map) if map.is_subuid() => Some(map), Ok(_) => None, Err(e) => { eprintln!("skipping: subordinate ids are not available: {e}"); None } } } /// Spawns the helper and writes the request to its stdin. Returns the child, not yet waited on. fn spawn_helper(req: &SandboxRequest) -> std::process::Child { let mut child = Command::new(env!("CARGO_BIN_EXE_forge-runner")) .arg("__sandbox") .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() .expect("spawn helper"); let json = serde_json::to_vec(req).expect("serialize request"); let mut stdin = child.stdin.take().expect("helper stdin"); stdin.write_all(&json).expect("write request"); drop(stdin); child } fn run_helper(req: &SandboxRequest) -> Output { spawn_helper(req).wait_with_output().expect("helper output") } fn stdout_of(out: &Output) -> String { String::from_utf8_lossy(&out.stdout).into_owned() } fn stderr_of(out: &Output) -> String { String::from_utf8_lossy(&out.stderr).into_owned() } #[test] fn job_runs_as_root_with_no_capabilities() { if !can_run() { return; } let dir = tempfile::tempdir().expect("tempdir"); let rootfs = dir.path().join("rootfs"); build_rootfs(&rootfs); let run = "echo hi; id -u; grep -E '^(CapEff|NoNewPrivs|Seccomp):' /proc/self/status; \ touch /workspace/f; echo \"$CI $FORGE_CI $HOME $FORGE_TEST\""; let req = request(dir.path(), &rootfs, run, false); let out = run_helper(&req); let stdout = stdout_of(&out); let stderr = stderr_of(&out); assert_eq!(out.status.code(), Some(0), "stderr: {stderr}"); assert!(stdout.lines().any(|l| l == "hi"), "stdout: {stdout}"); assert!(stdout.lines().any(|l| l == "0"), "uid is not 0: {stdout}"); assert!( stdout.contains("CapEff:\t0000000000000000"), "capabilities not empty: {stdout}" ); assert!( stdout.contains("NoNewPrivs:\t1"), "no_new_privs off: {stdout}" ); // Seccomp mode 2 is a filter. The profile is installed by `sandbox::exec::FilteredExecutor`. assert!( stdout.contains("Seccomp:\t2"), "seccomp filter not active: {stdout}" ); assert!( stdout.lines().any(|l| l == "true true /root yes"), "env wrong: {stdout}" ); assert!( req.workspace.join("f").exists(), "workspace file missing on host" ); assert!(!req.job_dir.join("bundle/config.json").exists()); assert!(!req.job_dir.exists(), "job dir was not removed"); } #[test] fn exit_code_is_passed_through() { if !can_run() { return; } let dir = tempfile::tempdir().expect("tempdir"); let rootfs = dir.path().join("rootfs"); build_rootfs(&rootfs); let out = run_helper(&request(dir.path(), &rootfs, "exit 7", false)); assert_eq!(out.status.code(), Some(7), "stderr: {}", stderr_of(&out)); } #[test] fn unshare_is_denied_inside_the_job() { if !can_run() { return; } let dir = tempfile::tempdir().expect("tempdir"); let rootfs = dir.path().join("rootfs"); build_rootfs(&rootfs); // Creating a user namespace works on the host. The seccomp filter makes unshare(2) fail with // EPERM inside the job, so this must print "denied". let run = "if unshare --user true 2>/dev/null; then echo allowed; else echo denied; fi"; let out = run_helper(&request(dir.path(), &rootfs, run, false)); let stdout = stdout_of(&out); assert_eq!(out.status.code(), Some(0), "stderr: {}", stderr_of(&out)); assert!(stdout.lines().any(|l| l == "denied"), "stdout: {stdout}"); assert!(!stdout.lines().any(|l| l == "allowed"), "stdout: {stdout}"); } #[test] fn network_false_has_only_loopback() { if !can_run() { return; } let dir = tempfile::tempdir().expect("tempdir"); let rootfs = dir.path().join("rootfs"); build_rootfs(&rootfs); let out = run_helper(&request(dir.path(), &rootfs, "cat /proc/net/dev", false)); let stdout = stdout_of(&out); assert_eq!(out.status.code(), Some(0), "stderr: {}", stderr_of(&out)); let interfaces: Vec<&str> = stdout .lines() .filter_map(|l| l.split_once(':').map(|(name, _)| name.trim())) .collect(); assert_eq!(interfaces, ["lo"], "unexpected interfaces: {stdout}"); } #[test] fn network_true_shares_the_host_network() { if !can_run() { return; } let dir = tempfile::tempdir().expect("tempdir"); let rootfs = dir.path().join("rootfs"); build_rootfs(&rootfs); let out = run_helper(&request(dir.path(), &rootfs, "cat /proc/net/dev", true)); let stdout = stdout_of(&out); assert_eq!(out.status.code(), Some(0), "stderr: {}", stderr_of(&out)); let names = |text: &str| -> Vec { let mut names: Vec = text .lines() .filter_map(|l| l.split_once(':').map(|(name, _)| name.trim().to_owned())) .collect(); names.sort(); names }; let host = fs::read_to_string("/proc/net/dev").expect("host /proc/net/dev"); assert_eq!( names(&stdout), names(&host), "job does not see the host network" ); } #[test] fn host_sysctl_is_read_only() { if !can_run() { return; } let dir = tempfile::tempdir().expect("tempdir"); let rootfs = dir.path().join("rootfs"); build_rootfs(&rootfs); let run = "if echo x > /proc/sys/kernel/hostname; then echo WROTE; else echo DENIED; fi"; let out = run_helper(&request(dir.path(), &rootfs, run, false)); let stdout = stdout_of(&out); assert_eq!(out.status.code(), Some(0), "stderr: {}", stderr_of(&out)); assert!(stdout.contains("DENIED"), "write was not denied: {stdout}"); assert!(!stdout.contains("WROTE"), "write was allowed: {stdout}"); } #[test] fn writes_land_in_the_overlay_not_the_image_rootfs() { if !can_run() { return; } let dir = tempfile::tempdir().expect("tempdir"); let rootfs = dir.path().join("rootfs"); build_rootfs(&rootfs); let out = run_helper(&request( dir.path(), &rootfs, "touch /etc/forge-marker", false, )); assert_eq!(out.status.code(), Some(0), "stderr: {}", stderr_of(&out)); assert!( !rootfs.join("etc/forge-marker").exists(), "the job wrote into the image rootfs" ); } #[test] fn sigterm_kills_the_container_and_the_helper_exits() { if !can_run() { return; } let dir = tempfile::tempdir().expect("tempdir"); let rootfs = dir.path().join("rootfs"); build_rootfs(&rootfs); let req = request(dir.path(), &rootfs, "sleep 60", false); let child = spawn_helper(&req); // Give the container time to start, so that SIGTERM hits a running job. std::thread::sleep(Duration::from_secs(2)); let pid = nix::unistd::Pid::from_raw(i32::try_from(child.id()).expect("pid fits i32")); nix::sys::signal::kill(pid, nix::sys::signal::Signal::SIGTERM).expect("send SIGTERM"); let started = Instant::now(); let out = child.wait_with_output().expect("helper output"); assert!( started.elapsed() < Duration::from_secs(20), "helper did not exit after SIGTERM" ); assert_eq!(out.status.code(), Some(137), "stderr: {}", stderr_of(&out)); } #[test] fn setup_failure_exits_125_with_a_reason() { let dir = tempfile::tempdir().expect("tempdir"); let missing = dir.path().join("no-such-rootfs"); let req = request(dir.path(), &missing, "echo never", false); let out = run_helper(&req); assert_eq!(out.status.code(), Some(125)); assert!( stderr_of(&out).contains("forge-sandbox: "), "stderr: {}", stderr_of(&out) ); } #[test] fn rootfs_is_copied_when_the_overlay_cannot_be_mounted() { if !can_run() { return; } let dir = tempfile::tempdir().expect("tempdir"); // A comma in a lower dir can't be written in overlay options, so the overlay mount fails // and the helper falls back to copying the rootfs. let rootfs = dir.path().join("rootfs,copy"); build_rootfs(&rootfs); let out = run_helper(&request( dir.path(), &rootfs, "echo copied; touch /etc/forge-marker", false, )); let stdout = stdout_of(&out); let stderr = stderr_of(&out); assert_eq!(out.status.code(), Some(0), "stderr: {stderr}"); assert!(stdout.lines().any(|l| l == "copied"), "stdout: {stdout}"); assert!( stderr.contains("overlay mount failed"), "no fallback warning: {stderr}" ); assert!(!rootfs.join("etc/forge-marker").exists()); } /// A request that runs under `map` (subuid mode when `map` has a range). fn subuid_request(dir: &Path, rootfs: &Path, run: &str, map: &IdMap) -> SandboxRequest { let mut req = request(dir, rootfs, run, false); req.idmap = map.clone(); req } #[test] fn subuid_job_can_chown_to_a_subordinate_id() { let Some(map) = subuid_map() else { return }; if !can_run() { return; } let dir = tempfile::tempdir().expect("tempdir"); let rootfs = dir.path().join("rootfs"); build_rootfs(&rootfs); let out = run_helper(&subuid_request( dir.path(), &rootfs, "touch /tmp/f && chown 1000:1000 /tmp/f && stat -c '%u %g' /tmp/f", &map, )); assert_eq!(out.status.code(), Some(0), "stderr: {}", stderr_of(&out)); assert_eq!(stdout_of(&out).trim(), "1000 1000"); } #[test] fn subuid_uid_and_gid_maps_have_one_plus_the_range() { let Some(map) = subuid_map() else { return }; if !can_run() { return; } let dir = tempfile::tempdir().expect("tempdir"); let rootfs = dir.path().join("rootfs"); build_rootfs(&rootfs); let out = run_helper(&subuid_request( dir.path(), &rootfs, "cat /proc/self/uid_map; cat /proc/self/gid_map", &map, )); let stdout = stdout_of(&out); assert_eq!(out.status.code(), Some(0), "stderr: {}", stderr_of(&out)); // Each line is `inside outside count`. The job's root plus the subordinate range make the // map's size, which is `1 + n`, in one or two lines. Sum the counts for each map. let sizes: Vec = stdout .lines() .filter_map(|l| l.split_whitespace().nth(2)?.parse().ok()) .collect(); assert_eq!(sizes.len(), 4, "stdout: {stdout}"); let (uid_sizes, gid_sizes) = sizes.split_at(2); assert_eq!( uid_sizes.iter().sum::(), map.size(), "stdout: {stdout}" ); assert_eq!( gid_sizes.iter().sum::(), map.size(), "stdout: {stdout}" ); assert!(map.size() > 1, "the range is empty"); } #[test] fn subuid_job_files_are_owned_by_the_runner_afterwards() { let Some(map) = subuid_map() else { return }; if !can_run() { return; } let dir = tempfile::tempdir().expect("tempdir"); let rootfs = dir.path().join("rootfs"); build_rootfs(&rootfs); let req = subuid_request( dir.path(), &rootfs, "touch /workspace/a && chown 1000:1000 /workspace/a \ && mkdir /workspace/d && chown 1000:1000 /workspace/d \ && touch /workspace/d/x && chown 1000:1000 /workspace/d/x", &map, ); let out = run_helper(&req); assert_eq!(out.status.code(), Some(0), "stderr: {}", stderr_of(&out)); for path in ["a", "d", "d/x"] { let path = req.workspace.join(path); let meta = fs::symlink_metadata(&path).expect("stat workspace entry"); assert_eq!( (meta.uid(), meta.gid()), (map.uid, map.gid), "{} is not owned by the runner", path.display() ); } } #[test] fn subuid_job_dir_is_removed_afterwards() { let Some(map) = subuid_map() else { return }; if !can_run() { return; } let dir = tempfile::tempdir().expect("tempdir"); let rootfs = dir.path().join("rootfs"); build_rootfs(&rootfs); let req = subuid_request( dir.path(), &rootfs, "touch /workspace/a && chown 1000:1000 /workspace/a", &map, ); let out = run_helper(&req); assert_eq!(out.status.code(), Some(0), "stderr: {}", stderr_of(&out)); assert!(!req.job_dir.exists(), "the job dir was left behind"); }