//! The OCI runtime spec for one job. [`build_spec`] is pure, so the hardening can be unit tested. //! //! Single mode gives the job its own user namespace, which maps container uid/gid 0 to the //! helper's uid/gid 0. The helper has already entered its user namespace, in which it is root //! (see `helper.rs`). In subuid mode the spec has no user namespace, and the job runs in the //! helper's, whose ids are the [`IdMap`] (the runner's uid, then the subordinate range). use std::collections::HashSet; use std::path::{Path, PathBuf}; use oci_spec::runtime::{ Capability, LinuxBuilder, LinuxCapabilitiesBuilder, LinuxCpuBuilder, LinuxIdMappingBuilder, LinuxMemoryBuilder, LinuxNamespaceBuilder, LinuxNamespaceType, LinuxPidsBuilder, LinuxResourcesBuilder, Mount, MountBuilder, PosixRlimitBuilder, PosixRlimitType, ProcessBuilder, RootBuilder, Spec, SpecBuilder, UserBuilder, }; use super::{SandboxError, SandboxRequest}; use crate::config::Limits; use crate::executor::validate_env_name; /// The PATH used when the image config does not set one. pub const DEFAULT_PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; /// The job's working directory inside the container. pub const WORKSPACE: &str = "/workspace"; /// The container's hostname (the UTS namespace is private). pub const HOSTNAME: &str = "forge-job"; /// RLIMIT_NOFILE, for both soft and hard limits. pub const NOFILE_LIMIT: u64 = 65_536; /// The CFS period used to turn `cpus` into a quota. pub const CPU_PERIOD_US: u64 = 100_000; /// Paths hidden by an empty bind mount, as Docker does. pub const MASKED_PATHS: &[&str] = &[ "/proc/acpi", "/proc/asound", "/proc/kcore", "/proc/keys", "/proc/latency_stats", "/proc/timer_list", "/proc/timer_stats", "/proc/sched_debug", "/proc/scsi", "/sys/firmware", "/sys/devices/virtual/powercap", ]; /// Paths that stay visible but read-only. pub const READONLY_PATHS: &[&str] = &[ "/proc/bus", "/proc/fs", "/proc/irq", "/proc/sys", "/proc/sysrq-trigger", ]; /// Host device nodes bind mounted into `/dev`. `tty` is left out: binding it opens the host's /// controlling terminal, and the runner has none (ENXIO). pub const DEVICES: &[&str] = &["null", "zero", "full", "random", "urandom"]; /// Builds the OCI spec for `req`. `rootfs_mount` is the directory libcontainer pivots into. Single /// mode gives the job its own user namespace with one id. Subuid mode has no user namespace in the /// spec: the job runs in the helper's, which maps the subordinate range (see `build_linux`). pub fn build_spec(req: &SandboxRequest, rootfs_mount: &Path) -> Result { let env = build_env(req)?; let args = vec![ "sh".to_owned(), "-eu".to_owned(), "-c".to_owned(), req.run.clone(), ]; let process = ProcessBuilder::default() .terminal(false) .user( UserBuilder::default() .uid(0u32) .gid(0u32) .build() .map_err(spec_err)?, ) .args(args) .env(env) .cwd(PathBuf::from(WORKSPACE)) .capabilities(job_capabilities(req)?) .rlimits(vec![ rlimit(PosixRlimitType::RlimitNproc, req.limits.pids.into())?, rlimit(PosixRlimitType::RlimitNofile, NOFILE_LIMIT)?, ]) .no_new_privileges(true) .build() .map_err(spec_err)?; let root = RootBuilder::default() .path(rootfs_mount) .readonly(false) .build() .map_err(spec_err)?; let linux = build_linux(req)?; SpecBuilder::default() .version("1.0.2") .root(root) .hostname(HOSTNAME) .process(process) .mounts(build_mounts(req)?) .linux(linux) .build() .map_err(spec_err) } /// The job env in the order the container sees it: PATH, HOME, CI, FORGE_CI from the image and /// defaults, then the image env, then the job env and secrets. Later entries win. fn build_env(req: &SandboxRequest) -> Result, SandboxError> { let mut out: Vec = Vec::new(); for entry in &req.image_env { let (name, _) = entry .split_once('=') .ok_or_else(|| SandboxError::Invalid(format!("image env without '=': {entry:?}")))?; validate_env_name(name).map_err(|e| SandboxError::Invalid(e.to_string()))?; check_nul(entry)?; out.push(entry.clone()); } if !req.image_env.iter().any(|e| e.starts_with("PATH=")) { out.push(format!("PATH={DEFAULT_PATH}")); } out.push("HOME=/root".to_owned()); out.push("CI=true".to_owned()); out.push("FORGE_CI=true".to_owned()); for (name, value) in &req.env { validate_env_name(name).map_err(|e| SandboxError::Invalid(e.to_string()))?; let entry = format!("{name}={value}"); check_nul(&entry)?; out.push(entry); } Ok(out) } fn check_nul(entry: &str) -> Result<(), SandboxError> { if entry.contains('\0') { // The name is safe to show, the value is not. let name = entry.split('=').next().unwrap_or_default(); return Err(SandboxError::Invalid(format!("env {name} contains NUL"))); } Ok(()) } /// The capabilities a subuid job's root gets, the same kind a rootless container gets. They act /// only inside the job's user namespace, on the ids it maps, so they can't touch the host. Package /// managers need them to change file ownership and to drop privileges. const SUBUID_CAPABILITIES: [Capability; 13] = [ Capability::Chown, Capability::DacOverride, Capability::Fowner, Capability::Fsetid, Capability::Kill, Capability::Setgid, Capability::Setuid, Capability::Setpcap, Capability::Setfcap, Capability::NetBindService, Capability::SysChroot, Capability::AuditWrite, Capability::Mknod, ]; /// The job's capability sets. Single mode has none. Subuid mode has [`SUBUID_CAPABILITIES`] in /// the bounding, effective and permitted sets. Inheritable and ambient stay empty, so an exec /// doesn't gain more. fn job_capabilities( req: &SandboxRequest, ) -> Result { if !req.idmap.is_subuid() { return no_capabilities(); } let set: HashSet = SUBUID_CAPABILITIES.into_iter().collect(); LinuxCapabilitiesBuilder::default() .bounding(set.clone()) .effective(set.clone()) .permitted(set) .inheritable(HashSet::new()) .ambient(HashSet::new()) .build() .map_err(spec_err) } fn no_capabilities() -> Result { let empty: HashSet = HashSet::new(); LinuxCapabilitiesBuilder::default() .bounding(empty.clone()) .effective(empty.clone()) .inheritable(empty.clone()) .permitted(empty.clone()) .ambient(empty) .build() .map_err(spec_err) } fn rlimit( typ: PosixRlimitType, value: u64, ) -> Result { PosixRlimitBuilder::default() .typ(typ) .soft(value) .hard(value) .build() .map_err(spec_err) } fn build_linux(req: &SandboxRequest) -> Result { // In subuid mode the job runs in the helper's own user namespace, which already maps the // runner's uid and the subordinate range. libcontainer writes only one id range for a new // user namespace, and the kernel refuses a multi-id root map written from the parent. So the // job can't get its own namespace with the full range, and it uses the helper's. let subuid = req.idmap.is_subuid(); let mut namespaces = Vec::new(); if !subuid { namespaces.push(ns(LinuxNamespaceType::User)?); } namespaces.extend([ ns(LinuxNamespaceType::Mount)?, ns(LinuxNamespaceType::Pid)?, ns(LinuxNamespaceType::Ipc)?, ns(LinuxNamespaceType::Uts)?, ns(LinuxNamespaceType::Cgroup)?, ]); if !req.network { namespaces.push(ns(LinuxNamespaceType::Network)?); } let mut linux = LinuxBuilder::default().namespaces(namespaces); if !subuid { // Single mode: container root maps to the helper's root, which is the runner's uid. let uid_map = LinuxIdMappingBuilder::default() .container_id(0u32) .host_id(0u32) .size(1u32) .build() .map_err(spec_err)?; let gid_map = LinuxIdMappingBuilder::default() .container_id(0u32) .host_id(0u32) .size(1u32) .build() .map_err(spec_err)?; linux = linux .uid_mappings(vec![uid_map]) .gid_mappings(vec![gid_map]); } // No seccomp section: libcontainer's libseccomp support is off, so it would ignore one. The // profile is installed by `exec::FilteredExecutor` instead. linux .resources(build_resources(&req.limits)?) .masked_paths( MASKED_PATHS .iter() .map(|s| (*s).to_owned()) .collect::>(), ) .readonly_paths( READONLY_PATHS .iter() .map(|s| (*s).to_owned()) .collect::>(), ) .build() .map_err(spec_err) } fn ns(typ: LinuxNamespaceType) -> Result { LinuxNamespaceBuilder::default() .typ(typ) .build() .map_err(spec_err) } fn build_resources(limits: &Limits) -> Result { let memory = parse_bytes(&limits.memory)?; let cpus = parse_cpus(&limits.cpus)?; let quota = (cpus * CPU_PERIOD_US as f64).round() as i64; LinuxResourcesBuilder::default() .memory( LinuxMemoryBuilder::default() .limit(memory) .build() .map_err(spec_err)?, ) .cpu( LinuxCpuBuilder::default() .quota(quota) .period(CPU_PERIOD_US) .build() .map_err(spec_err)?, ) .pids( LinuxPidsBuilder::default() .limit(i64::from(limits.pids)) .build() .map_err(spec_err)?, ) .build() .map_err(spec_err) } /// Parses `4g`, `512m`, `64k`, `1024` or `10b` into bytes. pub fn parse_bytes(text: &str) -> Result { let bad = || SandboxError::Invalid(format!("bad memory limit {text:?}")); let lower = text.trim().to_ascii_lowercase(); let (digits, multiplier) = match lower.chars().last() { Some('k') => (&lower[..lower.len() - 1], 1024_i64), Some('m') => (&lower[..lower.len() - 1], 1024 * 1024), Some('g') => (&lower[..lower.len() - 1], 1024 * 1024 * 1024), Some('b') => (&lower[..lower.len() - 1], 1), _ => (lower.as_str(), 1), }; let n: i64 = digits.parse().map_err(|_| bad())?; let bytes = n.checked_mul(multiplier).ok_or_else(bad)?; if bytes <= 0 { return Err(bad()); } Ok(bytes) } /// Parses `cpus` such as `2` or `1.5` into a positive CPU count. pub fn parse_cpus(text: &str) -> Result { let value: f64 = text .trim() .parse() .map_err(|_| SandboxError::Invalid(format!("bad cpus limit {text:?}")))?; if !value.is_finite() || value <= 0.0 { return Err(SandboxError::Invalid(format!("bad cpus limit {text:?}"))); } Ok(value) } fn build_mounts(req: &SandboxRequest) -> Result, SandboxError> { let mut mounts = vec![ mount("/proc", "proc", "proc", &["nosuid", "noexec", "nodev"])?, mount( "/dev", "tmpfs", "tmpfs", &["nosuid", "strictatime", "mode=755", "size=65536k"], )?, mount( "/dev/pts", "devpts", "devpts", &[ "nosuid", "noexec", "newinstance", "ptmxmode=0666", "mode=0620", ], )?, mount( "/dev/shm", "tmpfs", "shm", &["nosuid", "noexec", "nodev", "mode=1777", "size=65536k"], )?, ]; for dev in DEVICES { let host = format!("/dev/{dev}"); mounts.push(mount(&host, "bind", &host, &["bind", "nosuid", "noexec"])?); } if req.network { // A sysfs mount needs a network namespace owned by the user namespace, and the shared // host one is not. A read-only recursive bind of the host's /sys is the fallback. mounts.push(mount( "/sys", "bind", "/sys", &["rbind", "nosuid", "noexec", "nodev", "ro"], )?); mounts.push(mount( "/etc/resolv.conf", "bind", "/etc/resolv.conf", &["bind", "ro"], )?); } else { mounts.push(mount( "/sys", "sysfs", "sysfs", &["nosuid", "noexec", "nodev", "ro"], )?); } mounts.push(mount_path( WORKSPACE, "bind", &req.workspace, &["rbind", "rw", "nosuid", "nodev"], )?); mounts.push(mount( "/tmp", "tmpfs", "tmpfs", &["nosuid", "nodev", "mode=1777"], )?); mounts.push(mount_path( "/etc/hosts", "bind", &req.job_dir.join("etc/hosts"), &["bind", "ro"], )?); mounts.push(mount_path( "/etc/hostname", "bind", &req.job_dir.join("etc/hostname"), &["bind", "ro"], )?); Ok(mounts) } fn mount(dest: &str, typ: &str, source: &str, options: &[&str]) -> Result { mount_path(dest, typ, Path::new(source), options) } fn mount_path( dest: &str, typ: &str, source: &Path, options: &[&str], ) -> Result { MountBuilder::default() .destination(PathBuf::from(dest)) .typ(typ) .source(source) .options(options.iter().map(|s| (*s).to_owned()).collect::>()) .build() .map_err(spec_err) } fn spec_err(e: impl std::fmt::Display) -> SandboxError { SandboxError::Setup(format!("cannot build OCI spec: {e}")) } #[cfg(test)] mod tests { use super::*; use crate::sandbox::idmap::IdMap; use std::collections::HashSet; fn request(network: bool) -> SandboxRequest { SandboxRequest { job_id: "job_01".to_owned(), rootfs: PathBuf::from("/var/lib/forge/rootfs"), job_dir: PathBuf::from("/var/lib/forge/jobs/job_01"), workspace: PathBuf::from("/var/lib/forge/jobs/job_01/ws"), image_env: vec!["PATH=/opt/bin:/bin".to_owned(), "LANG=C.UTF-8".to_owned()], env: vec![ ("BUILD".to_owned(), "1".to_owned()), ("TOKEN".to_owned(), "s3cret-token".to_owned()), ], run: "echo hi".to_owned(), network, limits: Limits { memory: "2g".to_owned(), cpus: "1.5".to_owned(), pids: 512, image_max_gb: 20, }, idmap: IdMap::single(), } } fn spec_for(network: bool) -> Spec { build_spec(&request(network), Path::new("/rootfs")).expect("spec") } fn namespace_types(spec: &Spec) -> HashSet { spec.linux() .as_ref() .and_then(|l| l.namespaces().as_ref()) .map(|list| list.iter().map(|n| n.typ().to_string()).collect()) .unwrap_or_default() } #[test] fn namespaces_include_network_only_when_disabled() { let isolated = namespace_types(&spec_for(false)); for expected in ["user", "mnt", "pid", "ipc", "uts", "cgroup", "net"] { assert!(isolated.contains(expected), "missing {expected}"); } let shared = namespace_types(&spec_for(true)); assert!(!shared.contains("net")); assert!(shared.contains("user")); } #[test] fn single_mode_maps_only_container_root() { let spec = spec_for(false); let linux = spec.linux().as_ref().expect("linux"); let uid = &linux.uid_mappings().as_ref().expect("uid map")[0]; assert_eq!((uid.container_id(), uid.host_id(), uid.size()), (0, 0, 1)); let gid = &linux.gid_mappings().as_ref().expect("gid map")[0]; assert_eq!((gid.container_id(), gid.host_id(), gid.size()), (0, 0, 1)); let user = spec.process().as_ref().expect("process").user(); assert_eq!((user.uid(), user.gid()), (0, 0)); } #[test] fn subuid_mode_runs_in_the_helpers_user_namespace() { let mut req = request(false); req.idmap = IdMap { uid: 1000, gid: 1000, sub_uid: Some((524_288, 65_536)), sub_gid: Some((524_288, 65_536)), }; let spec = build_spec(&req, Path::new("/rootfs")).expect("spec"); assert!(!namespace_types(&spec).contains("user")); let linux = spec.linux().as_ref().expect("linux"); assert!(linux.uid_mappings().is_none()); assert!(linux.gid_mappings().is_none()); let caps = spec .process() .as_ref() .and_then(|p| p.capabilities().clone()) .expect("capabilities"); let effective = caps.effective().clone().unwrap_or_default(); assert!(effective.contains(&Capability::Chown)); assert!(!effective.contains(&Capability::SysAdmin)); } #[test] fn no_capabilities_in_any_set() { let spec = spec_for(false); let caps = spec .process() .as_ref() .and_then(|p| p.capabilities().clone()) .expect("capabilities"); for set in [ caps.bounding(), caps.effective(), caps.inheritable(), caps.permitted(), caps.ambient(), ] { assert!(set.as_ref().is_some_and(HashSet::is_empty)); } } #[test] fn no_new_privileges_is_set() { let spec = spec_for(false); assert_eq!( spec.process() .as_ref() .expect("process") .no_new_privileges(), Some(true) ); } #[test] fn spec_has_no_seccomp_section() { // The seccomp filter is built in `seccomp.rs` and installed by `exec.rs`. A seccomp // section here would be dead config that libcontainer ignores. let spec = spec_for(false); assert!(spec.linux().as_ref().expect("linux").seccomp().is_none()); } #[test] fn mounts_cover_the_standard_layout_and_the_workspace() { let spec = spec_for(false); let dests: Vec = spec .mounts() .as_ref() .expect("mounts") .iter() .map(|m| m.destination().display().to_string()) .collect(); for expected in [ "/proc", "/dev", "/dev/pts", "/dev/shm", "/dev/null", "/sys", "/workspace", "/tmp", "/etc/hosts", "/etc/hostname", ] { assert!( dests.iter().any(|d| d == expected), "missing mount {expected}" ); } assert!(!dests.iter().any(|d| d == "/etc/resolv.conf")); let ws = spec .mounts() .as_ref() .and_then(|m| m.iter().find(|m| m.destination() == Path::new(WORKSPACE))) .expect("workspace mount"); assert_eq!( ws.source().as_deref(), Some(Path::new("/var/lib/forge/jobs/job_01/ws")) ); } #[test] fn resolv_conf_is_mounted_only_with_network() { let spec = spec_for(true); let dests: Vec = spec .mounts() .as_ref() .expect("mounts") .iter() .map(|m| m.destination().display().to_string()) .collect(); assert!(dests.iter().any(|d| d == "/etc/resolv.conf")); } #[test] fn resource_limits_are_applied() { let spec = spec_for(false); let res = spec .linux() .as_ref() .and_then(|l| l.resources().clone()) .expect("resources"); assert_eq!( res.memory().as_ref().and_then(|m| m.limit()), Some(2 * 1024 * 1024 * 1024) ); let cpu = res.cpu().as_ref().expect("cpu"); assert_eq!(cpu.period(), Some(CPU_PERIOD_US)); assert_eq!(cpu.quota(), Some(150_000)); assert_eq!(res.pids().as_ref().map(|p| p.limit()), Some(512)); let rlimits = spec .process() .as_ref() .expect("process") .rlimits() .clone() .expect("rlimits"); let nproc = rlimits .iter() .find(|r| r.typ() == PosixRlimitType::RlimitNproc) .expect("nproc"); assert_eq!((nproc.soft(), nproc.hard()), (512, 512)); let nofile = rlimits .iter() .find(|r| r.typ() == PosixRlimitType::RlimitNofile) .expect("nofile"); assert_eq!(nofile.soft(), NOFILE_LIMIT); } #[test] fn env_order_is_image_then_defaults_then_job_then_secrets() { let spec = spec_for(false); let env = spec .process() .as_ref() .expect("process") .env() .clone() .expect("env"); let pos = |needle: &str| env.iter().position(|e| e == needle).expect(needle); assert!(pos("PATH=/opt/bin:/bin") < pos("HOME=/root")); assert!(pos("LANG=C.UTF-8") < pos("HOME=/root")); assert!(pos("FORGE_CI=true") < pos("BUILD=1")); assert!(pos("BUILD=1") < pos("TOKEN=s3cret-token")); // The image PATH is used, so no default PATH is added. assert_eq!(env.iter().filter(|e| e.starts_with("PATH=")).count(), 1); } #[test] fn default_path_is_used_when_the_image_has_none() { let mut req = request(false); req.image_env.clear(); let spec = build_spec(&req, Path::new("/rootfs")).expect("spec"); let env = spec .process() .as_ref() .expect("process") .env() .clone() .expect("env"); assert!(env.iter().any(|e| e == &format!("PATH={DEFAULT_PATH}"))); } #[test] fn secrets_never_appear_in_args() { let spec = spec_for(false); let args = spec .process() .as_ref() .expect("process") .args() .clone() .expect("args"); assert_eq!(args[..3], ["sh", "-eu", "-c"]); assert!(!args.iter().any(|a| a.contains("s3cret-token"))); assert_eq!(args[3], "echo hi"); } #[test] fn cwd_is_the_workspace_and_no_tty() { let spec = spec_for(false); let process = spec.process().as_ref().expect("process"); assert_eq!(process.cwd(), Path::new(WORKSPACE)); assert_eq!(process.terminal(), Some(false)); } #[test] fn masked_and_readonly_paths_are_set() { let spec = spec_for(false); let linux = spec.linux().as_ref().expect("linux"); let masked = linux.masked_paths().as_ref().expect("masked"); assert!(masked.iter().any(|p| p == "/proc/kcore")); let ro = linux.readonly_paths().as_ref().expect("readonly"); assert!(ro.iter().any(|p| p == "/proc/sys")); } #[test] fn parse_bytes_accepts_suffixes_and_rejects_junk() { assert_eq!(parse_bytes("4g").expect("g"), 4 * 1024 * 1024 * 1024); assert_eq!(parse_bytes("512M").expect("m"), 512 * 1024 * 1024); assert_eq!(parse_bytes("64k").expect("k"), 64 * 1024); assert_eq!(parse_bytes("1024").expect("plain"), 1024); assert!(parse_bytes("lots").is_err()); assert!(parse_bytes("0g").is_err()); assert!(parse_bytes("-1").is_err()); } #[test] fn parse_cpus_rejects_zero_and_junk() { assert_eq!(parse_cpus("2").expect("2"), 2.0); assert!(parse_cpus("0").is_err()); assert!(parse_cpus("nan").is_err()); assert!(parse_cpus("two").is_err()); } #[test] fn bad_env_names_are_rejected() { let mut req = request(false); req.env.push(("A=B".to_owned(), "x".to_owned())); assert!(build_spec(&req, Path::new("/rootfs")).is_err()); } #[test] fn nul_in_env_value_is_rejected() { let mut req = request(false); req.env.push(("OK".to_owned(), "a\0b".to_owned())); assert!(build_spec(&req, Path::new("/rootfs")).is_err()); } }