//! Garbage collection of LFS objects (`admin lfs-gc`). An object file no repo links is deleted once //! it is older than the minimum age. Upload temp files of the same age are abandoned uploads and //! are deleted too. The age gate keeps a fresh upload from being collected before its link row is //! written. use std::path::{Path, PathBuf}; use std::time::{Duration, SystemTime}; use forge_core::Store; use serde::{Deserialize, Serialize}; use super::{is_valid_oid, lfs_root, tmp_dir}; /// The age an unlinked file must reach before it is deleted. pub const GC_MIN_AGE: Duration = Duration::from_secs(24 * 60 * 60); /// What one run did. #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct GcSummary { /// Object files looked at. pub checked: u64, /// Object files deleted. pub removed: u64, /// Bytes in the deleted object files. pub bytes_freed: u64, /// Abandoned upload temp files deleted. pub temp_removed: u64, } /// An object file found on disk. struct Candidate { oid: String, path: PathBuf, size: u64, modified: SystemTime, } /// Deletes unlinked objects and abandoned temp files older than `min_age`. pub async fn gc( store: &Store, data_dir: &Path, min_age: Duration, ) -> forge_core::Result { let now = SystemTime::now(); let mut summary = GcSummary::default(); let temp = tmp_dir(data_dir); let temp_files = tokio::task::spawn_blocking(move || list_files(&temp)) .await .map_err(|e| forge_core::Error::Io(std::io::Error::other(e)))??; for (path, modified) in temp_files { if is_older_than(modified, now, min_age) && remove_if_present(&path).await? { summary.temp_removed += 1; } } let root = lfs_root(data_dir); let objects = tokio::task::spawn_blocking(move || collect_objects(&root)) .await .map_err(|e| forge_core::Error::Io(std::io::Error::other(e)))??; for candidate in objects { summary.checked += 1; if !is_older_than(candidate.modified, now, min_age) { continue; } if store.lfs_oid_is_linked(&candidate.oid).await? { continue; } // Look again just before deleting: an upload may have renamed the file into place and // linked it since the listing was taken. let fresh = tokio::fs::metadata(&candidate.path).await?; if !is_older_than(fresh.modified()?, now, min_age) || store.lfs_oid_is_linked(&candidate.oid).await? { continue; } if remove_if_present(&candidate.path).await? { summary.removed += 1; summary.bytes_freed += candidate.size; } } Ok(summary) } /// True when `modified` is at least `min_age` before `now`. A time in the future is never old. fn is_older_than(modified: SystemTime, now: SystemTime, min_age: Duration) -> bool { now.duration_since(modified).is_ok_and(|age| age >= min_age) } /// Deletes `path`. `Ok(false)` when it was already gone. async fn remove_if_present(path: &Path) -> std::io::Result { match tokio::fs::remove_file(path).await { Ok(()) => Ok(true), Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false), Err(e) => Err(e), } } /// The regular files directly inside `dir`, with their modification times. A missing directory is /// empty. fn list_files(dir: &Path) -> forge_core::Result> { let mut found = Vec::new(); let Ok(entries) = std::fs::read_dir(dir) else { return Ok(found); }; for entry in entries { let entry = entry?; let meta = entry.metadata()?; if meta.is_file() { found.push((entry.path(), meta.modified()?)); } } Ok(found) } /// Every object file under `root`, found through the `aa/bb/` layout. Anything that does not /// match that layout (the temp directory, stray files) is skipped. fn collect_objects(root: &Path) -> forge_core::Result> { let mut found = Vec::new(); let Ok(first) = std::fs::read_dir(root) else { return Ok(found); }; for a in first { let a = a?; let Some(a_name) = fan_out_name(&a)? else { continue; }; for b in std::fs::read_dir(a.path())? { let b = b?; let Some(b_name) = fan_out_name(&b)? else { continue; }; for file in std::fs::read_dir(b.path())? { let file = file?; let meta = file.metadata()?; let Some(oid) = file.file_name().to_str().map(str::to_owned) else { continue; }; if !meta.is_file() || !is_valid_oid(&oid) || !oid.starts_with(&format!("{a_name}{b_name}")) { continue; } found.push(Candidate { oid, path: file.path(), size: meta.len(), modified: meta.modified()?, }); } } } Ok(found) } /// The name of a two-character fan-out directory, or `None` for anything else. fn fan_out_name(entry: &std::fs::DirEntry) -> forge_core::Result> { if !entry.file_type()?.is_dir() { return Ok(None); } Ok(entry .file_name() .to_str() .filter(|name| name.len() == 2) .map(str::to_owned)) } #[cfg(test)] mod tests { use super::*; use forge_core::Visibility; const LINKED: &str = "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393"; const ORPHAN: &str = "a665a45920422f9d417e4867efdc4fb8a04a1f3fff1fa07e998e86f7f7a27ae3"; fn write_object(data: &Path, oid: &str, bytes: &[u8]) -> PathBuf { let path = super::super::object_path(data, oid); std::fs::create_dir_all(path.parent().unwrap()).unwrap(); std::fs::write(&path, bytes).unwrap(); path } #[tokio::test] async fn removes_unlinked_objects_and_keeps_linked_ones() { let dir = tempfile::tempdir().unwrap(); let data = dir.path(); let store = Store::open(&data.join("forge.db")).await.unwrap(); let user = store.create_user("alice", None, "h", false).await.unwrap(); let repo = store .create_repo(&user.id, "demo", "", Visibility::Public, "main") .await .unwrap(); let linked = write_object(data, LINKED, b"kept"); let orphan = write_object(data, ORPHAN, b"12345"); store.insert_lfs_object(&repo.id, LINKED, 4).await.unwrap(); let summary = gc(&store, data, Duration::ZERO).await.unwrap(); assert_eq!(summary.checked, 2); assert_eq!(summary.removed, 1); assert_eq!(summary.bytes_freed, 5); assert!(linked.exists()); assert!(!orphan.exists()); } #[tokio::test] async fn young_files_and_temp_files_survive_the_age_gate() { let dir = tempfile::tempdir().unwrap(); let data = dir.path(); let store = Store::open(&data.join("forge.db")).await.unwrap(); let orphan = write_object(data, ORPHAN, b"fresh"); std::fs::create_dir_all(tmp_dir(data)).unwrap(); let temp = tmp_dir(data).join("upload.part"); std::fs::write(&temp, b"partial").unwrap(); let summary = gc(&store, data, GC_MIN_AGE).await.unwrap(); assert_eq!(summary.removed, 0); assert_eq!(summary.temp_removed, 0); assert!(orphan.exists()); assert!(temp.exists()); } #[tokio::test] async fn abandoned_temp_files_are_removed_once_old() { let dir = tempfile::tempdir().unwrap(); let data = dir.path(); let store = Store::open(&data.join("forge.db")).await.unwrap(); std::fs::create_dir_all(tmp_dir(data)).unwrap(); let temp = tmp_dir(data).join("upload.part"); std::fs::write(&temp, b"partial").unwrap(); let summary = gc(&store, data, Duration::ZERO).await.unwrap(); assert_eq!(summary.temp_removed, 1); assert!(!temp.exists()); } #[tokio::test] async fn missing_store_is_a_no_op() { let dir = tempfile::tempdir().unwrap(); let store = Store::open(&dir.path().join("forge.db")).await.unwrap(); let summary = gc(&store, dir.path(), GC_MIN_AGE).await.unwrap(); assert_eq!(summary, GcSummary::default()); } }