//! Checks a backup set end to end. Every problem found is recorded in the report, so one run shows //! all of them. Only a set that cannot be read at all, such as a missing or unparseable manifest, //! is an error. use std::collections::BTreeSet; use std::path::{Path, PathBuf}; use forge_core::store::snapshot::verify_snapshot; use forge_core::{Error, Result}; use forge_git::GitEnv; use super::{ DB_FILE, FORMAT_VERSION, FileEntry, KEY_FILE, MANIFEST_FILE, Manifest, Scope, VerifyReport, create_private_dir, empty_sha256, hash_file, io_err, is_known_path, }; /// Directories a set may contain. Anything else is reported as unexpected. const DIRS: [&str; 3] = ["db", "repos", "keys"]; /// Verifies the set at `path`. `git` runs the bundle checks. Scratch repos go in a private /// directory next to `path`, never in `/tmp`, and are removed before this returns. pub async fn verify_backup(path: &Path, git: &GitEnv) -> Result { let path = std::path::absolute(path).map_err(io_err)?; let bytes = std::fs::read(path.join(MANIFEST_FILE)).map_err(io_err)?; let manifest: Manifest = serde_json::from_slice(&bytes) .map_err(|e| Error::Invalid(format!("manifest.json does not parse: {e}")))?; if manifest.format != FORMAT_VERSION { return Err(Error::Invalid(format!( "unsupported manifest format {} (this binary reads {FORMAT_VERSION})", manifest.format ))); } let mut problems = Vec::new(); check_layout(&path, &manifest, &mut problems)?; let total_bytes = check_files(&path, &manifest, &mut problems).await?; let schema_version = check_db(&path, &manifest, &mut problems).await; let bundles_verified = check_bundles(&path, &manifest, git, &mut problems).await?; Ok(VerifyReport { path, scope: manifest.scope, files: manifest.files.len(), bytes: total_bytes, repos: manifest.repos.len(), bundles_verified, empty_repos: manifest.repos.iter().filter(|r| r.bundle.is_none()).count(), schema_version, problems, }) } /// Every file on disk must be listed, and the scope, key and repo entries must agree with each /// other. fn check_layout(root: &Path, manifest: &Manifest, problems: &mut Vec) -> Result<()> { let mut on_disk = Vec::new(); walk(root, root, &mut on_disk, problems)?; let listed: BTreeSet<&str> = manifest.files.iter().map(|f| f.path.as_str()).collect(); for rel in &on_disk { if rel != MANIFEST_FILE && !listed.contains(rel.as_str()) { problems.push(format!("unlisted file in the set: {rel}")); } } for f in &manifest.files { if !is_known_path(&f.path) { problems.push(format!("manifest lists an unknown path: {}", f.path)); } } let has_key = listed.contains(KEY_FILE); if manifest.key_included != has_key { problems.push(format!( "key_included is {} but the set {} keys/forge.key", manifest.key_included, if has_key { "contains" } else { "does not contain" } )); } let has_db = listed.contains(DB_FILE); let want_db = manifest.scope != Scope::ReposOnly; if has_db != want_db || has_db != manifest.db_snapshot.is_some() { problems.push(format!( "scope {:?} does not match the database entries (file listed: {has_db})", manifest.scope )); } let bundles: Vec<&str> = manifest .repos .iter() .filter_map(|r| r.bundle.as_deref()) .collect(); if manifest.scope == Scope::DbOnly && (!bundles.is_empty() || !manifest.repos.is_empty()) { problems.push("a db-only set must not list repos".into()); } let mut ids = BTreeSet::new(); for repo in &manifest.repos { if !ids.insert(repo.id.as_str()) { problems.push(format!("repo {} is listed twice", repo.id)); } match &repo.bundle { Some(b) => { if *b != format!("repos/{}.bundle", repo.id) || !listed.contains(b.as_str()) { problems.push(format!( "repo {}: bundle {b} is not listed as a file", repo.id )); } } None => { if repo.refs_sha256 != empty_sha256() { problems.push(format!( "repo {} has no bundle but its refs are not empty", repo.id )); } } } // The wiki, when the set has one, follows the same rules under its own file name. if let Some(wiki) = &repo.wiki { match &wiki.bundle { Some(b) => { if *b != format!("repos/{}.wiki.bundle", repo.id) || !listed.contains(b.as_str()) { problems.push(format!( "repo {}: wiki bundle {b} is not listed as a file", repo.id )); } } None => { if wiki.refs_sha256 != empty_sha256() { problems.push(format!( "repo {} has no wiki bundle but its wiki refs are not empty", repo.id )); } } } } } Ok(()) } /// Collects regular files under `dir` as paths relative to `root`. Unexpected directories and /// special files are reported. fn walk(root: &Path, dir: &Path, out: &mut Vec, problems: &mut Vec) -> Result<()> { for entry in std::fs::read_dir(dir).map_err(io_err)? { let entry = entry.map_err(io_err)?; let full = entry.path(); let rel = full .strip_prefix(root) .map_err(|e| Error::Invalid(e.to_string()))? .to_string_lossy() .into_owned(); let kind = entry.file_type().map_err(io_err)?; if kind.is_dir() { let top_level = dir == root; if top_level && DIRS.contains(&rel.as_str()) { walk(root, &full, out, problems)?; } else { problems.push(format!("unexpected directory in the set: {rel}")); } } else if kind.is_file() { out.push(rel); } else { problems.push(format!("unexpected entry in the set: {rel}")); } } Ok(()) } /// Recomputes size and sha256 for every listed file. Returns the total size of the listed files. async fn check_files(root: &Path, manifest: &Manifest, problems: &mut Vec) -> Result { let mut total = 0u64; for entry in &manifest.files { let FileEntry { path: rel, bytes, sha256, } = entry; total += bytes; if !is_known_path(rel) { continue; } let full = root.join(rel); if !full.is_file() { problems.push(format!("missing file: {rel}")); continue; } let (actual_bytes, actual_sha) = tokio::task::spawn_blocking(move || hash_file(&full)) .await .map_err(|e| Error::Io(std::io::Error::other(e)))??; if actual_bytes != *bytes { problems.push(format!( "{rel}: size mismatch (manifest {bytes}, file {actual_bytes})" )); } if actual_sha != *sha256 { problems.push(format!( "{rel}: sha256 mismatch (manifest {sha256}, file {actual_sha})" )); } } Ok(total) } /// Opens the DB snapshot read-only, runs `integrity_check`, and compares its schema version and /// repo rows with the manifest. The repo comparison runs only for full sets, because a db-only /// set lists no repos. Returns the snapshot's schema version when it could be read. async fn check_db(root: &Path, manifest: &Manifest, problems: &mut Vec) -> Option { let snap = root.join(DB_FILE); if !snap.is_file() { return None; } let info = match verify_snapshot(&snap).await { Ok(info) => info, Err(e) => { problems.push(format!("database snapshot failed verification: {e}")); return None; } }; if let Some(d) = &manifest.db_snapshot && d.schema_version != info.schema_version { problems.push(format!( "manifest says schema {} but the snapshot is schema {}", d.schema_version, info.schema_version )); } if manifest.scope == Scope::All { let mut want: Vec<(String, String, String)> = manifest .repos .iter() .map(|r| (r.id.clone(), r.owner.clone(), r.name.clone())) .collect(); want.sort(); let mut have = info.repos.clone(); have.sort(); if want != have { problems.push(format!( "manifest repo list ({}) does not match the database snapshot's repos table ({})", want.len(), have.len() )); } } Some(info.schema_version) } /// Imports every bundle into its own fresh scratch repo and checks the imported refs against the /// manifest. Returns how many bundles passed. Scratch space is removed before returning. async fn check_bundles( root: &Path, manifest: &Manifest, git: &GitEnv, problems: &mut Vec, ) -> Result { let bundles: Vec<(usize, &super::RepoEntry, &str)> = manifest .repos .iter() .enumerate() .filter_map(|(i, r)| r.bundle.as_deref().map(|b| (i, r, b))) .collect(); if bundles.is_empty() { return Ok(0); } let parent = root .parent() .ok_or_else(|| Error::Invalid("the backup set has no parent directory".into()))?; let suffix = format!( "{}-{}", std::process::id(), time::OffsetDateTime::now_utc().unix_timestamp_nanos() ); let scratch_root: PathBuf = parent.join(format!(".partial-verify-{suffix}")); create_private_dir(&scratch_root)?; let mut passed = 0; for (idx, repo, bundle) in bundles { let scratch = scratch_root.join(format!("{idx}.git")); let outcome = async { forge_git::repo::init_bare(git, &scratch, "main").await?; forge_git::backup::bundle_verify(git, &scratch, &root.join(bundle)).await } .await; match outcome { Ok(imported) if imported == repo.refs_sha256 => passed += 1, Ok(imported) => problems.push(format!( "repo {}: bundle refs {imported} do not match the manifest {}", repo.id, repo.refs_sha256 )), Err(e) => problems.push(format!("repo {}: bundle failed verification: {e}", repo.id)), } } if let Err(e) = std::fs::remove_dir_all(&scratch_root) { problems.push(format!( "could not remove scratch space {}: {e}", scratch_root.display() )); } Ok(passed) }