//! Pull and push mirrors: `git fetch` from a remote URL into a repo, and `git push` of a repo's //! refs to a remote URL. Only `refs/heads/*` and `refs/tags/*` are synced, with prune. //! //! Safety for every remote call: protocols are limited to https (and http when the caller allows //! it), redirects are not followed, no credential helper runs, and a prompt is never shown. The //! credentials travel only in the child environment (`GIT_CONFIG_*` for an `http.extraHeader`), //! never in argv, the URL or an error. Errors are scrubbed before they leave this module. use std::fmt; use std::future::Future; use std::net::IpAddr; use std::path::Path; use std::time::Duration; use base64::Engine; use base64::engine::general_purpose::STANDARD; use crate::command::{GitCommand, GitEnv}; use crate::error::{Error, Result}; /// The refspecs both directions sync. Never `refs/pull/*`. pub const MIRROR_REFSPECS: [&str; 2] = ["+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*"]; /// Most git stderr kept in an error, in bytes. const MAX_STDERR_BYTES: usize = 2 * 1024; const REDACTED: &str = "[redacted]"; /// A username and password or token for a remote. The secret is never printed: the `Debug` /// impl redacts it. #[derive(Clone, PartialEq, Eq)] pub struct RemoteAuth { pub username: String, pub secret: Vec, } impl fmt::Debug for RemoteAuth { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("RemoteAuth") .field("username", &self.username) .field("secret", &REDACTED) .finish() } } /// A host and the one address it was checked to resolve to. Git is told to connect to that address /// (`http.curloptResolve`), so the connection cannot be sent somewhere a second DNS lookup returns. #[derive(Clone, Debug, PartialEq, Eq)] pub struct PinnedHost { pub host: String, pub port: u16, pub ip: IpAddr, } impl PinnedHost { /// The `-c` pair for git: `http.curloptResolve=host:port:ip`, with an IPv6 address in brackets. /// The host must be a plain DNS name, so the value cannot smuggle in another option. pub fn config_pair(&self) -> Result { let plain_host = !self.host.is_empty() && self .host .bytes() .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'.'); if !plain_host { return Err(Error::InvalidInput( "pinned host must be a plain DNS name".into(), )); } let ip = match self.ip { IpAddr::V4(v4) => v4.to_string(), IpAddr::V6(v6) => format!("[{v6}]"), }; Ok(format!( "http.curloptResolve={}:{}:{ip}", self.host, self.port )) } } impl RemoteAuth { /// The `Authorization` header value, Basic over `username:secret`. fn header_value(&self) -> String { let mut raw = self.username.as_bytes().to_vec(); raw.push(b':'); raw.extend_from_slice(&self.secret); format!("Authorization: Basic {}", STANDARD.encode(raw)) } } /// Fetches the mirror refs of `url` into the bare repo at `repo`, then points HEAD at the remote's /// default branch when our HEAD's branch is missing. With `pin`, git connects to the pinned /// address. Times out after `timeout`, killing git. pub async fn mirror_fetch( env: &GitEnv, repo: &Path, url: &str, auth: Option<&RemoteAuth>, allow_http: bool, pin: Option<&PinnedHost>, timeout: Duration, ) -> Result<()> { check_url(url, allow_http)?; with_timeout(timeout, fetch(env, repo, url, auth, allow_http, pin, false)).await } /// Pushes the mirror refs of the bare repo at `repo` to `url`, with prune. With `pin`, git connects /// to the pinned address. Times out after `timeout`, killing git. pub async fn mirror_push( env: &GitEnv, repo: &Path, url: &str, auth: Option<&RemoteAuth>, allow_http: bool, pin: Option<&PinnedHost>, timeout: Duration, ) -> Result<()> { check_url(url, allow_http)?; with_timeout(timeout, push(env, repo, url, auth, allow_http, pin, false)).await } /// Only https, or http when allowed. No userinfo, whitespace or control characters. Never /// `file://`, `ssh://` or `ext::`. fn check_url(url: &str, allow_http: bool) -> Result<()> { let rest = if let Some(rest) = url.strip_prefix("https://") { rest } else if let Some(rest) = url.strip_prefix("http://").filter(|_| allow_http) { rest } else { return Err(Error::InvalidInput( "mirror URL must use https (or http when allowed)".into(), )); }; let authority = rest.split('/').next().unwrap_or_default(); if authority.is_empty() || authority.contains('@') { return Err(Error::InvalidInput( "mirror URL needs a host and must not contain credentials".into(), )); } if url.chars().any(|c| c.is_whitespace() || c.is_control()) { return Err(Error::InvalidInput( "mirror URL contains whitespace or control characters".into(), )); } Ok(()) } async fn with_timeout(timeout: Duration, fut: impl Future>) -> Result { match tokio::time::timeout(timeout, fut).await { Ok(result) => result, Err(_) => Err(Error::Failed(format!( "git timed out after {} seconds", timeout.as_secs() ))), } } /// The git invocation with the safety flags, the optional repo and the optional credentials. /// `local` allows the file transport; only unit tests pass true. fn base_command( env: &GitEnv, repo: Option<&Path>, subcommand: &'static str, auth: Option<&RemoteAuth>, allow_http: bool, pin: Option<&PinnedHost>, local: bool, ) -> Result { let mut cmd = GitCommand::new(env, subcommand) .config("protocol.allow=never") .config("protocol.https.allow=always") .config("http.followRedirects=false") .config("credential.helper="); if allow_http { cmd = cmd.config("protocol.http.allow=always"); } if let Some(pin) = pin { cmd = cmd.config_checked(pin.config_pair()?)?; } if let Some(repo) = repo { cmd = cmd.git_dir(repo); } if local { cmd = cmd.allow_local_transport(); } if let Some(auth) = auth { cmd = cmd .env("GIT_CONFIG_COUNT", "1") .env("GIT_CONFIG_KEY_0", "http.extraHeader") .env("GIT_CONFIG_VALUE_0", auth.header_value()); } Ok(cmd) } async fn fetch( env: &GitEnv, repo: &Path, url: &str, auth: Option<&RemoteAuth>, allow_http: bool, pin: Option<&PinnedHost>, local: bool, ) -> Result<()> { let mut revs = vec![url]; revs.extend(MIRROR_REFSPECS); let cmd = base_command(env, Some(repo), "fetch", auth, allow_http, pin, local)? .arg("--prune") .arg("--no-recurse-submodules") .user_revs(&revs)?; run_checked(cmd, "fetch", auth, url).await?; let head = base_command(env, Some(repo), "ls-remote", auth, allow_http, pin, local)? .arg("--symref") .user_revs(&[url, "HEAD"])?; let listing = run_checked(head, "ls-remote", auth, url).await?; let listing = String::from_utf8_lossy(&listing); let Some(default_ref) = remote_default_branch(&listing) else { return Ok(()); }; point_head_at_default(env, repo, &default_ref).await; Ok(()) } /// The `refs/heads/...` target of the remote HEAD symref in `ls-remote --symref` output. fn remote_default_branch(listing: &str) -> Option { listing.lines().find_map(|line| { let (target, name) = line.strip_prefix("ref: ")?.split_once('\t')?; (name == "HEAD" && target.starts_with("refs/heads/")).then(|| target.to_string()) }) } /// Sets HEAD to `default_ref` when HEAD's branch does not exist and `default_ref` does. Best /// effort: failures leave HEAD as it was. async fn point_head_at_default(env: &GitEnv, repo: &Path, default_ref: &str) { let current = GitCommand::new(env, "symbolic-ref") .git_dir(repo) .arg("-q") .arg("HEAD") .output() .await .ok() .map(|out| String::from_utf8_lossy(&out).trim().to_string()); if let Some(current) = current && ref_exists(env, repo, ¤t).await { return; } if !ref_exists(env, repo, default_ref).await { return; } let set = GitCommand::new(env, "symbolic-ref") .git_dir(repo) .arg("HEAD") .user_arg(default_ref); if let Ok(cmd) = set { let _ = cmd.output().await; } } /// True when `refname` is a valid ref name that exists in `repo`. async fn ref_exists(env: &GitEnv, repo: &Path, refname: &str) -> bool { let Ok(cmd) = GitCommand::new(env, "show-ref") .git_dir(repo) .arg("--verify") .arg("--quiet") .user_arg(refname) else { return false; }; cmd.output().await.is_ok() } async fn push( env: &GitEnv, repo: &Path, url: &str, auth: Option<&RemoteAuth>, allow_http: bool, pin: Option<&PinnedHost>, local: bool, ) -> Result<()> { let mut revs = vec![url]; revs.extend(MIRROR_REFSPECS); let cmd = base_command(env, Some(repo), "push", auth, allow_http, pin, local)? .arg("--prune") .arg("--force") .user_revs(&revs)?; run_checked(cmd, "push", auth, url).await?; Ok(()) } /// Runs the command and returns stdout. A failure carries git's scrubbed stderr. async fn run_checked( cmd: GitCommand, what: &str, auth: Option<&RemoteAuth>, url: &str, ) -> Result> { let out = cmd.into_tokio().output().await?; if out.status.success() { return Ok(out.stdout); } let stderr = scrub(&String::from_utf8_lossy(&out.stderr), auth, url); Err(Error::Failed(format!( "git {what} exited with {}: {}", out.status, truncate(stderr.trim()) ))) } /// Removes credentials from text that may reach an error: the secret and its encoded form, any /// `Authorization:` line, and any `user:pass@` in a URL. fn scrub(text: &str, auth: Option<&RemoteAuth>, url: &str) -> String { let mut out = text.to_string(); if let Some(auth) = auth { let secret = String::from_utf8_lossy(&auth.secret).into_owned(); if !secret.is_empty() { out = out.replace(&secret, REDACTED); } let header = auth.header_value(); out = out.replace(&header, REDACTED); if let Some(encoded) = header.strip_prefix("Authorization: Basic ") { out = out.replace(encoded, REDACTED); } } out = out .lines() .map(|line| { if line.to_ascii_lowercase().contains("authorization:") { format!("Authorization: {REDACTED}") } else { line.to_string() } }) .collect::>() .join("\n"); let mut stripped = strip_userinfo(&out); if url.contains('@') { stripped = stripped.replace(url, REDACTED); } stripped } /// Replaces `user:pass@` after each `://` with `***@`. fn strip_userinfo(text: &str) -> String { let mut out = String::with_capacity(text.len()); let mut rest = text; while let Some(pos) = rest.find("://") { let (head, tail) = rest.split_at(pos + 3); out.push_str(head); let authority_end = tail .find(|c: char| c == '/' || c.is_whitespace()) .unwrap_or(tail.len()); let authority = &tail[..authority_end]; match authority.rfind('@') { Some(at) => { out.push_str("***"); out.push_str(&authority[at..]); } None => out.push_str(authority), } rest = &tail[authority_end..]; } out.push_str(rest); out } /// At most [`MAX_STDERR_BYTES`], cut at a character boundary. fn truncate(text: &str) -> String { if text.len() <= MAX_STDERR_BYTES { return text.to_string(); } let mut end = MAX_STDERR_BYTES; while !text.is_char_boundary(end) { end -= 1; } format!("{}...", &text[..end]) } #[cfg(test)] mod tests { use super::*; fn env(dir: &Path) -> GitEnv { GitEnv { git_bin: "git".into(), data_dir: dir.to_path_buf(), } } fn auth() -> RemoteAuth { RemoteAuth { username: "mirror-bot".into(), secret: b"s3cret-token-XYZ".to_vec(), } } /// Runs plain git against a git dir for test setup. fn plumbing(git_dir: &Path, args: &[&str]) -> String { let out = std::process::Command::new("git") .arg(format!("--git-dir={}", git_dir.display())) .args(args) .env("GIT_AUTHOR_NAME", "t") .env("GIT_AUTHOR_EMAIL", "t@example.com") .env("GIT_COMMITTER_NAME", "t") .env("GIT_COMMITTER_EMAIL", "t@example.com") .output() .expect("git runs"); assert!(out.status.success(), "git {args:?}: {out:?}"); String::from_utf8_lossy(&out.stdout).trim().to_string() } /// A bare repo whose `main` and tag `v1` point at one empty commit. fn source_repo(dir: &Path) -> std::path::PathBuf { let git_dir = dir.join("source.git"); plumbing(dir, &["init", "--bare", "-q", git_dir.to_str().unwrap()]); let tree = plumbing(&git_dir, &["hash-object", "-t", "tree", "-w", "--stdin"]); let commit = plumbing(&git_dir, &["commit-tree", &tree, "-m", "first"]); plumbing(&git_dir, &["update-ref", "refs/heads/main", &commit]); plumbing(&git_dir, &["symbolic-ref", "HEAD", "refs/heads/main"]); plumbing(&git_dir, &["update-ref", "refs/tags/v1", &commit]); git_dir } fn file_url(path: &Path) -> String { path.to_str().unwrap().to_string() } fn refs(git_dir: &Path) -> String { plumbing(git_dir, &["for-each-ref", "--format=%(refname)"]) } #[test] fn urls_are_limited_to_https_without_credentials() { assert!(check_url("https://example.com/a.git", false).is_ok()); assert!(check_url("http://example.com/a.git", false).is_err()); assert!(check_url("http://example.com/a.git", true).is_ok()); assert!(check_url("file:///etc/passwd", true).is_err()); assert!(check_url("ssh://example.com/a.git", true).is_err()); assert!(check_url("ext::sh -c x", true).is_err()); assert!(check_url("https://user:pw@example.com/a.git", false).is_err()); assert!(check_url("https://example.com/a b.git", false).is_err()); } #[test] fn credentials_stay_out_of_argv() { let dir = tempfile::tempdir().unwrap(); let a = auth(); let cmd = base_command( &env(dir.path()), Some(dir.path()), "fetch", Some(&a), false, None, false, ) .expect("command builds") .arg("--prune") .user_revs(&["https://example.com/a.git", "+refs/heads/*:refs/heads/*"]) .unwrap(); let std_cmd = cmd.into_tokio(); let std_cmd = std_cmd.as_std(); let argv: Vec = std_cmd .get_args() .map(|a| a.to_string_lossy().into_owned()) .collect(); let joined = argv.join(" "); assert!(!joined.contains("s3cret"), "{joined}"); assert!(!joined.contains("Authorization"), "{joined}"); assert!(!joined.contains(&STANDARD.encode(b"mirror-bot:s3cret-token-XYZ"))); // The header travels in the environment instead. let env_value = std_cmd .get_envs() .find(|(k, _)| *k == "GIT_CONFIG_VALUE_0") .and_then(|(_, v)| v.map(|v| v.to_string_lossy().into_owned())) .expect("header in env"); assert_eq!(env_value, a.header_value()); assert!(env_value.starts_with("Authorization: Basic ")); } #[test] fn scrub_removes_secret_header_and_userinfo() { let a = auth(); let encoded = STANDARD.encode(b"mirror-bot:s3cret-token-XYZ"); let stderr = format!( "fatal: unable to access 'https://mirror-bot:s3cret-token-XYZ@h.example/x.git/': \ Authorization: Basic {encoded}\nerror: s3cret-token-XYZ rejected" ); let scrubbed = scrub(&stderr, Some(&a), "https://h.example/x.git"); assert!(!scrubbed.contains("s3cret"), "{scrubbed}"); assert!(!scrubbed.contains(&encoded), "{scrubbed}"); assert!(!scrubbed.contains("Basic"), "{scrubbed}"); assert!(scrubbed.contains("h.example"), "{scrubbed}"); assert!(scrubbed.contains(REDACTED), "{scrubbed}"); } #[test] fn debug_hides_the_secret() { let shown = format!("{:?}", auth()); assert!(!shown.contains("s3cret"), "{shown}"); assert!(shown.contains("mirror-bot"), "{shown}"); } #[test] fn truncation_is_bounded_and_on_a_boundary() { let long = "é".repeat(4000); let t = truncate(&long); assert!(t.len() <= MAX_STDERR_BYTES + 3); assert!(t.ends_with("...")); } #[test] fn remote_head_symref_is_parsed() { let listing = "ref: refs/heads/trunk\tHEAD\nabc\tHEAD\nabc\trefs/heads/trunk\n"; assert_eq!( remote_default_branch(listing).as_deref(), Some("refs/heads/trunk") ); assert_eq!(remote_default_branch("abc\tHEAD\n"), None); } #[tokio::test] async fn mirror_fetch_and_push_sync_heads_and_tags_with_prune() { let dir = tempfile::tempdir().unwrap(); let source = source_repo(dir.path()); let env = env(dir.path()); let timeout = Duration::from_secs(60); // Pull: fetch the source into an empty bare repo, through the local transport that // only unit tests may use. let pulled = dir.path().join("pulled.git"); plumbing( dir.path(), &["init", "--bare", "-q", pulled.to_str().unwrap()], ); fetch(&env, &pulled, &file_url(&source), None, false, None, true) .await .unwrap(); let fetched = refs(&pulled); assert!(fetched.contains("refs/heads/main"), "{fetched}"); assert!(fetched.contains("refs/tags/v1"), "{fetched}"); assert_eq!( plumbing(&pulled, &["symbolic-ref", "HEAD"]), "refs/heads/main" ); // A branch that the remote no longer has is pruned by the next fetch. let head = plumbing(&source, &["rev-parse", "refs/heads/main"]); plumbing(&pulled, &["update-ref", "refs/heads/gone", &head]); plumbing(&source, &["update-ref", "refs/heads/new", &head]); fetch(&env, &pulled, &file_url(&source), None, false, None, true) .await .unwrap(); let fetched = refs(&pulled); assert!(fetched.contains("refs/heads/new"), "{fetched}"); assert!(!fetched.contains("refs/heads/gone"), "{fetched}"); // Push: mirror our refs to an empty bare remote, then prune a stale branch there. let remote = dir.path().join("remote.git"); plumbing( dir.path(), &["init", "--bare", "-q", remote.to_str().unwrap()], ); plumbing( &remote, &[ "fetch", "-q", source.to_str().unwrap(), "refs/heads/main:refs/heads/stale", ], ); push(&env, &pulled, &file_url(&remote), None, false, None, true) .await .unwrap(); let pushed = refs(&remote); assert!(pushed.contains("refs/heads/main"), "{pushed}"); assert!(pushed.contains("refs/tags/v1"), "{pushed}"); assert!(!pushed.contains("refs/heads/stale"), "{pushed}"); // The public entry points reject the file transport outright. assert!( mirror_fetch( &env, &pulled, &file_url(&source), None, false, None, timeout ) .await .is_err() ); assert!( mirror_push( &env, &pulled, &file_url(&remote), None, false, None, timeout ) .await .is_err() ); } #[tokio::test] async fn failure_text_never_carries_the_secret() { let dir = tempfile::tempdir().unwrap(); let env = env(dir.path()); let pulled = dir.path().join("pulled.git"); plumbing( dir.path(), &["init", "--bare", "-q", pulled.to_str().unwrap()], ); let a = auth(); // Port 9 on loopback refuses the connection, so git fails with its own error text. let url = "https://127.0.0.1:9/x.git"; let err = mirror_fetch( &env, &pulled, url, Some(&a), false, None, Duration::from_secs(60), ) .await .unwrap_err(); let text = err.to_string(); assert!(!text.contains("s3cret"), "{text}"); assert!( !text.contains(&STANDARD.encode(b"mirror-bot:s3cret-token-XYZ")), "{text}" ); } fn pin(ip: &str) -> PinnedHost { PinnedHost { host: "pin.invalid".into(), port: 443, ip: ip.parse().expect("ip"), } } #[test] fn pinned_config_pair_formats_ipv4_and_ipv6() { let v4 = PinnedHost { host: "example.com".into(), port: 443, ip: "93.184.216.34".parse().expect("ip"), }; assert_eq!( v4.config_pair().expect("pair"), "http.curloptResolve=example.com:443:93.184.216.34" ); let v6 = PinnedHost { host: "example.com".into(), port: 80, ip: "2001:db8::1".parse().expect("ip"), }; assert_eq!( v6.config_pair().expect("pair"), "http.curloptResolve=example.com:80:[2001:db8::1]" ); } #[test] fn pinned_host_must_be_a_plain_name() { for host in ["", "a b", "[::1]", "x,y", "evil\nname"] { let bad = PinnedHost { host: host.into(), port: 443, ip: "93.184.216.34".parse().expect("ip"), }; assert!(bad.config_pair().is_err(), "{host:?}"); } } #[test] fn pin_becomes_a_checked_config_arg() { let dir = tempfile::tempdir().expect("tempdir"); let cmd = base_command( &env(dir.path()), None, "ls-remote", None, false, Some(&pin("127.0.0.1")), false, ) .expect("command builds"); let std_cmd = cmd.into_tokio(); let argv: Vec = std_cmd .as_std() .get_args() .map(|a| a.to_string_lossy().into_owned()) .collect(); let at = argv .iter() .position(|a| a == "http.curloptResolve=pin.invalid:443:127.0.0.1") .expect("pinned config arg present"); assert_eq!(argv[at - 1], "-c", "{argv:?}"); } #[tokio::test] async fn fetch_connects_to_the_pinned_address() { let dir = tempfile::tempdir().expect("tempdir"); let env = env(dir.path()); let pulled = dir.path().join("pulled.git"); plumbing( dir.path(), &["init", "--bare", "-q", pulled.to_str().expect("utf8 path")], ); // The host never resolves, so only the pin can get git past name lookup. Nothing listens // on loopback port 443, so the run fails at connect, not at resolve. let err = mirror_fetch( &env, &pulled, "https://pin.invalid/x.git", None, false, Some(&pin("127.0.0.1")), Duration::from_secs(60), ) .await .unwrap_err() .to_string(); assert!(!err.contains("Could not resolve host"), "{err}"); } }