//! Raw-origin route `/{owner}/{repo}/-/media/{ref}/{path...}`. The literal `-` keeps it apart //! from raw paths, since no ref name may start with `-`. It reads the Git LFS pointer at //! `ref:path` and streams the object it names, when the repo links that object. Access follows //! the raw route in `raw.rs`: public repos are open, and private repos need a signed `?token=`. use forge_core::raw_token::verify_raw; use forge_core::{Repo, Visibility}; use futures_util::TryStreamExt; use http_body::Frame; use http_body_util::StreamBody; use serde::Deserialize; use tokio_util::io::ReaderStream; use topcoat::context::{Cx, app_context}; use topcoat::router::request::method; use topcoat::router::response::Response; use topcoat::router::{ Body, Method, RouteFuture, StatusCode, parse_query_params, path_param_segment, path_param_segments, }; use crate::lfs::{has_object, object_path}; use crate::lfs_pointer; use crate::raw::{ CACHE_NONE, CACHE_PRIVATE, CACHE_PUBLIC, LINK_TTL_SECS, Lookup, Meta, OCTET, encode_path, finish, lookup, now_unix, plain, }; use crate::state::AppState; #[derive(Deserialize)] struct TokenQuery { token: Option, } /// Absolute URL of the media route for `path` at `rev` on the user-content origin. Private repos /// get a signed `?token=`, the same as raw URLs. The route is `/{owner}/{repo}/-/media/...`. pub fn media_url(state: &AppState, repo: &Repo, rev: &str, path: &str) -> String { let base = state.config.user_content_base_url.trim_end_matches('/'); let mut url = format!( "{base}/{}/{}/-/media/{}/{}", encode_path(&repo.owner_name), encode_path(&repo.name), encode_path(rev), encode_path(path) ); if matches!(repo.visibility, Visibility::Private) { let expires = now_unix().saturating_add(LINK_TTL_SECS); url.push_str("?token="); url.push_str(&forge_core::raw_token::sign_raw( &state.key, &repo.id, expires, )); } url } pub(crate) fn media_file(cx: &Cx, _body: Body) -> RouteFuture<'_> { Box::pin(async move { Ok(serve(cx).await) }) } /// Serves one LFS object. Every failure is turned into a response here, so the route never errors. async fn serve(cx: &Cx) -> Response { let state = app_context::(cx).clone(); let owner = path_param_segment(cx, "owner").to_owned(); let name = path_param_segment(cx, "repo").to_owned(); let rest = path_param_segments(cx, "rest") .collect::>() .join("/"); let is_head = method(cx) == Method::HEAD; let token = parse_query_params::(cx) .ok() .and_then(|query| query.token); let repo = match state.store.repo_by_owner_and_name(&owner, &name).await { Ok(Some(repo)) => repo, // A missing repo and a private repo without a valid token look the same. Ok(None) => return plain(StatusCode::NOT_FOUND, CACHE_NONE, None), Err(e) => { tracing::error!(error = %e, "media: repo lookup failed"); return plain(StatusCode::INTERNAL_SERVER_ERROR, CACHE_NONE, None); } }; let cache = match repo.visibility { Visibility::Public => CACHE_PUBLIC, Visibility::Private => { let valid = token .as_deref() .is_some_and(|t| verify_raw(&state.key, t, &repo.id, now_unix())); if !valid { return plain(StatusCode::NOT_FOUND, CACHE_NONE, None); } CACHE_PRIVATE } }; let repo_dir = match forge_git::repo::repo_path(&state.git.data_dir, &repo.id) { Ok(path) => path, Err(e) => { tracing::error!(error = %e, "media: repo path failed"); return plain(StatusCode::INTERNAL_SERVER_ERROR, CACHE_NONE, None); } }; let looked = tokio::task::spawn_blocking(move || lookup(&repo_dir, &rest, None)).await; // Pointers are a few hundred bytes, so they always come back inline. Anything else is not // a pointer, and a symlink is never one. let data = match looked { Ok(Ok(Lookup::Inline { symlink: false, data, .. })) => data, Ok(Ok(_)) => return plain(StatusCode::NOT_FOUND, CACHE_NONE, None), Ok(Err(e)) => { tracing::error!(error = %e, "media: lookup failed"); return plain(StatusCode::INTERNAL_SERVER_ERROR, CACHE_NONE, None); } Err(e) => { tracing::error!(error = %e, "media: lookup task failed"); return plain(StatusCode::INTERNAL_SERVER_ERROR, CACHE_NONE, None); } }; let Some(pointer) = lfs_pointer::parse(&data) else { return plain(StatusCode::NOT_FOUND, CACHE_NONE, None); }; // The repo must link the object, even when the file is on disk: objects are shared by every repo. let size = match has_object(&state, &repo.id, &pointer.oid).await { Ok(Some(size)) => size, Ok(None) => return plain(StatusCode::NOT_FOUND, CACHE_NONE, None), Err(e) => { tracing::error!(error = %e, "media: object link lookup failed"); return plain(StatusCode::INTERNAL_SERVER_ERROR, CACHE_NONE, None); } }; let file = match tokio::fs::File::open(object_path(&state.config.data_dir, &pointer.oid)).await { Ok(file) => file, Err(e) if e.kind() == std::io::ErrorKind::NotFound => { return plain(StatusCode::NOT_FOUND, CACHE_NONE, None); } Err(e) => { tracing::error!(error = %e, "media: failed to open object"); return plain(StatusCode::INTERNAL_SERVER_ERROR, CACHE_NONE, None); } }; // The Content-Length comes from the link, so a file of another size would break the response. let on_disk = match file.metadata().await { Ok(meta) => meta.len(), Err(e) => { tracing::error!(error = %e, "media: object metadata failed"); return plain(StatusCode::INTERNAL_SERVER_ERROR, CACHE_NONE, None); } }; if on_disk != size { tracing::error!(oid = %pointer.oid, on_disk, size, "media: object size differs from its link"); return plain(StatusCode::INTERNAL_SERVER_ERROR, CACHE_NONE, None); } let meta = Meta { content_type: OCTET, attachment: true, len: size, }; if is_head { return finish( StatusCode::OK, Body::empty(), &pointer.oid, cache, Some(meta), ); } let chunks = ReaderStream::new(file).map_ok(Frame::data); let body = Body::new(StreamBody::new(chunks)); finish(StatusCode::OK, body, &pointer.oid, cache, Some(meta)) } #[cfg(test)] mod tests { use super::*; use forge_core::raw_token::sign_raw; use sha2::{Digest, Sha256}; use topcoat::router::Router; use topcoat::router::request::Request; use crate::raw::router; const POINTER_HEADER: &str = "version https://git-lfs.github.com/spec/v1"; struct Fixture { _tmp: tempfile::TempDir, state: AppState, router: Router, public: Repo, private: Repo, object: Vec, oid: String, } fn sha256_hex(data: &[u8]) -> String { hex::encode(Sha256::digest(data)) } fn pointer_text(oid: &str, size: usize) -> Vec { format!("{POINTER_HEADER}\noid sha256:{oid}\nsize {size}\n").into_bytes() } fn git(dir: &std::path::Path, args: &[&str]) { let out = std::process::Command::new("git") .arg("-C") .arg(dir) .args(["-c", "commit.gpgsign=false"]) .args(args) .env("GIT_AUTHOR_NAME", "test") .env("GIT_AUTHOR_EMAIL", "test@example.com") .env("GIT_COMMITTER_NAME", "test") .env("GIT_COMMITTER_EMAIL", "test@example.com") .output() .expect("git runs"); assert!( out.status.success(), "git {args:?} failed: {}", String::from_utf8_lossy(&out.stderr) ); } /// Commits `files` on `main` in a work tree and pushes it to `bare`. fn populate(work: &std::path::Path, bare: &std::path::Path, files: &[(&str, Vec)]) { std::fs::create_dir_all(work).expect("mkdir work"); git(work, &["init", "-q", "-b", "main"]); for (rel, contents) in files { std::fs::write(work.join(rel), contents).expect("write file"); } git(work, &["add", "-A"]); git(work, &["commit", "-q", "-m", "init"]); git(work, &["push", "-q", bare.to_str().expect("utf8"), "main"]); } async fn fixture() -> Fixture { let tmp = tempfile::tempdir().expect("tempdir"); let data_dir = tmp.path().join("data"); std::fs::create_dir_all(&data_dir).expect("data dir"); let data_dir = data_dir.canonicalize().expect("canonical data dir"); let mut config = crate::test_support::test_config(&data_dir); config.user_content_base_url = "http://127.0.0.1:7661".into(); let state = AppState { config: std::sync::Arc::new(config), ..crate::test_support::test_state(&data_dir).await }; let store = state.store.clone(); let owner = store .create_user("alice", None, "unused", false) .await .expect("user"); let public = store .create_repo(&owner.id, "pub", "", forge_core::Visibility::Public, "main") .await .expect("public repo"); let private = store .create_repo( &owner.id, "priv", "", forge_core::Visibility::Private, "main", ) .await .expect("private repo"); for repo in [&public, &private] { let bare = forge_git::repo::repo_path(&data_dir, &repo.id).expect("repo path"); forge_git::repo::init_bare(&state.git, &bare, "main") .await .expect("init bare"); } let object = b"lfs object bytes\n".to_vec(); let oid = sha256_hex(&object); let missing_oid = sha256_hex(b"never stored"); let stored_pointer = pointer_text(&oid, object.len()); let missing_pointer = pointer_text(&missing_oid, 12); let files: Vec<(&str, Vec)> = vec![ ("big.bin", stored_pointer.clone()), ("missing.bin", missing_pointer), ("plain.txt", b"just text\n".to_vec()), ]; let pub_bare = forge_git::repo::repo_path(&data_dir, &public.id).expect("path"); populate(&tmp.path().join("work-pub"), &pub_bare, &files); let priv_bare = forge_git::repo::repo_path(&data_dir, &private.id).expect("path"); populate( &tmp.path().join("work-priv"), &priv_bare, &[("big.bin", stored_pointer)], ); // The object is linked to both repos and written once to the shared store. for repo in [&public, &private] { store .insert_lfs_object(&repo.id, &oid, object.len() as u64) .await .expect("link object"); } let path = object_path(&data_dir, &oid); std::fs::create_dir_all(path.parent().expect("parent")).expect("mkdir objects"); std::fs::write(&path, &object).expect("write object"); let router = router(state.clone()); Fixture { _tmp: tmp, state, router, public, private, object, oid, } } async fn get(router: &Router, method: &str, uri: &str) -> Response { let req = Request::builder() .method(method) .uri(uri) .body(Body::empty()) .expect("request"); router.handle(req).await } fn header_str(res: &Response, name: &str) -> String { res.headers() .get(name) .and_then(|v| v.to_str().ok()) .unwrap_or_default() .to_owned() } async fn body_bytes(res: Response) -> Vec { topcoat::router::to_bytes(res.into_body(), usize::MAX) .await .expect("body") .to_vec() } #[tokio::test] async fn media_serves_the_linked_object_as_a_download() { let fx = fixture().await; let res = get(&fx.router, "GET", "/alice/pub/-/media/main/big.bin").await; assert_eq!(res.status(), StatusCode::OK); assert_eq!(header_str(&res, "content-disposition"), "attachment"); assert_eq!(header_str(&res, "x-content-type-options"), "nosniff"); assert_eq!(header_str(&res, "content-type"), OCTET); assert_eq!( header_str(&res, "content-length"), fx.object.len().to_string() ); assert_eq!(header_str(&res, "etag"), format!("\"{}\"", fx.oid)); assert_eq!(body_bytes(res).await, fx.object); } #[tokio::test] async fn head_has_headers_and_no_body() { let fx = fixture().await; let res = get(&fx.router, "HEAD", "/alice/pub/-/media/main/big.bin").await; assert_eq!(res.status(), StatusCode::OK); assert_eq!( header_str(&res, "content-length"), fx.object.len().to_string() ); assert!(body_bytes(res).await.is_empty()); } #[tokio::test] async fn missing_object_is_404() { let fx = fixture().await; // The pointer exists, but the repo does not link its object. let res = get(&fx.router, "GET", "/alice/pub/-/media/main/missing.bin").await; assert_eq!(res.status(), StatusCode::NOT_FOUND); } #[tokio::test] async fn non_pointer_and_missing_path_are_404() { let fx = fixture().await; let text = get(&fx.router, "GET", "/alice/pub/-/media/main/plain.txt").await; assert_eq!(text.status(), StatusCode::NOT_FOUND); let absent = get(&fx.router, "GET", "/alice/pub/-/media/main/nope.bin").await; assert_eq!(absent.status(), StatusCode::NOT_FOUND); let unknown_repo = get(&fx.router, "GET", "/alice/nope/-/media/main/big.bin").await; assert_eq!(unknown_repo.status(), StatusCode::NOT_FOUND); } #[tokio::test] async fn private_repo_needs_a_valid_token() { let fx = fixture().await; let path = "/alice/priv/-/media/main/big.bin"; let none = get(&fx.router, "GET", path).await; assert_eq!(none.status(), StatusCode::NOT_FOUND); let now = now_unix(); let valid = sign_raw(&fx.state.key, &fx.private.id, now + 60); let ok = get(&fx.router, "GET", &format!("{path}?token={valid}")).await; assert_eq!(ok.status(), StatusCode::OK); assert_eq!(body_bytes(ok).await, fx.object); let expired = sign_raw(&fx.state.key, &fx.private.id, now - 1); let res = get(&fx.router, "GET", &format!("{path}?token={expired}")).await; assert_eq!(res.status(), StatusCode::NOT_FOUND); // A token signed for the public repo does not open the private one. let other = sign_raw(&fx.state.key, &fx.public.id, now + 60); let res = get(&fx.router, "GET", &format!("{path}?token={other}")).await; assert_eq!(res.status(), StatusCode::NOT_FOUND); } #[tokio::test] async fn media_url_encodes_and_signs_private_repos_only() { let fx = fixture().await; let public = media_url(&fx.state, &fx.public, "main", "a b/big.bin"); assert_eq!( public, "http://127.0.0.1:7661/alice/pub/-/media/main/a%20b/big.bin" ); let private = media_url(&fx.state, &fx.private, "main", "big.bin"); assert!( private.starts_with("http://127.0.0.1:7661/alice/priv/-/media/main/big.bin?token=") ); let token = private .split_once("?token=") .map(|(_, t)| t) .unwrap_or_default(); assert!(verify_raw(&fx.state.key, token, &fx.private.id, now_unix())); } /// A branch named `media` must not shadow the media route. The raw route serves its files, /// and the URL from `media_url` still serves the linked LFS object. #[tokio::test] async fn branch_named_media_does_not_shadow_the_media_route() { let fx = fixture().await; let work = fx._tmp.path().join("work-pub"); let bare = forge_git::repo::repo_path(&fx.state.config.data_dir, &fx.public.id) .expect("repo path"); git(&work, &["checkout", "-q", "-b", "media"]); std::fs::write(work.join("README.md"), "# On a media branch\n").expect("write README"); git(&work, &["add", "-A"]); git(&work, &["commit", "-q", "-m", "media branch"]); git( &work, &["push", "-q", bare.to_str().expect("utf8"), "media"], ); let readme = get(&fx.router, "GET", "/alice/pub/media/README.md").await; assert_eq!(readme.status(), StatusCode::OK); assert_eq!(body_bytes(readme).await, b"# On a media branch\n".to_vec()); // `-` is never a ref, so a raw path under it is not resolved. let dash = get(&fx.router, "GET", "/alice/pub/-/README.md").await; assert_eq!(dash.status(), StatusCode::NOT_FOUND); let url = media_url(&fx.state, &fx.public, "main", "big.bin"); let path = url .strip_prefix("http://127.0.0.1:7661") .expect("configured base"); let media = get(&fx.router, "GET", path).await; assert_eq!(media.status(), StatusCode::OK); assert_eq!(body_bytes(media).await, fx.object); } }