From c5f00bd431681e18abecaa909c04d25c19defd3d Mon Sep 17 00:00:00 2001 From: Mshehu5 Date: Tue, 13 Oct 2026 17:27:59 +0100 Subject: [PATCH] Fetch missing history in shallow clones Fetch missing explicit SHAs and deepen ancestry until the selected range is complete. Stop when fetching fails or makes no progress, and preserve checkout state, refs, and FETCH_HEAD. Use Git for fetching so existing credentials and HTTP settings apply. Include Git in the Nix package and cover shallow merge histories through the CLI. --- nix/package.nix | 3 +- src/history.rs | 138 ++++++++++++++-- src/lib.rs | 8 + tests/cli.rs | 420 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 551 insertions(+), 18 deletions(-) diff --git a/nix/package.nix b/nix/package.nix index 5d12a4c..307b74d 100644 --- a/nix/package.nix +++ b/nix/package.nix @@ -10,6 +10,7 @@ commonArgs = { src = craneLib.cleanCargoSource ../.; strictDeps = true; + nativeCheckInputs = [ pkgs.git ]; }; cargoArtifacts = craneLib.buildDepsOnly commonArgs; validate-commits = craneLib.buildPackage ( @@ -19,7 +20,7 @@ nativeBuildInputs = [ pkgs.makeWrapper ]; postInstall = '' wrapProgram "$out/bin/validate-commits" \ - --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.nix ]} + --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.nix pkgs.git ]} ''; } ); diff --git a/src/history.rs b/src/history.rs index 2898358..ea03282 100644 --- a/src/history.rs +++ b/src/history.rs @@ -1,13 +1,13 @@ -use gix::{ObjectId, Repository, revision::plumbing::Spec}; use std::collections::{HashMap, HashSet}; +use std::process::{Command, Stdio}; + +use gix::objs::Exists; +use gix::{ObjectId, Repository, revision::plumbing::Spec}; use crate::{CommitId, Error}; pub(crate) fn resolve_commits(repo: &Repository, input: &str) -> Result, Error> { - let spec = repo - .rev_parse(input) - .map_err(|source| Error::ParseRevision(Box::new(source)))? - .detach(); + let spec = parse(repo, input)?; let mut selection = Selection::default(); match spec { Spec::Exclude(_) => {} @@ -31,6 +31,9 @@ pub(crate) fn resolve_commits(repo: &Repository, input: &str) -> Result = selection @@ -39,17 +42,30 @@ pub(crate) fn resolve_commits(repo: &Repository, input: &str) -> Result Result { + if let Ok(spec) = repo.rev_parse(input) { + return Ok(spec.detach()); + } + // Recover only missing explicit SHAs. Parent-qualified revisions must + // already resolve, as in the CI helper that accepts a base SHA. + if repo.is_shallow() { + for revision in input.split("..").map(|part| part.trim_start_matches('.')) { + if let Ok(id) = ObjectId::from_hex(revision.as_bytes()) + && !repo.objects.exists(&id) + { + fetch(repo, &[id], FetchDepth::Initial)?; + } + } + } + repo.rev_parse(input) + .map(|spec| spec.detach()) + .map_err(|source| Error::ParseRevision(Box::new(source))) +} + fn peel(repo: &Repository, id: ObjectId) -> Result { repo.find_object(id) .map_err(|source| Error::FindObject(Box::new(source)))? @@ -67,13 +103,60 @@ fn peel(repo: &Repository, id: ObjectId) -> Result { .map_err(|source| Error::PeelToCommit(Box::new(source))) } +enum FetchDepth { + Initial, + Deepen(u32), +} + +fn fetch(repo: &Repository, tips: &[ObjectId], depth: FetchDepth) -> Result<(), Error> { + let before = boundaries(repo)?; + let depth_flag = match depth { + FetchDepth::Initial => "--depth=1".to_owned(), + FetchDepth::Deepen(count) => format!("--deepen={count}"), + }; + let status = Command::new("git") + .current_dir(repo.workdir().ok_or(Error::BareRepository)?) + .args([ + "fetch", + "--no-tags", + "--no-write-fetch-head", + "--refmap=", + "--no-auto-maintenance", + "--no-recurse-submodules", + ]) + .arg(depth_flag) + .arg("origin") + .args(tips.iter().map(ToString::to_string)) + .stdout(Stdio::null()) + .status() + .map_err(Error::SpawnGit)?; + if !status.success() { + return Err(Error::FetchFailed { status }); + } + if matches!(depth, FetchDepth::Deepen(_)) && boundaries(repo)? == before { + return Err(Error::NoHistoryProgress); + } + Ok(()) +} + +fn boundaries(repo: &Repository) -> Result, Error> { + Ok(repo + .shallow_commits() + .map_err(Error::ReadShallowCommits)? + .map(|commits| commits.iter().copied().collect()) + .unwrap_or_default()) +} + struct AvailableHistory { parents: HashMap>, + truncated: HashSet, } impl AvailableHistory { fn read(repo: &Repository, tips: &[ObjectId]) -> Result { + let shallow = boundaries(repo)?; let mut parents = HashMap::new(); + let mut truncated = HashSet::new(); let mut pending = tips.to_vec(); while let Some(id) = pending.pop() { if parents.contains_key(&id) { @@ -82,15 +165,19 @@ impl AvailableHistory { let commit = repo .find_commit(id) .map_err(|source| Error::FindCommit(Box::new(source)))?; - let ancestors: Vec<_> = commit + let mut ancestors: Vec<_> = commit .decode() .map_err(Error::DecodeCommit)? .parents() .collect(); + if shallow.contains(&id) && !ancestors.is_empty() { + truncated.insert(id); + ancestors.clear(); + } pending.extend(ancestors.iter().copied()); parents.insert(id, ancestors); } - Ok(Self { parents }) + Ok(Self { parents, truncated }) } fn reachable(&self, tips: &[ObjectId]) -> HashSet { @@ -104,6 +191,23 @@ impl AvailableHistory { seen } + fn needs_deepening(&self, included: &HashSet, excluded: &HashSet) -> bool { + if !included.is_disjoint(&self.truncated) { + return true; + } + let hidden_boundaries: HashSet<_> = + excluded.intersection(&self.truncated).copied().collect(); + // Each earliest included commit must descend from every hidden boundary. + // Otherwise the missing base history could still exclude that commit. + !hidden_boundaries.is_empty() + && included.iter().any(|id| { + self.parents[id] + .iter() + .all(|parent| excluded.contains(parent)) + && !hidden_boundaries.is_subset(&self.reachable(&[*id])) + }) + } + fn select(&self, tips: &[ObjectId], excluded: &HashSet) -> Vec { let mut seen = HashSet::new(); let mut commits = Vec::new(); diff --git a/src/lib.rs b/src/lib.rs index 96f3698..bb7f642 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -125,6 +125,14 @@ pub enum Error { FindCommit(#[source] Box), #[error("could not decode a selected commit: {0}")] DecodeCommit(#[source] gix::objs::decode::Error), + #[error("could not read shallow history: {0}")] + ReadShallowCommits(#[source] gix::shallow::read::Error), + #[error("could not run git fetch: {0}")] + SpawnGit(#[source] std::io::Error), + #[error("git fetch failed{}", status_suffix(.status))] + FetchFailed { status: ExitStatus }, + #[error("fetch did not deepen history; origin may itself be shallow")] + NoHistoryProgress, #[error("could not write output: {0}")] WriteOutput(#[source] std::io::Error), #[error("could not start Nix while checking commit {commit}: {source}")] diff --git a/tests/cli.rs b/tests/cli.rs index 2c3ed97..79e1023 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -1,5 +1,6 @@ #![cfg(unix)] +use gix::objs::Exists; use gix::{ObjectId, Repository}; use std::env; use std::fs; @@ -266,3 +267,422 @@ fn orders_merge_parents_with_skewed_dates() { ); assert_eq!(fixture.calls(), expected_calls(&[fork, left, a, b, head])); } + +fn history_commit(fixture: &Fixture, message: &str, parents: &[ObjectId]) -> ObjectId { + let tree = fixture + .repository + .find_commit(fixture.commits[0]) + .unwrap() + .tree_id() + .unwrap() + .detach(); + commit_as( + &fixture.repository, + None, + message, + tree, + parents.iter().copied(), + 100, + ) +} + +fn git_at(repo: &std::path::Path, args: &[&str]) -> String { + let output = Command::new("git") + .args(args) + .current_dir(repo) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap().trim().to_owned() +} + +fn clone_shallow(fixture: &mut Fixture, depth: usize, branch: &str) { + let remote = fixture.root.path().join("remote"); + fs::rename(&fixture.repo, &remote).unwrap(); + git_at( + fixture.root.path(), + &[ + "clone", + "--quiet", + &format!("--depth={depth}"), + &format!("--branch={branch}"), + &format!("file://{}", remote.display()), + "repo", + ], + ); + fixture.repository = gix::open(&fixture.repo).unwrap(); +} + +fn shallow_fixture(depth: usize) -> (Fixture, ObjectId, Vec) { + let mut fixture = Fixture::new(); + for i in 0..100 { + fixture.commit(&format!("Main {i}")); + } + let base = *fixture.commits.last().unwrap(); + let mut side = base; + let mut selected = Vec::new(); + for i in 0..40 { + side = history_commit(&fixture, &format!("Side {i}"), &[side]); + selected.push(side); + } + let head = history_commit(&fixture, "Merge", &[base, side]); + selected.push(head); + git_at( + &fixture.repo, + &["update-ref", "refs/heads/queue", &head.to_string()], + ); + clone_shallow(&mut fixture, depth, "queue"); + (fixture, base, selected) +} + +#[test] +fn deepens_missing_base_and_truncated_merge_parents() { + for depth in [1, 2] { + let (fixture, base, selected) = shallow_fixture(depth); + let before = fixture.dirty_checkout(); + let refs = git_at(&fixture.repo, &["show-ref"]); + let fetch_head = fixture.repo.join(".git/FETCH_HEAD"); + fs::write(&fetch_head, "preserve fetch head\n").unwrap(); + let output = fixture.run(&format!("{base}..HEAD"), None); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let mut actual = fixture.calls(); + let mut expected = expected_calls(&selected); + actual.sort(); + expected.sort(); + assert_eq!(actual, expected); + assert_eq!( + git_at(&fixture.repo, &["rev-parse", "--is-shallow-repository"]), + "true" + ); + fixture.assert_checkout(&before); + assert_eq!(git_at(&fixture.repo, &["show-ref"]), refs); + assert_eq!( + fs::read_to_string(&fetch_head).unwrap(), + "preserve fetch head\n" + ); + } +} + +#[test] +fn complete_and_empty_shallow_ranges_do_not_fetch() { + let (fixture, base, selected) = shallow_fixture(42); + git_at( + &fixture.repo, + &[ + "remote", + "set-url", + "origin", + "/nonexistent/validate-commits", + ], + ); + assert!(fixture.run("HEAD..HEAD", None).status.success()); + assert!(fixture.calls().is_empty()); + let output = fixture.run(&format!("{base}..HEAD"), None); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(fixture.calls(), expected_calls(&selected)); +} + +#[test] +fn shallow_origin_without_progress_stops_before_nix() { + let (fixture, _, _) = shallow_fixture(1); + let source = fixture.root.path().join("remote"); + let shallow_origin = fixture.root.path().join("shallow-origin"); + git_at( + fixture.root.path(), + &[ + "clone", + "--quiet", + "--depth=1", + "--branch=queue", + &format!("file://{}", source.display()), + "shallow-origin", + ], + ); + git_at( + &fixture.repo, + &[ + "remote", + "set-url", + "origin", + &format!("file://{}", shallow_origin.display()), + ], + ); + let output = fixture.run("HEAD", None); + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("fetch did not deepen history"), "{stderr}"); + assert!(fixture.calls().is_empty()); +} + +#[test] +fn fetch_failures_and_invalid_ranges_stop_before_nix() { + // Depth 1 requires fetching the base; depth 2 requires deepening ancestry. + for depth in [1, 2] { + let (fixture, base, _) = shallow_fixture(depth); + git_at( + &fixture.repo, + &[ + "remote", + "set-url", + "origin", + "/nonexistent/validate-commits", + ], + ); + let output = fixture.run(&format!("{base}..HEAD"), None); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("git fetch failed")); + assert!(fixture.calls().is_empty()); + assert!(!fixture.run("missing..HEAD", None).status.success()); + assert!(fixture.calls().is_empty()); + } +} + +#[test] +fn deepens_base_history_to_exclude_shared_fork() { + let mut fixture = Fixture::new(); + for i in 0..100 { + fixture.commit(&format!("Shared {i}")); + } + let shared = *fixture.commits.last().unwrap(); + let fork = history_commit(&fixture, "Fork", &[shared]); + let branch = history_commit(&fixture, "Branch", &[shared]); + let mut main = fork; + for i in 0..20 { + main = history_commit(&fixture, &format!("Main {i}"), &[main]); + } + let base = history_commit(&fixture, "Merge branch", &[main, branch]); + let pr = history_commit(&fixture, "PR", &[fork]); + let head = history_commit(&fixture, "Queue", &[base, pr]); + git_at( + &fixture.repo, + &["update-ref", "refs/heads/queue", &head.to_string()], + ); + clone_shallow(&mut fixture, 1, "queue"); + let output = fixture.run(&format!("{base}..HEAD"), None); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let mut actual = fixture.calls(); + actual.sort(); + let mut expected = expected_calls(&[pr, head]); + expected.sort(); + assert_eq!(actual, expected); + assert_eq!( + git_at(&fixture.repo, &["rev-parse", "--is-shallow-repository"]), + "true" + ); +} + +#[test] +fn deepening_can_reach_full_history() { + let mut fixture = Fixture::new(); + let base = fixture.commits[0]; + let branch = fixture.repository.head_name().unwrap().unwrap(); + clone_shallow(&mut fixture, 1, branch.shorten().to_string().as_str()); + let output = fixture.run(&format!("{base}..HEAD"), None); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(fixture.calls(), expected_calls(&fixture.commits[1..])); + assert_eq!( + git_at(&fixture.repo, &["rev-parse", "--is-shallow-repository"]), + "false" + ); +} + +#[test] +fn real_roots_in_complete_ranges_do_not_fetch_unrelated_shallow_history() { + let fixture = Fixture::new(); + let base = fixture.commits[0]; + let independent = history_commit(&fixture, "Independent root", &[]); + let head = history_commit(&fixture, "Merge independent history", &[base, independent]); + let boundary = fixture.commits[3]; + fs::write(fixture.repo.join(".git/shallow"), format!("{boundary}\n")).unwrap(); + // The shallow boundary belongs to another branch; origin is unnecessary. + let output = fixture.run(&format!("{base}..{head}"), None); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(fixture.calls(), expected_calls(&[independent, head])); + assert_eq!( + fs::read_to_string(fixture.repo.join(".git/shallow")).unwrap(), + format!("{boundary}\n") + ); +} + +#[test] +fn real_roots_are_not_selected_without_proving_base_history_complete() { + let fixture = Fixture::new(); + let base = fixture.commits[3]; + fs::write(fixture.repo.join(".git/shallow"), format!("{base}\n")).unwrap(); + let output = fixture.run(&format!("{base}..{}", fixture.commits[0]), None); + assert!(!output.status.success()); + assert!(fixture.calls().is_empty()); +} + +#[test] +fn orders_merge_parents_with_skewed_dates_in_a_shallow_clone() { + let mut fixture = Fixture::new(); + for i in 0..20 { + fixture.commit(&format!("Main {i}")); + } + let base = *fixture.commits.last().unwrap(); + let tree = fixture + .repository + .find_commit(base) + .unwrap() + .tree_id() + .unwrap() + .detach(); + let fork = commit_as(&fixture.repository, None, "Fork", tree, [base], 900); + let left = commit_as(&fixture.repository, None, "Left", tree, [fork], 10); + let a = history_commit(&fixture, "A", &[left]); + let b = history_commit(&fixture, "B", &[a]); + let head = history_commit(&fixture, "Merge", &[b, fork]); + git_at( + &fixture.repo, + &["update-ref", "refs/heads/queue", &head.to_string()], + ); + clone_shallow(&mut fixture, 6, "queue"); + let output = fixture.run(&format!("{base}..{head}"), None); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(fixture.calls(), expected_calls(&[fork, left, a, b, head])); +} + +#[test] +fn single_commit_and_empty_shallow_selections_do_not_fetch() { + let (fixture, _, selected) = shallow_fixture(1); + git_at( + &fixture.repo, + &[ + "remote", + "set-url", + "origin", + "/nonexistent/validate-commits", + ], + ); + for revision in ["HEAD...HEAD", "^HEAD", "HEAD.."] { + let output = fixture.run(revision, None); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(fixture.calls().is_empty()); + } + let output = fixture.run("HEAD^!", None); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + fixture.calls(), + expected_calls(&[*selected.last().unwrap()]) + ); +} + +#[test] +fn single_revision_and_parent_selections_deepen_shallow_history() { + for revision in ["HEAD", "HEAD^@"] { + let (fixture, _, _) = shallow_fixture(1); + let remote = fixture.root.path().join("remote"); + let remote_revision = revision.replace("HEAD", "queue"); + let expected: Vec<_> = git_at(&remote, &["rev-list", &remote_revision]) + .lines() + .map(|line| ObjectId::from_hex(line.as_bytes()).unwrap()) + .collect(); + let output = fixture.run(revision, None); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let mut actual = fixture.calls(); + actual.sort(); + let mut expected = expected_calls(&expected); + expected.sort(); + assert_eq!(actual, expected); + } +} + +#[test] +fn symmetric_shallow_ranges_deepen_until_shared_history_is_excluded() { + let mut fixture = Fixture::new(); + let shared = fixture.commits[3]; + let mut left = shared; + let mut right = shared; + let mut selected = Vec::new(); + for i in 0..3 { + left = history_commit(&fixture, &format!("Left {i}"), &[left]); + selected.push(left); + } + for i in 0..20 { + right = history_commit(&fixture, &format!("Right {i}"), &[right]); + selected.push(right); + } + git_at( + &fixture.repo, + &["update-ref", "refs/heads/queue", &left.to_string()], + ); + git_at( + &fixture.repo, + &["update-ref", "refs/heads/right", &right.to_string()], + ); + clone_shallow(&mut fixture, 1, "queue"); + let output = fixture.run(&format!("HEAD...{right}"), None); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let mut actual = fixture.calls(); + actual.sort(); + let mut expected = expected_calls(&selected); + expected.sort(); + assert_eq!(actual, expected); +} + +#[test] +fn unavailable_parent_revisions_fail_without_fetching() { + let (fixture, base, _) = shallow_fixture(1); + let shallow = fs::read(fixture.repo.join(".git/shallow")).unwrap(); + let before = fixture.dirty_checkout(); + for range in ["HEAD~1..HEAD".to_owned(), format!("{base}~1..HEAD")] { + let output = fixture.run(&range, None); + assert!(!output.status.success(), "{range}"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(!stderr.contains("Deepening history"), "{range}: {stderr}"); + assert!(fixture.calls().is_empty()); + assert_eq!( + fs::read(fixture.repo.join(".git/shallow")).unwrap(), + shallow + ); + fixture.assert_checkout(&before); + } + // The missing explicit SHA was not fetched merely to retry parent notation. + assert!(!fixture.repo.join(".git/FETCH_HEAD").exists()); + assert!(!fixture.repository.objects.exists(&base)); +}