diff --git a/README.md b/README.md index 4a9ca25..98f1c0d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # clone -Go library for programs that keep local checkouts of HTTPS Git repositories. It shells out to the `git` binary, which must be on `PATH`. The package supports Go 1.25 or later. For in-process object parsing or history walking, use a library such as [go-git](https://github.com/go-git/go-git). +Go library for programs that keep local checkouts of HTTPS Git repositories. Clone, fetch, remote queries, and command retries shell out to the `git` binary, which must be on `PATH`. Blob reads use [go-git](https://github.com/go-git/go-git) in process. The package supports Go 1.25 or later. ## Install @@ -57,9 +57,9 @@ if err := cache.EnsureCommit(ctx, url, commit); err != nil { ## Read a file from a commit -`InspectBlob` runs `git show :` and reads at most `maxBytes+1`. The extra byte distinguishes content exactly at the limit from truncated content. Complete reads use `magic.Detect`; truncated reads use `magic.DetectPrefix` so the result can report that later bytes may change the classification. The returned content is retained for text, binary, and unknown results. +`InspectBlob` reads the object in process and returns at most `maxBytes`. The object's size distinguishes content exactly at the limit from truncated content. Complete reads use `magic.Detect`; truncated reads use `magic.DetectPrefix` so the result can report that later bytes may change the classification. The returned content is retained for text, binary, and unknown results. Repositories using object formats unsupported by go-git fall back to `git show`. -Both blob functions validate commits and paths before invoking Git. `ValidCommit` and `SanitizePath` are also available when callers need to validate input earlier: +Both blob functions validate commits and paths before reading the repository. `ValidCommit` and `SanitizePath` are also available when callers need to validate input earlier: ```go path, ok := clone.SanitizePath("cmd/tool/main.go") diff --git a/blob.go b/blob.go index 5d39538..eaef105 100644 --- a/blob.go +++ b/blob.go @@ -22,7 +22,8 @@ type BlobResult struct { // InspectBlob reads path from commit in dir and classifies the returned bytes. // It uses prefix detection when maxBytes truncates the blob. commit and path -// are validated with ValidCommit and SanitizePath before reaching Git. +// are validated with ValidCommit and SanitizePath before reading the +// repository. func InspectBlob(ctx context.Context, dir, commit, blobPath string, maxBytes int64) (BlobResult, error) { content, truncated, err := readBlob(ctx, dir, commit, blobPath, maxBytes) if err != nil { @@ -45,7 +46,7 @@ func InspectBlob(ctx context.Context, dir, commit, blobPath string, maxBytes int // Blob reads path from commit in dir. It caps content at maxBytes and reports // whether the blob is binary or was truncated. commit and path are validated -// with ValidCommit and SanitizePath before reaching Git. +// with ValidCommit and SanitizePath before reading the repository. func Blob(ctx context.Context, dir, commit, blobPath string, maxBytes int64) (content []byte, binary, truncated bool, err error) { content, truncated, err = readBlob(ctx, dir, commit, blobPath, maxBytes) if err != nil { @@ -71,7 +72,48 @@ func readBlob(ctx context.Context, dir, commit, blobPath string, maxBytes int64) if !ok { return nil, false, fmt.Errorf("invalid path %q", blobPath) } + if err := ctx.Err(); err != nil { + return nil, false, err + } + + raw, truncated, goGitErr := readBlobWithGoGit(ctx, dir, commit, clean, maxBytes) + if goGitErr == nil { + return raw, truncated, nil + } + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, false, ctxErr + } + // Native Git remains the compatibility path for object formats and + // repository layouts that go-git v5 cannot read. This also covers + // abbreviated SHA-256 object IDs, whose length alone does not identify the + // repository's object format. + raw, truncated, gitErr := readBlobWithGit(ctx, dir, commit, clean, maxBytes) + if gitErr != nil { + return nil, false, errors.Join( + fmt.Errorf("go-git blob read: %w", goGitErr), + fmt.Errorf("git blob read: %w", gitErr), + ) + } + return raw, truncated, nil +} + +type contextReader struct { + ctx context.Context + reader io.Reader +} + +func (r contextReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + n, err := r.reader.Read(p) + if err == nil { + err = r.ctx.Err() + } + return n, err +} +func readBlobWithGit(ctx context.Context, dir, commit, clean string, maxBytes int64) (content []byte, truncated bool, err error) { // --end-of-options stops a commit or path that somehow slipped past the // validators from being parsed as a git-show flag. commit is validated to // hex above, so this is defence in depth rather than the primary guard. diff --git a/blob_test.go b/blob_test.go index 6d4006c..41fff04 100644 --- a/blob_test.go +++ b/blob_test.go @@ -3,7 +3,9 @@ package clone import ( "bytes" "context" + "errors" "os" + "os/exec" "path/filepath" "strings" "testing" @@ -18,17 +20,22 @@ func seedBlobRepository(t testing.TB) (string, string) { dir := t.TempDir() runGitTest(t, dir, "init", "--quiet", "-b", "main") files := map[string][]byte{ - "exact.txt": []byte("12345"), - "big.txt": bytes.Repeat([]byte("a"), 128<<10), - "binary": {'a', 0, 'b'}, - "late-nul": {'a', 'b', 'c', 0}, - "empty": {}, - "png": []byte("\x89PNG\r\n\x1a\n"), - "invalid": {0xff, 'a'}, - "utf16le": {0xff, 0xfe, 'h', 0, 'i', 0}, + "exact.txt": []byte("12345"), + "big.txt": bytes.Repeat([]byte("a"), 128<<10), + "binary": {'a', 0, 'b'}, + "late-nul": {'a', 'b', 'c', 0}, + "empty": {}, + "png": []byte("\x89PNG\r\n\x1a\n"), + "invalid": {0xff, 'a'}, + "utf16le": {0xff, 0xfe, 'h', 0, 'i', 0}, + "nested/file.txt": []byte("nested"), } for name, content := range files { - if err := os.WriteFile(filepath.Join(dir, name), content, 0o644); err != nil { + path := filepath.Join(dir, name) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, content, 0o644); err != nil { t.Fatal(err) } } @@ -225,6 +232,158 @@ func TestBlobReadsTextAtLimit(t *testing.T) { } } +func TestBlobReadsWithoutGitOnPath(t *testing.T) { + dir, commit := seedBlobRepository(t) + t.Setenv("PATH", t.TempDir()) + + content, binary, truncated, err := Blob(context.Background(), dir, commit, "nested/file.txt", 6) + if err != nil { + t.Fatalf("Blob: %v", err) + } + if string(content) != "nested" || binary || truncated { + t.Errorf("Blob = (%q, %v, %v), want exact text", content, binary, truncated) + } +} + +func TestBlobReadsPackedObject(t *testing.T) { + dir, commit := seedBlobRepository(t) + runGitTest(t, dir, "gc", "--quiet", "--prune=now") + t.Setenv("PATH", t.TempDir()) + + content, binary, truncated, err := Blob(context.Background(), dir, commit, "big.txt", 32) + if err != nil { + t.Fatalf("Blob: %v", err) + } + if !bytes.Equal(content, bytes.Repeat([]byte("a"), 32)) || binary || !truncated { + t.Errorf("Blob = (%q, %v, %v), want truncated text", content, binary, truncated) + } +} + +func TestBlobReadsLinkedWorktree(t *testing.T) { + dir, commit := seedBlobRepository(t) + worktree := filepath.Join(t.TempDir(), "checkout") + runGitTest(t, dir, "worktree", "add", "--quiet", "--detach", worktree, commit) + gitFile := filepath.Join(worktree, ".git") + resolved, err := resolveGitFile(gitFile) + if err != nil { + t.Fatal(err) + } + relative, err := filepath.Rel(worktree, resolved) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(gitFile, []byte("gitdir: "+relative+"\n"), 0o644); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", t.TempDir()) + + content, binary, truncated, err := Blob(context.Background(), worktree, commit, "exact.txt", 5) + if err != nil { + t.Fatalf("Blob: %v", err) + } + if string(content) != "12345" || binary || truncated { + t.Errorf("Blob = (%q, %v, %v), want exact text", content, binary, truncated) + } +} + +func TestBlobResolvesAbbreviatedCommitFromSubdirectory(t *testing.T) { + dir, commit := seedBlobRepository(t) + nested := filepath.Join(dir, "nested") + t.Setenv("PATH", t.TempDir()) + + content, binary, truncated, err := Blob(context.Background(), nested, commit[:7], "exact.txt", 5) + if err != nil { + t.Fatalf("Blob: %v", err) + } + if string(content) != "12345" || binary || truncated { + t.Errorf("Blob = (%q, %v, %v), want exact text", content, binary, truncated) + } +} + +func TestBlobReadsBareRepositoryWithoutGitOnPath(t *testing.T) { + dir, commit := seedBlobRepository(t) + bare := filepath.Join(t.TempDir(), "repo.git") + runGitTest(t, dir, "clone", "--quiet", "--bare", dir, bare) + t.Setenv("PATH", t.TempDir()) + + content, binary, truncated, err := Blob(context.Background(), bare, commit, "exact.txt", 5) + if err != nil { + t.Fatalf("Blob: %v", err) + } + if string(content) != "12345" || binary || truncated { + t.Errorf("Blob = (%q, %v, %v), want exact text", content, binary, truncated) + } +} + +func TestBlobPeelsAnnotatedTagWithoutGitOnPath(t *testing.T) { + dir, _ := seedBlobRepository(t) + runGitTest(t, dir, "tag", "-a", "blob-test", "-m", "blob test") + tag := runGitTest(t, dir, "rev-parse", "blob-test^{tag}") + t.Setenv("PATH", t.TempDir()) + + content, binary, truncated, err := Blob(context.Background(), dir, tag, "exact.txt", 5) + if err != nil { + t.Fatalf("Blob: %v", err) + } + if string(content) != "12345" || binary || truncated { + t.Errorf("Blob = (%q, %v, %v), want exact text", content, binary, truncated) + } +} + +func TestBlobHonorsCanceledContext(t *testing.T) { + dir, commit := seedBlobRepository(t) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, _, _, err := Blob(ctx, dir, commit, "exact.txt", 5) + if !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v, want context.Canceled", err) + } +} + +func TestBlobPreservesGoGitAndGitErrors(t *testing.T) { + dir := t.TempDir() + t.Setenv("PATH", t.TempDir()) + + _, _, _, err := Blob(context.Background(), dir, strings.Repeat("a", 40), "file.txt", 5) + if !errors.Is(err, exec.ErrNotFound) { + t.Errorf("error = %v, want exec.ErrNotFound", err) + } + if err == nil || !strings.Contains(err.Error(), dir) { + t.Errorf("error = %v, want starting path %q", err, dir) + } +} + +func TestBlobReadsSHA256RepositoryWithGitFallback(t *testing.T) { + requireGit(t) + dir := t.TempDir() + cmd := exec.Command("git", "init", "--quiet", "--object-format=sha256", "-b", "main") + cmd.Dir = dir + cmd.Env = gitTestEnv() + if out, err := cmd.CombinedOutput(); err != nil { + t.Skipf("git does not support SHA-256 repositories: %s", out) + } + if err := os.WriteFile(filepath.Join(dir, "file.txt"), []byte("content"), 0o644); err != nil { + t.Fatal(err) + } + runGitTest(t, dir, "add", "file.txt") + runGitTest(t, dir, "commit", "--quiet", "-m", "file") + commit := runGitTest(t, dir, "rev-parse", "HEAD") + if len(commit) != 64 { + t.Fatalf("SHA-256 commit length = %d, want 64", len(commit)) + } + + for _, revision := range []string{commit, commit[:12]} { + content, binary, truncated, err := Blob(context.Background(), dir, revision, "file.txt", 7) + if err != nil { + t.Fatalf("Blob(%q): %v", revision, err) + } + if string(content) != "content" || binary || truncated { + t.Errorf("Blob(%q) = (%q, %v, %v), want exact text", revision, content, binary, truncated) + } + } +} + func TestBlobCapsAndDrainsLargeOutput(t *testing.T) { dir, commit := seedBlobRepository(t) content, binary, truncated, err := Blob(context.Background(), dir, commit, "big.txt", 32) diff --git a/doc.go b/doc.go index df247c3..d449156 100644 --- a/doc.go +++ b/doc.go @@ -1,8 +1,6 @@ -// Package clone keeps local checkouts of HTTPS Git repositories. It shells -// out to the git binary, which must be on PATH. It provides shallow -// clone-or-fetch, bounded retries for network failures, a persistent cache, -// and capped reads and content classification for files from commits. -// -// Applications that need to parse Git objects or walk history in process can -// use a library such as github.com/go-git/go-git. +// Package clone keeps local checkouts of HTTPS Git repositories. It provides +// shallow clone-or-fetch, bounded retries for network failures, a persistent +// cache, and capped reads and content classification for files from commits. +// Blob reads use go-git in process. Clone, fetch, remote queries, and command +// retries shell out to the git binary, which must be on PATH. package clone diff --git a/go.mod b/go.mod index 530ab32..dadcd32 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,19 @@ module github.com/git-pkgs/clone go 1.25.6 -require github.com/git-pkgs/magic v0.1.0 +require ( + github.com/git-pkgs/magic v0.1.0 + github.com/go-git/go-billy/v5 v5.9.0 + github.com/go-git/go-git/v5 v5.19.2 +) + +require ( + github.com/cyphar/filepath-securejoin v0.6.1 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/pjbgf/sha1cd v0.6.0 // indirect + golang.org/x/net v0.56.0 // indirect + golang.org/x/sys v0.46.0 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect +) diff --git a/go.sum b/go.sum index 1c7e6dc..094569f 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,72 @@ +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw= +github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= +github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= +github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= +github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= +github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/git-pkgs/magic v0.1.0 h1:xLrqq7CMXB9g5bJnmJyKw17Rvlh0GFiEmO6e5RFsoeY= github.com/git-pkgs/magic v0.1.0/go.mod h1:3ndidt+yvFaI1M0aEkkzkOlFnLPkeVQASIUojazcxCI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.9.0 h1:jItGXszUDRtR/AlferWPTMN4j38BQ88XnXKbilmmBPA= +github.com/go-git/go-billy/v5 v5.9.0/go.mod h1:jCnQMLj9eUgGU7+ludSTYoZL/GGmii14RxKFj7ROgHw= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/go-git/go-git/v5 v5.19.2 h1:wkfn7vOlUBu8ivAWKBWisTiwJK4jYHzTF8Ndv1LyGqY= +github.com/go-git/go-git/v5 v5.19.2/go.mod h1:QqCBE1EFN5ddFmrliLQ3/ntRCUjZU3EJuwuB/jWEHjk= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= +github.com/pjbgf/sha1cd v0.6.0 h1:3WJ8Wz8gvDz29quX1OcEmkAlUg9diU4GxJHqs0/XiwU= +github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= +github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= +golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/object_store.go b/object_store.go new file mode 100644 index 0000000..8ee63d2 --- /dev/null +++ b/object_store.go @@ -0,0 +1,363 @@ +package clone + +import ( + "bufio" + "context" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/go-git/go-billy/v5" + "github.com/go-git/go-billy/v5/osfs" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/cache" + "github.com/go-git/go-git/v5/storage/filesystem" + "github.com/go-git/go-git/v5/storage/filesystem/dotgit" +) + +const goGitV5SHA1HexLength = 40 + +func readBlobWithGoGit(ctx context.Context, dir, commit, blobPath string, maxBytes int64) ([]byte, bool, error) { + if len(commit) > goGitV5SHA1HexLength { + return nil, false, fmt.Errorf("go-git v5 does not support %d-character object IDs", len(commit)) + } + + store, err := openObjectStore(dir, maxBytes) + if err != nil { + return nil, false, err + } + commitHash, err := resolveCommitHash(ctx, store, commit) + if err != nil { + return nil, false, err + } + treeHash, err := readHeaderHash(ctx, store, plumbing.CommitObject, commitHash, "tree") + if err != nil { + return nil, false, err + } + blob, err := findBlobObject(ctx, store, treeHash, blobPath) + if err != nil { + return nil, false, err + } + if blob.Size() < 0 { + return nil, false, fmt.Errorf("blob %q has invalid size %d", blobPath, blob.Size()) + } + + readSize := min(blob.Size(), maxBytes) + if uint64(readSize) > uint64(maxInt()) { + return nil, false, fmt.Errorf("blob read size %d exceeds platform limit", readSize) + } + content := make([]byte, int(readSize)) + if readSize == 0 { + return content, blob.Size() > maxBytes, nil + } + + reader, err := blob.Reader() + if err != nil { + return nil, false, err + } + _, readErr := io.ReadFull(contextReader{ctx: ctx, reader: reader}, content) + closeErr := reader.Close() + if readErr != nil { + return nil, false, readErr + } + if closeErr != nil { + return nil, false, closeErr + } + if err := ctx.Err(); err != nil { + return nil, false, err + } + return content, blob.Size() > maxBytes, nil +} + +func maxInt() int { + return int(^uint(0) >> 1) +} + +func openObjectStore(dir string, maxBytes int64) (*filesystem.Storage, error) { + gitDir, commonDir, err := findGitDirs(dir) + if err != nil { + return nil, err + } + + dotGitFS := osfs.New(gitDir) + var repositoryFS billy.Filesystem + if commonDir != "" && commonDir != gitDir { + repositoryFS = dotgit.NewRepositoryFilesystem(dotGitFS, osfs.New(commonDir)) + } else { + repositoryFS = dotGitFS + } + largeObjectThreshold := max(maxBytes, 1) + return filesystem.NewStorageWithOptions( + repositoryFS, + cache.NewObjectLRUDefault(), + filesystem.Options{LargeObjectThreshold: largeObjectThreshold}, + ), nil +} + +func findGitDirs(dir string) (gitDir, commonDir string, err error) { + start, err := filepath.Abs(dir) + if err != nil { + return "", "", err + } + info, err := os.Stat(start) + if err != nil { + return "", "", err + } + if !info.IsDir() { + return "", "", fmt.Errorf("repository path %q is not a directory", dir) + } + + for current := start; ; current = filepath.Dir(current) { + dotGit := filepath.Join(current, ".git") + if info, statErr := os.Stat(dotGit); statErr == nil { + if info.IsDir() { + return gitDirectories(dotGit) + } + resolved, resolveErr := resolveGitFile(dotGit) + if resolveErr != nil { + return "", "", resolveErr + } + return gitDirectories(resolved) + } else if !errors.Is(statErr, os.ErrNotExist) { + return "", "", statErr + } + + if current == start && isBareGitDir(current) { + return gitDirectories(current) + } + parent := filepath.Dir(current) + if parent == current { + return "", "", fmt.Errorf("no Git repository found from %q", start) + } + } +} + +func resolveGitFile(path string) (string, error) { + content, err := os.ReadFile(path) + if err != nil { + return "", err + } + value := strings.TrimSpace(string(content)) + const prefix = "gitdir:" + if !strings.HasPrefix(value, prefix) { + return "", fmt.Errorf("invalid gitfile %q", path) + } + gitDir := strings.TrimSpace(strings.TrimPrefix(value, prefix)) + if gitDir == "" { + return "", fmt.Errorf("invalid gitfile %q", path) + } + if !filepath.IsAbs(gitDir) { + gitDir = filepath.Join(filepath.Dir(path), gitDir) + } + return filepath.Clean(gitDir), nil +} + +func gitDirectories(gitDir string) (string, string, error) { + content, err := os.ReadFile(filepath.Join(gitDir, "commondir")) + if errors.Is(err, os.ErrNotExist) { + return gitDir, "", nil + } + if err != nil { + return "", "", err + } + commonDir := strings.TrimSpace(string(content)) + if commonDir == "" { + return "", "", fmt.Errorf("empty commondir in %q", gitDir) + } + if !filepath.IsAbs(commonDir) { + commonDir = filepath.Join(gitDir, commonDir) + } + return gitDir, filepath.Clean(commonDir), nil +} + +func isBareGitDir(dir string) bool { + if info, err := os.Stat(filepath.Join(dir, "objects")); err != nil || !info.IsDir() { + return false + } + info, err := os.Stat(filepath.Join(dir, "HEAD")) + return err == nil && !info.IsDir() +} + +func resolveCommitHash(ctx context.Context, store *filesystem.Storage, revision string) (plumbing.Hash, error) { + hashes, err := resolveObjectPrefix(store, revision) + if err != nil { + return plumbing.ZeroHash, err + } + var resolved plumbing.Hash + found := false + for _, hash := range hashes { + commit, commitErr := peelCommitHash(ctx, store, hash) + if commitErr != nil { + continue + } + if found { + return plumbing.ZeroHash, fmt.Errorf("ambiguous commit prefix %q", revision) + } + resolved = commit + found = true + } + if !found { + return plumbing.ZeroHash, fmt.Errorf("commit %q not found", revision) + } + return resolved, nil +} + +func peelCommitHash(ctx context.Context, store *filesystem.Storage, hash plumbing.Hash) (plumbing.Hash, error) { + const maxTagDepth = 16 + for range maxTagDepth { + encoded, err := store.EncodedObject(plumbing.AnyObject, hash) + if err != nil { + return plumbing.ZeroHash, err + } + switch encoded.Type() { + case plumbing.CommitObject: + return hash, nil + case plumbing.TagObject: + hash, err = readEncodedHeaderHash(ctx, encoded, "object") + if err != nil { + return plumbing.ZeroHash, err + } + default: + return plumbing.ZeroHash, fmt.Errorf("object %s is not a commit", hash) + } + } + return plumbing.ZeroHash, fmt.Errorf("tag chain for %s is too deep", hash) +} + +func readHeaderHash( + ctx context.Context, + store *filesystem.Storage, + typeName plumbing.ObjectType, + hash plumbing.Hash, + header string, +) (plumbing.Hash, error) { + encoded, err := store.EncodedObject(typeName, hash) + if err != nil { + return plumbing.ZeroHash, err + } + return readEncodedHeaderHash(ctx, encoded, header) +} + +func readEncodedHeaderHash(ctx context.Context, encoded plumbing.EncodedObject, header string) (plumbing.Hash, error) { + reader, err := encoded.Reader() + if err != nil { + return plumbing.ZeroHash, err + } + line, readErr := bufio.NewReader(contextReader{ctx: ctx, reader: reader}).ReadString('\n') + closeErr := reader.Close() + if readErr != nil { + return plumbing.ZeroHash, readErr + } + if closeErr != nil { + return plumbing.ZeroHash, closeErr + } + value, ok := strings.CutPrefix(strings.TrimSuffix(line, "\n"), header+" ") + if !ok || !plumbing.IsHash(value) { + return plumbing.ZeroHash, fmt.Errorf("object has invalid %s header", header) + } + return plumbing.NewHash(value), nil +} + +func findBlobObject( + ctx context.Context, + store *filesystem.Storage, + treeHash plumbing.Hash, + blobPath string, +) (*storedBlob, error) { + parts := strings.Split(blobPath, "/") + for index, part := range parts { + tree, err := store.EncodedObject(plumbing.TreeObject, treeHash) + if err != nil { + return nil, err + } + mode, entryHash, err := findTreeEntry(ctx, tree, part) + if err != nil { + return nil, err + } + if index == len(parts)-1 { + encoded, encodedErr := store.EncodedObject(plumbing.BlobObject, entryHash) + if encodedErr != nil { + return nil, encodedErr + } + return &storedBlob{EncodedObject: encoded}, nil + } + if mode != "40000" && mode != "040000" { + return nil, fmt.Errorf("path component %q is not a directory", part) + } + treeHash = entryHash + } + return nil, fmt.Errorf("path %q does not exist", blobPath) +} + +type storedBlob struct { + plumbing.EncodedObject +} + +func findTreeEntry( + ctx context.Context, + encoded plumbing.EncodedObject, + want string, +) (mode string, hash plumbing.Hash, err error) { + reader, err := encoded.Reader() + if err != nil { + return "", plumbing.ZeroHash, err + } + defer func() { + if closeErr := reader.Close(); err == nil { + err = closeErr + } + }() + + buffered := bufio.NewReader(contextReader{ctx: ctx, reader: reader}) + for { + mode, err = buffered.ReadString(' ') + if errors.Is(err, io.EOF) { + return "", plumbing.ZeroHash, fmt.Errorf("path %q does not exist", want) + } + if err != nil { + return "", plumbing.ZeroHash, err + } + mode = strings.TrimSuffix(mode, " ") + name, nameErr := buffered.ReadString(0) + if nameErr != nil { + return "", plumbing.ZeroHash, nameErr + } + name = strings.TrimSuffix(name, "\x00") + if _, err = io.ReadFull(buffered, hash[:]); err != nil { + return "", plumbing.ZeroHash, err + } + if name == want { + return mode, hash, nil + } + } +} + +func resolveObjectPrefix(store *filesystem.Storage, revision string) ([]plumbing.Hash, error) { + if len(revision) == goGitV5SHA1HexLength { + return []plumbing.Hash{plumbing.NewHash(revision)}, nil + } + + evenHex := revision[:len(revision)&^1] + prefix, err := hex.DecodeString(evenHex) + if err != nil { + return nil, err + } + hashes, err := store.HashesWithPrefix(prefix) + if err != nil { + return nil, err + } + if len(evenHex) == len(revision) { + return hashes, nil + } + filtered := hashes[:0] + for _, hash := range hashes { + if strings.HasPrefix(hash.String(), revision) { + filtered = append(filtered, hash) + } + } + return filtered, nil +}