From bdc10155853c313c0188bdce7822c1de8a5a456d Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sun, 16 Aug 2026 22:27:25 +0100 Subject: [PATCH] Add .conda package reader The v2 conda format is an uncompressed zip containing metadata.json, pkg-.tar.zst and info-.tar.zst. Both inner tarballs already store their entries with the paths that appear in the equivalent v1 .tar.bz2 package (info/ is a prefix inside the tar), so the reader opens each .tar.zst member through openTar and concatenates the entries into a single tar view. Hash covers the outer .conda bytes, matching what anaconda.org publishes in repodata.json. Decompressed size is capped cumulatively across all members so a zip with many pkg-*/info-* entries cannot exceed maxDecompressedSize by staying under it per member. Closes #24 --- README.md | 3 +- archives.go | 6 ++ archives_test.go | 1 + conda.go | 83 ++++++++++++++++++++ conda_test.go | 198 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 290 insertions(+), 1 deletion(-) create mode 100644 conda.go create mode 100644 conda_test.go diff --git a/README.md b/README.md index 42a6f5f..c0a976b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # archives -A Go library for reading and browsing archive files in memory. Supports ZIP, TAR (with gzip, bzip2, xz, zstd compression), and Ruby gem formats. +A Go library for reading and browsing archive files in memory. Supports ZIP, TAR (with gzip, bzip2, xz, zstd compression), Ruby gem, and conda formats. ## Installation @@ -117,6 +117,7 @@ for _, f := range result.Files { - `.zip`, `.jar`, `.whl`, `.nupkg`, `.egg`, `.vsix` (ZIP-based) - `.tar`, `.tar.gz`, `.tgz`, `.crate`, `.tar.bz2`, `.tar.xz`, `.tar.zst` - `.gem` (Ruby gems with nested data.tar.gz) +- `.conda` (v2 conda packages: zip of `pkg-*.tar.zst` and `info-*.tar.zst`, presented as one merged tar) - `.apk` (routed by content: Android packages open as ZIP, Alpine packages open as gzipped tar) Filenames without a recognised extension are opened by inspecting the first bytes for a ZIP, tar, gzip, bzip2, xz, or zstd signature. diff --git a/archives.go b/archives.go index 891a3e6..6c41eeb 100644 --- a/archives.go +++ b/archives.go @@ -4,6 +4,7 @@ // - ZIP (.zip, .jar, .whl, .nupkg, .egg, .vsix) // - TAR (.tar, .tar.gz, .tgz, .crate, .tar.bz2, .tar.xz, .tar.zst) // - GEM (.gem - Ruby gems with nested tar structure) +// - CONDA (.conda - v2 conda packages, zip of zstd tarballs) // // The .apk extension is routed by content since Android packages are ZIP // and Alpine packages are gzipped tar. Filenames without a recognised @@ -33,6 +34,7 @@ const ( formatTarXZ = "tar.xz" formatTarZstd = "tar.zst" formatGem = "gem" + formatConda = "conda" contentSniffSize = 512 ) @@ -135,6 +137,8 @@ func openRaw(format string, raw []byte) (Reader, error) { return openTar(raw, "zstd") case formatGem: return openGem(raw) + case formatConda: + return openConda(raw) default: return nil, fmt.Errorf("unsupported format: %s", format) } @@ -231,6 +235,8 @@ func detectFormat(filename string) string { return formatTGZ case ".gem": return formatGem + case ".conda": + return formatConda default: return "" } diff --git a/archives_test.go b/archives_test.go index e4eb4e7..524e674 100644 --- a/archives_test.go +++ b/archives_test.go @@ -34,6 +34,7 @@ func TestDetectFormat(t *testing.T) { {"package.tar.xz", "tar.xz"}, {"package.tar.zst", "tar.zst"}, {"package.gem", "gem"}, + {"package.conda", "conda"}, {"package.vsix", "zip"}, {"package.crate", "tgz"}, {"package.apk", ""}, // Ambiguous: routed by content sniff diff --git a/conda.go b/conda.go new file mode 100644 index 0000000..dba636f --- /dev/null +++ b/conda.go @@ -0,0 +1,83 @@ +package archives + +import ( + "archive/zip" + "bytes" + "fmt" + "io" + "strings" +) + +// openConda handles the v2 .conda format used by anaconda.org and +// conda-forge: an uncompressed zip containing metadata.json plus two +// zstd-compressed tarballs, pkg-.tar.zst holding the installed file +// tree and info-.tar.zst holding index.json, paths.json and the +// recipe. Both tarballs already store their entries with the paths that +// appear in the equivalent v1 .tar.bz2 package (info/ is a prefix inside +// the tar, not something to add), so the reader presents them as a single +// merged tarReader with raw pointing at the outer .conda bytes so Hash +// matches the digest anaconda.org publishes in repodata.json. +func openConda(raw []byte) (*tarReader, error) { + zr, err := zip.NewReader(bytes.NewReader(raw), int64(len(raw))) + if err != nil { + return nil, fmt.Errorf("opening conda zip: %w", err) + } + + var files []tarFileEntry + var total int64 + for _, f := range zr.File { + if !strings.HasSuffix(f.Name, ".tar.zst") { + continue + } + if !strings.HasPrefix(f.Name, "pkg-") && !strings.HasPrefix(f.Name, "info-") { + continue + } + entries, size, err := readCondaMember(f) + if err != nil { + return nil, err + } + total += size + if total > maxDecompressedSize { + return nil, fmt.Errorf("%w: exceeds %d bytes", ErrDecompressLimit, maxDecompressedSize) + } + files = append(files, entries...) + } + if files == nil { + return nil, fmt.Errorf("no pkg-*.tar.zst or info-*.tar.zst member in conda package") + } + + index := make(map[string]int, len(files)) + for i, f := range files { + if _, seen := index[f.info.Path]; !seen { + index[f.info.Path] = i + } + } + + return &tarReader{raw: raw, files: files, index: index}, nil +} + +func readCondaMember(f *zip.File) ([]tarFileEntry, int64, error) { + rc, err := f.Open() + if err != nil { + return nil, 0, fmt.Errorf("opening %s: %w", f.Name, err) + } + defer func() { _ = rc.Close() }() + + data, err := io.ReadAll(io.LimitReader(rc, maxDecompressedSize+1)) + if err != nil { + return nil, 0, fmt.Errorf("reading %s: %w", f.Name, err) + } + if int64(len(data)) > maxDecompressedSize { + return nil, 0, fmt.Errorf("%w: %s exceeds %d bytes", ErrDecompressLimit, f.Name, maxDecompressedSize) + } + + tr, err := openTar(data, "zstd") + if err != nil { + return nil, 0, fmt.Errorf("opening %s: %w", f.Name, err) + } + var size int64 + for _, e := range tr.files { + size += int64(len(e.data)) + } + return tr.files, size, nil +} diff --git a/conda_test.go b/conda_test.go new file mode 100644 index 0000000..1b09dcf --- /dev/null +++ b/conda_test.go @@ -0,0 +1,198 @@ +package archives + +import ( + "archive/tar" + "archive/zip" + "bytes" + "errors" + "io" + "strings" + "testing" + + "github.com/klauspost/compress/zstd" +) + +func writeTarZst(t *testing.T, files map[string]string) []byte { + t.Helper() + var buf bytes.Buffer + enc, err := zstd.NewWriter(&buf) + if err != nil { + t.Fatal(err) + } + tw := tar.NewWriter(enc) + for name, content := range files { + _ = tw.WriteHeader(&tar.Header{ + Name: name, + Size: int64(len(content)), + Mode: 0644, + }) + _, _ = tw.Write([]byte(content)) + } + _ = tw.Close() + _ = enc.Close() + return buf.Bytes() +} + +func createTestConda(t *testing.T) []byte { + t.Helper() + + pkg := writeTarZst(t, map[string]string{ + "site-packages/six.py": "print('six')", + "info/licenses/LICENSE": "MIT", + "site-packages/six-1.0/META": "meta", + }) + info := writeTarZst(t, map[string]string{ + "info/index.json": `{"name":"six"}`, + "info/paths.json": `{"paths":[]}`, + }) + + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for _, m := range []struct { + name string + data []byte + }{ + {"metadata.json", []byte(`{"conda_pkg_format_version": 2}`)}, + {"pkg-six-1.0-py_0.tar.zst", pkg}, + {"info-six-1.0-py_0.tar.zst", info}, + } { + w, err := zw.CreateHeader(&zip.FileHeader{Name: m.name, Method: zip.Store}) + if err != nil { + t.Fatal(err) + } + if _, err := w.Write(m.data); err != nil { + t.Fatal(err) + } + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func TestCondaReader(t *testing.T) { + data := createTestConda(t) + reader, err := Open("six-1.0-py_0.conda", bytes.NewReader(data)) + if err != nil { + t.Fatalf("Open conda failed: %v", err) + } + defer func() { _ = reader.Close() }() + + if _, ok := reader.(*tarReader); !ok { + t.Fatalf("reader = %T, want *tarReader", reader) + } + + files, err := reader.List() + if err != nil { + t.Fatalf("List failed: %v", err) + } + if len(files) != 5 { + t.Errorf("List returned %d files, want 5", len(files)) + } + + // Extract from the pkg tarball + rc, err := reader.Extract("site-packages/six.py") + if err != nil { + t.Fatalf("Extract pkg file failed: %v", err) + } + content, _ := io.ReadAll(rc) + _ = rc.Close() + if string(content) != "print('six')" { + t.Errorf("pkg content = %q", string(content)) + } + + // Extract from the info tarball + rc, err = reader.Extract("info/index.json") + if err != nil { + t.Fatalf("Extract info file failed: %v", err) + } + content, _ = io.ReadAll(rc) + _ = rc.Close() + if string(content) != `{"name":"six"}` { + t.Errorf("info content = %q", string(content)) + } + + // ListDir should merge both tarballs: root has site-packages/ and info/ + root, err := reader.ListDir("") + if err != nil { + t.Fatalf("ListDir failed: %v", err) + } + assertNoDuplicates(t, "conda ListDir root", root) + dirs := map[string]bool{} + for _, f := range root { + if f.IsDir { + dirs[f.Name] = true + } + } + if !dirs["site-packages"] || !dirs["info"] { + t.Errorf("ListDir root dirs = %v, want site-packages and info", dirs) + } +} + +func TestCondaHashIsOuterArchive(t *testing.T) { + data := createTestConda(t) + reader, err := openConda(data) + if err != nil { + t.Fatalf("openConda failed: %v", err) + } + defer func() { _ = reader.Close() }() + + want := expectedDigests(data)[SHA256] + got, err := reader.Hash(SHA256) + if err != nil { + t.Fatalf("Hash failed: %v", err) + } + if got != want { + t.Errorf("conda Hash = %s, want outer archive digest %s", got, want) + } +} + +func TestOpenCondaMissingMembers(t *testing.T) { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, _ := zw.Create("metadata.json") + _, _ = w.Write([]byte(`{"conda_pkg_format_version": 2}`)) + _ = zw.Close() + + _, err := openConda(buf.Bytes()) + if err == nil { + t.Fatal("conda zip with no tar.zst members was accepted") + } +} + +func TestOpenCondaRejectsCumulativeOverflow(t *testing.T) { + oldMax := maxDecompressedSize + maxDecompressedSize = 1024 + defer func() { maxDecompressedSize = oldMax }() + + // Each member decompresses to 768 bytes, under the 1024 limit on its + // own; together they exceed it. + member := writeTarZst(t, map[string]string{"blob": strings.Repeat("x", 768)}) + + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for _, name := range []string{"pkg-a-1.tar.zst", "info-a-1.tar.zst"} { + w, _ := zw.CreateHeader(&zip.FileHeader{Name: name, Method: zip.Store}) + _, _ = w.Write(member) + } + _ = zw.Close() + + _, err := openConda(buf.Bytes()) + if err == nil { + t.Fatal("expected error when cumulative decompressed size exceeds limit") + } + if !errors.Is(err, ErrDecompressLimit) { + t.Fatalf("expected ErrDecompressLimit, got: %v", err) + } +} + +func TestOpenDoesNotInferConda(t *testing.T) { + reader, err := OpenBytes("artifact", createTestConda(t)) + if err != nil { + t.Fatal(err) + } + defer func() { _ = reader.Close() }() + if _, ok := reader.(*zipReader); !ok { + t.Fatalf("reader = %T, want generic ZIP reader", reader) + } +}