Add .conda package reader

README.md

@@ -1,6 +1,6 @@
 # archives
 
-A Go library for reading and browsing archive files in memory. Supports ZIP, TAR (with gzip, bzip2, xz, zstd compression), and Ruby gem formats.
+A Go library for reading and browsing archive files in memory. Supports ZIP, TAR (with gzip, bzip2, xz, zstd compression), Ruby gem, and conda formats.
 
 ## Installation
 
@@ -117,6 +117,7 @@ for _, f := range result.Files {
 - `.zip`, `.jar`, `.whl`, `.nupkg`, `.egg`, `.vsix` (ZIP-based)
 - `.tar`, `.tar.gz`, `.tgz`, `.crate`, `.tar.bz2`, `.tar.xz`, `.tar.zst`
 - `.gem` (Ruby gems with nested data.tar.gz)
+- `.conda` (v2 conda packages: zip of `pkg-*.tar.zst` and `info-*.tar.zst`, presented as one merged tar)
 - `.apk` (routed by content: Android packages open as ZIP, Alpine packages open as gzipped tar)
 
 Filenames without a recognised extension are opened by inspecting the first bytes for a ZIP, tar, gzip, bzip2, xz, or zstd signature.

archives.go

@@ -4,6 +4,7 @@
 //   - ZIP (.zip, .jar, .whl, .nupkg, .egg, .vsix)
 //   - TAR (.tar, .tar.gz, .tgz, .crate, .tar.bz2, .tar.xz, .tar.zst)
 //   - GEM (.gem - Ruby gems with nested tar structure)
+//   - CONDA (.conda - v2 conda packages, zip of zstd tarballs)
 //
 // The .apk extension is routed by content since Android packages are ZIP
 // and Alpine packages are gzipped tar. Filenames without a recognised
@@ -33,6 +34,7 @@ const (
 	formatTarXZ      = "tar.xz"
 	formatTarZstd    = "tar.zst"
 	formatGem        = "gem"
+	formatConda      = "conda"
 	contentSniffSize = 512
 )
 
@@ -135,6 +137,8 @@ func openRaw(format string, raw []byte) (Reader, error) {
 		return openTar(raw, "zstd")
 	case formatGem:
 		return openGem(raw)
+	case formatConda:
+		return openConda(raw)
 	default:
 		return nil, fmt.Errorf("unsupported format: %s", format)
 	}
@@ -231,6 +235,8 @@ func detectFormat(filename string) string {
 		return formatTGZ
 	case ".gem":
 		return formatGem
+	case ".conda":
+		return formatConda
 	default:
 		return ""
 	}

archives_test.go

@@ -34,6 +34,7 @@ func TestDetectFormat(t *testing.T) {
 		{"package.tar.xz", "tar.xz"},
 		{"package.tar.zst", "tar.zst"},
 		{"package.gem", "gem"},
+		{"package.conda", "conda"},
 		{"package.vsix", "zip"},
 		{"package.crate", "tgz"},
 		{"package.apk", ""}, // Ambiguous: routed by content sniff

conda.go

@@ -0,0 +1,83 @@
+package archives
+
+import (
+	"archive/zip"
+	"bytes"
+	"fmt"
+	"io"
+	"strings"
+)
+
+// openConda handles the v2 .conda format used by anaconda.org and
+// conda-forge: an uncompressed zip containing metadata.json plus two
+// zstd-compressed tarballs, pkg-<name>.tar.zst holding the installed file
+// tree and info-<name>.tar.zst holding index.json, paths.json and the
+// recipe. Both tarballs already store their entries with the paths that
+// appear in the equivalent v1 .tar.bz2 package (info/ is a prefix inside
+// the tar, not something to add), so the reader presents them as a single
+// merged tarReader with raw pointing at the outer .conda bytes so Hash
+// matches the digest anaconda.org publishes in repodata.json.
+func openConda(raw []byte) (*tarReader, error) {
+	zr, err := zip.NewReader(bytes.NewReader(raw), int64(len(raw)))
+	if err != nil {
+		return nil, fmt.Errorf("opening conda zip: %w", err)
+	}
+
+	var files []tarFileEntry
+	var total int64
+	for _, f := range zr.File {
+		if !strings.HasSuffix(f.Name, ".tar.zst") {
+			continue
+		}
+		if !strings.HasPrefix(f.Name, "pkg-") && !strings.HasPrefix(f.Name, "info-") {
+			continue
+		}
+		entries, size, err := readCondaMember(f)
+		if err != nil {
+			return nil, err
+		}
+		total += size
+		if total > maxDecompressedSize {
+			return nil, fmt.Errorf("%w: exceeds %d bytes", ErrDecompressLimit, maxDecompressedSize)
+		}
+		files = append(files, entries...)
+	}
+	if files == nil {
+		return nil, fmt.Errorf("no pkg-*.tar.zst or info-*.tar.zst member in conda package")
+	}
+
+	index := make(map[string]int, len(files))
+	for i, f := range files {
+		if _, seen := index[f.info.Path]; !seen {
+			index[f.info.Path] = i
+		}
+	}
+
+	return &tarReader{raw: raw, files: files, index: index}, nil
+}
+
+func readCondaMember(f *zip.File) ([]tarFileEntry, int64, error) {
+	rc, err := f.Open()
+	if err != nil {
+		return nil, 0, fmt.Errorf("opening %s: %w", f.Name, err)
+	}
+	defer func() { _ = rc.Close() }()
+
+	data, err := io.ReadAll(io.LimitReader(rc, maxDecompressedSize+1))
+	if err != nil {
+		return nil, 0, fmt.Errorf("reading %s: %w", f.Name, err)
+	}
+	if int64(len(data)) > maxDecompressedSize {
+		return nil, 0, fmt.Errorf("%w: %s exceeds %d bytes", ErrDecompressLimit, f.Name, maxDecompressedSize)
+	}
+
+	tr, err := openTar(data, "zstd")
+	if err != nil {
+		return nil, 0, fmt.Errorf("opening %s: %w", f.Name, err)
+	}
+	var size int64
+	for _, e := range tr.files {
+		size += int64(len(e.data))
+	}
+	return tr.files, size, nil
+}

conda_test.go

@@ -0,0 +1,198 @@
+package archives
+
+import (
+	"archive/tar"
+	"archive/zip"
+	"bytes"
+	"errors"
+	"io"
+	"strings"
+	"testing"
+
+	"github.com/klauspost/compress/zstd"
+)
+
+func writeTarZst(t *testing.T, files map[string]string) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	enc, err := zstd.NewWriter(&buf)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tw := tar.NewWriter(enc)
+	for name, content := range files {
+		_ = tw.WriteHeader(&tar.Header{
+			Name: name,
+			Size: int64(len(content)),
+			Mode: 0644,
+		})
+		_, _ = tw.Write([]byte(content))
+	}
+	_ = tw.Close()
+	_ = enc.Close()
+	return buf.Bytes()
+}
+
+func createTestConda(t *testing.T) []byte {
+	t.Helper()
+
+	pkg := writeTarZst(t, map[string]string{
+		"site-packages/six.py":       "print('six')",
+		"info/licenses/LICENSE":      "MIT",
+		"site-packages/six-1.0/META": "meta",
+	})
+	info := writeTarZst(t, map[string]string{
+		"info/index.json": `{"name":"six"}`,
+		"info/paths.json": `{"paths":[]}`,
+	})
+
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	for _, m := range []struct {
+		name string
+		data []byte
+	}{
+		{"metadata.json", []byte(`{"conda_pkg_format_version": 2}`)},
+		{"pkg-six-1.0-py_0.tar.zst", pkg},
+		{"info-six-1.0-py_0.tar.zst", info},
+	} {
+		w, err := zw.CreateHeader(&zip.FileHeader{Name: m.name, Method: zip.Store})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if _, err := w.Write(m.data); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := zw.Close(); err != nil {
+		t.Fatal(err)
+	}
+	return buf.Bytes()
+}
+
+func TestCondaReader(t *testing.T) {
+	data := createTestConda(t)
+	reader, err := Open("six-1.0-py_0.conda", bytes.NewReader(data))
+	if err != nil {
+		t.Fatalf("Open conda failed: %v", err)
+	}
+	defer func() { _ = reader.Close() }()
+
+	if _, ok := reader.(*tarReader); !ok {
+		t.Fatalf("reader = %T, want *tarReader", reader)
+	}
+
+	files, err := reader.List()
+	if err != nil {
+		t.Fatalf("List failed: %v", err)
+	}
+	if len(files) != 5 {
+		t.Errorf("List returned %d files, want 5", len(files))
+	}
+
+	// Extract from the pkg tarball
+	rc, err := reader.Extract("site-packages/six.py")
+	if err != nil {
+		t.Fatalf("Extract pkg file failed: %v", err)
+	}
+	content, _ := io.ReadAll(rc)
+	_ = rc.Close()
+	if string(content) != "print('six')" {
+		t.Errorf("pkg content = %q", string(content))
+	}
+
+	// Extract from the info tarball
+	rc, err = reader.Extract("info/index.json")
+	if err != nil {
+		t.Fatalf("Extract info file failed: %v", err)
+	}
+	content, _ = io.ReadAll(rc)
+	_ = rc.Close()
+	if string(content) != `{"name":"six"}` {
+		t.Errorf("info content = %q", string(content))
+	}
+
+	// ListDir should merge both tarballs: root has site-packages/ and info/
+	root, err := reader.ListDir("")
+	if err != nil {
+		t.Fatalf("ListDir failed: %v", err)
+	}
+	assertNoDuplicates(t, "conda ListDir root", root)
+	dirs := map[string]bool{}
+	for _, f := range root {
+		if f.IsDir {
+			dirs[f.Name] = true
+		}
+	}
+	if !dirs["site-packages"] || !dirs["info"] {
+		t.Errorf("ListDir root dirs = %v, want site-packages and info", dirs)
+	}
+}
+
+func TestCondaHashIsOuterArchive(t *testing.T) {
+	data := createTestConda(t)
+	reader, err := openConda(data)
+	if err != nil {
+		t.Fatalf("openConda failed: %v", err)
+	}
+	defer func() { _ = reader.Close() }()
+
+	want := expectedDigests(data)[SHA256]
+	got, err := reader.Hash(SHA256)
+	if err != nil {
+		t.Fatalf("Hash failed: %v", err)
+	}
+	if got != want {
+		t.Errorf("conda Hash = %s, want outer archive digest %s", got, want)
+	}
+}
+
+func TestOpenCondaMissingMembers(t *testing.T) {
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	w, _ := zw.Create("metadata.json")
+	_, _ = w.Write([]byte(`{"conda_pkg_format_version": 2}`))
+	_ = zw.Close()
+
+	_, err := openConda(buf.Bytes())
+	if err == nil {
+		t.Fatal("conda zip with no tar.zst members was accepted")
+	}
+}
+
+func TestOpenCondaRejectsCumulativeOverflow(t *testing.T) {
+	oldMax := maxDecompressedSize
+	maxDecompressedSize = 1024
+	defer func() { maxDecompressedSize = oldMax }()
+
+	// Each member decompresses to 768 bytes, under the 1024 limit on its
+	// own; together they exceed it.
+	member := writeTarZst(t, map[string]string{"blob": strings.Repeat("x", 768)})
+
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	for _, name := range []string{"pkg-a-1.tar.zst", "info-a-1.tar.zst"} {
+		w, _ := zw.CreateHeader(&zip.FileHeader{Name: name, Method: zip.Store})
+		_, _ = w.Write(member)
+	}
+	_ = zw.Close()
+
+	_, err := openConda(buf.Bytes())
+	if err == nil {
+		t.Fatal("expected error when cumulative decompressed size exceeds limit")
+	}
+	if !errors.Is(err, ErrDecompressLimit) {
+		t.Fatalf("expected ErrDecompressLimit, got: %v", err)
+	}
+}
+
+func TestOpenDoesNotInferConda(t *testing.T) {
+	reader, err := OpenBytes("artifact", createTestConda(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = reader.Close() }()
+	if _, ok := reader.(*zipReader); !ok {
+		t.Fatalf("reader = %T, want generic ZIP reader", reader)
+	}
+}