feat: support extraction of multiple package types

cmd/chisel/cmd_debug_check_release_archives.go

@@ -13,7 +13,6 @@ import (
 
 	"github.com/canonical/chisel/internal/archive"
 	"github.com/canonical/chisel/internal/cache"
-	"github.com/canonical/chisel/internal/deb"
 	"github.com/canonical/chisel/internal/setup"
 )
 
@@ -150,7 +149,7 @@ func computePathObservations(release *setup.Release, archives map[string]archive
 			if err != nil {
 				return nil, err
 			}
-			dataReader, err := deb.DataReader(pkgReader)
+			dataReader, err := pkgReader.TarStream()
 			if err != nil {
 				return nil, err
 			}

internal/archive/archive.go

@@ -16,11 +16,12 @@ import (
 	"github.com/canonical/chisel/internal/control"
 	"github.com/canonical/chisel/internal/deb"
 	"github.com/canonical/chisel/internal/pgputil"
+	"github.com/canonical/chisel/internal/tarball"
 )
 
 type Archive interface {
 	Options() *Options
-	Fetch(pkg string) (io.ReadSeekCloser, *PackageInfo, error)
+	Fetch(pkg string) (tarball.TarStreamer, *PackageInfo, error)
 	Exists(pkg string) bool
 	Info(pkg string) (*PackageInfo, error)
 }
@@ -140,7 +141,7 @@ func (a *ubuntuArchive) selectPackage(pkg string) (control.Section, *ubuntuIndex
 	return selectedSection, selectedIndex, nil
 }
 
-func (a *ubuntuArchive) Fetch(pkg string) (io.ReadSeekCloser, *PackageInfo, error) {
+func (a *ubuntuArchive) Fetch(pkg string) (tarball.TarStreamer, *PackageInfo, error) {
 	section, index, err := a.selectPackage(pkg)
 	if err != nil {
 		return nil, nil, err
@@ -153,7 +154,7 @@ func (a *ubuntuArchive) Fetch(pkg string) (io.ReadSeekCloser, *PackageInfo, erro
 		return nil, nil, err
 	}
 	info := sectionPackageInfo(section)
-	return reader, info, nil
+	return deb.Open(reader), info, nil
 }
 
 func (a *ubuntuArchive) Info(pkg string) (*PackageInfo, error) {

internal/archive/archive_test.go

@@ -1,9 +1,7 @@
 package archive_test
 
 import (
-	"golang.org/x/crypto/openpgp/packet"
-	. "gopkg.in/check.v1"
-
+	"archive/tar"
 	"crypto/sha256"
 	"crypto/sha512"
 	"debug/elf"
@@ -18,6 +16,9 @@ import (
 	"path/filepath"
 	"strings"
 
+	"golang.org/x/crypto/openpgp/packet"
+	. "gopkg.in/check.v1"
+
 	"github.com/canonical/chisel/internal/archive"
 	"github.com/canonical/chisel/internal/archive/testarchive"
 	"github.com/canonical/chisel/internal/tarball"
@@ -255,7 +256,7 @@ func (s *httpSuite) TestFetchPackage(c *C) {
 		Name:    "mypkg1",
 		Version: "1.1",
 		Arch:    "amd64",
-		SHA256:  "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05",
+		SHA256:  "ff175644a17301e047ac757681f6e16b1c410228d9fe50441bba8438a7c45fa2",
 	})
 	c.Assert(read(pkg), Equals, "mypkg1 1.1 data")
 
@@ -266,7 +267,7 @@ func (s *httpSuite) TestFetchPackage(c *C) {
 		Name:    "mypkg4",
 		Version: "1.4",
 		Arch:    "amd64",
-		SHA256:  "54af70097b30b33cfcbb6911ad3d0df86c2d458928169e348fa7873e4fc678e4",
+		SHA256:  "fe0b0023af4cd5786a2563faadf6ec31e48079a9b176356b008261c1590b6df9",
 	})
 	c.Assert(read(pkg), Equals, "mypkg4 1.4 data")
 }
@@ -322,13 +323,16 @@ func (s *httpSuite) TestFetchBothDigests(c *C) {
 		Name:    "mypkg1",
 		Version: "1.1",
 		Arch:    "amd64",
-		SHA256:  "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05",
+		SHA256:  "ff175644a17301e047ac757681f6e16b1c410228d9fe50441bba8438a7c45fa2",
 	})
 	c.Assert(read(pkg), Equals, "mypkg1 1.1 data")
 
 	// Pin the cache key: with both digests advertised, the package is cached
 	// under its strongest digest.
-	sha512Digest := fmt.Sprintf("%x", sha512.Sum512([]byte("mypkg1 1.1 data")))
+	sha512Digest := fmt.Sprintf("%x", sha512.Sum512(testutil.MustMakeDeb([]testutil.TarEntry{
+		testutil.Dir(0o755, "./"),
+		testutil.Reg(0o644, "./data", "mypkg1 1.1 data"),
+	})))
 	_, err = os.Stat(filepath.Join(options.CacheDir, "sha512", sha512Digest))
 	c.Assert(err, IsNil)
 }
@@ -359,7 +363,7 @@ func (s *httpSuite) TestFetchPortsPackage(c *C) {
 		Name:    "mypkg1",
 		Version: "1.1",
 		Arch:    "arm64",
-		SHA256:  "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05",
+		SHA256:  "ff175644a17301e047ac757681f6e16b1c410228d9fe50441bba8438a7c45fa2",
 	})
 	c.Assert(read(pkg), Equals, "mypkg1 1.1 data")
 
@@ -370,7 +374,7 @@ func (s *httpSuite) TestFetchPortsPackage(c *C) {
 		Name:    "mypkg4",
 		Version: "1.4",
 		Arch:    "arm64",
-		SHA256:  "54af70097b30b33cfcbb6911ad3d0df86c2d458928169e348fa7873e4fc678e4",
+		SHA256:  "fe0b0023af4cd5786a2563faadf6ec31e48079a9b176356b008261c1590b6df9",
 	})
 	c.Assert(read(pkg), Equals, "mypkg4 1.4 data")
 }
@@ -382,7 +386,10 @@ func (s *httpSuite) TestFetchSecurityPackage(c *C) {
 		err := release.Walk(func(item testarchive.Item) error {
 			if p, ok := item.(*testarchive.Package); ok && p.Name == "mypkg1" {
 				p.Version = fmt.Sprintf("%s.%d", p.Version, i)
-				p.Data = []byte("package from " + suite)
+				p.Data = testutil.MustMakeDeb([]testutil.TarEntry{
+					testutil.Dir(0o755, "./"),
+					testutil.Reg(0o644, "./data", "package from "+suite),
+				})
 			}
 			return nil
 		})
@@ -410,7 +417,7 @@ func (s *httpSuite) TestFetchSecurityPackage(c *C) {
 		Name:    "mypkg1",
 		Version: "1.1.2.2",
 		Arch:    "amd64",
-		SHA256:  "5448585bdd916e5023eff2bc1bc3b30bcc6ee9db9c03e531375a6a11ddf0913c",
+		SHA256:  "e3732bc52b8a11c8e749266c1eee5548ab02cbaaf84c826277e9ae245d1099b7",
 	})
 	c.Assert(read(pkg), Equals, "package from jammy-security")
 
@@ -420,7 +427,7 @@ func (s *httpSuite) TestFetchSecurityPackage(c *C) {
 		Name:    "mypkg2",
 		Version: "1.2",
 		Arch:    "amd64",
-		SHA256:  "a4b4f3f3a8fa09b69e3ba23c60a41a1f8144691fd371a2455812572fd02e6f79",
+		SHA256:  "0d229011ec711ef268779580130dc034409e42124d3c25b01e0b71eac90284ad",
 	})
 	c.Assert(read(pkg), Equals, "mypkg2 1.2 data")
 }
@@ -665,7 +672,7 @@ var packageInfoTests = []struct {
 		Name:    "mypkg1",
 		Version: "1.1",
 		Arch:    "amd64",
-		SHA256:  "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05",
+		SHA256:  "ff175644a17301e047ac757681f6e16b1c410228d9fe50441bba8438a7c45fa2",
 	},
 }, {
 	summary: "Package not found in archive",
@@ -700,12 +707,29 @@ func (s *httpSuite) TestPackageInfo(c *C) {
 	}
 }
 
-func read(r io.Reader) string {
-	data, err := io.ReadAll(r)
+func read(pkg tarball.TarStreamer) string {
+	tarStream, err := pkg.TarStream()
 	if err != nil {
 		panic(err)
 	}
-	return string(data)
+	defer tarStream.Close()
+	tarReader := tar.NewReader(tarStream)
+	for {
+		tarHeader, err := tarReader.Next()
+		if err == io.EOF {
+			panic("no data file in package")
+		}
+		if err != nil {
+			panic(err)
+		}
+		if tarHeader.Name == "./data" {
+			data, err := io.ReadAll(tarReader)
+			if err != nil {
+				panic(err)
+			}
+			return string(data)
+		}
+	}
 }
 
 // fetchRequestStatus checks whether a request was made whose URL path

internal/archive/testarchive/testarchive.go

@@ -106,7 +106,10 @@ func (p *Package) Section() []byte {
 
 func (p *Package) Content() []byte {
 	if len(p.Data) == 0 {
-		return []byte(p.Name + " " + p.Version + " data")
+		return testutil.MustMakeDeb([]testutil.TarEntry{
+			testutil.Dir(0o755, "./"),
+			testutil.Reg(0o644, "./data", p.Name+" "+p.Version+" data"),
+		})
 	}
 	return p.Data
 }

internal/bin/extract.go

@@ -0,0 +1,31 @@
+package bin
+
+import (
+	"io"
+
+	"github.com/ulikunitz/xz"
+)
+
+type Pkg struct {
+	reader io.ReadSeekCloser
+}
+
+func Open(reader io.ReadSeekCloser) *Pkg {
+	return &Pkg{reader: reader}
+}
+
+func (p *Pkg) TarStream() (io.ReadCloser, error) {
+	_, err := p.reader.Seek(0, io.SeekStart)
+	if err != nil {
+		return nil, err
+	}
+	xzReader, err := xz.NewReader(p.reader)
+	if err != nil {
+		return nil, err
+	}
+	return io.NopCloser(xzReader), nil
+}
+
+func (p *Pkg) Close() error {
+	return p.reader.Close()
+}

internal/bin/extract_test.go

@@ -0,0 +1,38 @@
+package bin_test
+
+import (
+	"archive/tar"
+
+	. "gopkg.in/check.v1"
+
+	"github.com/canonical/chisel/internal/bin"
+	"github.com/canonical/chisel/internal/tarball"
+	"github.com/canonical/chisel/internal/testutil"
+)
+
+var _ tarball.TarStreamer = (*bin.Pkg)(nil)
+
+func (s *S) TestPkgTarStream(c *C) {
+	pkg := testutil.NewBinPkg(testutil.MustMakeBin([]testutil.TarEntry{
+		testutil.Dir(0o755, "./"),
+		testutil.Reg(0o644, "./file", "content"),
+	}))
+
+	// Each call returns a fresh stream over the same content.
+	for range 2 {
+		tarStream, err := pkg.TarStream()
+		c.Assert(err, IsNil)
+		tarReader := tar.NewReader(tarStream)
+		_, err = tarReader.Next()
+		c.Assert(err, IsNil)
+		err = tarStream.Close()
+		c.Assert(err, IsNil)
+	}
+}
+
+func (s *S) TestPkgTarStreamInvalid(c *C) {
+	pkg := testutil.NewBinPkg([]byte("not an xz stream"))
+
+	_, err := pkg.TarStream()
+	c.Assert(err, ErrorMatches, "xz.*")
+}

internal/bin/log.go

@@ -0,0 +1,53 @@
+package bin
+
+import (
+	"fmt"
+	"sync"
+)
+
+// Avoid importing the log type information unnecessarily.  There's a small cost
+// associated with using an interface rather than the type.  Depending on how
+// often the logger is plugged in, it would be worth using the type instead.
+type log_Logger interface {
+	Output(calldepth int, s string) error
+}
+
+var globalLoggerLock sync.Mutex
+var globalLogger log_Logger
+var globalDebug bool
+
+// Specify the *log.Logger object where log messages should be sent to.
+func SetLogger(logger log_Logger) {
+	globalLoggerLock.Lock()
+	globalLogger = logger
+	globalLoggerLock.Unlock()
+}
+
+// Enable the delivery of debug messages to the logger.  Only meaningful
+// if a logger is also set.
+func SetDebug(debug bool) {
+	globalLoggerLock.Lock()
+	globalDebug = debug
+	globalLoggerLock.Unlock()
+}
+
+// logf sends to the logger registered via SetLogger the string resulting
+// from running format and args through Sprintf.
+func logf(format string, args ...any) {
+	globalLoggerLock.Lock()
+	defer globalLoggerLock.Unlock()
+	if globalLogger != nil {
+		globalLogger.Output(2, fmt.Sprintf(format, args...))
+	}
+}
+
+// debugf sends to the logger registered via SetLogger the string resulting
+// from running format and args through Sprintf, but only if debugging was
+// enabled via SetDebug.
+func debugf(format string, args ...any) {
+	globalLoggerLock.Lock()
+	defer globalLoggerLock.Unlock()
+	if globalDebug && globalLogger != nil {
+		globalLogger.Output(2, fmt.Sprintf(format, args...))
+	}
+}

internal/bin/suite_test.go

@@ -0,0 +1,25 @@
+package bin_test
+
+import (
+	"testing"
+
+	. "gopkg.in/check.v1"
+
+	"github.com/canonical/chisel/internal/bin"
+)
+
+func Test(t *testing.T) { TestingT(t) }
+
+type S struct{}
+
+var _ = Suite(&S{})
+
+func (s *S) SetUpTest(c *C) {
+	bin.SetDebug(true)
+	bin.SetLogger(c)
+}
+
+func (s *S) TearDownTest(c *C) {
+	bin.SetDebug(false)
+	bin.SetLogger(nil)
+}

internal/deb/extract.go

@@ -10,10 +10,22 @@ import (
 	"github.com/ulikunitz/xz"
 )
 
-// DataReader takes a Reader for the ar file belonging to a Debian package and
-// returns a Reader to the inner tarball.
-func DataReader(pkgReader io.ReadSeeker) (io.ReadCloser, error) {
-	arReader := ar.NewReader(pkgReader)
+type Pkg struct {
+	reader io.ReadSeekCloser
+}
+
+func Open(reader io.ReadSeekCloser) *Pkg {
+	return &Pkg{reader: reader}
+}
+
+// TarStream returns a ReadCloser to the inner tarball of
+// a Debian package.
+func (p *Pkg) TarStream() (io.ReadCloser, error) {
+	_, err := p.reader.Seek(0, io.SeekStart)
+	if err != nil {
+		return nil, err
+	}
+	arReader := ar.NewReader(p.reader)
 	var dataReader io.ReadCloser
 	for dataReader == nil {
 		arHeader, err := arReader.Next()
@@ -47,3 +59,7 @@ func DataReader(pkgReader io.ReadSeeker) (io.ReadCloser, error) {
 
 	return dataReader, nil
 }
+
+func (p *Pkg) Close() error {
+	return p.reader.Close()
+}