diff --git a/internal/archive/archive.go b/internal/archive/archive.go index d301494b5..97c5c5644 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -29,15 +29,16 @@ type PackageInfo struct { Name string Version string Arch string - SHA256 string + Digests map[cache.DigestKind]string } -func (p *PackageInfo) PkgName() string { return p.Name } -func (p *PackageInfo) PkgVersion() string { return p.Version } -func (p *PackageInfo) PkgRevision() int { return 0 } -func (p *PackageInfo) PkgArch() string { return p.Arch } -func (p *PackageInfo) PkgDigestKind() cache.DigestKind { return cache.SHA256 } -func (p *PackageInfo) PkgDigest() string { return p.SHA256 } +func (p *PackageInfo) PkgName() string { return p.Name } +func (p *PackageInfo) PkgVersion() string { return p.Version } +func (p *PackageInfo) PkgRevision() int { return 0 } +func (p *PackageInfo) PkgArch() string { return p.Arch } +func (p *PackageInfo) PkgDigests() map[cache.DigestKind]string { + return p.Digests +} type Options struct { Label string @@ -146,13 +147,17 @@ func (a *ubuntuArchive) Fetch(pkg string) (io.ReadSeekCloser, *PackageInfo, erro return nil, nil, err } path := section.Get("Filename") - digest, digestKind := packageDigest(section) + digests, strongestKind, ok := packageDigests(section) + if !ok { + return nil, nil, fmt.Errorf("cannot find digest for package %q", pkg) + } + digest := digests[strongestKind] logf("Fetching %s...", path) - reader, err := index.fetch(path, digest, digestKind, fetchBulk) + reader, err := index.fetch(path, digest, strongestKind, fetchBulk) if err != nil { return nil, nil, err } - info := sectionPackageInfo(section) + info := sectionPackageInfo(section, digests) return reader, info, nil } @@ -161,7 +166,11 @@ func (a *ubuntuArchive) Info(pkg string) (*PackageInfo, error) { if err != nil { return nil, err } - info := sectionPackageInfo(section) + digests, _, ok := packageDigests(section) + if !ok { + return nil, fmt.Errorf("cannot find digest for package %q", pkg) + } + info := sectionPackageInfo(section, digests) return info, nil } @@ -338,48 +347,49 @@ func (index *ubuntuIndex) fetchRelease() error { return nil } -// digestField is an archive checksum field Chisel can verify. Its name -// doubles as the by-hash directory name in the archive layout. +// digestField is an archive checksum field Chisel can verify, along with +// the digest kind it carries. type digestField struct { name string kind cache.DigestKind } -// digestFields lists the checksum fields Chisel can verify, in order of -// preference: strongest first. The order also matches the by-hash archive -// layout, where only the by-hash directory of the strongest advertised hash -// is guaranteed to exist. +// digestFields lists the checksum fields Chisel looks up in archive index +// and package files, in order of preference: strongest first. Digest kinds +// weaker than SHA256 (e.g. MD5) are not looked up. var digestFields = []digestField{ {"SHA512", cache.SHA512}, {"SHA256", cache.SHA256}, } -// findDigest returns the digest recorded for path in the release, along with -// the field it was found in, trying the given fields in order. -func findDigest(release control.Section, path string, order []digestField) (digest string, field digestField) { - for _, f := range order { - if d, _, ok := control.ParsePathInfo(release.Get(f.name), path); ok { - return d, f +func findDigest(release control.Section, path string) (digest string, field digestField, ok bool) { + for _, f := range digestFields { + if d, _, found := control.ParsePathInfo(release.Get(f.name), path); found { + return d, f, true } } - return "", digestField{} + return "", digestField{}, false } -func packageDigest(section control.Section) (digest string, kind cache.DigestKind) { +func packageDigests(section control.Section) (all map[cache.DigestKind]string, strongest cache.DigestKind, ok bool) { + all = make(map[cache.DigestKind]string) for _, f := range digestFields { if d := section.Get(f.name); d != "" { - return d, f.kind + all[f.kind] = d + if !ok { + // digestFields is ordered strongest first, so the first + // digest found is the strongest. + strongest, ok = f.kind, true + } } } - // No digest advertised; fall back to SHA256 so the package can still be - // cached and retrieved by its computed digest. - return "", cache.SHA256 + return all, strongest, ok } func (index *ubuntuIndex) fetchIndex() error { packagesPath := fmt.Sprintf("%s/binary-%s/Packages", index.component, index.arch) - packagesDigest, field := findDigest(index.release, packagesPath, digestFields) - if packagesDigest == "" { + packagesDigest, field, ok := findDigest(index.release, packagesPath) + if !ok { return fmt.Errorf("%s is missing from %s %s component digests", packagesPath, index.suite, index.component) } @@ -396,8 +406,8 @@ func (index *ubuntuIndex) fetchIndex() error { // hash the archive advertises, which is what findDigest prefers. If // the archive advertises a hash stronger than any Chisel knows, the // URL may 404 and the named-path fallback below applies. - packagesGzDigest, byHashField := findDigest(index.release, packagesGzPath, digestFields) - if packagesGzDigest != "" { + packagesGzDigest, byHashField, ok := findDigest(index.release, packagesGzPath) + if ok { packagesByHashPath := fmt.Sprintf("%s/binary-%s/by-hash/%s/%s", index.component, index.arch, byHashField.name, packagesGzDigest) r, err := index.fetch(index.distPath(packagesByHashPath), packagesDigest, field.kind, fetchBulk|fetchGzip) if err != nil && err != errNotFound { @@ -516,12 +526,12 @@ func (index *ubuntuIndex) fetch(path, digest string, digestKind cache.DigestKind return index.archive.cache.Open(digestKind, writer.Digest()) } -func sectionPackageInfo(section control.Section) *PackageInfo { +func sectionPackageInfo(section control.Section, digests map[cache.DigestKind]string) *PackageInfo { return &PackageInfo{ Name: section.Get("Package"), Version: section.Get("Version"), Arch: section.Get("Architecture"), - SHA256: section.Get("SHA256"), + Digests: digests, } } diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index b89a8d71d..f30303ea7 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -5,7 +5,6 @@ import ( . "gopkg.in/check.v1" "crypto/sha256" - "crypto/sha512" "debug/elf" "errors" "flag" @@ -20,6 +19,7 @@ import ( "github.com/canonical/chisel/internal/archive" "github.com/canonical/chisel/internal/archive/testarchive" + "github.com/canonical/chisel/internal/cache" "github.com/canonical/chisel/internal/tarball" "github.com/canonical/chisel/internal/testutil" ) @@ -255,7 +255,7 @@ func (s *httpSuite) TestFetchPackage(c *C) { Name: "mypkg1", Version: "1.1", Arch: "amd64", - SHA256: "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05", + Digests: map[cache.DigestKind]string{cache.SHA256: "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05"}, }) c.Assert(read(pkg), Equals, "mypkg1 1.1 data") @@ -266,7 +266,7 @@ func (s *httpSuite) TestFetchPackage(c *C) { Name: "mypkg4", Version: "1.4", Arch: "amd64", - SHA256: "54af70097b30b33cfcbb6911ad3d0df86c2d458928169e348fa7873e4fc678e4", + Digests: map[cache.DigestKind]string{cache.SHA256: "54af70097b30b33cfcbb6911ad3d0df86c2d458928169e348fa7873e4fc678e4"}, }) c.Assert(read(pkg), Equals, "mypkg4 1.4 data") } @@ -274,12 +274,12 @@ func (s *httpSuite) TestFetchPackage(c *C) { func (s *httpSuite) TestFetchSHA512Digests(c *C) { // Ubuntu 26.10+ publishes SHA512-only indices (no SHA256 section), so both // the index digest and the package digest must be read from SHA512. - s.prepareArchiveAdjustRelease("stonking", "25.10", "amd64", []string{"main", "universe"}, + s.prepareArchiveAdjustRelease("stonking", "26.10", "amd64", []string{"main", "universe"}, []string{"SHA512"}, nil) options := archive.Options{ Label: "ubuntu", - Version: "25.10", + Version: "26.10", Arch: "amd64", Suites: []string{"stonking"}, Components: []string{"main", "universe"}, @@ -290,22 +290,27 @@ func (s *httpSuite) TestFetchSHA512Digests(c *C) { testArchive, err := archive.Open(&options) c.Assert(err, IsNil) - pkg, _, err := testArchive.Fetch("mypkg1") + pkg, info, err := testArchive.Fetch("mypkg1") c.Assert(err, IsNil) + c.Assert(info, DeepEquals, &archive.PackageInfo{ + Name: "mypkg1", + Version: "1.1", + Arch: "amd64", + Digests: map[cache.DigestKind]string{cache.SHA512: "27c6e88def3d3848f4a068040bddbf908ab90e33bf93fc24fd02af7ed6a1953151302f2c59306313f065163143b51f1000cd22d102b7a58d7efd6430f5e162fb"}, + }) c.Assert(read(pkg), Equals, "mypkg1 1.1 data") } -func (s *httpSuite) TestFetchBothDigests(c *C) { - // An archive publishing both SHA256 and SHA512 sections (index table and - // package fields) must be handled, with the strongest digest preferred - // for verification and caching. PackageInfo.SHA256 still surfaces: it is - // read from the package section directly, not from the preference order. - s.prepareArchiveAdjustRelease("stonking", "25.10", "amd64", []string{"main", "universe"}, +func (s *httpSuite) TestFetchMultipleDigests(c *C) { + // An archive publishing SHA256 and SHA512 sections (index table and + // package fields) must be handled. All published digests are recorded in + // the manifest; the strongest one is used for verification and caching. + s.prepareArchiveAdjustRelease("stonking", "26.10", "amd64", []string{"main", "universe"}, []string{"SHA256", "SHA512"}, nil) options := archive.Options{ Label: "ubuntu", - Version: "25.10", + Version: "26.10", Arch: "amd64", Suites: []string{"stonking"}, Components: []string{"main", "universe"}, @@ -322,14 +327,17 @@ func (s *httpSuite) TestFetchBothDigests(c *C) { Name: "mypkg1", Version: "1.1", Arch: "amd64", - SHA256: "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05", + Digests: map[cache.DigestKind]string{ + cache.SHA256: "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05", + cache.SHA512: "27c6e88def3d3848f4a068040bddbf908ab90e33bf93fc24fd02af7ed6a1953151302f2c59306313f065163143b51f1000cd22d102b7a58d7efd6430f5e162fb", + }, }) c.Assert(read(pkg), Equals, "mypkg1 1.1 data") - // Pin the cache key: with both digests advertised, the package is cached + // Pin the cache key: with multiple digests advertised, the package is cached // under its strongest digest. - sha512Digest := fmt.Sprintf("%x", sha512.Sum512([]byte("mypkg1 1.1 data"))) - _, err = os.Stat(filepath.Join(options.CacheDir, "sha512", sha512Digest)) + _, err = os.Stat(filepath.Join(options.CacheDir, "sha512", + "27c6e88def3d3848f4a068040bddbf908ab90e33bf93fc24fd02af7ed6a1953151302f2c59306313f065163143b51f1000cd22d102b7a58d7efd6430f5e162fb")) c.Assert(err, IsNil) } @@ -359,7 +367,7 @@ func (s *httpSuite) TestFetchPortsPackage(c *C) { Name: "mypkg1", Version: "1.1", Arch: "arm64", - SHA256: "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05", + Digests: map[cache.DigestKind]string{cache.SHA256: "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05"}, }) c.Assert(read(pkg), Equals, "mypkg1 1.1 data") @@ -370,7 +378,7 @@ func (s *httpSuite) TestFetchPortsPackage(c *C) { Name: "mypkg4", Version: "1.4", Arch: "arm64", - SHA256: "54af70097b30b33cfcbb6911ad3d0df86c2d458928169e348fa7873e4fc678e4", + Digests: map[cache.DigestKind]string{cache.SHA256: "54af70097b30b33cfcbb6911ad3d0df86c2d458928169e348fa7873e4fc678e4"}, }) c.Assert(read(pkg), Equals, "mypkg4 1.4 data") } @@ -410,7 +418,7 @@ func (s *httpSuite) TestFetchSecurityPackage(c *C) { Name: "mypkg1", Version: "1.1.2.2", Arch: "amd64", - SHA256: "5448585bdd916e5023eff2bc1bc3b30bcc6ee9db9c03e531375a6a11ddf0913c", + Digests: map[cache.DigestKind]string{cache.SHA256: "5448585bdd916e5023eff2bc1bc3b30bcc6ee9db9c03e531375a6a11ddf0913c"}, }) c.Assert(read(pkg), Equals, "package from jammy-security") @@ -420,7 +428,7 @@ func (s *httpSuite) TestFetchSecurityPackage(c *C) { Name: "mypkg2", Version: "1.2", Arch: "amd64", - SHA256: "a4b4f3f3a8fa09b69e3ba23c60a41a1f8144691fd371a2455812572fd02e6f79", + Digests: map[cache.DigestKind]string{cache.SHA256: "a4b4f3f3a8fa09b69e3ba23c60a41a1f8144691fd371a2455812572fd02e6f79"}, }) c.Assert(read(pkg), Equals, "mypkg2 1.2 data") } @@ -665,7 +673,7 @@ var packageInfoTests = []struct { Name: "mypkg1", Version: "1.1", Arch: "amd64", - SHA256: "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05", + Digests: map[cache.DigestKind]string{cache.SHA256: "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05"}, }, }, { summary: "Package not found in archive", diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 3b5621ecc..10a2e3a0f 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -9,6 +9,7 @@ import ( "io" "os" "path/filepath" + "slices" "time" "golang.org/x/crypto/sha3" @@ -96,11 +97,20 @@ type DigestKind string const ( SHA256 DigestKind = "sha256" - SHA384 DigestKind = "sha384" SHA512 DigestKind = "sha512" + SHA384 DigestKind = "sha384" ) -var digestKinds = []DigestKind{SHA256, SHA384, SHA512} +// digestKinds lists the digest kinds the cache supports, in order of +// strength: strongest first. +var digestKinds = []DigestKind{SHA384, SHA512, SHA256} + +func ValidateDigestKind(kind DigestKind) error { + if !slices.Contains(digestKinds, kind) { + return fmt.Errorf("unsupported digest kind: %q", kind) + } + return nil +} var ErrMiss = fmt.Errorf("not cached") diff --git a/internal/manifestutil/manifestutil.go b/internal/manifestutil/manifestutil.go index 3cf4db75d..932425646 100644 --- a/internal/manifestutil/manifestutil.go +++ b/internal/manifestutil/manifestutil.go @@ -26,8 +26,7 @@ type PackageInfo interface { // revisions. PkgRevision() int PkgArch() string - PkgDigestKind() cache.DigestKind - PkgDigest() string + PkgDigests() map[cache.DigestKind]string } const DefaultFilename = "manifest.wall" @@ -85,13 +84,17 @@ func Write(options *WriteOptions, writer io.Writer) error { func manifestAddPackages(dbw *jsonwall.DBWriter, infos []PackageInfo) error { for _, info := range infos { - err := dbw.Add(&manifest.Package{ - Kind: "package", + digests := make(map[string]string, len(info.PkgDigests())) + for kind, digest := range info.PkgDigests() { + digests[string(kind)] = digest + } + pkg := manifest.NewPackage(&manifest.PackageOptions{ Name: info.PkgName(), Version: info.PkgVersion(), - Digest: info.PkgDigest(), Arch: info.PkgArch(), + Digests: digests, }) + err := dbw.Add(pkg) if err != nil { return err } @@ -272,13 +275,18 @@ func validatePackage(pkg PackageInfo) (err error) { if pkg.PkgArch() == "" { return fmt.Errorf("package %q missing arch", name) } - // The manifest records the package digest as a SHA256 one. Fail rather - // than recording a digest of another kind under that name. - // TODO: record packages whose digest is not a SHA256 one, such as the - // ones coming from a store. This requires recording the digest kind in - // the manifest as well. - if pkg.PkgDigestKind() != cache.SHA256 || pkg.PkgDigest() == "" { - return fmt.Errorf("package %q missing sha256", name) + digests := pkg.PkgDigests() + if len(digests) == 0 { + return fmt.Errorf("package %q missing digests", name) + } + for kind, digest := range digests { + err = cache.ValidateDigestKind(kind) + if err != nil { + return fmt.Errorf("package %q: %s", name, err) + } + if digest == "" { + return fmt.Errorf("package %q has empty %s digest", name, kind) + } } if pkg.PkgVersion() == "" { return fmt.Errorf("package %q missing version", name) diff --git a/internal/manifestutil/manifestutil_test.go b/internal/manifestutil/manifestutil_test.go index e7e68752d..19eaaf43b 100644 --- a/internal/manifestutil/manifestutil_test.go +++ b/internal/manifestutil/manifestutil_test.go @@ -13,6 +13,7 @@ import ( "github.com/canonical/chisel/internal/apachetestutil" "github.com/canonical/chisel/internal/archive" + "github.com/canonical/chisel/internal/cache" "github.com/canonical/chisel/internal/manifestutil" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/public/manifest" @@ -153,13 +154,13 @@ var generateManifestTests = []struct { Name: "package1", Version: "v1", Arch: "a1", - SHA256: "s1", + Digests: map[cache.DigestKind]string{cache.SHA256: "s1"}, }, &archive.PackageInfo{ Name: "package2", Version: "v2", Arch: "a2", - SHA256: "s2", + Digests: map[cache.DigestKind]string{cache.SHA256: "s2"}, }, }, expected: &apachetestutil.ManifestContents{ @@ -183,12 +184,14 @@ var generateManifestTests = []struct { Name: "package1", Version: "v1", Digest: "s1", + Digests: map[string]string{"sha256": "s1"}, Arch: "a1", }, { Kind: "package", Name: "package2", Version: "v2", Digest: "s2", + Digests: map[string]string{"sha256": "s2"}, Arch: "a2", }}, Slices: []*manifest.Slice{{ @@ -212,6 +215,97 @@ var generateManifestTests = []struct { Path: "/link", }}, }, +}, { + summary: "SHA512 package digest", + selection: []*setup.Slice{slice1}, + report: &manifestutil.Report{ + Root: "/", + Entries: map[string]manifestutil.ReportEntry{ + "/file": { + Path: "/file", + Mode: 0o456, + Slices: map[*setup.Slice]bool{slice1: true}, + }, + }, + }, + packageInfo: []manifestutil.PackageInfo{ + &archive.PackageInfo{ + Name: "package1", + Version: "v1", + Arch: "a1", + Digests: map[cache.DigestKind]string{cache.SHA512: "s512"}, + }, + }, + expected: &apachetestutil.ManifestContents{ + Paths: []*manifest.Path{{ + Kind: "path", + Path: "/file", + Mode: "0456", + Slices: []string{"package1_slice1"}, + }}, + Packages: []*manifest.Package{{ + Kind: "package", + Name: "package1", + Version: "v1", + Digest: "s512", + Digests: map[string]string{"sha512": "s512"}, + Arch: "a1", + }}, + Slices: []*manifest.Slice{{ + Kind: "slice", + Name: "package1_slice1", + }}, + Contents: []*manifest.Content{{ + Kind: "content", + Slice: "package1_slice1", + Path: "/file", + }}, + }, +}, { + summary: "SHA384 package digest", + selection: []*setup.Slice{slice1}, + report: &manifestutil.Report{ + Root: "/", + Entries: map[string]manifestutil.ReportEntry{ + "/file": { + Path: "/file", + Mode: 0o456, + Slices: map[*setup.Slice]bool{slice1: true}, + }, + }, + }, + packageInfo: []manifestutil.PackageInfo{ + &archive.PackageInfo{ + Name: "package1", + Version: "v1", + Arch: "a1", + Digests: map[cache.DigestKind]string{cache.SHA384: "s384"}, + }, + }, + expected: &apachetestutil.ManifestContents{ + Paths: []*manifest.Path{{ + Kind: "path", + Path: "/file", + Mode: "0456", + Slices: []string{"package1_slice1"}, + }}, + Packages: []*manifest.Package{{ + Kind: "package", + Name: "package1", + Version: "v1", + Digests: map[string]string{"sha384": "s384"}, + Arch: "a1", + }}, + Slices: []*manifest.Slice{{ + Kind: "slice", + Name: "package1_slice1", + }}, + Contents: []*manifest.Content{{ + Kind: "content", + Slice: "package1_slice1", + Path: "/file", + }}, + }, }, { summary: "Missing slice", report: &manifestutil.Report{ @@ -403,7 +497,7 @@ var generateManifestTests = []struct { Name: "package1", Version: "v1", Arch: "a1", - SHA256: "s1", + Digests: map[cache.DigestKind]string{cache.SHA256: "s1"}, }, }, expected: &apachetestutil.ManifestContents{ @@ -431,6 +525,7 @@ var generateManifestTests = []struct { Name: "package1", Version: "v1", Digest: "s1", + Digests: map[string]string{"sha256": "s1"}, Arch: "a1", }}, Slices: []*manifest.Slice{{ @@ -503,7 +598,7 @@ var generateManifestTests = []struct { &archive.PackageInfo{ Version: "v1", Arch: "a1", - SHA256: "s1", + Digests: map[cache.DigestKind]string{cache.SHA256: "s1"}, }, }, error: `internal error: invalid manifest: package name not set`, @@ -511,9 +606,9 @@ var generateManifestTests = []struct { summary: "Invalid package: missing version", packageInfo: []manifestutil.PackageInfo{ &archive.PackageInfo{ - Name: "package-1", - Arch: "a1", - SHA256: "s1", + Name: "package-1", + Arch: "a1", + Digests: map[cache.DigestKind]string{cache.SHA256: "s1"}, }, }, error: `internal error: invalid manifest: package "package-1" missing version`, @@ -523,12 +618,12 @@ var generateManifestTests = []struct { &archive.PackageInfo{ Name: "package-1", Version: "v1", - SHA256: "s1", + Digests: map[cache.DigestKind]string{cache.SHA256: "s1"}, }, }, error: `internal error: invalid manifest: package "package-1" missing arch`, }, { - summary: "Invalid package: missing sha256", + summary: "Invalid package: missing digests", packageInfo: []manifestutil.PackageInfo{ &archive.PackageInfo{ Name: "package-1", @@ -536,7 +631,29 @@ var generateManifestTests = []struct { Arch: "a1", }, }, - error: `internal error: invalid manifest: package "package-1" missing sha256`, + error: `internal error: invalid manifest: package "package-1" missing digests`, +}, { + summary: "Invalid package: unsupported digest kind", + packageInfo: []manifestutil.PackageInfo{ + &archive.PackageInfo{ + Name: "package-1", + Version: "v1", + Arch: "a1", + Digests: map[cache.DigestKind]string{cache.DigestKind("md5"): "s1"}, + }, + }, + error: `internal error: invalid manifest: package "package-1": unsupported digest kind: "md5"`, +}, { + summary: "Invalid package: empty digest", + packageInfo: []manifestutil.PackageInfo{ + &archive.PackageInfo{ + Name: "package-1", + Version: "v1", + Arch: "a1", + Digests: map[cache.DigestKind]string{cache.SHA256: ""}, + }, + }, + error: `internal error: invalid manifest: package "package-1" has empty sha256 digest`, }} func (s *S) TestGenerateManifests(c *C) { @@ -551,7 +668,7 @@ func (s *S) TestGenerateManifests(c *C) { Name: "package1", Version: "v1", Arch: "a1", - SHA256: "s1", + Digests: map[cache.DigestKind]string{cache.SHA256: "s1"}, }, } } @@ -639,6 +756,27 @@ var validateManifestTests = []struct { {"kind":"slice","name":"pkg1_myslice"} `, error: `invalid manifest: content path /dir/ has no matching entry in paths`, +}, { + summary: "Package with sha512 digest", + input: ` + {"jsonwall":"1.0","schema":"1.0","count":2} + {"kind":"package","name":"pkg1","version":"v1","sha512":"hash1","arch":"arch1"} + {"kind":"slice","name":"pkg1_myslice"} + `, +}, { + summary: "Package with sha384 digest", + input: ` + {"jsonwall":"1.0","schema":"1.0","count":2} + {"kind":"package","name":"pkg1","version":"v1","sha384":"hash1","arch":"arch1"} + {"kind":"slice","name":"pkg1_myslice"} + `, +}, { + summary: "Package with multiple digests", + input: ` + {"jsonwall":"1.0","schema":"1.0","count":2} + {"kind":"package","name":"pkg1","version":"v1","sha256":"hash1","sha512":"hash2","arch":"arch1"} + {"kind":"slice","name":"pkg1_myslice"} + `, }, { summary: "Malformed jsonwall", input: ` diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index d6ef9ca0d..d4b226aa0 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -15,6 +15,7 @@ import ( . "gopkg.in/check.v1" "github.com/canonical/chisel/internal/archive" + "github.com/canonical/chisel/internal/cache" "github.com/canonical/chisel/internal/manifestutil" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" @@ -783,25 +784,25 @@ var slicerTests = []slicerTest{{ slices: []setup.SliceKey{{"test-package", "myslice"}, {"other-package", "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", - Hash: "h1", + Digests: map[cache.DigestKind]string{cache.SHA256: "h1"}, Version: "v1", Arch: "a1", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Reg(0644, "./file", "from foo"), + testutil.Reg(0o644, "./file", "from foo"), }), Archives: []string{"foo"}, }, { Name: "test-package", - Hash: "h2", + Digests: map[cache.DigestKind]string{cache.SHA256: "h2"}, Version: "v2", Arch: "a2", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Reg(0644, "./file", "from bar"), + testutil.Reg(0o644, "./file", "from bar"), }), Archives: []string{"bar"}, }, { Name: "other-package", - Hash: "h3", + Digests: map[cache.DigestKind]string{cache.SHA256: "h3"}, Version: "v3", Arch: "a3", Data: testutil.MustMakeDeb([]testutil.TarEntry{ @@ -859,24 +860,24 @@ var slicerTests = []slicerTest{{ "/other-file": "file 0644 fa0c9cdb {other-package_myslice}", }, manifestPkgs: map[string]string{ - "test-package": "test-package v1 a1 h1", - "other-package": "other-package v3 a3 h3", + "test-package": "test-package v1 a1 sha256=h1", + "other-package": "other-package v3 a3 sha256=h3", }, }, { summary: "Pinned archive bypasses higher priority", slices: []setup.SliceKey{{"test-package", "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", - Hash: "h1", + Digests: map[cache.DigestKind]string{cache.SHA256: "h1"}, Version: "v1", Arch: "a1", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Reg(0644, "./file", "from foo"), + testutil.Reg(0o644, "./file", "from foo"), }), Archives: []string{"foo"}, }, { Name: "test-package", - Hash: "h2", + Digests: map[cache.DigestKind]string{cache.SHA256: "h2"}, Version: "v2", Arch: "a2", Data: testutil.MustMakeDeb([]testutil.TarEntry{ @@ -928,7 +929,7 @@ var slicerTests = []slicerTest{{ "/file": "file 0644 fa0c9cdb {test-package_myslice}", }, manifestPkgs: map[string]string{ - "test-package": "test-package v2 a2 h2", + "test-package": "test-package v2 a2 sha256=h2", }, }, { summary: "Pinned archive does not have the package", @@ -1055,7 +1056,7 @@ var slicerTests = []slicerTest{{ slices: []setup.SliceKey{{"test-package", "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", - Hash: "h1", + Digests: map[cache.DigestKind]string{cache.SHA256: "h1"}, Version: "v1", Arch: "a1", Data: testutil.MustMakeDeb([]testutil.TarEntry{ @@ -1097,7 +1098,70 @@ var slicerTests = []slicerTest{{ "/file": "file 0644 7a3e00f5 {test-package_myslice}", }, manifestPkgs: map[string]string{ - "test-package": "test-package v1 a1 h1", + "test-package": "test-package v1 a1 sha256=h1", + }, +}, { + summary: "Package with sha512 digest is recorded in the manifest", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + pkgs: []*testutil.TestPackage{{ + Name: "test-package", + Digests: map[cache.DigestKind]string{cache.SHA512: "h1"}, + Version: "v1", + Arch: "a1", + Data: testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Reg(0o644, "./file", "from foo"), + }), + }}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /file: + `, + }, + filesystem: map[string]string{ + "/file": "file 0644 7a3e00f5", + }, + manifestPaths: map[string]string{ + "/file": "file 0644 7a3e00f5 {test-package_myslice}", + }, + manifestPkgs: map[string]string{ + "test-package": "test-package v1 a1 sha512=h1", + }, +}, { + summary: "Package with multiple digests is recorded in the manifest", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + pkgs: []*testutil.TestPackage{{ + Name: "test-package", + Digests: map[cache.DigestKind]string{ + cache.SHA256: "h256", + cache.SHA512: "h512", + }, + Version: "v1", + Arch: "a1", + Data: testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Reg(0o644, "./file", "from foo"), + }), + }}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /file: + `, + }, + filesystem: map[string]string{ + "/file": "file 0644 7a3e00f5", + }, + manifestPaths: map[string]string{ + "/file": "file 0644 7a3e00f5 {test-package_myslice}", + }, + manifestPkgs: map[string]string{ + "test-package": "test-package v1 a1 sha256=h256,sha512=h512", }, }, { summary: "Multiple slices of same package", @@ -1360,13 +1424,13 @@ var slicerTests = []slicerTest{{ }, pkgs: []*testutil.TestPackage{{ Name: "test-package", - Hash: "h1", + Digests: map[cache.DigestKind]string{cache.SHA256: "h1"}, Version: "v1", Arch: "a1", Data: testutil.PackageData["test-package"], }, { Name: "other-package", - Hash: "h2", + Digests: map[cache.DigestKind]string{cache.SHA256: "h2"}, Version: "v2", Arch: "a2", Data: testutil.PackageData["other-package"], @@ -1386,8 +1450,8 @@ var slicerTests = []slicerTest{{ `, }, manifestPkgs: map[string]string{ - "test-package": "test-package v1 a1 h1", - "other-package": "other-package v2 a2 h2", + "test-package": "test-package v1 a1 sha256=h1", + "other-package": "other-package v2 a2 sha256=h2", }, }, { summary: "Two packages, only one is selected and recorded", @@ -1396,13 +1460,13 @@ var slicerTests = []slicerTest{{ }, pkgs: []*testutil.TestPackage{{ Name: "test-package", - Hash: "h1", + Digests: map[cache.DigestKind]string{cache.SHA256: "h1"}, Version: "v1", Arch: "a1", Data: testutil.PackageData["test-package"], }, { Name: "other-package", - Hash: "h2", + Digests: map[cache.DigestKind]string{cache.SHA256: "h2"}, Version: "v2", Arch: "a2", Data: testutil.PackageData["other-package"], @@ -1422,7 +1486,7 @@ var slicerTests = []slicerTest{{ `, }, manifestPkgs: map[string]string{ - "test-package": "test-package v1 a1 h1", + "test-package": "test-package v1 a1 sha256=h1", }, }, { summary: "Relative paths are properly trimmed during extraction", @@ -2054,8 +2118,8 @@ func runSlicerTests(s *S, c *C, tests []slicerTest) { if pkg.Arch == "" { pkg.Arch = "arch" } - if pkg.Hash == "" { - pkg.Hash = "hash" + if pkg.Digests == nil { + pkg.Digests = map[cache.DigestKind]string{cache.SHA256: "hash"} } if pkg.Version == "" { pkg.Version = "version" @@ -2220,7 +2284,16 @@ func treeDumpManifestPaths(mfest *manifest.Manifest) (map[string]string, error) func dumpManifestPkgs(mfest *manifest.Manifest) (map[string]string, error) { result := map[string]string{} err := mfest.IteratePackages(func(pkg *manifest.Package) error { - result[pkg.Name] = fmt.Sprintf("%s %s %s %s", pkg.Name, pkg.Version, pkg.Arch, pkg.Digest) + kinds := make([]string, 0, len(pkg.Digests)) + for kind := range pkg.Digests { + kinds = append(kinds, kind) + } + sort.Strings(kinds) + digests := make([]string, 0, len(kinds)) + for _, kind := range kinds { + digests = append(digests, kind+"="+pkg.Digests[kind]) + } + result[pkg.Name] = fmt.Sprintf("%s %s %s %s", pkg.Name, pkg.Version, pkg.Arch, strings.Join(digests, ",")) return nil }) if err != nil { diff --git a/internal/testutil/archive.go b/internal/testutil/archive.go index d06fd1b0c..7c6e57c4c 100644 --- a/internal/testutil/archive.go +++ b/internal/testutil/archive.go @@ -6,6 +6,7 @@ import ( "io" "github.com/canonical/chisel/internal/archive" + "github.com/canonical/chisel/internal/cache" ) type TestArchive struct { @@ -16,7 +17,7 @@ type TestArchive struct { type TestPackage struct { Name string Version string - Hash string + Digests map[cache.DigestKind]string Arch string Data []byte Archives []string @@ -34,7 +35,7 @@ func (a *TestArchive) Fetch(pkgName string) (io.ReadSeekCloser, *archive.Package info := &archive.PackageInfo{ Name: pkg.Name, Version: pkg.Version, - SHA256: pkg.Hash, + Digests: pkg.Digests, Arch: pkg.Arch, } return ReadSeekNopCloser(bytes.NewReader(pkg.Data)), info, nil @@ -53,7 +54,7 @@ func (a *TestArchive) Info(pkgName string) (*archive.PackageInfo, error) { return &archive.PackageInfo{ Name: pkg.Name, Version: pkg.Version, - SHA256: pkg.Hash, + Digests: pkg.Digests, Arch: pkg.Arch, }, nil } diff --git a/public/manifest/manifest.go b/public/manifest/manifest.go index 1e4809b8b..fe6a225c0 100644 --- a/public/manifest/manifest.go +++ b/public/manifest/manifest.go @@ -3,8 +3,10 @@ package manifest import ( + "encoding/json" "fmt" "io" + "maps" "github.com/canonical/chisel/public/jsonwall" ) @@ -12,13 +14,99 @@ import ( const Schema = "1.0" type Package struct { + Kind string + Name string + Version string + // Digest holds the sha256 digest when present, and the sha512 digest + // otherwise. It is empty when neither is recorded. + Digest string + // Digests holds the digests of the package, keyed by digest kind + // (e.g. "sha256"). + Digests map[string]string + Arch string +} + +type packageJSON struct { Kind string `json:"kind"` Name string `json:"name,omitempty"` Version string `json:"version,omitempty"` - Digest string `json:"sha256,omitempty"` + SHA256 string `json:"sha256,omitempty"` + SHA512 string `json:"sha512,omitempty"` + SHA384 string `json:"sha384,omitempty"` Arch string `json:"arch,omitempty"` } +type PackageOptions struct { + Name string + Version string + Arch string + Digests map[string]string +} + +func NewPackage(opts *PackageOptions) *Package { + digests := make(map[string]string, len(opts.Digests)) + maps.Copy(digests, opts.Digests) + digest := digests["sha256"] + if digest == "" { + digest = digests["sha512"] + } + return &Package{ + Kind: "package", + Name: opts.Name, + Version: opts.Version, + Digest: digest, + Digests: digests, + Arch: opts.Arch, + } +} + +func (p *Package) MarshalJSON() ([]byte, error) { + pj := packageJSON{ + Kind: p.Kind, + Name: p.Name, + Version: p.Version, + Arch: p.Arch, + } + for kind, digest := range p.Digests { + switch kind { + case "sha256": + pj.SHA256 = digest + case "sha512": + pj.SHA512 = digest + case "sha384": + pj.SHA384 = digest + default: + return nil, fmt.Errorf("cannot marshal package %q: unsupported digest kind %q", p.Name, kind) + } + } + return json.Marshal(pj) +} + +func (p *Package) UnmarshalJSON(data []byte) error { + var pj packageJSON + err := json.Unmarshal(data, &pj) + if err != nil { + return err + } + digests := make(map[string]string) + for kind, digest := range map[string]string{ + "sha256": pj.SHA256, + "sha512": pj.SHA512, + "sha384": pj.SHA384, + } { + if digest != "" { + digests[kind] = digest + } + } + *p = *NewPackage(&PackageOptions{ + Name: pj.Name, + Version: pj.Version, + Arch: pj.Arch, + Digests: digests, + }) + return nil +} + type Slice struct { Kind string `json:"kind"` Name string `json:"name,omitempty"` diff --git a/public/manifest/manifest_test.go b/public/manifest/manifest_test.go index d710e121d..5e85d9c6a 100644 --- a/public/manifest/manifest_test.go +++ b/public/manifest/manifest_test.go @@ -3,6 +3,7 @@ package manifest_test import ( + "encoding/json" "os" "path" "slices" @@ -52,8 +53,8 @@ var readManifestTests = []struct { {Kind: "path", Path: "/manifest/manifest.wall", Mode: "0644", Slices: []string{"pkg1_manifest"}, SHA256: "", FinalSHA256: "", Size: 0x0, Link: ""}, }, Packages: []*manifest.Package{ - {Kind: "package", Name: "pkg1", Version: "v1", Digest: "hash1", Arch: "arch1"}, - {Kind: "package", Name: "pkg2", Version: "v2", Digest: "hash2", Arch: "arch2"}, + {Kind: "package", Name: "pkg1", Version: "v1", Digests: map[string]string{"sha256": "hash1"}, Digest: "hash1", Arch: "arch1"}, + {Kind: "package", Name: "pkg2", Version: "v2", Digests: map[string]string{"sha256": "hash2"}, Digest: "hash2", Arch: "arch2"}, }, Slices: []*manifest.Slice{ {Kind: "slice", Name: "pkg1_manifest"}, @@ -70,6 +71,39 @@ var readManifestTests = []struct { {Kind: "content", Slice: "pkg2_myotherslice", Path: "/dir/foo/bar/"}, }, }, +}, { + summary: "SHA512 package digest", + input: ` + {"jsonwall":"1.0","schema":"1.0","count":1} + {"kind":"package","name":"pkg1","version":"v1","sha512":"hash1","arch":"arch1"} + `, + mfest: &apachetestutil.ManifestContents{ + Packages: []*manifest.Package{ + {Kind: "package", Name: "pkg1", Version: "v1", Digests: map[string]string{"sha512": "hash1"}, Digest: "hash1", Arch: "arch1"}, + }, + }, +}, { + summary: "SHA384 package digest", + input: ` + {"jsonwall":"1.0","schema":"1.0","count":1} + {"kind":"package","name":"pkg1","version":"v1","sha384":"hash1","arch":"arch1"} + `, + mfest: &apachetestutil.ManifestContents{ + Packages: []*manifest.Package{ + {Kind: "package", Name: "pkg1", Version: "v1", Digests: map[string]string{"sha384": "hash1"}, Arch: "arch1"}, + }, + }, +}, { + summary: "Multiple digests recorded", + input: ` + {"jsonwall":"1.0","schema":"1.0","count":1} + {"kind":"package","name":"pkg1","version":"v1","sha256":"hash1","sha512":"hash2","arch":"arch1"} + `, + mfest: &apachetestutil.ManifestContents{ + Packages: []*manifest.Package{ + {Kind: "package", Name: "pkg1", Version: "v1", Digests: map[string]string{"sha256": "hash1", "sha512": "hash2"}, Digest: "hash1", Arch: "arch1"}, + }, + }, }, { summary: "Unknown schema", input: ` @@ -108,6 +142,13 @@ func (s *S) TestManifestRead(c *C) { defer r.Close() mfest, err := manifest.Read(r) + if err == nil { + // Entry-level errors surface while iterating, as the manifest + // is not fully decoded on read. + err = mfest.IteratePackages(func(pkg *manifest.Package) error { + return nil + }) + } if test.error != "" { c.Assert(err, ErrorMatches, test.error) continue @@ -118,3 +159,72 @@ func (s *S) TestManifestRead(c *C) { } } } + +var marshalPackageTests = []struct { + summary string + pkg *manifest.Package + expected string + error string +}{{ + summary: "SHA256 digest", + pkg: &manifest.Package{ + Kind: "package", + Name: "pkg1", + Version: "v1", + Digests: map[string]string{"sha256": "hash1"}, + Arch: "arch1", + }, + expected: `{"kind":"package","name":"pkg1","version":"v1","sha256":"hash1","arch":"arch1"}`, +}, { + summary: "SHA512 digest", + pkg: &manifest.Package{ + Kind: "package", + Name: "pkg1", + Version: "v1", + Digests: map[string]string{"sha512": "hash1"}, + Arch: "arch1", + }, + expected: `{"kind":"package","name":"pkg1","version":"v1","sha512":"hash1","arch":"arch1"}`, +}, { + summary: "No digest recorded", + pkg: &manifest.Package{ + Kind: "package", + Name: "pkg1", + Version: "v1", + Arch: "arch1", + }, + expected: `{"kind":"package","name":"pkg1","version":"v1","arch":"arch1"}`, +}, { + summary: "Multiple digests recorded", + pkg: &manifest.Package{ + Kind: "package", + Name: "pkg1", + Version: "v1", + Digests: map[string]string{"sha256": "hash1", "sha512": "hash2"}, + Arch: "arch1", + }, + expected: `{"kind":"package","name":"pkg1","version":"v1","sha256":"hash1","sha512":"hash2","arch":"arch1"}`, +}, { + summary: "Unsupported digest kind", + pkg: &manifest.Package{ + Kind: "package", + Name: "pkg1", + Version: "v1", + Digests: map[string]string{"md5": "hash1"}, + Arch: "arch1", + }, + error: `json: error calling MarshalJSON for type \*manifest\.Package: cannot marshal package "pkg1": unsupported digest kind "md5"`, +}} + +func (s *S) TestMarshalPackage(c *C) { + for _, test := range marshalPackageTests { + c.Logf("Summary: %s", test.summary) + data, err := json.Marshal(test.pkg) + if test.error != "" { + c.Assert(err, ErrorMatches, test.error) + continue + } + c.Assert(err, IsNil) + c.Assert(string(data), Equals, test.expected) + } +}