From baee503a246c313876ed4d762406fc5006c771b5 Mon Sep 17 00:00:00 2001 From: "Calvin A. Allen" Date: Thu, 12 Nov 2026 20:18:03 -0500 Subject: [PATCH 1/3] feat(mirror): publish python.org's Windows builds at their own location Every python.org build the mirror assembles is now also published at python//python.org/.tar.gz, alongside the version's default build. collectJobs dedupes on the metadata key rather than version and platform, so python-build-standalone keeps the default location and python.org's build still gets its own. Versions python-build-standalone lacks keep getting python.org's build at the default location, as before. From 3.14 on python.org's per-component MSIs carry no OpenPGP signature, so windows-amd64 builds from 3.14 come from the zips python.org's install manager installs, verified against the SHA-256 in python.org's index-windows.json. --- .../mirror-binaries/python_installmanager.go | 175 +++++++++++++++++ .../python_installmanager_test.go | 179 ++++++++++++++++++ scripts/mirror-binaries/python_winbuild.go | 86 ++++++--- .../python_winbuild_network_test.go | 35 ++++ .../mirror-binaries/python_winbuild_test.go | 73 ++++--- scripts/mirror-binaries/sources.go | 20 +- scripts/mirror-binaries/sources_test.go | 31 ++- 7 files changed, 534 insertions(+), 65 deletions(-) create mode 100644 scripts/mirror-binaries/python_installmanager.go create mode 100644 scripts/mirror-binaries/python_installmanager_test.go diff --git a/scripts/mirror-binaries/python_installmanager.go b/scripts/mirror-binaries/python_installmanager.go new file mode 100644 index 0000000..bf24ad8 --- /dev/null +++ b/scripts/mirror-binaries/python_installmanager.go @@ -0,0 +1,175 @@ +package main + +// python.org's builds of Python 3.14 and later for windows-amd64. +// +// From 3.14 on python.org signs its release files with Sigstore only, and +// its per-component MSIs carry no signature of their own, so PythonMSISource +// cannot verify them. python.org's Python install manager installs from +// full-install zips instead, listed with their SHA-256 in an index on +// python.org. This source repackages those zips, checked against that +// index, into the same python/ layout as the other Windows builds. + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "net/url" + "regexp" + "strings" +) + +const ( + // pythonInstallManagerIndexURL is the first page of the index the Python + // install manager reads. Each page names the next in "next". + pythonInstallManagerIndexURL = "https://www.python.org/ftp/python/index-windows.json" + + // maxInstallManagerIndexPages bounds how many "next" links are followed. + maxInstallManagerIndexPages = 10 +) + +// installManagerAMD64ID matches the index id of the regular 64-bit x86 +// build of a release ("pythoncore-3.14-64"), not the free-threaded ("3.14t") +// or 32-bit ones. +var installManagerAMD64ID = regexp.MustCompile(`^pythoncore-(\d+\.\d+)-64$`) + +// PythonInstallManagerSource repackages python.org's install-manager zips +// for windows-amd64, from firstSigstoreOnlyPython on. +type PythonInstallManagerSource struct { + // indexURL is the index's first page; tests point it at a local server. + indexURL string + // downloadPrefix is the only place zips may come from; tests point it + // at a local server. + downloadPrefix string +} + +func (s *PythonInstallManagerSource) Name() string { + return "python.org install manager" +} + +// installManagerIndex is one page of the install manager's index. +type installManagerIndex struct { + Versions []installManagerEntry `json:"versions"` + Next string `json:"next"` +} + +type installManagerEntry struct { + ID string `json:"id"` + SortVersion string `json:"sort-version"` + URL string `json:"url"` + Hash struct { + SHA256 string `json:"sha256"` + } `json:"hash"` +} + +// FetchVersions returns jobs for every x.y.z release from +// firstSigstoreOnlyPython on that the index lists for 64-bit x86, from +// python.org, with a SHA-256. +func (s *PythonInstallManagerSource) FetchVersions() ([]MirrorJob, error) { + entries, err := s.fetchIndex() + if err != nil { + return nil, err + } + + prefix := s.downloadPrefix + if prefix == "" { + prefix = pythonFTPBaseURL + } + + var jobs []MirrorJob + seen := make(map[string]bool) + for _, e := range entries { + m := installManagerAMD64ID.FindStringSubmatch(e.ID) + if m == nil { + continue + } + parts, ok := parseVersion(e.SortVersion) + if !ok || compareVersions(parts, firstSigstoreOnlyPython) < 0 { + continue + } + if !strings.HasPrefix(e.SortVersion, m[1]+".") { + continue + } + if !strings.HasPrefix(e.URL, prefix) || !strings.HasSuffix(e.URL, ".zip") || !isSHA256Hex(e.Hash.SHA256) { + continue + } + if seen[e.SortVersion] { + continue + } + seen[e.SortVersion] = true + + zipURL, want := e.URL, strings.ToLower(e.Hash.SHA256) + jobs = append(jobs, pythonOrgJobs(e.SortVersion, "windows-amd64", zipURL, func() ([]byte, error) { + return buildFromInstallManagerZip(zipURL, want) + })...) + } + return jobs, nil +} + +// fetchIndex reads every page of the index, following "next" links. +func (s *PythonInstallManagerSource) fetchIndex() ([]installManagerEntry, error) { + page := s.indexURL + if page == "" { + page = pythonInstallManagerIndexURL + } + + var entries []installManagerEntry + visited := make(map[string]bool) + for page != "" { + if visited[page] { + return nil, fmt.Errorf("index page %s links back to itself", page) + } + if len(visited) == maxInstallManagerIndexPages { + return nil, fmt.Errorf("index has more than %d pages", maxInstallManagerIndexPages) + } + visited[page] = true + + body, err := download(page) + if err != nil { + return nil, err + } + var index installManagerIndex + if err := json.Unmarshal(body, &index); err != nil { + return nil, fmt.Errorf("parsing %s: %w", page, err) + } + entries = append(entries, index.Versions...) + + if index.Next == "" { + break + } + base, err := url.Parse(page) + if err != nil { + return nil, err + } + next, err := base.Parse(index.Next) + if err != nil { + return nil, fmt.Errorf("next page %q in %s: %w", index.Next, page, err) + } + page = next.String() + } + return entries, nil +} + +// buildFromInstallManagerZip downloads an install-manager zip, verifies it +// against the SHA-256 the index lists and repackages it. +func buildFromInstallManagerZip(zipURL, wantSHA256 string) ([]byte, error) { + pkg, err := download(zipURL) + if err != nil { + return nil, err + } + sum := sha256.Sum256(pkg) + if got := hex.EncodeToString(sum[:]); got != wantSHA256 { + return nil, fmt.Errorf("%s: SHA-256 mismatch: expected %s, got %s", zipURL, wantSHA256, got) + } + // The zips hold tkinter but not the Tcl/Tk script library. + return archiveZipTree(pkg, "", false) +} + +// isSHA256Hex reports whether s is a hex-encoded SHA-256 digest. +func isSHA256Hex(s string) bool { + if len(s) != sha256.Size*2 { + return false + } + _, err := hex.DecodeString(s) + return err == nil +} diff --git a/scripts/mirror-binaries/python_installmanager_test.go b/scripts/mirror-binaries/python_installmanager_test.go new file mode 100644 index 0000000..2e3fe5e --- /dev/null +++ b/scripts/mirror-binaries/python_installmanager_test.go @@ -0,0 +1,179 @@ +package main + +import ( + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" +) + +// installManagerZipEntries is a full install as the install-manager zips +// lay it out: everything at the root of the zip. +func installManagerZipEntries() map[string]string { + return map[string]string{ + "__install__.json": "{}", + "python.exe": "exe", + "python315.dll": "dll", + "Lib/os.py": "os", + "Lib/venv/__init__.py": "venv", + "Lib/ensurepip/__init__.py": "ensurepip", + "Lib/script%20(dev).tmpl": "not escaped", + "include/Python.h": "header", + "libs/python315.lib": "lib", + "DLLs/_tkinter.pyd": "tkinter", + "Lib/tkinter/__init__.py": "tkinter", + "Lib/site-packages/README.md": "readme", + } +} + +// indexEntry is one install-manager index entry. +func indexEntry(id, version, url, sha string) map[string]any { + return map[string]any{ + "id": id, + "sort-version": version, + "url": url, + "hash": map[string]string{"sha256": sha}, + } +} + +// installManagerServer serves index pages at / and files at +// /ftp/. +func installManagerServer(t *testing.T, pages map[string]map[string]any, files map[string][]byte) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if name, ok := strings.CutPrefix(r.URL.Path, "/ftp/"); ok { + if body, ok := files[name]; ok { + _, _ = w.Write(body) + return + } + } + if page, ok := pages[strings.TrimPrefix(r.URL.Path, "/")]; ok { + _ = json.NewEncoder(w).Encode(page) + return + } + http.NotFound(w, r) + })) + t.Cleanup(srv.Close) + return srv +} + +func TestPythonInstallManagerSource_FetchVersions(t *testing.T) { + var srv *httptest.Server + good := strings.Repeat("ab", 32) + ftp := func(name string) string { return srv.URL + "/ftp/" + name } + pages := map[string]map[string]any{} + srv = installManagerServer(t, pages, nil) + + pages["index-windows.json"] = map[string]any{ + "versions": []any{ + indexEntry("pythoncore-3.15-64", "3.15.2", ftp("python-3.15.2-amd64.zip"), good), + // Other architectures, free-threaded builds and pre-releases. + indexEntry("pythoncore-3.15-arm64", "3.15.2", ftp("python-3.15.2-arm64.zip"), good), + indexEntry("pythoncore-3.15t-64", "3.15.2", ftp("python-3.15.2t-amd64.zip"), good), + indexEntry("pythoncore-3.15-32", "3.15.2", ftp("python-3.15.2-win32.zip"), good), + indexEntry("pythoncore-3.16-64", "3.16.0a2", ftp("python-3.16.0a2-amd64.zip"), good), + // From elsewhere, without a usable hash, or not a zip. + indexEntry("pythoncore-3.14-64", "3.14.9", "https://example.invalid/python-3.14.9-amd64.zip", good), + indexEntry("pythoncore-3.14-64", "3.14.8", ftp("python-3.14.8-amd64.zip"), "not-a-hash"), + indexEntry("pythoncore-3.14-64", "3.14.7", ftp("python-3.14.7-amd64.nupkg"), good), + // An id that disagrees with its version. + indexEntry("pythoncore-3.13-64", "3.14.6", ftp("python-3.14.6-amd64.zip"), good), + }, + "next": "index-windows-recent.json", + } + pages["index-windows-recent.json"] = map[string]any{ + "versions": []any{ + indexEntry("pythoncore-3.14-64", "3.14.0", ftp("python-3.14.0-amd64.zip"), strings.ToUpper(good)), + // Covered by the MSIs and older than this source's range. + indexEntry("pythoncore-3.13-64", "3.13.9", ftp("python-3.13.9-amd64.zip"), good), + // Listed again on a later page. + indexEntry("pythoncore-3.15-64", "3.15.2", ftp("python-3.15.2-amd64-dup.zip"), good), + }, + } + + src := &PythonInstallManagerSource{indexURL: srv.URL + "/index-windows.json", downloadPrefix: srv.URL + "/ftp/"} + jobs, err := src.FetchVersions() + if err != nil { + t.Fatalf("FetchVersions() error: %v", err) + } + + versions := checkPythonOrgJobPairs(t, jobs) + if want := []string{"3.15.2", "3.14.0"}; !reflect.DeepEqual(versions, want) { + t.Errorf("FetchVersions() versions = %v, want %v", versions, want) + } + for _, j := range jobs { + if j.Platform != "windows-amd64" { + t.Errorf("job %s: platform %q, want windows-amd64", j.Version, j.Platform) + } + } + if want := ftp("python-3.15.2-amd64.zip"); jobs[0].URL != want { + t.Errorf("3.15.2 job URL = %q, want %q", jobs[0].URL, want) + } +} + +func TestPythonInstallManagerSource_IndexErrors(t *testing.T) { + pages := map[string]map[string]any{ + "loop.json": {"versions": []any{}, "next": "loop.json"}, + } + srv := installManagerServer(t, pages, nil) + + for name, indexURL := range map[string]string{ + "page links to itself": srv.URL + "/loop.json", + "missing page": srv.URL + "/missing.json", + } { + src := &PythonInstallManagerSource{indexURL: indexURL, downloadPrefix: srv.URL + "/ftp/"} + if _, err := src.FetchVersions(); err == nil { + t.Errorf("%s: FetchVersions() = nil error, want error", name) + } + } +} + +func TestBuildFromInstallManagerZip(t *testing.T) { + full := nupkg(t, installManagerZipEntries()) + embeddableEntries := installManagerZipEntries() + embeddableEntries["python315._pth"] = "python315.zip" + embeddable := nupkg(t, embeddableEntries) + + srv := installManagerServer(t, nil, map[string][]byte{"full.zip": full, "embeddable.zip": embeddable}) + + archive, err := buildFromInstallManagerZip(srv.URL+"/ftp/full.zip", sha256Hex(full)) + if err != nil { + t.Fatalf("buildFromInstallManagerZip() error: %v", err) + } + entries := tarEntries(t, archive) + for rel, want := range map[string]string{ + "python.exe": "exe", + "__install__.json": "{}", + "include/Python.h": "header", + // Entry names are used as they are, not URI-unescaped. + "Lib/script%20(dev).tmpl": "not escaped", + } { + if got, ok := entries["python/"+rel]; !ok || got != want { + t.Errorf("archive entry python/%s = %q (present %v), want %q", rel, got, ok, want) + } + } + + if _, err := buildFromInstallManagerZip(srv.URL+"/ftp/full.zip", sha256Hex([]byte("other"))); err == nil || !strings.Contains(err.Error(), "SHA-256 mismatch") { + t.Errorf("buildFromInstallManagerZip(wrong hash) error = %v, want a SHA-256 mismatch", err) + } + if _, err := buildFromInstallManagerZip(srv.URL+"/ftp/embeddable.zip", sha256Hex(embeddable)); !errors.Is(err, errNotFullBuild) { + t.Errorf("buildFromInstallManagerZip(embeddable) error = %v, want errNotFullBuild", err) + } +} + +func TestIsSHA256Hex(t *testing.T) { + for s, want := range map[string]bool{ + strings.Repeat("0a", 32): true, + strings.Repeat("0A", 32): true, + strings.Repeat("0a", 31): false, + strings.Repeat("zz", 32): false, + "": false, + } { + if got := isSHA256Hex(s); got != want { + t.Errorf("isSHA256Hex(%q) = %v, want %v", s, got, want) + } + } +} diff --git a/scripts/mirror-binaries/python_winbuild.go b/scripts/mirror-binaries/python_winbuild.go index b71ed72..fe14983 100644 --- a/scripts/mirror-binaries/python_winbuild.go +++ b/scripts/mirror-binaries/python_winbuild.go @@ -21,6 +21,15 @@ package main // Either way the result is a .tar.gz with everything under python/, the // layout of python-build-standalone's Windows archives, so dtvem installs it // without changes. +// +// Each python.org build is published twice over: at the version's default +// location (python//.tar.gz), which it only gets when no +// earlier source covers that version, and always at the python.org location +// (python//python.org/.tar.gz). The python.org location +// is what `dtvem install python --signed` installs: python.org's +// own build, whose binaries carry the Python Software Foundation's +// Authenticode signature, where the default build is python-build- +// standalone's unsigned one. import ( "archive/tar" @@ -64,8 +73,40 @@ const ( // msiDiscoveryWorkers bounds concurrent requests to python.org while // checking which releases have signed MSIs. msiDiscoveryWorkers = 8 + + // pythonOrgBuildDir is the folder under python// that holds + // python.org's build of that version for each platform. + pythonOrgBuildDir = "python.org" ) +// pythonOrgJobs returns the two jobs that publish one python.org build: +// at the version's default location, which collectJobs drops when an +// earlier source already covers the version, and at the python.org +// location. Each job runs build itself, so a build both jobs mirror is +// assembled twice rather than held in memory between them. +func pythonOrgJobs(version, platform, sourceURL string, build func() ([]byte, error)) []MirrorJob { + return []MirrorJob{ + { + Runtime: "python", + Version: version, + Platform: platform, + URL: sourceURL, + R2Key: fmt.Sprintf("python/%s/%s.tar.gz", version, platform), + MetaKey: fmt.Sprintf("python/%s/%s.meta.json", version, platform), + Build: build, + }, + { + Runtime: "python", + Version: version, + Platform: platform, + URL: sourceURL, + R2Key: fmt.Sprintf("python/%s/%s/%s.tar.gz", version, pythonOrgBuildDir, platform), + MetaKey: fmt.Sprintf("python/%s/%s/%s.meta.json", version, pythonOrgBuildDir, platform), + Build: build, + }, + } +} + // pythonWindowsSigningKey is the armored public key for // pythonWindowsSigningFingerprint, committed so the mirror never depends on // a keyserver. @@ -177,17 +218,9 @@ func (s *PythonMSISource) FetchVersions() ([]MirrorJob, error) { continue } dirURL := base + v + "/amd64/" - jobs = append(jobs, MirrorJob{ - Runtime: "python", - Version: v, - Platform: "windows-amd64", - URL: dirURL, - R2Key: fmt.Sprintf("python/%s/windows-amd64.tar.gz", v), - MetaKey: fmt.Sprintf("python/%s/windows-amd64.meta.json", v), - Build: func() ([]byte, error) { - return buildFromMSIs(dirURL, verifier, s.extract) - }, - }) + jobs = append(jobs, pythonOrgJobs(v, "windows-amd64", dirURL, func() ([]byte, error) { + return buildFromMSIs(dirURL, verifier, s.extract) + })...) } return jobs, nil } @@ -311,17 +344,9 @@ func (s *PythonNuGetSource) FetchVersions() ([]MirrorJob, error) { id := pkg.id nupkgURL := fmt.Sprintf("%s%s/%s/%s.%s.nupkg", flat, id, v, id, v) leafURL := fmt.Sprintf("%s%s/%s.json", registration, id, v) - jobs = append(jobs, MirrorJob{ - Runtime: "python", - Version: v, - Platform: pkg.platform, - URL: nupkgURL, - R2Key: fmt.Sprintf("python/%s/%s.tar.gz", v, pkg.platform), - MetaKey: fmt.Sprintf("python/%s/%s.meta.json", v, pkg.platform), - Build: func() ([]byte, error) { - return buildFromNuGet(nupkgURL, leafURL) - }, - }) + jobs = append(jobs, pythonOrgJobs(v, pkg.platform, nupkgURL, func() ([]byte, error) { + return buildFromNuGet(nupkgURL, leafURL) + })...) } } return jobs, nil @@ -469,6 +494,14 @@ func archiveDir(dir string, requireTk bool) ([]byte, error) { // archiveNuGetTools packs the full install under tools/ in a NuGet package, // dropping the package's own files. func archiveNuGetTools(pkg []byte) ([]byte, error) { + // NuGet stores part names URI-escaped ("script%20(dev).tmpl"). + return archiveZipTree(pkg, "tools/", true) +} + +// archiveZipTree packs the full install found under prefix in a zip ("" for +// the whole zip), dropping everything outside it. escaped says the zip's +// entry names are URI-escaped, as in a NuGet package. +func archiveZipTree(pkg []byte, prefix string, escaped bool) ([]byte, error) { zr, err := zip.NewReader(bytes.NewReader(pkg), int64(len(pkg))) if err != nil { return nil, fmt.Errorf("reading package: %w", err) @@ -478,11 +511,12 @@ func archiveNuGetTools(pkg []byte) ([]byte, error) { var files []string for _, f := range zr.File { name := f.Name - // NuGet stores part names URI-escaped ("script%20(dev).tmpl"). - if unescaped, err := url.PathUnescape(name); err == nil { - name = unescaped + if escaped { + if unescaped, err := url.PathUnescape(name); err == nil { + name = unescaped + } } - rel, ok := strings.CutPrefix(name, "tools/") + rel, ok := strings.CutPrefix(name, prefix) if !ok || rel == "" || strings.HasSuffix(rel, "/") { continue } diff --git a/scripts/mirror-binaries/python_winbuild_network_test.go b/scripts/mirror-binaries/python_winbuild_network_test.go index 54c01f2..3f37692 100644 --- a/scripts/mirror-binaries/python_winbuild_network_test.go +++ b/scripts/mirror-binaries/python_winbuild_network_test.go @@ -14,6 +14,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "testing" ) @@ -92,6 +93,40 @@ func TestNetwork_BuildFromNuGet(t *testing.T) { } } +func TestNetwork_InstallManager(t *testing.T) { + requireNetworkTests(t) + + jobs, err := (&PythonInstallManagerSource{}).FetchVersions() + if err != nil { + t.Fatalf("PythonInstallManagerSource.FetchVersions: %v", err) + } + have := make(map[string]bool) + for _, j := range jobs { + have[j.Version] = true + if parts, _ := parseVersion(j.Version); compareVersions(parts, firstSigstoreOnlyPython) < 0 { + t.Errorf("install-manager source has a job for %s, which the MSIs cover", j.Version) + } + } + t.Logf("python.org install manager: %d x64 releases", len(have)) + for _, v := range []string{"3.14.0", "3.15.2"} { + if !have[v] { + t.Errorf("install-manager source has no job for %s", v) + } + } + + // The first 3.14 release; its python.exe should carry python.org's + // Authenticode signature (check on Windows with MIRROR_NETWORK_OUT). + for _, j := range jobs { + if j.Version == "3.14.0" && strings.Contains(j.R2Key, "/python.org/") { + archive, err := j.Build() + if err != nil { + t.Fatalf("building 3.14.0: %v", err) + } + checkRealArchive(t, "python-3.14.0-windows-amd64-python.org.tar.gz", archive, false) + } + } +} + func TestNetwork_FetchVersions(t *testing.T) { requireNetworkTests(t) diff --git a/scripts/mirror-binaries/python_winbuild_test.go b/scripts/mirror-binaries/python_winbuild_test.go index 45bb5e3..8ae5da0 100644 --- a/scripts/mirror-binaries/python_winbuild_test.go +++ b/scripts/mirror-binaries/python_winbuild_test.go @@ -200,17 +200,10 @@ func TestPythonMSISource_FetchVersions(t *testing.T) { t.Fatalf("FetchVersions() error: %v", err) } - var versions []string + versions := checkPythonOrgJobPairs(t, jobs) for _, j := range jobs { - versions = append(versions, j.Version) - if j.Platform != "windows-amd64" || j.Build == nil { - t.Errorf("job %s: platform %q, Build set %v; want windows-amd64 with a Build step", j.Version, j.Platform, j.Build != nil) - } - if want := fmt.Sprintf("python/%s/windows-amd64.tar.gz", j.Version); j.R2Key != want { - t.Errorf("job %s: R2Key %q, want %q", j.Version, j.R2Key, want) - } - if want := fmt.Sprintf("python/%s/windows-amd64.meta.json", j.Version); j.MetaKey != want { - t.Errorf("job %s: MetaKey %q, want %q", j.Version, j.MetaKey, want) + if j.Platform != "windows-amd64" { + t.Errorf("job %s: platform %q, want windows-amd64", j.Version, j.Platform) } if want := srv.URL + "/" + j.Version + "/amd64/"; j.URL != want { t.Errorf("job %s: URL %q, want %q", j.Version, j.URL, want) @@ -227,6 +220,40 @@ func TestPythonMSISource_FetchVersions(t *testing.T) { } } +// checkPythonOrgJobPairs checks that jobs come in the pairs pythonOrgJobs +// makes, one at the default location and one at the python.org location, +// and returns the version of each pair. +func checkPythonOrgJobPairs(t *testing.T, jobs []MirrorJob) []string { + t.Helper() + if len(jobs)%2 != 0 { + t.Fatalf("got %d jobs, want pairs of default and python.org jobs", len(jobs)) + } + var versions []string + for i := 0; i < len(jobs); i += 2 { + def, org := jobs[i], jobs[i+1] + v, p := def.Version, def.Platform + versions = append(versions, v) + + if org.Version != v || org.Platform != p || org.URL != def.URL { + t.Errorf("pair %d: python.org job %s/%s from %s, want %s/%s from %s", i/2, org.Version, org.Platform, org.URL, v, p, def.URL) + } + if def.Build == nil || org.Build == nil { + t.Errorf("pair %s/%s: Build set %v and %v, want both", v, p, def.Build != nil, org.Build != nil) + } + for _, c := range []struct{ got, want string }{ + {def.R2Key, fmt.Sprintf("python/%s/%s.tar.gz", v, p)}, + {def.MetaKey, fmt.Sprintf("python/%s/%s.meta.json", v, p)}, + {org.R2Key, fmt.Sprintf("python/%s/python.org/%s.tar.gz", v, p)}, + {org.MetaKey, fmt.Sprintf("python/%s/python.org/%s.meta.json", v, p)}, + } { + if c.got != c.want { + t.Errorf("pair %s/%s: key %q, want %q", v, p, c.got, c.want) + } + } + } + return versions +} + // pathRecorder records requested paths; test servers handle requests on // several goroutines at once. type pathRecorder struct { @@ -579,22 +606,18 @@ func TestPythonNuGetSource_FetchVersions(t *testing.T) { t.Fatalf("FetchVersions() error: %v", err) } - var got []string + got := checkPythonOrgJobPairs(t, jobs) for _, j := range jobs { - got = append(got, j.Version+"/"+j.Platform) - if j.Build == nil { - t.Errorf("job %s/%s has no Build step", j.Version, j.Platform) - } - if want := fmt.Sprintf("python/%s/%s.tar.gz", j.Version, j.Platform); j.R2Key != want { - t.Errorf("job %s: R2Key %q, want %q", j.Version, j.R2Key, want) + if j.Platform != "windows-arm64" { + t.Errorf("job %s: platform %q, want windows-arm64", j.Version, j.Platform) } } - want := []string{"3.9.7/windows-arm64", "3.11.4/windows-arm64"} + want := []string{"3.9.7", "3.11.4"} if !reflect.DeepEqual(got, want) { - t.Errorf("FetchVersions() = %v, want %v", got, want) + t.Errorf("FetchVersions() versions = %v, want %v", got, want) } - if want := srv.URL + "/flat/pythonarm64/3.11.4/pythonarm64.3.11.4.nupkg"; jobs[1].URL != want { - t.Errorf("arm64 job URL = %q, want %q", jobs[1].URL, want) + if want := srv.URL + "/flat/pythonarm64/3.11.4/pythonarm64.3.11.4.nupkg"; jobs[2].URL != want { + t.Errorf("arm64 job URL = %q, want %q", jobs[2].URL, want) } for _, p := range requested.list() { if strings.HasPrefix(p, "/flat/python/") { @@ -634,8 +657,8 @@ func TestBuildFromNuGet(t *testing.T) { ) src := &PythonNuGetSource{flatContainerURL: srv.URL + "/flat/", registrationURL: srv.URL + "/registration/"} jobs, err := src.FetchVersions() - if err != nil || len(jobs) != 1 { - t.Fatalf("FetchVersions() = %d jobs, %v; want 1 job", len(jobs), err) + if err != nil || len(jobs) != 2 { + t.Fatalf("FetchVersions() = %d jobs, %v; want a default and a python.org job", len(jobs), err) } archive, err := jobs[0].Build() @@ -657,8 +680,8 @@ func TestBuildFromNuGet_HashMismatch(t *testing.T) { ) src := &PythonNuGetSource{flatContainerURL: srv.URL + "/flat/", registrationURL: srv.URL + "/registration/"} jobs, err := src.FetchVersions() - if err != nil || len(jobs) != 1 { - t.Fatalf("FetchVersions() = %d jobs, %v; want 1 job", len(jobs), err) + if err != nil || len(jobs) != 2 { + t.Fatalf("FetchVersions() = %d jobs, %v; want a default and a python.org job", len(jobs), err) } if _, err := jobs[0].Build(); err == nil || !strings.Contains(err.Error(), "SHA-512 mismatch") { diff --git a/scripts/mirror-binaries/sources.go b/scripts/mirror-binaries/sources.go index 33a28a7..ab93113 100644 --- a/scripts/mirror-binaries/sources.go +++ b/scripts/mirror-binaries/sources.go @@ -84,10 +84,12 @@ func getUpstreamSources(runtime string) ([]UpstreamSource, error) { case "python": return []UpstreamSource{ &PythonStandaloneSource{}, - // Full Windows builds for versions python-build-standalone - // lacks, in place of python.org's embeddable zips. + // python.org's full Windows builds: published at the + // python.org location for `dtvem install --signed`, and at the + // default location for versions python-build-standalone lacks. &PythonMSISource{}, &PythonNuGetSource{}, + &PythonInstallManagerSource{}, }, nil case "ruby": return []UpstreamSource{ @@ -112,10 +114,15 @@ func fetchJobsFromUpstream(runtime string) ([]MirrorJob, error) { // collectJobs fetches mirror jobs from each source in order. A failing source // does not stop the others: jobs from the sources that succeeded are returned // together with an error describing every source that failed. +// +// When several jobs publish to the same place (the same metadata key), the +// first source's job wins. A version's default build is at +// //; python.org's builds also have a place of +// their own, which python-build-standalone's never claim. func collectJobs(sources []UpstreamSource) ([]MirrorJob, error) { var allJobs []MirrorJob var errs []error - seen := make(map[string]bool) // Track version+platform to avoid duplicates + seen := make(map[string]bool) // Metadata keys already claimed for _, source := range sources { fmt.Printf(" Fetching from %s...\n", source.Name()) @@ -129,14 +136,13 @@ func collectJobs(sources []UpstreamSource) ([]MirrorJob, error) { // Add jobs, avoiding duplicates (first source wins) added := 0 for _, job := range jobs { - key := fmt.Sprintf("%s/%s", job.Version, job.Platform) - if !seen[key] { - seen[key] = true + if !seen[job.MetaKey] { + seen[job.MetaKey] = true allJobs = append(allJobs, job) added++ } } - fmt.Printf(" Found %d versions from %s (%d new)\n", len(jobs), source.Name(), added) + fmt.Printf(" Found %d jobs from %s (%d new)\n", len(jobs), source.Name(), added) } return allJobs, errors.Join(errs...) diff --git a/scripts/mirror-binaries/sources_test.go b/scripts/mirror-binaries/sources_test.go index dde17e8..51ff9d2 100644 --- a/scripts/mirror-binaries/sources_test.go +++ b/scripts/mirror-binaries/sources_test.go @@ -79,17 +79,23 @@ func TestCollectJobs(t *testing.T) { primary := &fakeSource{ name: "primary", jobs: []MirrorJob{ - {Version: "3.13.16", Platform: "windows-amd64", URL: "primary"}, - {Version: "3.13.16", Platform: "linux-amd64", URL: "primary"}, + {Version: "3.13.16", Platform: "windows-amd64", URL: "primary", MetaKey: "python/3.13.16/windows-amd64.meta.json"}, + {Version: "3.13.16", Platform: "linux-amd64", URL: "primary", MetaKey: "python/3.13.16/linux-amd64.meta.json"}, }, } fallback := &fakeSource{ name: "fallback", jobs: []MirrorJob{ - {Version: "3.13.16", Platform: "windows-amd64", URL: "fallback"}, - {Version: "3.11.17", Platform: "windows-amd64", URL: "fallback"}, + {Version: "3.13.16", Platform: "windows-amd64", URL: "fallback", MetaKey: "python/3.13.16/windows-amd64.meta.json"}, + {Version: "3.11.17", Platform: "windows-amd64", URL: "fallback", MetaKey: "python/3.11.17/windows-amd64.meta.json"}, }, } + // python.org's builds also go to a place of their own, which an earlier + // source covering the same version does not take away. + pythonOrg := &fakeSource{ + name: "python.org", + jobs: pythonOrgJobs("3.13.16", "windows-amd64", "python.org", nil), + } broken := &fakeSource{name: "broken", err: errors.New("fetching releases: HTTP 504")} tests := []struct { @@ -103,6 +109,16 @@ func TestCollectJobs(t *testing.T) { sources: []UpstreamSource{primary, fallback}, wantURLs: []string{"primary", "primary", "fallback"}, }, + { + name: "python.org location kept when default is covered", + sources: []UpstreamSource{primary, pythonOrg}, + wantURLs: []string{"primary", "primary", "python.org"}, + }, + { + name: "python.org fills the default location when uncovered", + sources: []UpstreamSource{pythonOrg, primary}, + wantURLs: []string{"python.org", "python.org", "primary"}, + }, { name: "failing source keeps jobs from healthy sources", sources: []UpstreamSource{broken, fallback}, @@ -140,8 +156,9 @@ func TestCollectJobs(t *testing.T) { } // The first source to offer a version and platform wins, so the order sets -// which build a version gets: python-build-standalone where it has one, then -// python.org's MSIs (full x64 builds with tkinter), then its NuGet packages. +// which build a version gets by default: python-build-standalone where it +// has one, then python.org's MSIs (full x64 builds with tkinter), then its +// NuGet packages, then its install-manager zips (x64 from 3.14 on). func TestGetUpstreamSources_PythonOrder(t *testing.T) { sources, err := getUpstreamSources("python") if err != nil { @@ -152,7 +169,7 @@ func TestGetUpstreamSources_PythonOrder(t *testing.T) { for _, s := range sources { got = append(got, s.Name()) } - want := []string{"python-build-standalone", "python.org MSIs", "python.org NuGet packages"} + want := []string{"python-build-standalone", "python.org MSIs", "python.org NuGet packages", "python.org install manager"} if strings.Join(got, ",") != strings.Join(want, ",") { t.Errorf("python sources = %v, want %v", got, want) } From 832fd857fc97d60fca3bfb93c58a56ec15675054 Mon Sep 17 00:00:00 2001 From: "Calvin A. Allen" Date: Thu, 12 Nov 2026 20:18:04 -0500 Subject: [PATCH 2/3] feat(manifest): label Python builds and list python.org builds as alternates The manifest generator labels each Python entry with the build it is (python-build-standalone or python.org, from the source URL in its metadata) and nests the python.org build under the default entry's alternates. Released CLIs ignore both fields and install the same URL as before. The generator now probes R2 for a binary's extension when the source URL's extension is not there (a repacked install-manager zip), and its manifest assembly is a pure function with tests, which the mirror tool workflow now runs. The CLI's manifest types gain Build and Alternates. Its cache files gain a format number, and a cache written by an older CLI, which dropped the new fields, is fetched again. --- .github/workflows/mirror-tool.yml | 30 ++- schemas/manifest.schema.json | 31 ++- scripts/generate-manifests-from-r2/main.go | 226 ++++++++++++++---- .../generate-manifests-from-r2/main_test.go | 195 +++++++++++++++ src/internal/manifest/build_test.go | 197 +++++++++++++++ src/internal/manifest/cache.go | 14 +- src/internal/manifest/manifest.go | 53 ++++ 7 files changed, 689 insertions(+), 57 deletions(-) create mode 100644 scripts/generate-manifests-from-r2/main_test.go create mode 100644 src/internal/manifest/build_test.go diff --git a/.github/workflows/mirror-tool.yml b/.github/workflows/mirror-tool.yml index 9b9a726..1708811 100644 --- a/.github/workflows/mirror-tool.yml +++ b/.github/workflows/mirror-tool.yml @@ -1,18 +1,21 @@ name: Mirror Tool -# Tests for scripts/mirror-binaries, which is its own Go module and so is -# not covered by the Build & Test workflow. +# Tests for scripts/mirror-binaries and scripts/generate-manifests-from-r2, +# which are Go modules of their own and so are not covered by the Build & +# Test workflow. on: pull_request: branches: [ main ] paths: - 'scripts/mirror-binaries/**' + - 'scripts/generate-manifests-from-r2/**' - '.github/workflows/mirror-tool.yml' push: branches: [ main ] paths: - 'scripts/mirror-binaries/**' + - 'scripts/generate-manifests-from-r2/**' - '.github/workflows/mirror-tool.yml' permissions: @@ -43,3 +46,26 @@ jobs: # MIRROR_NETWORK_TESTS=1, so this needs neither network nor msitools. - name: Test run: go test -race ./... + + test-generator: + name: Test manifest generator + runs-on: ubuntu-latest + defaults: + run: + working-directory: scripts/generate-manifests-from-r2 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: 'scripts/generate-manifests-from-r2/go.mod' + cache-dependency-path: 'scripts/generate-manifests-from-r2/go.sum' + + - name: Vet + run: go vet ./... + + - name: Test + run: go test -race ./... diff --git a/schemas/manifest.schema.json b/schemas/manifest.schema.json index 1703ba0..b0e1fae 100644 --- a/schemas/manifest.schema.json +++ b/schemas/manifest.schema.json @@ -68,6 +68,24 @@ "source": { "type": "string", "description": "Build source indicator: 'built-from-source' when the binary was compiled by dtvem rather than obtained from upstream" + }, + "build": { + "type": "string", + "enum": ["python-build-standalone", "python.org"], + "description": "Whose build of the runtime this is, for runtimes with more than one. Python: 'python-build-standalone', or 'python.org' for python.org's own build (on Windows, Authenticode-signed by the Python Software Foundation)" + }, + "alternates": { + "type": "array", + "description": "Other builds of the same version for the same platform, installed only when asked for (dtvem install python --signed). Released CLIs that predate this field ignore it and install the entry's own url", + "items": { + "allOf": [ + { "$ref": "#/$defs/download" }, + { + "required": ["build"], + "not": { "required": ["alternates"] } + } + ] + } } } } @@ -78,9 +96,18 @@ "versions": { "3.13.1": { "windows-amd64": { - "url": "https://builds.dtvem.io/python/3.13.1/windows-amd64.zip", + "url": "https://builds.dtvem.io/python/3.13.1/windows-amd64.tar.gz", "sha256": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", - "sha256_source": "upstream" + "sha256_source": "upstream", + "build": "python-build-standalone", + "alternates": [ + { + "url": "https://builds.dtvem.io/python/3.13.1/python.org/windows-amd64.tar.gz", + "sha256": "c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4", + "sha256_source": "dtvem", + "build": "python.org" + } + ] }, "darwin-arm64": null, "linux-amd64": { diff --git a/scripts/generate-manifests-from-r2/main.go b/scripts/generate-manifests-from-r2/main.go index 86a4ada..b41adeb 100644 --- a/scripts/generate-manifests-from-r2/main.go +++ b/scripts/generate-manifests-from-r2/main.go @@ -6,6 +6,7 @@ import ( "flag" "fmt" "io" + "net/url" "os" "path/filepath" "regexp" @@ -33,6 +34,27 @@ type ManifestDownload struct { SHA256 string `json:"sha256,omitempty"` SHA256Source string `json:"sha256_source,omitempty"` Source string `json:"source,omitempty"` // "built-from-source" when not from upstream + + // Build names whose build of the runtime this is, for runtimes with + // more than one (Python: "python-build-standalone" or "python.org"). + Build string `json:"build,omitempty"` + + // Alternates are other builds of the same version for the same + // platform, which dtvem installs only when asked for by name. CLIs + // released before alternates existed ignore them. + Alternates []*ManifestDownload `json:"alternates,omitempty"` +} + +// Build names, as recorded in ManifestDownload.Build. +const ( + buildPythonStandalone = "python-build-standalone" + buildPythonOrg = "python.org" +) + +// alternateBuildDirs are the folders under // that hold +// builds other than the default, mapped to the build each holds. +var alternateBuildDirs = map[string]string{ + "python/python.org": buildPythonOrg, } // Manifest represents the output manifest structure @@ -52,8 +74,10 @@ var ( dryRun = flag.Bool("dry-run", false, "Report what would be generated without writing files") ) -// metaKeyPattern matches paths like "node/20.18.0/linux-amd64.meta.json" -var metaKeyPattern = regexp.MustCompile(`^([^/]+)/([^/]+)/([^/]+)\.meta\.json$`) +// metaKeyPattern matches a version's default build, like +// "node/20.18.0/linux-amd64.meta.json", and a build in a folder of its own, +// like "python/3.12.10/python.org/windows-amd64.meta.json". +var metaKeyPattern = regexp.MustCompile(`^([^/]+)/([^/]+)/(?:([^/]+)/)?([^/]+)\.meta\.json$`) func main() { flag.Parse() @@ -139,11 +163,6 @@ func createS3Client() (*s3.Client, error) { } func generateManifest(client *s3.Client, runtime string) (*Manifest, error) { - manifest := &Manifest{ - Version: 1, - Versions: make(map[string]map[string]*ManifestDownload), - } - // List all files for this runtime, separating meta files from binary keys prefix := runtime + "/" paginator := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{ @@ -170,61 +189,165 @@ func generateManifest(client *s3.Client, runtime string) (*Manifest, error) { fmt.Printf(" Found %d metadata files\n", len(metaFiles)) - // Process each metadata file + metas := make(map[string]*BinaryMeta) for _, metaKey := range metaFiles { - // Parse the key to extract runtime, version, platform + meta, err := downloadMeta(client, metaKey) + if err != nil { + fmt.Printf(" Warning: failed to read metadata for %s: %v\n", metaKey, err) + continue + } + metas[metaKey] = meta + } + + return assembleManifest(runtime, *baseURL, metas, allKeys), nil +} + +// manifestEntry is one build found in R2: a metadata file and where its +// binary is. +type manifestEntry struct { + version string + platform string + // keyBase is the binary's key without its extension, e.g. + // "python/3.12.10/python.org/windows-amd64". + keyBase string + // build is the build this entry is, for builds in a folder of their own + // ("" for a version's default build). + build string + meta *BinaryMeta +} + +// assembleManifest builds a runtime's manifest from the metadata files in +// R2 (keyed by their R2 key) and the set of every key under the runtime. +// +// Each version and platform gets its default build, labeled with whose +// build it is where that is known. Builds in a folder of their own are +// listed as alternates of the default; where there is no default build, +// one of them takes its place, and one that is the same build as the +// default is left out. +func assembleManifest(runtime, baseURL string, metas map[string]*BinaryMeta, allKeys map[string]bool) *Manifest { + manifest := &Manifest{ + Version: 1, + Versions: make(map[string]map[string]*ManifestDownload), + } + + var defaults, alternates []manifestEntry + for metaKey, meta := range metas { matches := metaKeyPattern.FindStringSubmatch(metaKey) if matches == nil { fmt.Printf(" Warning: skipping invalid meta key: %s\n", metaKey) continue } - - rt := matches[1] - version := matches[2] - platform := matches[3] - + rt, version, dir, platform := matches[1], matches[2], matches[3], matches[4] if rt != runtime { continue } - // Download and parse metadata - meta, err := downloadMeta(client, metaKey) - if err != nil { - fmt.Printf(" Warning: failed to read metadata for %s: %v\n", metaKey, err) + entry := manifestEntry{version: version, platform: platform, meta: meta} + if dir == "" { + entry.keyBase = fmt.Sprintf("%s/%s/%s", runtime, version, platform) + defaults = append(defaults, entry) continue } - - // Determine the binary file extension from source URL - ext := getExtension(meta.SourceURL) - if !isValidArchiveExt(ext) { - // Source URL doesn't contain a valid extension (e.g., "built-from-source"), - // probe R2 for the actual binary file - ext = findBinaryExtension(allKeys, runtime, version, platform) - if ext == "" { - fmt.Printf(" Warning: no valid binary found for %s/%s/%s, skipping\n", runtime, version, platform) - continue - } + build, ok := alternateBuildDirs[runtime+"/"+dir] + if !ok { + fmt.Printf(" Warning: skipping %s, in unknown build folder %q\n", metaKey, dir) + continue } - binaryURL := fmt.Sprintf("%s/%s/%s/%s%s", *baseURL, runtime, version, platform, ext) + entry.keyBase = fmt.Sprintf("%s/%s/%s/%s", runtime, version, dir, platform) + entry.build = build + alternates = append(alternates, entry) + } - // Add to manifest - if manifest.Versions[version] == nil { - manifest.Versions[version] = make(map[string]*ManifestDownload) + for _, e := range defaults { + download := newManifestDownload(runtime, baseURL, e, allKeys) + if download == nil { + continue } + if manifest.Versions[e.version] == nil { + manifest.Versions[e.version] = make(map[string]*ManifestDownload) + } + manifest.Versions[e.version][e.platform] = download + } - download := &ManifestDownload{ - URL: binaryURL, - SHA256: meta.SHA256, - SHA256Source: meta.SHA256Source, + // Attach alternates in a fixed order, so the output does not depend on + // map iteration. + sort.Slice(alternates, func(i, j int) bool { return alternates[i].keyBase < alternates[j].keyBase }) + for _, e := range alternates { + download := newManifestDownload(runtime, baseURL, e, allKeys) + if download == nil { + continue + } + if manifest.Versions[e.version] == nil { + manifest.Versions[e.version] = make(map[string]*ManifestDownload) } - if meta.SourceURL == "built-from-source" { - download.Source = "built-from-source" + def := manifest.Versions[e.version][e.platform] + switch { + case def == nil: + manifest.Versions[e.version][e.platform] = download + case def.Build == download.Build: + // The default already is this build. + default: + def.Alternates = append(def.Alternates, download) } + } - manifest.Versions[version][platform] = download + return manifest +} + +// newManifestDownload returns the manifest entry for one build, or nil when +// its binary cannot be found in R2. +func newManifestDownload(runtime, baseURL string, e manifestEntry, allKeys map[string]bool) *ManifestDownload { + // The source URL's extension is the binary's, unless dtvem built or + // repacked it (built from source, or a Windows Python assembled from + // python.org's files); then R2 has the binary under another extension. + ext := getExtension(e.meta.SourceURL) + if !isValidArchiveExt(ext) || !allKeys[e.keyBase+ext] { + probed := findBinaryExtension(allKeys, e.keyBase) + switch { + case probed != "": + ext = probed + case !isValidArchiveExt(ext): + fmt.Printf(" Warning: no valid binary found for %s, skipping\n", e.keyBase) + return nil + } } - return manifest, nil + download := &ManifestDownload{ + URL: fmt.Sprintf("%s/%s%s", baseURL, e.keyBase, ext), + SHA256: e.meta.SHA256, + SHA256Source: e.meta.SHA256Source, + Build: e.build, + } + if e.meta.SourceURL == "built-from-source" { + download.Source = "built-from-source" + } + if download.Build == "" { + download.Build = buildFromSourceURL(runtime, e.meta.SourceURL) + } + return download +} + +// buildFromSourceURL names whose build a binary is, from the upstream URL +// the mirror took it from. It returns "" for runtimes with a single build. +func buildFromSourceURL(runtime, sourceURL string) string { + if runtime != "python" { + return "" + } + u, err := url.Parse(sourceURL) + if err != nil { + return "" + } + switch { + case strings.Contains(u.Path, "/python-build-standalone/"): + return buildPythonStandalone + case u.Host == "www.python.org", u.Host == "python.org": + // The MSIs, install-manager zips and embeddable zips + return buildPythonOrg + case u.Host == "api.nuget.org" && strings.Contains(u.Path, "/python"): + // python.org's NuGet packages + return buildPythonOrg + } + return "" } func downloadMeta(client *s3.Client, key string) (*BinaryMeta, error) { @@ -250,25 +373,25 @@ func downloadMeta(client *s3.Client, key string) (*BinaryMeta, error) { return &meta, nil } -func getExtension(url string) string { +func getExtension(sourceURL string) string { // Handle common archive extensions - if strings.HasSuffix(url, ".tar.gz") { + if strings.HasSuffix(sourceURL, ".tar.gz") { return ".tar.gz" } - if strings.HasSuffix(url, ".tar.xz") { + if strings.HasSuffix(sourceURL, ".tar.xz") { return ".tar.xz" } - if strings.HasSuffix(url, ".tar.bz2") { + if strings.HasSuffix(sourceURL, ".tar.bz2") { return ".tar.bz2" } - if strings.HasSuffix(url, ".zip") { + if strings.HasSuffix(sourceURL, ".zip") { return ".zip" } - if strings.HasSuffix(url, ".7z") { + if strings.HasSuffix(sourceURL, ".7z") { return ".7z" } // Fallback: extract from URL - base := filepath.Base(url) + base := filepath.Base(sourceURL) if idx := strings.Index(base, "."); idx != -1 { return base[idx:] } @@ -283,10 +406,11 @@ func isValidArchiveExt(ext string) bool { return false } -func findBinaryExtension(allKeys map[string]bool, runtime, version, platform string) string { - prefix := fmt.Sprintf("%s/%s/%s", runtime, version, platform) +// findBinaryExtension returns the extension of the binary at keyBase +// ("python/3.12.10/windows-amd64") in R2, or "" when there is none. +func findBinaryExtension(allKeys map[string]bool, keyBase string) string { for _, ext := range []string{".tar.gz", ".tar.xz", ".zip", ".7z"} { - if allKeys[prefix+ext] { + if allKeys[keyBase+ext] { return ext } } diff --git a/scripts/generate-manifests-from-r2/main_test.go b/scripts/generate-manifests-from-r2/main_test.go new file mode 100644 index 0000000..7770f92 --- /dev/null +++ b/scripts/generate-manifests-from-r2/main_test.go @@ -0,0 +1,195 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" +) + +const testBaseURL = "https://builds.dtvem.io" + +const ( + pbsURL = "https://github.com/astral-sh/python-build-standalone/releases/download/20250529/cpython-3.12.10%2B20250529-x86_64-pc-windows-msvc-install_only.tar.gz" + pbsLinuxURL = "https://github.com/astral-sh/python-build-standalone/releases/download/20250529/cpython-3.12.10%2B20250529-x86_64-unknown-linux-gnu-install_only.tar.gz" +) + +func meta(sourceURL, sha string) *BinaryMeta { + return &BinaryMeta{SHA256: sha, SHA256Source: "dtvem", SourceURL: sourceURL} +} + +func keys(list ...string) map[string]bool { + m := make(map[string]bool) + for _, k := range list { + m[k] = true + } + return m +} + +func TestAssembleManifest_Python(t *testing.T) { + metas := map[string]*BinaryMeta{ + // python-build-standalone default with a python.org alternate. + "python/3.12.10/windows-amd64.meta.json": meta(pbsURL, "pbs"), + "python/3.12.10/linux-amd64.meta.json": meta(pbsLinuxURL, "linux"), + "python/3.12.10/python.org/windows-amd64.meta.json": meta("https://www.python.org/ftp/python/3.12.10/amd64/", "org"), + // The default already is python.org's build. + "python/3.9.13/windows-amd64.meta.json": meta("https://www.python.org/ftp/python/3.9.13/amd64/", "org-default"), + "python/3.9.13/python.org/windows-amd64.meta.json": meta("https://www.python.org/ftp/python/3.9.13/amd64/", "org-alt"), + // Only a python.org build, from an install-manager zip it repacked. + "python/3.14.99/python.org/windows-amd64.meta.json": meta("https://www.python.org/ftp/python/3.14.99/python-3.14.99-amd64.zip", "org-only"), + // A default repacked from an install-manager zip. + "python/3.15.0/windows-amd64.meta.json": meta("https://www.python.org/ftp/python/3.15.0/python-3.15.0-amd64.zip", "zip-default"), + // python.org's NuGet package for arm64. + "python/3.11.4/windows-arm64.meta.json": meta("https://api.nuget.org/v3-flatcontainer/pythonarm64/3.11.4/pythonarm64.3.11.4.nupkg", "nuget"), + // An embeddable zip, kept as it is. + "python/3.5.2/windows-amd64.meta.json": meta("https://www.python.org/ftp/python/3.5.2/python-3.5.2-embed-amd64.zip", "embed"), + // A folder that holds no known build. + "python/3.12.10/other/windows-amd64.meta.json": meta("https://example.invalid/x.tar.gz", "other"), + // Another runtime's file. + "node/22.0.0/linux-amd64.meta.json": meta("https://nodejs.org/dist/v22.0.0/node-v22.0.0-linux-x64.tar.xz", "node"), + } + allKeys := keys( + "python/3.12.10/windows-amd64.tar.gz", + "python/3.12.10/linux-amd64.tar.gz", + "python/3.12.10/python.org/windows-amd64.tar.gz", + "python/3.9.13/windows-amd64.tar.gz", + "python/3.9.13/windows-amd64.zip", + "python/3.9.13/python.org/windows-amd64.tar.gz", + "python/3.14.99/python.org/windows-amd64.tar.gz", + "python/3.15.0/windows-amd64.tar.gz", + "python/3.11.4/windows-arm64.tar.gz", + "python/3.5.2/windows-amd64.zip", + "python/3.12.10/other/windows-amd64.tar.gz", + ) + + m := assembleManifest("python", testBaseURL, metas, allKeys) + + tests := []struct { + version, platform string + wantURL, wantSHA string + wantBuild string + wantAlternates []string // URLs + }{ + {"3.12.10", "windows-amd64", testBaseURL + "/python/3.12.10/windows-amd64.tar.gz", "pbs", buildPythonStandalone, + []string{testBaseURL + "/python/3.12.10/python.org/windows-amd64.tar.gz"}}, + {"3.12.10", "linux-amd64", testBaseURL + "/python/3.12.10/linux-amd64.tar.gz", "linux", buildPythonStandalone, nil}, + {"3.9.13", "windows-amd64", testBaseURL + "/python/3.9.13/windows-amd64.tar.gz", "org-default", buildPythonOrg, nil}, + {"3.14.99", "windows-amd64", testBaseURL + "/python/3.14.99/python.org/windows-amd64.tar.gz", "org-only", buildPythonOrg, nil}, + {"3.15.0", "windows-amd64", testBaseURL + "/python/3.15.0/windows-amd64.tar.gz", "zip-default", buildPythonOrg, nil}, + {"3.11.4", "windows-arm64", testBaseURL + "/python/3.11.4/windows-arm64.tar.gz", "nuget", buildPythonOrg, nil}, + {"3.5.2", "windows-amd64", testBaseURL + "/python/3.5.2/windows-amd64.zip", "embed", buildPythonOrg, nil}, + } + for _, tt := range tests { + t.Run(tt.version+"/"+tt.platform, func(t *testing.T) { + d := m.Versions[tt.version][tt.platform] + if d == nil { + t.Fatalf("no entry for %s %s", tt.version, tt.platform) + } + if d.URL != tt.wantURL || d.SHA256 != tt.wantSHA || d.Build != tt.wantBuild { + t.Errorf("entry = {%s %s %s}, want {%s %s %s}", d.URL, d.SHA256, d.Build, tt.wantURL, tt.wantSHA, tt.wantBuild) + } + var alts []string + for _, a := range d.Alternates { + alts = append(alts, a.URL) + if a.Build != buildPythonOrg || len(a.Alternates) != 0 { + t.Errorf("alternate %s: build %q with %d alternates, want python.org with none", a.URL, a.Build, len(a.Alternates)) + } + } + if strings.Join(alts, ",") != strings.Join(tt.wantAlternates, ",") { + t.Errorf("alternates = %v, want %v", alts, tt.wantAlternates) + } + }) + } + + if alt := m.Versions["3.12.10"]["windows-amd64"].Alternates[0]; alt.SHA256 != "org" || alt.SHA256Source != "dtvem" { + t.Errorf("3.12.10 python.org alternate = {%s %s}, want {org dtvem}", alt.SHA256, alt.SHA256Source) + } + if _, ok := m.Versions["22.0.0"]; ok { + t.Error("python manifest includes node's 22.0.0") + } + if n := len(m.Versions["3.12.10"]); n != 2 { + t.Errorf("3.12.10 has %d platforms, want 2", n) + } +} + +// Runtimes with one build get no build labels, so their manifests are +// unchanged. +func TestAssembleManifest_SingleBuildRuntimes(t *testing.T) { + metas := map[string]*BinaryMeta{ + "node/22.0.0/linux-amd64.meta.json": meta("https://nodejs.org/dist/v22.0.0/node-v22.0.0-linux-x64.tar.xz", "node"), + "node/22.0.0/windows-amd64.meta.json": meta("https://nodejs.org/dist/v22.0.0/node-v22.0.0-win-x64.zip", "node-win"), + // The source URL's extension wins when R2 has it, as before. + "node/22.0.0/darwin-arm64.meta.json": meta("https://nodejs.org/dist/v22.0.0/node-v22.0.0-darwin-arm64.tar.gz", "node-mac"), + } + allKeys := keys( + "node/22.0.0/linux-amd64.tar.xz", + "node/22.0.0/windows-amd64.zip", + "node/22.0.0/darwin-arm64.tar.gz", + "node/22.0.0/darwin-arm64.zip", + ) + m := assembleManifest("node", testBaseURL, metas, allKeys) + + for platform, want := range map[string]string{ + "linux-amd64": testBaseURL + "/node/22.0.0/linux-amd64.tar.xz", + "windows-amd64": testBaseURL + "/node/22.0.0/windows-amd64.zip", + "darwin-arm64": testBaseURL + "/node/22.0.0/darwin-arm64.tar.gz", + } { + d := m.Versions["22.0.0"][platform] + if d == nil || d.URL != want || d.Build != "" { + t.Errorf("node 22.0.0 %s = %+v, want URL %s and no build", platform, d, want) + } + } + + data, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(data), `"build"`) || strings.Contains(string(data), `"alternates"`) { + t.Errorf("node manifest has build fields: %s", data) + } + + rubyMetas := map[string]*BinaryMeta{ + "ruby/3.4.1/linux-amd64.meta.json": meta("built-from-source", "ruby"), + } + rm := assembleManifest("ruby", testBaseURL, rubyMetas, keys("ruby/3.4.1/linux-amd64.tar.gz")) + if d := rm.Versions["3.4.1"]["linux-amd64"]; d == nil || d.Source != "built-from-source" || d.URL != testBaseURL+"/ruby/3.4.1/linux-amd64.tar.gz" { + t.Errorf("ruby 3.4.1 = %+v, want a built-from-source .tar.gz", d) + } +} + +func TestAssembleManifest_MissingBinary(t *testing.T) { + metas := map[string]*BinaryMeta{ + // Neither the source URL nor R2 tells the extension: skipped. + "python/3.13.1/python.org/windows-amd64.meta.json": meta("https://www.python.org/ftp/python/3.13.1/amd64/", "x"), + // The source URL's extension is used even when R2 lacks it, as + // before. + "python/3.13.1/linux-amd64.meta.json": meta(pbsLinuxURL, "y"), + } + m := assembleManifest("python", testBaseURL, metas, keys()) + + if d := m.Versions["3.13.1"]["windows-amd64"]; d != nil { + t.Errorf("windows-amd64 = %+v, want no entry without a binary", d) + } + if d := m.Versions["3.13.1"]["linux-amd64"]; d == nil || !strings.HasSuffix(d.URL, "/python/3.13.1/linux-amd64.tar.gz") { + t.Errorf("linux-amd64 = %+v, want the source URL's extension", d) + } +} + +func TestBuildFromSourceURL(t *testing.T) { + tests := []struct { + runtime, url, want string + }{ + {"python", pbsURL, buildPythonStandalone}, + {"python", "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.8.18%2B20240107-x86_64-pc-windows-msvc-shared-install_only.tar.gz", buildPythonStandalone}, + {"python", "https://www.python.org/ftp/python/3.9.13/amd64/", buildPythonOrg}, + {"python", "https://www.python.org/ftp/python/3.15.2/python-3.15.2-amd64.zip", buildPythonOrg}, + {"python", "https://api.nuget.org/v3-flatcontainer/pythonarm64/3.11.4/pythonarm64.3.11.4.nupkg", buildPythonOrg}, + {"python", "built-from-source", ""}, + {"python", "https://example.invalid/python.tar.gz", ""}, + {"node", "https://www.python.org/ftp/python/3.9.13/amd64/", ""}, + } + for _, tt := range tests { + if got := buildFromSourceURL(tt.runtime, tt.url); got != tt.want { + t.Errorf("buildFromSourceURL(%s, %s) = %q, want %q", tt.runtime, tt.url, got, tt.want) + } + } +} diff --git a/src/internal/manifest/build_test.go b/src/internal/manifest/build_test.go new file mode 100644 index 0000000..041e774 --- /dev/null +++ b/src/internal/manifest/build_test.go @@ -0,0 +1,197 @@ +package manifest + +import ( + "encoding/json" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "time" +) + +// buildsManifestJSON has a python-build-standalone default with a +// python.org alternate (3.12.10), a python.org default (3.9.13), a version +// with no python.org build (3.12.12) and an entry from before build labels +// (3.11.0). +const buildsManifestJSON = `{ + "version": 1, + "versions": { + "3.12.10": { + "windows-amd64": { + "url": "https://builds.dtvem.io/python/3.12.10/windows-amd64.tar.gz", + "sha256": "3e0fd10e4bd60e66baad8be2a0e60775bb389e97e4f4a07138c34599c356aaed", + "sha256_source": "dtvem", + "build": "python-build-standalone", + "alternates": [ + { + "url": "https://builds.dtvem.io/python/3.12.10/python.org/windows-amd64.tar.gz", + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "sha256_source": "dtvem", + "build": "python.org" + } + ] + }, + "linux-amd64": { + "url": "https://builds.dtvem.io/python/3.12.10/linux-amd64.tar.gz", + "build": "python-build-standalone" + } + }, + "3.12.12": { + "windows-amd64": { + "url": "https://builds.dtvem.io/python/3.12.12/windows-amd64.tar.gz", + "sha256": "33a611265c1abfd245ea1fd23273f8535857883552be1bd8b62dfe7db9ef0694", + "sha256_source": "upstream", + "build": "python-build-standalone" + } + }, + "3.9.13": { + "windows-amd64": { + "url": "https://builds.dtvem.io/python/3.9.13/windows-amd64.tar.gz", + "sha256": "34981d11032b07867ad4af3c9a501aafb9602a8526abdfd652423c9edc491684", + "sha256_source": "dtvem", + "build": "python.org" + }, + "windows-arm64": null + }, + "3.11.0": { + "windows-amd64": { + "url": "https://builds.dtvem.io/python/3.11.0/windows-amd64.tar.gz" + } + } + } +}` + +func TestDownloadForBuild(t *testing.T) { + m, err := ParseManifest([]byte(buildsManifestJSON)) + if err != nil { + t.Fatalf("ParseManifest() error: %v", err) + } + + tests := []struct { + name string + version, platform string + build string + wantURL string + }{ + {"alternate", "3.12.10", "windows-amd64", BuildPythonOrg, "https://builds.dtvem.io/python/3.12.10/python.org/windows-amd64.tar.gz"}, + {"default is the build", "3.12.10", "windows-amd64", BuildPythonStandalone, "https://builds.dtvem.io/python/3.12.10/windows-amd64.tar.gz"}, + {"python.org default", "3.9.13", "windows-amd64", BuildPythonOrg, "https://builds.dtvem.io/python/3.9.13/windows-amd64.tar.gz"}, + {"no such build", "3.12.12", "windows-amd64", BuildPythonOrg, ""}, + {"unlabeled entry", "3.11.0", "windows-amd64", BuildPythonOrg, ""}, + {"no build asked for", "3.12.10", "windows-amd64", "", ""}, + {"unavailable platform", "3.9.13", "windows-arm64", BuildPythonOrg, ""}, + {"missing platform", "3.12.12", "linux-amd64", BuildPythonOrg, ""}, + {"missing version", "3.99.0", "windows-amd64", BuildPythonOrg, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := m.GetDownload(tt.version, tt.platform).ForBuild(tt.build) + gotURL := "" + if got != nil { + gotURL = got.URL + } + if gotURL != tt.wantURL { + t.Errorf("ForBuild(%q) URL = %q, want %q", tt.build, gotURL, tt.wantURL) + } + }) + } +} + +func TestManifestListBuildVersions(t *testing.T) { + m, err := ParseManifest([]byte(buildsManifestJSON)) + if err != nil { + t.Fatalf("ParseManifest() error: %v", err) + } + + got := m.ListBuildVersions("windows-amd64", BuildPythonOrg) + sort.Strings(got) + if want := "3.12.10,3.9.13"; strings.Join(got, ",") != want { + t.Errorf("ListBuildVersions(windows-amd64, python.org) = %v, want %s", got, want) + } + if got := m.ListBuildVersions("linux-amd64", BuildPythonOrg); len(got) != 0 { + t.Errorf("ListBuildVersions(linux-amd64, python.org) = %v, want none", got) + } +} + +// legacyDownload is Download as CLIs released before builds and alternates +// read it. +type legacyDownload struct { + URL string `json:"url"` + SHA256 string `json:"sha256"` + SHA256Source string `json:"sha256_source,omitempty"` +} + +// A manifest with builds and alternates still gives released CLIs the same +// default download as before. +func TestBuildsManifest_LegacyCLI(t *testing.T) { + var legacy struct { + Version int `json:"version"` + Versions map[string]map[string]*legacyDownload `json:"versions"` + } + if err := json.Unmarshal([]byte(buildsManifestJSON), &legacy); err != nil { + t.Fatalf("legacy parse error: %v", err) + } + got := legacy.Versions["3.12.10"]["windows-amd64"] + want := legacyDownload{ + URL: "https://builds.dtvem.io/python/3.12.10/windows-amd64.tar.gz", + SHA256: "3e0fd10e4bd60e66baad8be2a0e60775bb389e97e4f4a07138c34599c356aaed", + SHA256Source: "dtvem", + } + if got == nil || *got != want { + t.Errorf("legacy 3.12.10 windows-amd64 = %+v, want %+v", got, want) + } +} + +func TestCachedSource_KeepsBuilds(t *testing.T) { + m, err := ParseManifest([]byte(buildsManifestJSON)) + if err != nil { + t.Fatalf("ParseManifest() error: %v", err) + } + mock := newMockSource() + mock.manifests["python"] = m + source := NewCachedSource(mock, t.TempDir(), time.Hour) + + if _, err := source.GetManifest("python"); err != nil { + t.Fatalf("first GetManifest() error: %v", err) + } + cached, err := source.GetManifest("python") + if err != nil { + t.Fatalf("second GetManifest() error: %v", err) + } + if mock.callCount["python"] != 1 { + t.Errorf("source called %d times, want 1 (second call from cache)", mock.callCount["python"]) + } + if cached.GetDownload("3.12.10", "windows-amd64").ForBuild(BuildPythonOrg) == nil { + t.Error("cached manifest lost the python.org alternate") + } +} + +// A cache file written by a CLI that did not know about builds holds the +// manifest without them, so it is fetched again rather than used. +func TestCachedSource_IgnoresOlderFormat(t *testing.T) { + dir := t.TempDir() + old := `{"cached_at":"` + time.Now().Format(time.RFC3339Nano) + `","manifest":{"version":1,"versions":{"3.12.10":{"windows-amd64":{"url":"https://builds.dtvem.io/python/3.12.10/windows-amd64.tar.gz","sha256":"x"}}}}}` + if err := os.WriteFile(filepath.Join(dir, "python.cache.json"), []byte(old), 0o644); err != nil { + t.Fatal(err) + } + + m, err := ParseManifest([]byte(buildsManifestJSON)) + if err != nil { + t.Fatalf("ParseManifest() error: %v", err) + } + mock := newMockSource() + mock.manifests["python"] = m + source := NewCachedSource(mock, dir, time.Hour) + + got, err := source.GetManifest("python") + if err != nil { + t.Fatalf("GetManifest() error: %v", err) + } + if mock.callCount["python"] != 1 { + t.Errorf("source called %d times, want 1 (older-format cache ignored)", mock.callCount["python"]) + } + if got.GetDownload("3.12.10", "windows-amd64").ForBuild(BuildPythonOrg) == nil { + t.Error("manifest has no python.org alternate; the older-format cache was used") + } +} diff --git a/src/internal/manifest/cache.go b/src/internal/manifest/cache.go index d6c0585..4f5f4e9 100644 --- a/src/internal/manifest/cache.go +++ b/src/internal/manifest/cache.go @@ -18,8 +18,17 @@ type CachedSource struct { ttl time.Duration } +// cacheFormat is the format of the cache files this version writes. A cache +// file of another format is treated as missing: it holds the manifest as +// the CLI that wrote it parsed it, without the fields that CLI did not know. +// +// - 0: no format field; written before builds and alternates existed +// - 2: keeps Download.Build and Download.Alternates +const cacheFormat = 2 + // cacheEntry stores a manifest along with its cache timestamp. type cacheEntry struct { + Format int `json:"format,omitempty"` CachedAt time.Time `json:"cached_at"` Manifest *Manifest `json:"manifest"` } @@ -116,8 +125,8 @@ func (s *CachedSource) loadFromCache(runtime string) (*Manifest, error) { } // Check if cache is still valid - if time.Since(entry.CachedAt) > s.ttl { - return nil, os.ErrNotExist // Treat expired cache as not found + if entry.Format != cacheFormat || time.Since(entry.CachedAt) > s.ttl { + return nil, os.ErrNotExist // Treat expired or older-format cache as not found } return entry.Manifest, nil @@ -129,6 +138,7 @@ func (s *CachedSource) saveToCache(runtime string, manifest *Manifest) error { } entry := cacheEntry{ + Format: cacheFormat, CachedAt: time.Now(), Manifest: manifest, } diff --git a/src/internal/manifest/manifest.go b/src/internal/manifest/manifest.go index ee22741..5db32f9 100644 --- a/src/internal/manifest/manifest.go +++ b/src/internal/manifest/manifest.go @@ -33,6 +33,46 @@ type Download struct { // "dtvem" - checksum generated by dtvem during mirroring // Empty string for legacy manifests without this field SHA256Source string `json:"sha256_source,omitempty"` + + // Build names whose build of the runtime this is, for runtimes with + // more than one (BuildPythonStandalone, BuildPythonOrg). Empty when the + // manifest does not say. + Build string `json:"build,omitempty"` + + // Alternates are other builds of the same version for the same + // platform, installed only when asked for by name. CLIs released before + // this field existed ignore it and install the entry's own URL. + Alternates []*Download `json:"alternates,omitempty"` +} + +// Build names, as recorded in Download.Build. +const ( + // BuildPythonStandalone is python-build-standalone's build of Python, + // which dtvem installs by default. + BuildPythonStandalone = "python-build-standalone" + + // BuildPythonOrg is python.org's own build of Python. On Windows its + // binaries carry the Python Software Foundation's Authenticode + // signature. + BuildPythonOrg = "python.org" +) + +// ForBuild returns the download of the named build: d itself when it is +// that build, otherwise the alternate that is. It returns nil when there is +// none, and for a nil d. +func (d *Download) ForBuild(build string) *Download { + if d == nil || build == "" { + return nil + } + if d.Build == build { + return d + } + for _, alt := range d.Alternates { + if alt != nil && alt.Build == build { + return alt + } + } + return nil } // Availability represents whether a version is available for a platform. @@ -100,6 +140,19 @@ func (m *Manifest) ListAvailableVersions(platform string) []string { return versions } +// ListBuildVersions returns the versions that have the named build for the +// given platform, as their default download or as an alternate. The order +// is not guaranteed. +func (m *Manifest) ListBuildVersions(platform, build string) []string { + var versions []string + for v, platforms := range m.Versions { + if platforms[platform].ForBuild(build) != nil { + versions = append(versions, v) + } + } + return versions +} + // ParseManifest parses JSON data into a Manifest. func ParseManifest(data []byte) (*Manifest, error) { var m Manifest From ce4d2c2867c81c031461556c7526f405b570f2e2 Mon Sep 17 00:00:00 2001 From: "Calvin A. Allen" Date: Thu, 12 Nov 2026 20:18:06 -0500 Subject: [PATCH 3/3] feat(python): install python.org's signed build with --signed On Windows, dtvem install python --signed installs python.org's own build, Authenticode-signed by the Python Software Foundation, instead of python-build-standalone's unsigned one. A version python.org shipped no Windows build for is refused, naming the nearest version that has one; a partial version resolves to the newest version with a signed build, and dtvem warns when newer releases on that line have none. Bulk installs from runtimes.json take the flag too. Each install records its build in .dtvem-build.json. dtvem list shows which build each install is and whether it is signed, and dtvem list-all marks the versions with a signed build. list-all's check marks now initialise their style first; the installed marker was always empty. --- CLAUDE.md | 6 +- README.md | 13 ++ src/cmd/install.go | 197 ++++++++++++++++++++++-- src/cmd/list.go | 69 +++++---- src/cmd/listall.go | 48 +++++- src/cmd/signed_test.go | 185 +++++++++++++++++++++++ src/internal/runtime/signed.go | 66 ++++++++ src/internal/version/compare.go | 53 +++++++ src/internal/version/compare_test.go | 57 +++++++ src/runtimes/python/build.go | 163 ++++++++++++++++++++ src/runtimes/python/build_test.go | 216 +++++++++++++++++++++++++++ src/runtimes/python/provider_full.go | 86 +++++++---- 12 files changed, 1084 insertions(+), 75 deletions(-) create mode 100644 src/cmd/signed_test.go create mode 100644 src/internal/runtime/signed.go create mode 100644 src/internal/version/compare.go create mode 100644 src/internal/version/compare_test.go create mode 100644 src/runtimes/python/build.go create mode 100644 src/runtimes/python/build_test.go diff --git a/CLAUDE.md b/CLAUDE.md index a0f2197..1fbc9ba 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -327,9 +327,11 @@ cd src && go test -cover ./... # With coverage | `deploy-manifests.yml` | Push to main (manifests/) | Deploy manifest files to R2 | | `mirror-all.yml` | Manual dispatch | Mirror all runtime binaries to R2 | | `mirror-sync.yml` | Scheduled | Sync new versions to R2 mirror (installs msitools for Python) | -| `mirror-tool.yml` | PR, push to main (`scripts/mirror-binaries/`) | Vet and test the mirror tool, a separate Go module | +| `mirror-tool.yml` | PR, push to main (`scripts/mirror-binaries/`, `scripts/generate-manifests-from-r2/`) | Vet and test the mirror tool and the manifest generator, separate Go modules | -**Windows Python builds in the mirror**: python-build-standalone is the first source. For versions it lacks, the mirror assembles full builds instead of using python.org's embeddable zips (`scripts/mirror-binaries/python_winbuild.go`): `windows-amd64` from python.org's per-component MSIs (verified against the pinned Windows signing key in `keys/`, unpacked with `msiextract`), `windows-arm64` from python.org's `pythonarm64` NuGet packages (verified against nuget.org's SHA-512). Both are packed as a `.tar.gz` with everything under `python/`. Builds that turn out not to be full installs are skipped, not published. `MIRROR_NETWORK_TESTS=1 go test -run Network ./...` (needs msitools) builds real archives. +**Windows Python builds in the mirror**: python-build-standalone is the first source. For versions it lacks, the mirror assembles full builds instead of using python.org's embeddable zips (`scripts/mirror-binaries/python_winbuild.go`): `windows-amd64` from python.org's per-component MSIs up to 3.13 (verified against the pinned Windows signing key in `keys/`, unpacked with `msiextract`) and from python.org's install-manager zips from 3.14 on, where the MSIs carry no OpenPGP signature (verified against the SHA-256 in python.org's `index-windows.json`, `python_installmanager.go`); `windows-arm64` from python.org's `pythonarm64` NuGet packages (verified against nuget.org's SHA-512). All are packed as a `.tar.gz` with everything under `python/`. Builds that turn out not to be full installs are skipped, not published. `MIRROR_NETWORK_TESTS=1 go test -run Network ./...` (needs msitools) builds real archives. + +**Signed Python builds**: every python.org build is also published at `python//python.org/.tar.gz`, whatever the default build is, so `dtvem install python --signed` can install python.org's Authenticode-signed build (`collectJobs` dedupes on the metadata key, so python-build-standalone's default never displaces it). The manifest generator labels each Python entry with `build` (`python-build-standalone` or `python.org`, from the metadata's source URL) and nests the python.org build under the default entry's `alternates`; released CLIs ignore both fields. The CLI's manifest cache has a `format` (2); older cache files are refetched. Each install records its build in `.dtvem-build.json` in the version's directory (`src/runtimes/python/build.go`); `dtvem list` shows it, and `dtvem list-all` marks the versions with a signed build. ### Reusable Workflows (from `dtvem/.github` repo) diff --git a/README.md b/README.md index 21cbbd3..98074c6 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,19 @@ dtvem migrate python See [Getting Started](https://dtvem.io/docs/user-guide/getting-started) for more examples. +### 🔏 Signed Python Builds (Windows) + +On Windows, dtvem installs [python-build-standalone](https://github.com/astral-sh/python-build-standalone)'s build of Python by default, and its executables are not Authenticode-signed. To install python.org's own build instead, signed by the Python Software Foundation, add `--signed`: + +```powershell +dtvem install python 3.12.10 --signed # python.org's build of 3.12.10 +dtvem install python 3.12 --signed # the newest 3.12.x with a python.org build +dtvem list-all python # the Signed column shows which versions have one +dtvem list python # the Build column shows which build each install is +``` + +python.org ships no Windows binaries for security-only releases (3.12.11 onward, for example), so those versions have no signed build. dtvem says so and names the nearest version that has one, rather than installing an unsigned build. A signed build can therefore lack security fixes that newer releases on the same line carry, and dtvem warns when it does. + ## 📚 Documentation | Topic | Description | diff --git a/src/cmd/install.go b/src/cmd/install.go index e45736a..d7e3f2c 100644 --- a/src/cmd/install.go +++ b/src/cmd/install.go @@ -1,8 +1,10 @@ package cmd import ( + "errors" "fmt" "os" + "sort" "strings" "github.com/CodingWithCalvin/dtvem.cli/src/internal/config" @@ -14,7 +16,8 @@ import ( ) var ( - installYesFlag bool + installYesFlag bool + installSignedFlag bool ) var installCmd = &cobra.Command{ @@ -32,7 +35,18 @@ Single install (partial version - resolves to latest match): Bulk install (reads .dtvem/runtimes.json): dtvem install - dtvem install --yes # Skip confirmation prompt`, + dtvem install --yes # Skip confirmation prompt + +Signed Python builds (Windows): + dtvem install python 3.12.10 --signed # python.org's own build + dtvem install python 3.12 --signed # Newest 3.12.x with a python.org build + + On Windows dtvem installs python-build-standalone's build of Python by + default, which is not Authenticode-signed. --signed installs python.org's + build instead, signed by the Python Software Foundation. python.org ships + no Windows binaries for security-only releases (e.g. 3.12.11 on), so + those versions have no signed build. 'dtvem list-all python' shows which + versions do.`, Args: func(cmd *cobra.Command, args []string) error { if len(args) == 0 || len(args) == 2 { return nil @@ -53,6 +67,7 @@ Bulk install (reads .dtvem/runtimes.json): func init() { rootCmd.AddCommand(installCmd) installCmd.Flags().BoolVarP(&installYesFlag, "yes", "y", false, "Skip confirmation prompt") + installCmd.Flags().BoolVar(&installSignedFlag, "signed", false, "Install python.org's Authenticode-signed build (Python on Windows)") } // installSingle installs a single runtime/version @@ -69,8 +84,21 @@ func installSingle(runtimeName, versionInput string) { ui.Debug("Using provider: %s (%s)", provider.Name(), provider.DisplayName()) + var signed runtime.SignedBuildProvider + if installSignedFlag { + if signed, err = signedBuildProvider(provider); err != nil { + ui.Error("%v", err) + os.Exit(1) + } + } + // Resolve partial version to full version if needed - resolvedVersion, err := resolveVersionForProvider(provider, versionInput) + var resolvedVersion string + if signed != nil { + resolvedVersion, err = resolveSignedVersion(provider, signed, versionInput) + } else { + resolvedVersion, err = resolveVersionForProvider(provider, versionInput) + } if err != nil { ui.Debug("Version resolution failed: %v", err) ui.Error("%v", err) @@ -79,21 +107,127 @@ func installSingle(runtimeName, versionInput string) { // Inform user if version was resolved from partial input if resolvedVersion != versionInput { - ui.Info("Resolved %s to %s", versionInput, resolvedVersion) + if signed != nil { + ui.Info("Resolved %s to %s, the newest %s with a signed build", versionInput, resolvedVersion, versionInput) + } else { + ui.Info("Resolved %s to %s", versionInput, resolvedVersion) + } } - if err := provider.Install(resolvedVersion); err != nil { + if signed != nil { + err = signed.InstallSigned(resolvedVersion) + } else { + err = provider.Install(resolvedVersion) + } + if err != nil { ui.Debug("Installation failed: %v", err) ui.Error("%v", err) + showNoSignedBuildHelp(provider, err) os.Exit(1) } ui.Success("Successfully installed %s %s", provider.DisplayName(), resolvedVersion) + if signed != nil { + warnNewerUnsigned(provider, signed, resolvedVersion) + } // Auto-set global version if no global version is currently configured autoSetGlobalIfNeeded(provider, resolvedVersion) } +// signedBuildProvider returns provider's support for signed builds, or an +// error when it has none on this platform. +func signedBuildProvider(provider runtime.Provider) (runtime.SignedBuildProvider, error) { + signed, ok := provider.(runtime.SignedBuildProvider) + if !ok { + return nil, fmt.Errorf("--signed is only supported for Python, not %s", provider.DisplayName()) + } + if !signed.SignedBuildsSupported() { + return nil, errors.New("--signed is only supported on Windows: python.org's signed builds are Windows builds") + } + return signed, nil +} + +// resolveSignedVersion resolves a partial version to the newest version +// with a signed build. A full version is returned as-is; installing it +// reports when it has no signed build. +func resolveSignedVersion(provider runtime.Provider, signed runtime.SignedBuildProvider, input string) (string, error) { + if !version.IsPartialVersion(input) { + return strings.TrimPrefix(input, "v"), nil + } + + versions, err := signed.SignedVersions() + if err != nil { + return "", fmt.Errorf("failed to fetch signed versions: %w", err) + } + resolved, err := version.ResolvePartialVersion(input, versions) + if err != nil { + return "", fmt.Errorf("no %s version matching %q has a signed build; 'dtvem list-all %s' shows which versions do", provider.DisplayName(), input, provider.Name()) + } + return resolved, nil +} + +// showNoSignedBuildHelp says what to do instead when err is a +// *runtime.NoSignedBuildError. +func showNoSignedBuildHelp(provider runtime.Provider, err error) { + var noSigned *runtime.NoSignedBuildError + if !errors.As(err, &noSigned) { + return + } + if noSigned.Nearest != "" { + ui.Info("Install the signed build of %s with: dtvem install %s %s --signed", noSigned.Nearest, provider.Name(), noSigned.Nearest) + } + if !noSigned.NoneListed { + ui.Info("Or install %s %s without --signed to get its default build, which is not signed", provider.DisplayName(), noSigned.Version) + } +} + +// warnNewerUnsigned warns when newer releases on the major.minor line of a +// signed version have no signed build: they may carry security fixes the +// signed version lacks. +func warnNewerUnsigned(provider runtime.Provider, signed runtime.SignedBuildProvider, installed string) { + available, err := provider.ListAvailable() + if err != nil { + return + } + signedVersions, err := signed.SignedVersions() + if err != nil { + return + } + if newer := newerUnsigned(installed, available, signedVersions); len(newer) > 0 { + ui.Warning("%s", newerUnsignedWarning(provider.DisplayName(), installed, newer)) + } +} + +// newerUnsigned returns the versions in available that are newer than +// installed, on its major.minor line, and not in signedVersions. +func newerUnsigned(installed string, available []runtime.AvailableVersion, signedVersions []string) []string { + hasSigned := make(map[string]bool, len(signedVersions)) + for _, v := range signedVersions { + hasSigned[v] = true + } + var newer []string + for _, av := range available { + v := av.Version.Raw + if !hasSigned[v] && version.MajorMinor(v) == version.MajorMinor(installed) && version.Compare(v, installed) > 0 { + newer = append(newer, v) + } + } + return newer +} + +// newerUnsignedWarning words the warning that versions newer than installed +// on its line have no signed build. +func newerUnsignedWarning(displayName, installed string, newer []string) string { + sort.Slice(newer, func(i, j int) bool { return version.Compare(newer[i], newer[j]) > 0 }) + if len(newer) == 1 { + return fmt.Sprintf("%s %s is newer but has no signed build; it may carry security fixes %s lacks", + displayName, newer[0], installed) + } + return fmt.Sprintf("%s %s through %s are newer but have no signed build; they may carry security fixes %s lacks", + displayName, newer[len(newer)-1], newer[0], installed) +} + // autoSetGlobalIfNeeded sets the installed version as global if no global version exists func autoSetGlobalIfNeeded(provider runtime.Provider, ver string) { currentGlobal, err := provider.GlobalVersion() @@ -150,6 +284,8 @@ type installTask struct { version string provider runtime.Provider alreadyInstalled bool + // signed is set when --signed asks for this runtime's signed build. + signed runtime.SignedBuildProvider } // buildInstallTasks creates a list of install tasks from the config @@ -166,17 +302,37 @@ func buildInstallTasks(runtimes map[string]string) []installTask { alreadyInstalled := isVersionInstalled(provider, version) - tasks = append(tasks, installTask{ + task := installTask{ runtimeName: runtimeName, version: version, provider: provider, alreadyInstalled: alreadyInstalled, - }) + } + if installSignedFlag { + // --signed applies to the runtimes that have signed builds; + // the others install as usual. + task.signed, _ = provider.(runtime.SignedBuildProvider) + } + tasks = append(tasks, task) } return tasks } +// checkSignedTasks fails when --signed asks for signed builds this platform +// does not offer, rather than quietly installing unsigned ones. +func checkSignedTasks(tasks []installTask) error { + for _, task := range tasks { + if task.signed == nil { + continue + } + if _, err := signedBuildProvider(task.provider); err != nil { + return err + } + } + return nil +} + // isVersionInstalled checks if a specific version is already installed func isVersionInstalled(provider runtime.Provider, version string) bool { installedVersions, err := provider.ListInstalled() @@ -198,10 +354,17 @@ func showInstallationPlan(tasks []installTask) (toInstall, alreadyInstalled int) ui.Header("\nInstallation Plan:") for _, task := range tasks { - if task.alreadyInstalled { + switch { + case task.alreadyInstalled && task.signed != nil && !task.signed.InstalledBuild(task.version).Signed: + ui.Warning(" ✓ %s %s (already installed, not a signed build; uninstall it to install the signed build)", task.provider.DisplayName(), task.version) + alreadyInstalled++ + case task.alreadyInstalled: ui.Info(" ✓ %s %s (already installed)", task.provider.DisplayName(), task.version) alreadyInstalled++ - } else { + case task.signed != nil: + ui.Info(" → %s %s (will install the signed build)", task.provider.DisplayName(), task.version) + toInstall++ + default: ui.Info(" → %s %s (will install)", task.provider.DisplayName(), task.version) toInstall++ } @@ -237,12 +400,22 @@ func executeInstalls(tasks []installTask) (success, failures int, failureList [] ui.Progress("Installing %s %s...", task.provider.DisplayName(), task.version) - if err := task.provider.Install(task.version); err != nil { + var err error + if task.signed != nil { + err = task.signed.InstallSigned(task.version) + } else { + err = task.provider.Install(task.version) + } + if err != nil { ui.Error("Failed to install %s %s: %v", task.provider.DisplayName(), task.version, err) + showNoSignedBuildHelp(task.provider, err) failures++ failureList = append(failureList, fmt.Sprintf("%s %s", task.provider.DisplayName(), task.version)) } else { ui.Success("Installed %s %s", task.provider.DisplayName(), task.version) + if task.signed != nil { + warnNewerUnsigned(task.provider, task.signed, task.version) + } success++ // Auto-set global version if needed autoSetGlobalIfNeeded(task.provider, task.version) @@ -305,6 +478,10 @@ func installBulk() { // Build install tasks tasks := buildInstallTasks(runtimes) + if err := checkSignedTasks(tasks); err != nil { + ui.Error("%v", err) + os.Exit(1) + } // Show installation plan toInstallCount, alreadyInstalledCount := showInstallationPlan(tasks) diff --git a/src/cmd/list.go b/src/cmd/list.go index 3b3bb47..6eeaf8c 100644 --- a/src/cmd/list.go +++ b/src/cmd/list.go @@ -63,27 +63,7 @@ func listAllRuntimes() { } hasAny = true - runtimeName := provider.Name() - globalVersion, _ := provider.GlobalVersion() - localVersion, _ := config.LocalVersion(runtimeName) - - // Create table for this runtime with title - table := tui.NewTable("Version", "Status") - table.SetTitle(provider.DisplayName()) - - for _, v := range versions { - version := v.String() - status := getVersionStatus(version, globalVersion, localVersion) - isActive := isVersionActive(version, globalVersion, localVersion) - - if isActive { - table.AddActiveRow(version, status) - } else { - table.AddRow(version, status) - } - } - - fmt.Println(table.Render()) + fmt.Println(installedVersionsTable(provider, versions).Render()) } if !hasAny { @@ -111,26 +91,55 @@ func listSingleRuntime(runtimeName string) { return } + fmt.Println(installedVersionsTable(provider, versions).Render()) +} + +// installedVersionsTable lists a runtime's installed versions with their +// global/local status and, for runtimes with signed builds on this +// platform, which build each one is. +func installedVersionsTable(provider runtime.Provider, versions []runtime.InstalledVersion) *tui.Table { globalVersion, _ := provider.GlobalVersion() - localVersion, _ := config.LocalVersion(runtimeName) + localVersion, _ := config.LocalVersion(provider.Name()) + + signed, _ := provider.(runtime.SignedBuildProvider) + if signed != nil && !signed.SignedBuildsSupported() { + signed = nil + } - // Create table with title - table := tui.NewTable("Version", "Status") + var table *tui.Table + if signed != nil { + table = tui.NewTable("Version", "Status", "Build") + } else { + table = tui.NewTable("Version", "Status") + } table.SetTitle(provider.DisplayName()) for _, v := range versions { version := v.String() - status := getVersionStatus(version, globalVersion, localVersion) - isActive := isVersionActive(version, globalVersion, localVersion) + cells := []string{version, getVersionStatus(version, globalVersion, localVersion)} + if signed != nil { + cells = append(cells, describeBuild(signed.InstalledBuild(version))) + } - if isActive { - table.AddActiveRow(version, status) + if isVersionActive(version, globalVersion, localVersion) { + table.AddActiveRow(cells...) } else { - table.AddRow(version, status) + table.AddRow(cells...) } } + return table +} - fmt.Println(table.Render()) +// describeBuild says which build an install is and whether it is signed. +func describeBuild(info runtime.BuildInfo) string { + switch { + case info.Name == "": + return "unknown" + case info.Signed: + return info.Name + ", signed" + default: + return info.Name + ", not signed" + } } // getVersionStatus returns a status string for a version (global, local, or empty) diff --git a/src/cmd/listall.go b/src/cmd/listall.go index 9e95507..fa61acf 100644 --- a/src/cmd/listall.go +++ b/src/cmd/listall.go @@ -19,7 +19,9 @@ var listAllCmd = &cobra.Command{ Long: `Display all available versions of a runtime that can be installed. This command queries official sources to show all versions available for download. -Installed versions are marked with a ✓ indicator. +Installed versions are marked with a ✓ indicator. For Python on Windows, a +Signed column marks the versions with a signed python.org build, which +'dtvem install python --signed' installs. Examples: dtvem list-all python @@ -70,6 +72,9 @@ Examples: globalVersion, _ := provider.GlobalVersion() localVersion, _ := config.LocalVersion(runtimeName) + // Versions with a signed build, for runtimes that offer them here + signedVersions, showSigned := signedVersionSet(provider) + // Filter versions if requested filteredVersions := available if filter != "" { @@ -99,7 +104,12 @@ Examples: } // Create table for this page - table := tui.NewTable("", "Version", "Status") + var table *tui.Table + if showSigned { + table = tui.NewTable("", "Version", "Status", "Signed") + } else { + table = tui.NewTable("", "Version", "Status") + } table.SetTitle(provider.DisplayName()) for i := 0; i < pageSize; i++ { @@ -109,13 +119,21 @@ Examples: // Check if installed marker := "" if installedMap[version] { - marker = tui.CheckMark + marker = tui.GetCheckMark() } // Build status: combine global/local indicators with lifecycle status := getVersionStatusWithLifecycle(version, globalVersion, localVersion, v.LifecycleStatus) - table.AddRow(marker, version, status) + if showSigned { + signedMarker := "" + if signedVersions[version] { + signedMarker = tui.GetCheckMark() + } + table.AddRow(marker, version, status, signedMarker) + } else { + table.AddRow(marker, version, status) + } } fmt.Println() @@ -142,9 +160,31 @@ Examples: fmt.Println() ui.Info("Install a version with: dtvem install %s ", runtimeName) + if showSigned { + ui.Info("Install a signed build with: dtvem install %s --signed", runtimeName) + } }, } +// signedVersionSet returns the versions with a signed build, and whether +// the provider offers signed builds on this platform at all. +func signedVersionSet(provider runtime.Provider) (map[string]bool, bool) { + signed, ok := provider.(runtime.SignedBuildProvider) + if !ok || !signed.SignedBuildsSupported() { + return nil, false + } + versions, err := signed.SignedVersions() + if err != nil { + ui.Warning("Could not check which versions have signed builds: %v", err) + return nil, false + } + set := make(map[string]bool, len(versions)) + for _, v := range versions { + set[v] = true + } + return set, true +} + func init() { listAllCmd.Flags().StringP("filter", "f", "", "Filter versions by substring (e.g., '3.11' for Python 3.11.x)") listAllCmd.Flags().IntP("limit", "l", 50, "Number of versions to show per page") diff --git a/src/cmd/signed_test.go b/src/cmd/signed_test.go new file mode 100644 index 0000000..13c178c --- /dev/null +++ b/src/cmd/signed_test.go @@ -0,0 +1,185 @@ +package cmd + +import ( + "errors" + "strings" + "testing" + + "github.com/CodingWithCalvin/dtvem.cli/src/internal/runtime" +) + +// mockSignedProvider is a mockProvider that also offers signed builds. +type mockSignedProvider struct { + mockProvider + supported bool + signedVersions []string + signedErr error + installedBuild map[string]runtime.BuildInfo + signedCalls []string +} + +func (m *mockSignedProvider) SignedBuildsSupported() bool { return m.supported } +func (m *mockSignedProvider) SignedVersions() ([]string, error) { + return m.signedVersions, m.signedErr +} +func (m *mockSignedProvider) InstallSigned(version string) error { + m.signedCalls = append(m.signedCalls, version) + return nil +} +func (m *mockSignedProvider) InstalledBuild(version string) runtime.BuildInfo { + return m.installedBuild[version] +} + +func newMockPython(supported bool) *mockSignedProvider { + return &mockSignedProvider{ + mockProvider: mockProvider{ + name: "python", + displayName: "Python", + availableVersions: []runtime.AvailableVersion{ + {Version: runtime.NewVersion("3.13.9")}, + {Version: runtime.NewVersion("3.12.12")}, + {Version: runtime.NewVersion("3.12.11")}, + {Version: runtime.NewVersion("3.12.10")}, + {Version: runtime.NewVersion("3.11.9")}, + }, + }, + supported: supported, + signedVersions: []string{"3.13.9", "3.12.10", "3.12.9"}, + } +} + +func TestSignedBuildProvider(t *testing.T) { + node := &mockProvider{name: "node", displayName: "Node.js"} + if _, err := signedBuildProvider(node); err == nil || !strings.Contains(err.Error(), "only supported for Python, not Node.js") { + t.Errorf("signedBuildProvider(node) error = %v, want only-Python error", err) + } + + if _, err := signedBuildProvider(newMockPython(false)); err == nil || !strings.Contains(err.Error(), "only supported on Windows") { + t.Errorf("signedBuildProvider(python, unsupported) error = %v, want only-Windows error", err) + } + + python := newMockPython(true) + got, err := signedBuildProvider(python) + if err != nil || got != python { + t.Errorf("signedBuildProvider(python) = %v, %v; want the provider", got, err) + } +} + +func TestResolveSignedVersion(t *testing.T) { + python := newMockPython(true) + + tests := []struct { + input string + want string + wantErr string + }{ + // A full version is installed as asked; the install reports a + // missing signed build. + {input: "3.12.12", want: "3.12.12"}, + {input: "v3.12.10", want: "3.12.10"}, + // A partial version picks the newest signed build, not the newest + // release (3.12.12). + {input: "3.12", want: "3.12.10"}, + {input: "3", want: "3.13.9"}, + {input: "3.11", wantErr: `no Python version matching "3.11" has a signed build; 'dtvem list-all python' shows which versions do`}, + } + for _, tt := range tests { + got, err := resolveSignedVersion(python, python, tt.input) + if tt.wantErr != "" { + if err == nil || err.Error() != tt.wantErr { + t.Errorf("resolveSignedVersion(%q) error = %v, want %q", tt.input, err, tt.wantErr) + } + continue + } + if err != nil || got != tt.want { + t.Errorf("resolveSignedVersion(%q) = %q, %v; want %q", tt.input, got, err, tt.want) + } + } + + python.signedErr = errors.New("offline") + if _, err := resolveSignedVersion(python, python, "3.12"); err == nil || !strings.Contains(err.Error(), "offline") { + t.Errorf("resolveSignedVersion() with failing SignedVersions error = %v, want it passed on", err) + } +} + +func TestNewerUnsigned(t *testing.T) { + python := newMockPython(true) + python.availableVersions = append(python.availableVersions, runtime.AvailableVersion{Version: runtime.NewVersion("3.12.9")}) + + tests := []struct { + installed string + want string + }{ + // 3.12.10 has a signed build, so only 3.12.11 and 3.12.12 count. + {"3.12.9", "3.12.12,3.12.11"}, + {"3.12.10", "3.12.12,3.12.11"}, + {"3.13.9", ""}, + // Other lines never count. + {"3.11.9", ""}, + } + for _, tt := range tests { + got := newerUnsigned(tt.installed, python.availableVersions, python.signedVersions) + if strings.Join(got, ",") != tt.want { + t.Errorf("newerUnsigned(%s) = %v, want %s", tt.installed, got, tt.want) + } + } +} + +func TestNewerUnsignedWarning(t *testing.T) { + if got, want := newerUnsignedWarning("Python", "3.12.10", []string{"3.12.11", "3.12.12"}), + "Python 3.12.11 through 3.12.12 are newer but have no signed build; they may carry security fixes 3.12.10 lacks"; got != want { + t.Errorf("newerUnsignedWarning(two) = %q, want %q", got, want) + } + if got, want := newerUnsignedWarning("Python", "3.12.10", []string{"3.12.11"}), + "Python 3.12.11 is newer but has no signed build; it may carry security fixes 3.12.10 lacks"; got != want { + t.Errorf("newerUnsignedWarning(one) = %q, want %q", got, want) + } +} + +func TestDescribeBuild(t *testing.T) { + for _, tt := range []struct { + info runtime.BuildInfo + want string + }{ + {runtime.BuildInfo{Name: "python.org", Signed: true}, "python.org, signed"}, + {runtime.BuildInfo{Name: "python-build-standalone"}, "python-build-standalone, not signed"}, + {runtime.BuildInfo{}, "unknown"}, + } { + if got := describeBuild(tt.info); got != tt.want { + t.Errorf("describeBuild(%+v) = %q, want %q", tt.info, got, tt.want) + } + } +} + +func TestCheckSignedTasks(t *testing.T) { + node := &mockProvider{name: "node", displayName: "Node.js"} + + // Runtimes without signed builds install as usual. + ok := []installTask{{provider: node}, {provider: newMockPython(true), signed: newMockPython(true)}} + if err := checkSignedTasks(ok); err != nil { + t.Errorf("checkSignedTasks(supported) error = %v, want nil", err) + } + + unsupported := newMockPython(false) + bad := []installTask{{provider: node}, {provider: unsupported, signed: unsupported}} + if err := checkSignedTasks(bad); err == nil { + t.Error("checkSignedTasks(unsupported) error = nil, want an error rather than unsigned installs") + } +} + +func TestShowNoSignedBuildHelp_IgnoresOtherErrors(t *testing.T) { + // Only a *runtime.NoSignedBuildError gets help; this must not panic. + showNoSignedBuildHelp(newMockPython(true), errors.New("download failed")) + showNoSignedBuildHelp(newMockPython(true), &runtime.NoSignedBuildError{Runtime: "Python", Version: "3.12.12", Build: "python.org", Platform: "windows-amd64", Nearest: "3.12.10"}) +} + +func TestNoSignedBuildError(t *testing.T) { + err := &runtime.NoSignedBuildError{Runtime: "Python", Version: "3.12.12", Build: "python.org", Platform: "windows-amd64", Nearest: "3.12.10"} + if want := "Python 3.12.12 has no signed (python.org) build for windows-amd64; the nearest version with one is 3.12.10"; err.Error() != want { + t.Errorf("Error() = %q, want %q", err.Error(), want) + } + none := &runtime.NoSignedBuildError{Runtime: "Python", Build: "python.org", Platform: "windows-amd64", NoneListed: true} + if want := "the Python manifest lists no signed (python.org) builds for windows-amd64"; none.Error() != want { + t.Errorf("Error() = %q, want %q", none.Error(), want) + } +} diff --git a/src/internal/runtime/signed.go b/src/internal/runtime/signed.go new file mode 100644 index 0000000..1aa2168 --- /dev/null +++ b/src/internal/runtime/signed.go @@ -0,0 +1,66 @@ +//go:build !shim + +package runtime + +import "fmt" + +// NoSignedBuildError reports that there is no signed build of a version +// for a platform. +type NoSignedBuildError struct { + // Runtime is the runtime's display name, e.g. "Python". + Runtime string + Version string + // Build names the signed build, e.g. "python.org". + Build string + Platform string + // Nearest is the version nearest to Version on its major.minor line + // that has a signed build, or "" if none does. + Nearest string + // NoneListed is set when there are no signed builds at all for the + // platform. + NoneListed bool +} + +func (e *NoSignedBuildError) Error() string { + if e.NoneListed { + return fmt.Sprintf("the %s manifest lists no signed (%s) builds for %s", e.Runtime, e.Build, e.Platform) + } + msg := fmt.Sprintf("%s %s has no signed (%s) build for %s", e.Runtime, e.Version, e.Build, e.Platform) + if e.Nearest != "" { + msg += fmt.Sprintf("; the nearest version with one is %s", e.Nearest) + } + return msg +} + +// BuildInfo says which build of a runtime an installed version came from. +type BuildInfo struct { + // Name is the build, such as "python.org" or "python-build-standalone". + // Empty when dtvem cannot tell. + Name string + + // Signed reports whether the build's executables carry their + // publisher's Authenticode signature. + Signed bool +} + +// SignedBuildProvider is implemented by providers that can install a +// version's publisher-signed build in place of the build they install by +// default: Python on Windows, where python.org's build is Authenticode- +// signed by the Python Software Foundation and python-build-standalone's +// is not. +type SignedBuildProvider interface { + // SignedBuildsSupported reports whether signed builds are offered on + // this platform. + SignedBuildsSupported() bool + + // SignedVersions returns the versions that have a signed build for + // this platform, newest first. + SignedVersions() ([]string, error) + + // InstallSigned installs the signed build of version. It fails with a + // *NoSignedBuildError when version has no signed build. + InstallSigned(version string) error + + // InstalledBuild reports which build an installed version came from. + InstalledBuild(version string) BuildInfo +} diff --git a/src/internal/version/compare.go b/src/internal/version/compare.go new file mode 100644 index 0000000..8e434ed --- /dev/null +++ b/src/internal/version/compare.go @@ -0,0 +1,53 @@ +package version + +import ( + "strings" +) + +// Compare compares two version strings semantically. It returns a positive +// number if a > b, a negative one if a < b, and 0 if they are equal. A +// pre-release sorts below the release it precedes. +func Compare(a, b string) int { + return compareVersionStrings(a, b) +} + +// MajorMinor returns the major.minor line of a version ("3.12" for +// "3.12.10"), or the version itself when it has fewer parts. +func MajorMinor(v string) string { + v = strings.TrimPrefix(v, "v") + parts := strings.SplitN(v, ".", 3) + if len(parts) < 2 { + return v + } + return parts[0] + "." + parts[1] +} + +// Nearest returns the version in candidates nearest to target on target's +// major.minor line: target itself if present, else the highest candidate +// below it, else the lowest above it. It returns "" when no candidate is on +// that line. +func Nearest(target string, candidates []string) string { + line := MajorMinor(target) + var below, above string + for _, c := range candidates { + if MajorMinor(c) != line { + continue + } + switch cmp := Compare(c, target); { + case cmp == 0: + return c + case cmp < 0: + if below == "" || Compare(c, below) > 0 { + below = c + } + default: + if above == "" || Compare(c, above) < 0 { + above = c + } + } + } + if below != "" { + return below + } + return above +} diff --git a/src/internal/version/compare_test.go b/src/internal/version/compare_test.go new file mode 100644 index 0000000..49a837d --- /dev/null +++ b/src/internal/version/compare_test.go @@ -0,0 +1,57 @@ +package version + +import "testing" + +func TestCompare(t *testing.T) { + tests := []struct { + a, b string + want int // sign only + }{ + {"3.12.10", "3.12.9", 1}, + {"3.12.9", "3.12.10", -1}, + {"3.12.10", "3.12.10", 0}, + {"v22.0.0", "22.0.0", 0}, + {"4.0.0-preview2", "4.0.0", -1}, + } + for _, tt := range tests { + got := Compare(tt.a, tt.b) + if (got > 0) != (tt.want > 0) || (got < 0) != (tt.want < 0) { + t.Errorf("Compare(%q, %q) = %d, want sign %d", tt.a, tt.b, got, tt.want) + } + } +} + +func TestMajorMinor(t *testing.T) { + for in, want := range map[string]string{ + "3.12.10": "3.12", + "v22.1.0": "22.1", + "3.12": "3.12", + "3": "3", + } { + if got := MajorMinor(in); got != want { + t.Errorf("MajorMinor(%q) = %q, want %q", in, got, want) + } + } +} + +func TestNearest(t *testing.T) { + signed := []string{"3.12.0", "3.12.9", "3.12.10", "3.13.9", "3.9.13"} + tests := []struct { + target, want string + }{ + {"3.12.12", "3.12.10"}, // a security-only release: the last binary release below it + {"3.12.10", "3.12.10"}, // present + {"3.12.5", "3.12.0"}, // a gap: the highest below + {"3.13.0", "3.13.9"}, // nothing below on the line: the lowest above + {"3.11.4", ""}, // nothing on the line + {"3.1.0", ""}, // "3.1" is not "3.12" or "3.13" + } + for _, tt := range tests { + if got := Nearest(tt.target, signed); got != tt.want { + t.Errorf("Nearest(%q) = %q, want %q", tt.target, got, tt.want) + } + } + if got := Nearest("3.12.12", nil); got != "" { + t.Errorf("Nearest(no candidates) = %q, want \"\"", got) + } +} diff --git a/src/runtimes/python/build.go b/src/runtimes/python/build.go new file mode 100644 index 0000000..d7b5a53 --- /dev/null +++ b/src/runtimes/python/build.go @@ -0,0 +1,163 @@ +//go:build !shim + +// Which build of Python an install comes from. On Windows dtvem installs +// python-build-standalone's build by default, whose executables are not +// Authenticode-signed; `dtvem install python --signed` installs +// python.org's own build instead, signed by the Python Software Foundation, +// for the versions python.org shipped Windows binaries for. + +package python + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + goruntime "runtime" + "sort" + "sync" + + "github.com/CodingWithCalvin/dtvem.cli/src/internal/config" + "github.com/CodingWithCalvin/dtvem.cli/src/internal/constants" + "github.com/CodingWithCalvin/dtvem.cli/src/internal/manifest" + "github.com/CodingWithCalvin/dtvem.cli/src/internal/runtime" + "github.com/CodingWithCalvin/dtvem.cli/src/internal/version" +) + +// buildMarkerFile, in an install's directory, records which build was +// installed there. Installs made before dtvem wrote it have none. +const buildMarkerFile = ".dtvem-build.json" + +// buildMarker is the content of buildMarkerFile. +type buildMarker struct { + Build string `json:"build"` + URL string `json:"url"` + SHA256 string `json:"sha256,omitempty"` +} + +// SignedBuildsSupported reports whether signed builds are offered here: +// python.org's builds dtvem mirrors are Windows builds. +func (p *Provider) SignedBuildsSupported() bool { + return goruntime.GOOS == constants.OSWindows +} + +// SignedVersions returns the versions with a python.org build for this +// platform, newest first. +func (p *Provider) SignedVersions() ([]string, error) { + if !p.SignedBuildsSupported() { + return nil, nil + } + m, err := manifest.DefaultSource().GetManifest("python") + if err != nil { + return nil, fmt.Errorf("failed to load manifest: %w", err) + } + return signedVersions(m, manifest.CurrentPlatform()), nil +} + +// InstalledBuild reports which build an installed version came from: what +// the install recorded, or for an install from before dtvem recorded it, +// the build its version's default download is in the manifest built into +// dtvem (the default downloads of a version do not change). +func (p *Provider) InstalledBuild(ver string) runtime.BuildInfo { + name := readBuildMarker(config.RuntimeVersionPath("python", ver)) + if name == "" { + if m, err := embeddedPythonManifest(); err == nil { + name = defaultBuild(m, ver, manifest.CurrentPlatform()) + } + } + return runtime.BuildInfo{Name: name, Signed: name == manifest.BuildPythonOrg} +} + +// alreadyInstalledError explains why a version that is installed is not +// installed again, saying how to replace it with the build asked for. +func (p *Provider) alreadyInstalledError(ver, build string) error { + if build == "" { + return fmt.Errorf("Python %s is already installed", ver) + } + installed := p.InstalledBuild(ver).Name + if installed == build { + return fmt.Errorf("Python %s is already installed (%s build)", ver, build) + } + from := "dtvem cannot tell from which build" + if installed != "" { + from = fmt.Sprintf("from %s", installed) + } + return fmt.Errorf("Python %s is already installed, %s; to replace it with the %s build, run `dtvem uninstall python %s`, then install it again", ver, from, build, ver) +} + +// selectDownload returns the download of a version for a platform: the +// named build, or the default download when build is empty. +func selectDownload(m *manifest.Manifest, ver, platform, build string) (*manifest.Download, error) { + dl := m.GetDownload(ver, platform) + if build == "" { + if dl == nil { + return nil, fmt.Errorf("Python %s is not available for %s", ver, platform) + } + return dl, nil + } + + if b := dl.ForBuild(build); b != nil { + return b, nil + } + available := m.ListBuildVersions(platform, build) + return nil, &runtime.NoSignedBuildError{ + Runtime: "Python", + Version: ver, + Build: build, + Platform: platform, + Nearest: version.Nearest(ver, available), + NoneListed: len(available) == 0, + } +} + +// signedVersions returns the versions with a python.org build for a +// platform, newest first. +func signedVersions(m *manifest.Manifest, platform string) []string { + versions := m.ListBuildVersions(platform, manifest.BuildPythonOrg) + sort.Slice(versions, func(i, j int) bool { + return version.Compare(versions[i], versions[j]) > 0 + }) + return versions +} + +// defaultBuild returns the build of a version's default download for a +// platform, or "" when the manifest does not say. +func defaultBuild(m *manifest.Manifest, ver, platform string) string { + if dl := m.GetDownload(ver, platform); dl != nil { + return dl.Build + } + return "" +} + +// embeddedPythonManifest is the Python manifest built into dtvem, parsed +// once. +var embeddedPythonManifest = sync.OnceValues(func() (*manifest.Manifest, error) { + return manifest.NewEmbeddedSource().GetManifest("python") +}) + +// writeBuildMarker records in installPath which build dl is. It writes +// nothing when the manifest did not say. +func writeBuildMarker(installPath string, dl *manifest.Download) error { + if dl.Build == "" { + return nil + } + data, err := json.MarshalIndent(buildMarker{Build: dl.Build, URL: dl.URL, SHA256: dl.SHA256}, "", " ") + if err != nil { + return err + } + return os.WriteFile(filepath.Join(installPath, buildMarkerFile), data, 0o644) +} + +// readBuildMarker returns the build recorded in installPath, or "" when +// there is no readable record. +func readBuildMarker(installPath string) string { + data, err := os.ReadFile(filepath.Join(installPath, buildMarkerFile)) + if err != nil { + return "" + } + var m buildMarker + if err := json.Unmarshal(data, &m); err != nil { + return "" + } + return m.Build +} diff --git a/src/runtimes/python/build_test.go b/src/runtimes/python/build_test.go new file mode 100644 index 0000000..dd32dc5 --- /dev/null +++ b/src/runtimes/python/build_test.go @@ -0,0 +1,216 @@ +package python + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/CodingWithCalvin/dtvem.cli/src/internal/config" + "github.com/CodingWithCalvin/dtvem.cli/src/internal/manifest" + "github.com/CodingWithCalvin/dtvem.cli/src/internal/runtime" +) + +const testPlatform = "windows-amd64" + +// testBuildsManifest has a python-build-standalone default with a +// python.org alternate (3.12.10, 3.13.9), a python.org default (3.9.13) and +// versions with no python.org build (3.12.12, 3.11.0). +func testBuildsManifest() *manifest.Manifest { + pbs := func(v string) *manifest.Download { + return &manifest.Download{URL: "https://builds.dtvem.io/python/" + v + "/windows-amd64.tar.gz", SHA256: "pbs-" + v, Build: manifest.BuildPythonStandalone} + } + org := func(v string) *manifest.Download { + return &manifest.Download{URL: "https://builds.dtvem.io/python/" + v + "/python.org/windows-amd64.tar.gz", SHA256: "org-" + v, Build: manifest.BuildPythonOrg} + } + withOrg := func(v string) *manifest.Download { + d := pbs(v) + d.Alternates = []*manifest.Download{org(v)} + return d + } + return &manifest.Manifest{ + Version: 1, + Versions: map[string]map[string]*manifest.Download{ + "3.12.10": {testPlatform: withOrg("3.12.10"), "linux-amd64": pbs("3.12.10")}, + "3.12.12": {testPlatform: pbs("3.12.12")}, + "3.13.9": {testPlatform: withOrg("3.13.9")}, + "3.9.13": {testPlatform: { + URL: "https://builds.dtvem.io/python/3.9.13/windows-amd64.tar.gz", SHA256: "org-default", Build: manifest.BuildPythonOrg, + }}, + "3.11.0": {testPlatform: {URL: "https://builds.dtvem.io/python/3.11.0/windows-amd64.tar.gz"}}, + }, + } +} + +func TestSelectDownload(t *testing.T) { + m := testBuildsManifest() + + tests := []struct { + name string + version string + platform string + build string + wantURL string + wantNearest string // for a *NoSignedBuildError + wantErr string + }{ + {name: "default", version: "3.12.10", platform: testPlatform, wantURL: "https://builds.dtvem.io/python/3.12.10/windows-amd64.tar.gz"}, + {name: "default, unlabeled", version: "3.11.0", platform: testPlatform, wantURL: "https://builds.dtvem.io/python/3.11.0/windows-amd64.tar.gz"}, + {name: "default, missing", version: "3.99.0", platform: testPlatform, wantErr: "Python 3.99.0 is not available for windows-amd64"}, + {name: "signed alternate", version: "3.12.10", platform: testPlatform, build: manifest.BuildPythonOrg, wantURL: "https://builds.dtvem.io/python/3.12.10/python.org/windows-amd64.tar.gz"}, + {name: "signed default", version: "3.9.13", platform: testPlatform, build: manifest.BuildPythonOrg, wantURL: "https://builds.dtvem.io/python/3.9.13/windows-amd64.tar.gz"}, + {name: "no signed build: security-only release", version: "3.12.12", platform: testPlatform, build: manifest.BuildPythonOrg, wantNearest: "3.12.10", + wantErr: "Python 3.12.12 has no signed (python.org) build for windows-amd64; the nearest version with one is 3.12.10"}, + {name: "no signed build on the line", version: "3.11.0", platform: testPlatform, build: manifest.BuildPythonOrg, + wantErr: "Python 3.11.0 has no signed (python.org) build for windows-amd64"}, + {name: "none listed for platform", version: "3.12.10", platform: "linux-amd64", build: manifest.BuildPythonOrg, + wantErr: "the Python manifest lists no signed (python.org) builds for linux-amd64"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dl, err := selectDownload(m, tt.version, tt.platform, tt.build) + if tt.wantErr != "" { + if err == nil || err.Error() != tt.wantErr { + t.Fatalf("selectDownload() error = %v, want %q", err, tt.wantErr) + } + var noSigned *runtime.NoSignedBuildError + if tt.build != "" && (!errors.As(err, &noSigned) || noSigned.Nearest != tt.wantNearest) { + t.Errorf("selectDownload() error = %#v, want a *runtime.NoSignedBuildError with Nearest %q", err, tt.wantNearest) + } + return + } + if err != nil { + t.Fatalf("selectDownload() error = %v", err) + } + if dl.URL != tt.wantURL { + t.Errorf("selectDownload() URL = %q, want %q", dl.URL, tt.wantURL) + } + }) + } +} + +func TestSignedVersions(t *testing.T) { + got := signedVersions(testBuildsManifest(), testPlatform) + if want := "3.13.9,3.12.10,3.9.13"; strings.Join(got, ",") != want { + t.Errorf("signedVersions() = %v, want %s (newest first)", got, want) + } + if got := signedVersions(testBuildsManifest(), "linux-amd64"); len(got) != 0 { + t.Errorf("signedVersions(linux-amd64) = %v, want none", got) + } +} + +func TestDefaultBuild(t *testing.T) { + m := testBuildsManifest() + for _, tt := range []struct{ version, platform, want string }{ + {"3.12.10", testPlatform, manifest.BuildPythonStandalone}, + {"3.9.13", testPlatform, manifest.BuildPythonOrg}, + {"3.11.0", testPlatform, ""}, + {"3.99.0", testPlatform, ""}, + } { + if got := defaultBuild(m, tt.version, tt.platform); got != tt.want { + t.Errorf("defaultBuild(%s, %s) = %q, want %q", tt.version, tt.platform, got, tt.want) + } + } +} + +func TestBuildMarker(t *testing.T) { + dir := t.TempDir() + + if got := readBuildMarker(dir); got != "" { + t.Errorf("readBuildMarker(no marker) = %q, want \"\"", got) + } + + // The manifest did not say which build: nothing is recorded. + if err := writeBuildMarker(dir, &manifest.Download{URL: "https://example.invalid/x.tar.gz"}); err != nil { + t.Fatalf("writeBuildMarker(unlabeled) error: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, buildMarkerFile)); !os.IsNotExist(err) { + t.Errorf("writeBuildMarker(unlabeled) wrote %s (stat error %v)", buildMarkerFile, err) + } + + dl := testBuildsManifest().GetDownload("3.12.10", testPlatform).ForBuild(manifest.BuildPythonOrg) + if err := writeBuildMarker(dir, dl); err != nil { + t.Fatalf("writeBuildMarker() error: %v", err) + } + if got := readBuildMarker(dir); got != manifest.BuildPythonOrg { + t.Errorf("readBuildMarker() = %q, want %q", got, manifest.BuildPythonOrg) + } + + if err := os.WriteFile(filepath.Join(dir, buildMarkerFile), []byte("not json"), 0o644); err != nil { + t.Fatal(err) + } + if got := readBuildMarker(dir); got != "" { + t.Errorf("readBuildMarker(corrupt) = %q, want \"\"", got) + } +} + +// setTestRoot points dtvem at an empty root for the test. +func setTestRoot(t *testing.T) { + t.Helper() + t.Setenv("DTVEM_ROOT", t.TempDir()) + config.ResetPathsCache() + t.Cleanup(config.ResetPathsCache) +} + +// installFake makes an install directory for version holding a marker for +// build, or no marker when build is empty. +func installFake(t *testing.T, version, build string) { + t.Helper() + dir := config.RuntimeVersionPath("python", version) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if build != "" { + if err := writeBuildMarker(dir, &manifest.Download{URL: "https://example.invalid/x.tar.gz", Build: build}); err != nil { + t.Fatal(err) + } + } +} + +func TestInstalledBuild(t *testing.T) { + setTestRoot(t) + p := NewProvider() + + installFake(t, "3.12.10", manifest.BuildPythonOrg) + installFake(t, "3.12.12", manifest.BuildPythonStandalone) + + if got, want := p.InstalledBuild("3.12.10"), (runtime.BuildInfo{Name: manifest.BuildPythonOrg, Signed: true}); got != want { + t.Errorf("InstalledBuild(3.12.10) = %+v, want %+v", got, want) + } + if got, want := p.InstalledBuild("3.12.12"), (runtime.BuildInfo{Name: manifest.BuildPythonStandalone}); got != want { + t.Errorf("InstalledBuild(3.12.12) = %+v, want %+v", got, want) + } + // Without a marker the answer comes from the built-in manifest, which + // knows nothing of a version that does not exist. + installFake(t, "0.0.1", "") + if got := p.InstalledBuild("0.0.1"); got != (runtime.BuildInfo{}) { + t.Errorf("InstalledBuild(0.0.1) = %+v, want unknown", got) + } +} + +func TestAlreadyInstalledError(t *testing.T) { + setTestRoot(t) + p := NewProvider() + installFake(t, "3.12.10", manifest.BuildPythonOrg) + installFake(t, "3.12.12", manifest.BuildPythonStandalone) + installFake(t, "0.0.1", "") + + tests := []struct { + version, build, want string + }{ + {"3.12.12", "", "Python 3.12.12 is already installed"}, + {"3.12.10", manifest.BuildPythonOrg, "Python 3.12.10 is already installed (python.org build)"}, + {"3.12.12", manifest.BuildPythonOrg, "Python 3.12.12 is already installed, from python-build-standalone; to replace it with the python.org build, run `dtvem uninstall python 3.12.12`, then install it again"}, + {"0.0.1", manifest.BuildPythonOrg, "Python 0.0.1 is already installed, dtvem cannot tell from which build; to replace it with the python.org build, run `dtvem uninstall python 0.0.1`, then install it again"}, + } + for _, tt := range tests { + if got := p.alreadyInstalledError(tt.version, tt.build).Error(); got != tt.want { + t.Errorf("alreadyInstalledError(%s, %q) = %q, want %q", tt.version, tt.build, got, tt.want) + } + } +} + +func TestSignedProviderInterface(t *testing.T) { + var _ runtime.SignedBuildProvider = NewProvider() +} diff --git a/src/runtimes/python/provider_full.go b/src/runtimes/python/provider_full.go index 63c74ca..4ef69b7 100644 --- a/src/runtimes/python/provider_full.go +++ b/src/runtimes/python/provider_full.go @@ -26,8 +26,9 @@ import ( "github.com/CodingWithCalvin/dtvem.cli/src/internal/ui" ) -// downloadAndExtract downloads and extracts the Python archive. -func (p *Provider) downloadAndExtract(version, downloadURL, archiveName string) (extractDir string, cleanup func(), err error) { +// downloadAndExtract downloads and extracts the Python archive, checking it +// against expectedSHA256 unless that is empty. +func (p *Provider) downloadAndExtract(version, downloadURL, archiveName, expectedSHA256 string) (extractDir string, cleanup func(), err error) { ui.Progress("Downloading from %s", downloadURL) tempDir, err := config.CreateStagingDir("python", version) @@ -38,7 +39,12 @@ func (p *Provider) downloadAndExtract(version, downloadURL, archiveName string) cleanupFunc := func() { _ = os.RemoveAll(tempDir) } archivePath := filepath.Join(tempDir, archiveName) - if err := download.File(downloadURL, archivePath); err != nil { + if expectedSHA256 != "" { + err = download.FileVerified(downloadURL, archivePath, expectedSHA256) + } else { + err = download.File(downloadURL, archivePath) + } + if err != nil { cleanupFunc() return "", nil, fmt.Errorf("failed to download: %w", err) } @@ -100,28 +106,64 @@ func (p *Provider) installPipIfNeeded(version string) { } } -// Install downloads and installs a specific version. +// Install downloads and installs a specific version: its default build in +// the manifest, which is python-build-standalone's wherever it has one. func (p *Provider) Install(version string) error { - ui.Debug("Starting Python installation for version %s", version) + return p.install(version, "") +} + +// InstallSigned downloads and installs python.org's build of a version, +// whose executables carry the Python Software Foundation's Authenticode +// signature. It fails with a *runtime.NoSignedBuildError when the manifest +// has no python.org build of the version for this platform. +func (p *Provider) InstallSigned(version string) error { + if !p.SignedBuildsSupported() { + return fmt.Errorf("signed Python builds are only offered on Windows") + } + return p.install(version, manifest.BuildPythonOrg) +} + +// install installs the named build of a version, or its default build when +// build is empty. +func (p *Provider) install(version, build string) error { + ui.Debug("Starting Python installation for version %s (build %q)", version, build) if err := config.EnsureDirectories(); err != nil { return fmt.Errorf("failed to create dtvem directories: %w", err) } if installed, _ := p.IsInstalled(version); installed { - return fmt.Errorf("Python %s is already installed", version) + return p.alreadyInstalledError(version, build) } - ui.Header("Installing Python v%s...", version) + if build == "" { + ui.Header("Installing Python v%s...", version) + } else { + ui.Header("Installing Python v%s (%s build)...", version, build) + } - downloadURL, archiveName, err := p.getDownloadURL(version) + m, err := manifest.DefaultSource().GetManifest("python") if err != nil { - return fmt.Errorf("failed to get download URL: %w", err) + return fmt.Errorf("failed to get download URL: failed to load manifest: %w", err) + } + dl, err := selectDownload(m, version, manifest.CurrentPlatform(), build) + if err != nil { + if build == "" { + return fmt.Errorf("failed to get download URL: %w", err) + } + return err } - ui.Debug("Download URL: %s", downloadURL) + archiveName := filepath.Base(dl.URL) + ui.Debug("Download URL: %s", dl.URL) ui.Debug("Archive name: %s", archiveName) - extractDir, cleanup, err := p.downloadAndExtract(version, downloadURL, archiveName) + // Only a build asked for by name is checked against the manifest's + // checksum, so a default install downloads exactly as it always has. + expectedSHA256 := "" + if build != "" { + expectedSHA256 = dl.SHA256 + } + extractDir, cleanup, err := p.downloadAndExtract(version, dl.URL, archiveName, expectedSHA256) if err != nil { return err } @@ -142,6 +184,10 @@ func (p *Provider) Install(version string) error { return fmt.Errorf("failed to move to install location: %w", err) } + if err := writeBuildMarker(installPath, dl); err != nil { + ui.Warning("Could not record which build Python %s came from: %v", version, err) + } + // Install/configure pip first (so executables exist before creating shims) p.installPipIfNeeded(version) @@ -159,24 +205,6 @@ func (p *Provider) Install(version string) error { return nil } -// getDownloadURL returns the download URL and archive name for a given version. -func (p *Provider) getDownloadURL(version string) (string, string, error) { - m, err := manifest.DefaultSource().GetManifest("python") - if err != nil { - return "", "", fmt.Errorf("failed to load manifest: %w", err) - } - - platform := manifest.CurrentPlatform() - dl := m.GetDownload(version, platform) - if dl == nil { - return "", "", fmt.Errorf("Python %s is not available for %s", version, platform) - } - - archiveName := filepath.Base(dl.URL) - - return dl.URL, archiveName, nil -} - // createShims creates shims for Python executables and registers them in the // shim-map cache so subsequent shim invocations resolve via O(1) lookup rather // than falling back to the provider registry. The version is recorded in the