From 3e73d1b1f81b80ed517f81d8325e9a76c67d5bce Mon Sep 17 00:00:00 2001 From: "Calvin A. Allen" Date: Fri, 13 Nov 2026 09:47:58 -0500 Subject: [PATCH] fix(mirror): keep python.org builds out of default locations python.org's sources filled a version's default location (python//) whenever python-build-standalone had no build of it in that run. A sync skips anything already mirrored, so a release python.org shipped before python-build-standalone did, or one mirrored on a day the python-build-standalone listing failed, kept python.org's build as its default for good. python.org's builds now go only to the python.org location. For a version with no default build, the manifest generator already lists the python.org build as the default, and python-build-standalone's build takes over once it is mirrored. Defaults already mirrored from python.org (3.9.13, for example) are left as they are. --- CLAUDE.md | 6 +- .../generate-manifests-from-r2/main_test.go | 31 +++++++++ .../mirror-binaries/python_installmanager.go | 4 +- .../python_installmanager_test.go | 2 +- scripts/mirror-binaries/python_winbuild.go | 68 ++++++++----------- .../mirror-binaries/python_winbuild_test.go | 47 ++++++------- scripts/mirror-binaries/sources.go | 10 +-- scripts/mirror-binaries/sources_test.go | 29 +++++--- 8 files changed, 111 insertions(+), 86 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1fbc9ba..1c5440d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -329,9 +329,11 @@ cd src && go test -cover ./... # With coverage | `mirror-sync.yml` | Scheduled | Sync new versions to R2 mirror (installs msitools for Python) | | `mirror-tool.yml` | PR, push to main (`scripts/mirror-binaries/`, `scripts/generate-manifests-from-r2/`) | Vet and test the mirror tool and the manifest generator, separate Go modules | -**Windows Python builds in the mirror**: python-build-standalone is the first source. For versions it lacks, the mirror assembles full builds instead of using python.org's embeddable zips (`scripts/mirror-binaries/python_winbuild.go`): `windows-amd64` from python.org's per-component MSIs up to 3.13 (verified against the pinned Windows signing key in `keys/`, unpacked with `msiextract`) and from python.org's install-manager zips from 3.14 on, where the MSIs carry no OpenPGP signature (verified against the SHA-256 in python.org's `index-windows.json`, `python_installmanager.go`); `windows-arm64` from python.org's `pythonarm64` NuGet packages (verified against nuget.org's SHA-512). All are packed as a `.tar.gz` with everything under `python/`. Builds that turn out not to be full installs are skipped, not published. `MIRROR_NETWORK_TESTS=1 go test -run Network ./...` (needs msitools) builds real archives. +**Windows Python builds in the mirror**: python-build-standalone is the only source for a version's default location (`python//`). The mirror also assembles python.org's own full builds instead of using its embeddable zips (`scripts/mirror-binaries/python_winbuild.go`): `windows-amd64` from python.org's per-component MSIs up to 3.13 (verified against the pinned Windows signing key in `keys/`, unpacked with `msiextract`) and from python.org's install-manager zips from 3.14 on, where the MSIs carry no OpenPGP signature (verified against the SHA-256 in python.org's `index-windows.json`, `python_installmanager.go`); `windows-arm64` from python.org's `pythonarm64` NuGet packages (verified against nuget.org's SHA-512). All are packed as a `.tar.gz` with everything under `python/` and published only at `python//python.org/.tar.gz`. Builds that turn out not to be full installs are skipped, not published. `MIRROR_NETWORK_TESTS=1 go test -run Network ./...` (needs msitools) builds real archives. -**Signed Python builds**: every python.org build is also published at `python//python.org/.tar.gz`, whatever the default build is, so `dtvem install python --signed` can install python.org's Authenticode-signed build (`collectJobs` dedupes on the metadata key, so python-build-standalone's default never displaces it). The manifest generator labels each Python entry with `build` (`python-build-standalone` or `python.org`, from the metadata's source URL) and nests the python.org build under the default entry's `alternates`; released CLIs ignore both fields. The CLI's manifest cache has a `format` (2); older cache files are refetched. Each install records its build in `.dtvem-build.json` in the version's directory (`src/runtimes/python/build.go`); `dtvem list` shows it, and `dtvem list-all` marks the versions with a signed build. +Never let a python.org source write a default location: a sync skips anything already mirrored, so a python.org build there (from a release python.org shipped before python-build-standalone did, or a run where the python-build-standalone listing failed) would stay the default for good. Older releases the mirror filled that way before this rule (3.9.13, for example) keep python.org's build as their default. + +**Signed Python builds**: `dtvem install python --signed` installs the build at the python.org location, python.org's Authenticode-signed one. The manifest generator labels each Python entry with `build` (`python-build-standalone` or `python.org`, from the metadata's source URL) and nests the python.org build under the default entry's `alternates`; released CLIs ignore both fields. A version with no default build gets its python.org build as the default instead, until python-build-standalone's arrives. The CLI's manifest cache has a `format` (2); older cache files are refetched. Each install records its build in `.dtvem-build.json` in the version's directory (`src/runtimes/python/build.go`); `dtvem list` shows it, and `dtvem list-all` marks the versions with a signed build. ### Reusable Workflows (from `dtvem/.github` repo) diff --git a/scripts/generate-manifests-from-r2/main_test.go b/scripts/generate-manifests-from-r2/main_test.go index 7770f92..018f582 100644 --- a/scripts/generate-manifests-from-r2/main_test.go +++ b/scripts/generate-manifests-from-r2/main_test.go @@ -193,3 +193,34 @@ func TestBuildFromSourceURL(t *testing.T) { } } } + +// A release python.org ships before python-build-standalone does is listed +// with python.org's build as its default; once python-build-standalone's +// build is mirrored, that becomes the default and python.org's the +// alternate. +func TestAssembleManifest_DefaultArrivesLater(t *testing.T) { + org := meta("https://www.python.org/ftp/python/3.14.11/python-3.14.11-amd64.zip", "org") + before := assembleManifest("python", testBaseURL, + map[string]*BinaryMeta{"python/3.14.11/python.org/windows-amd64.meta.json": org}, + keys("python/3.14.11/python.org/windows-amd64.tar.gz")) + + d := before.Versions["3.14.11"]["windows-amd64"] + if d == nil || d.Build != buildPythonOrg || d.URL != testBaseURL+"/python/3.14.11/python.org/windows-amd64.tar.gz" || len(d.Alternates) != 0 { + t.Fatalf("before python-build-standalone: %+v, want python.org's build as the default", d) + } + + after := assembleManifest("python", testBaseURL, + map[string]*BinaryMeta{ + "python/3.14.11/python.org/windows-amd64.meta.json": org, + "python/3.14.11/windows-amd64.meta.json": meta("https://github.com/astral-sh/python-build-standalone/releases/download/20261120/cpython-3.14.11%2B20261120-x86_64-pc-windows-msvc-install_only.tar.gz", "pbs"), + }, + keys("python/3.14.11/python.org/windows-amd64.tar.gz", "python/3.14.11/windows-amd64.tar.gz")) + + d = after.Versions["3.14.11"]["windows-amd64"] + if d == nil || d.Build != buildPythonStandalone || d.URL != testBaseURL+"/python/3.14.11/windows-amd64.tar.gz" { + t.Fatalf("after python-build-standalone: %+v, want its build as the default", d) + } + if len(d.Alternates) != 1 || d.Alternates[0].Build != buildPythonOrg || d.Alternates[0].SHA256 != "org" { + t.Errorf("after python-build-standalone: alternates %+v, want python.org's build", d.Alternates) + } +} diff --git a/scripts/mirror-binaries/python_installmanager.go b/scripts/mirror-binaries/python_installmanager.go index bf24ad8..f98f049 100644 --- a/scripts/mirror-binaries/python_installmanager.go +++ b/scripts/mirror-binaries/python_installmanager.go @@ -99,9 +99,9 @@ func (s *PythonInstallManagerSource) FetchVersions() ([]MirrorJob, error) { seen[e.SortVersion] = true zipURL, want := e.URL, strings.ToLower(e.Hash.SHA256) - jobs = append(jobs, pythonOrgJobs(e.SortVersion, "windows-amd64", zipURL, func() ([]byte, error) { + jobs = append(jobs, pythonOrgJob(e.SortVersion, "windows-amd64", zipURL, func() ([]byte, error) { return buildFromInstallManagerZip(zipURL, want) - })...) + })) } return jobs, nil } diff --git a/scripts/mirror-binaries/python_installmanager_test.go b/scripts/mirror-binaries/python_installmanager_test.go index 2e3fe5e..7ffb7d7 100644 --- a/scripts/mirror-binaries/python_installmanager_test.go +++ b/scripts/mirror-binaries/python_installmanager_test.go @@ -100,7 +100,7 @@ func TestPythonInstallManagerSource_FetchVersions(t *testing.T) { t.Fatalf("FetchVersions() error: %v", err) } - versions := checkPythonOrgJobPairs(t, jobs) + versions := checkPythonOrgJobs(t, jobs) if want := []string{"3.15.2", "3.14.0"}; !reflect.DeepEqual(versions, want) { t.Errorf("FetchVersions() versions = %v, want %v", versions, want) } diff --git a/scripts/mirror-binaries/python_winbuild.go b/scripts/mirror-binaries/python_winbuild.go index fe14983..24d317d 100644 --- a/scripts/mirror-binaries/python_winbuild.go +++ b/scripts/mirror-binaries/python_winbuild.go @@ -1,7 +1,6 @@ package main -// Full Windows builds for Python versions python-build-standalone does not -// cover. +// python.org's own full Windows builds of Python. // // python.org's embeddable zips, which the mirror used to fall back to, are a // trimmed runtime meant to be shipped inside an application: a ._pth file @@ -22,14 +21,22 @@ package main // layout of python-build-standalone's Windows archives, so dtvem installs it // without changes. // -// Each python.org build is published twice over: at the version's default -// location (python//.tar.gz), which it only gets when no -// earlier source covers that version, and always at the python.org location -// (python//python.org/.tar.gz). The python.org location +// python.org's builds are published at the python.org location, +// python//python.org/.tar.gz, and never at a version's +// default location, which only python-build-standalone fills. That location // is what `dtvem install python --signed` installs: python.org's // own build, whose binaries carry the Python Software Foundation's // Authenticode signature, where the default build is python-build- -// standalone's unsigned one. +// standalone's unsigned one. For a version python-build-standalone has no +// build of, the manifest generator lists python.org's build as the default +// until it does. +// +// The mirror once filled default locations with python.org's builds for +// versions python-build-standalone lacked. A sync never replaces a file +// already mirrored, so a release python.org shipped before python-build- +// standalone did, or one mirrored while the python-build-standalone listing +// failed, would have kept python.org's build as its default for good. +// Those already mirrored (3.9.13, for example) stay as they are. import ( "archive/tar" @@ -79,31 +86,17 @@ const ( pythonOrgBuildDir = "python.org" ) -// pythonOrgJobs returns the two jobs that publish one python.org build: -// at the version's default location, which collectJobs drops when an -// earlier source already covers the version, and at the python.org -// location. Each job runs build itself, so a build both jobs mirror is -// assembled twice rather than held in memory between them. -func pythonOrgJobs(version, platform, sourceURL string, build func() ([]byte, error)) []MirrorJob { - return []MirrorJob{ - { - Runtime: "python", - Version: version, - Platform: platform, - URL: sourceURL, - R2Key: fmt.Sprintf("python/%s/%s.tar.gz", version, platform), - MetaKey: fmt.Sprintf("python/%s/%s.meta.json", version, platform), - Build: build, - }, - { - Runtime: "python", - Version: version, - Platform: platform, - URL: sourceURL, - R2Key: fmt.Sprintf("python/%s/%s/%s.tar.gz", version, pythonOrgBuildDir, platform), - MetaKey: fmt.Sprintf("python/%s/%s/%s.meta.json", version, pythonOrgBuildDir, platform), - Build: build, - }, +// pythonOrgJob returns the job that publishes one python.org build at the +// python.org location. +func pythonOrgJob(version, platform, sourceURL string, build func() ([]byte, error)) MirrorJob { + return MirrorJob{ + Runtime: "python", + Version: version, + Platform: platform, + URL: sourceURL, + R2Key: fmt.Sprintf("python/%s/%s/%s.tar.gz", version, pythonOrgBuildDir, platform), + MetaKey: fmt.Sprintf("python/%s/%s/%s.meta.json", version, pythonOrgBuildDir, platform), + Build: build, } } @@ -218,9 +211,9 @@ func (s *PythonMSISource) FetchVersions() ([]MirrorJob, error) { continue } dirURL := base + v + "/amd64/" - jobs = append(jobs, pythonOrgJobs(v, "windows-amd64", dirURL, func() ([]byte, error) { + jobs = append(jobs, pythonOrgJob(v, "windows-amd64", dirURL, func() ([]byte, error) { return buildFromMSIs(dirURL, verifier, s.extract) - })...) + })) } return jobs, nil } @@ -311,8 +304,7 @@ func (s *PythonNuGetSource) Name() string { } // FetchVersions returns a job for every release package from -// minWindowsPython onward. Sources registered earlier win for any version -// and platform they also provide. +// minWindowsPython onward. func (s *PythonNuGetSource) FetchVersions() ([]MirrorJob, error) { flat := s.flatContainerURL if flat == "" { @@ -344,9 +336,9 @@ func (s *PythonNuGetSource) FetchVersions() ([]MirrorJob, error) { id := pkg.id nupkgURL := fmt.Sprintf("%s%s/%s/%s.%s.nupkg", flat, id, v, id, v) leafURL := fmt.Sprintf("%s%s/%s.json", registration, id, v) - jobs = append(jobs, pythonOrgJobs(v, pkg.platform, nupkgURL, func() ([]byte, error) { + jobs = append(jobs, pythonOrgJob(v, pkg.platform, nupkgURL, func() ([]byte, error) { return buildFromNuGet(nupkgURL, leafURL) - })...) + })) } } return jobs, nil diff --git a/scripts/mirror-binaries/python_winbuild_test.go b/scripts/mirror-binaries/python_winbuild_test.go index 8ae5da0..a9868ed 100644 --- a/scripts/mirror-binaries/python_winbuild_test.go +++ b/scripts/mirror-binaries/python_winbuild_test.go @@ -200,7 +200,7 @@ func TestPythonMSISource_FetchVersions(t *testing.T) { t.Fatalf("FetchVersions() error: %v", err) } - versions := checkPythonOrgJobPairs(t, jobs) + versions := checkPythonOrgJobs(t, jobs) for _, j := range jobs { if j.Platform != "windows-amd64" { t.Errorf("job %s: platform %q, want windows-amd64", j.Version, j.Platform) @@ -220,34 +220,25 @@ func TestPythonMSISource_FetchVersions(t *testing.T) { } } -// checkPythonOrgJobPairs checks that jobs come in the pairs pythonOrgJobs -// makes, one at the default location and one at the python.org location, -// and returns the version of each pair. -func checkPythonOrgJobPairs(t *testing.T, jobs []MirrorJob) []string { +// checkPythonOrgJobs checks that every job publishes a python.org build at +// the python.org location, never at a version's default location, and +// returns the version of each job. +func checkPythonOrgJobs(t *testing.T, jobs []MirrorJob) []string { t.Helper() - if len(jobs)%2 != 0 { - t.Fatalf("got %d jobs, want pairs of default and python.org jobs", len(jobs)) - } var versions []string - for i := 0; i < len(jobs); i += 2 { - def, org := jobs[i], jobs[i+1] - v, p := def.Version, def.Platform + for _, j := range jobs { + v, p := j.Version, j.Platform versions = append(versions, v) - if org.Version != v || org.Platform != p || org.URL != def.URL { - t.Errorf("pair %d: python.org job %s/%s from %s, want %s/%s from %s", i/2, org.Version, org.Platform, org.URL, v, p, def.URL) - } - if def.Build == nil || org.Build == nil { - t.Errorf("pair %s/%s: Build set %v and %v, want both", v, p, def.Build != nil, org.Build != nil) + if j.Build == nil { + t.Errorf("job %s/%s has no Build step", v, p) } for _, c := range []struct{ got, want string }{ - {def.R2Key, fmt.Sprintf("python/%s/%s.tar.gz", v, p)}, - {def.MetaKey, fmt.Sprintf("python/%s/%s.meta.json", v, p)}, - {org.R2Key, fmt.Sprintf("python/%s/python.org/%s.tar.gz", v, p)}, - {org.MetaKey, fmt.Sprintf("python/%s/python.org/%s.meta.json", v, p)}, + {j.R2Key, fmt.Sprintf("python/%s/python.org/%s.tar.gz", v, p)}, + {j.MetaKey, fmt.Sprintf("python/%s/python.org/%s.meta.json", v, p)}, } { if c.got != c.want { - t.Errorf("pair %s/%s: key %q, want %q", v, p, c.got, c.want) + t.Errorf("job %s/%s: key %q, want %q", v, p, c.got, c.want) } } } @@ -606,7 +597,7 @@ func TestPythonNuGetSource_FetchVersions(t *testing.T) { t.Fatalf("FetchVersions() error: %v", err) } - got := checkPythonOrgJobPairs(t, jobs) + got := checkPythonOrgJobs(t, jobs) for _, j := range jobs { if j.Platform != "windows-arm64" { t.Errorf("job %s: platform %q, want windows-arm64", j.Version, j.Platform) @@ -616,8 +607,8 @@ func TestPythonNuGetSource_FetchVersions(t *testing.T) { if !reflect.DeepEqual(got, want) { t.Errorf("FetchVersions() versions = %v, want %v", got, want) } - if want := srv.URL + "/flat/pythonarm64/3.11.4/pythonarm64.3.11.4.nupkg"; jobs[2].URL != want { - t.Errorf("arm64 job URL = %q, want %q", jobs[2].URL, want) + if want := srv.URL + "/flat/pythonarm64/3.11.4/pythonarm64.3.11.4.nupkg"; jobs[1].URL != want { + t.Errorf("arm64 job URL = %q, want %q", jobs[1].URL, want) } for _, p := range requested.list() { if strings.HasPrefix(p, "/flat/python/") { @@ -657,8 +648,8 @@ func TestBuildFromNuGet(t *testing.T) { ) src := &PythonNuGetSource{flatContainerURL: srv.URL + "/flat/", registrationURL: srv.URL + "/registration/"} jobs, err := src.FetchVersions() - if err != nil || len(jobs) != 2 { - t.Fatalf("FetchVersions() = %d jobs, %v; want a default and a python.org job", len(jobs), err) + if err != nil || len(jobs) != 1 { + t.Fatalf("FetchVersions() = %d jobs, %v; want 1 job", len(jobs), err) } archive, err := jobs[0].Build() @@ -680,8 +671,8 @@ func TestBuildFromNuGet_HashMismatch(t *testing.T) { ) src := &PythonNuGetSource{flatContainerURL: srv.URL + "/flat/", registrationURL: srv.URL + "/registration/"} jobs, err := src.FetchVersions() - if err != nil || len(jobs) != 2 { - t.Fatalf("FetchVersions() = %d jobs, %v; want a default and a python.org job", len(jobs), err) + if err != nil || len(jobs) != 1 { + t.Fatalf("FetchVersions() = %d jobs, %v; want 1 job", len(jobs), err) } if _, err := jobs[0].Build(); err == nil || !strings.Contains(err.Error(), "SHA-512 mismatch") { diff --git a/scripts/mirror-binaries/sources.go b/scripts/mirror-binaries/sources.go index ab93113..c8e36ce 100644 --- a/scripts/mirror-binaries/sources.go +++ b/scripts/mirror-binaries/sources.go @@ -84,9 +84,9 @@ func getUpstreamSources(runtime string) ([]UpstreamSource, error) { case "python": return []UpstreamSource{ &PythonStandaloneSource{}, - // python.org's full Windows builds: published at the - // python.org location for `dtvem install --signed`, and at the - // default location for versions python-build-standalone lacks. + // python.org's full Windows builds, published only at the + // python.org location, for `dtvem install --signed`. The default + // location is python-build-standalone's alone. &PythonMSISource{}, &PythonNuGetSource{}, &PythonInstallManagerSource{}, @@ -117,8 +117,8 @@ func fetchJobsFromUpstream(runtime string) ([]MirrorJob, error) { // // When several jobs publish to the same place (the same metadata key), the // first source's job wins. A version's default build is at -// //; python.org's builds also have a place of -// their own, which python-build-standalone's never claim. +// //; python.org's builds are at a place of +// their own, so they never compete with python-build-standalone's. func collectJobs(sources []UpstreamSource) ([]MirrorJob, error) { var allJobs []MirrorJob var errs []error diff --git a/scripts/mirror-binaries/sources_test.go b/scripts/mirror-binaries/sources_test.go index 51ff9d2..9e296ce 100644 --- a/scripts/mirror-binaries/sources_test.go +++ b/scripts/mirror-binaries/sources_test.go @@ -90,11 +90,11 @@ func TestCollectJobs(t *testing.T) { {Version: "3.11.17", Platform: "windows-amd64", URL: "fallback", MetaKey: "python/3.11.17/windows-amd64.meta.json"}, }, } - // python.org's builds also go to a place of their own, which an earlier - // source covering the same version does not take away. + // python.org's builds go to a place of their own, never a version's + // default location. pythonOrg := &fakeSource{ name: "python.org", - jobs: pythonOrgJobs("3.13.16", "windows-amd64", "python.org", nil), + jobs: []MirrorJob{pythonOrgJob("3.13.16", "windows-amd64", "python.org", nil)}, } broken := &fakeSource{name: "broken", err: errors.New("fetching releases: HTTP 504")} @@ -110,14 +110,23 @@ func TestCollectJobs(t *testing.T) { wantURLs: []string{"primary", "primary", "fallback"}, }, { - name: "python.org location kept when default is covered", + name: "python.org build kept alongside the default", sources: []UpstreamSource{primary, pythonOrg}, wantURLs: []string{"primary", "primary", "python.org"}, }, { - name: "python.org fills the default location when uncovered", + name: "python.org build never takes the default location", sources: []UpstreamSource{pythonOrg, primary}, - wantURLs: []string{"python.org", "python.org", "primary"}, + wantURLs: []string{"python.org", "primary", "primary"}, + }, + { + // A failed python-build-standalone listing must not let a + // python.org build take a default location, where a sync would + // never replace it. + name: "python.org build stays in place when the default source fails", + sources: []UpstreamSource{broken, pythonOrg}, + wantURLs: []string{"python.org"}, + wantErrText: "broken: fetching releases: HTTP 504", }, { name: "failing source keeps jobs from healthy sources", @@ -155,10 +164,10 @@ func TestCollectJobs(t *testing.T) { } } -// The first source to offer a version and platform wins, so the order sets -// which build a version gets by default: python-build-standalone where it -// has one, then python.org's MSIs (full x64 builds with tkinter), then its -// NuGet packages, then its install-manager zips (x64 from 3.14 on). +// python-build-standalone fills the default location. python.org's MSIs +// (full x64 builds with tkinter, up to 3.13), NuGet packages (arm64) and +// install-manager zips (x64 from 3.14 on) fill the python.org location; for +// a version and platform two of them offer, the first wins. func TestGetUpstreamSources_PythonOrder(t *testing.T) { sources, err := getUpstreamSources("python") if err != nil {