From 44b840586ceb7f32aba06596b96bd0e55830371a Mon Sep 17 00:00:00 2001 From: "Calvin A. Allen" Date: Fri, 13 Nov 2026 12:10:22 -0500 Subject: [PATCH] fix(mirror): recover from a cut-off GitHub release listing fetchWithRetry replaces httpGetWithRetry and reads the whole body inside the retry, so a response GitHub drops part way through ("stream error: ... CANCEL") is fetched again instead of failing the source. Generate Manifests from R2 now also runs after a failed Mirror Sync, so one failed source or download no longer keeps everything else that synced out of the manifests. --- .../workflows/generate-manifests-from-r2.yml | 9 +- .github/workflows/mirror-sync.yml | 2 +- CLAUDE.md | 2 +- scripts/mirror-binaries/python.go | 5 +- scripts/mirror-binaries/python_test.go | 46 +++++- scripts/mirror-binaries/python_winbuild.go | 5 +- scripts/mirror-binaries/ruby.go | 10 +- scripts/mirror-binaries/sources.go | 55 +++++-- scripts/mirror-binaries/sources_test.go | 139 ++++++++++++++++++ 9 files changed, 238 insertions(+), 35 deletions(-) diff --git a/.github/workflows/generate-manifests-from-r2.yml b/.github/workflows/generate-manifests-from-r2.yml index cae937d..9727b75 100644 --- a/.github/workflows/generate-manifests-from-r2.yml +++ b/.github/workflows/generate-manifests-from-r2.yml @@ -32,7 +32,10 @@ jobs: generate: name: Generate Manifests runs-on: ubuntu-latest - if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + # Runs after a failed sync too: one source or download failing must not + # hold back everything the other jobs mirrored. Manifests only list what + # R2 holds, and a build's metadata is uploaded after its archive. + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' || github.event.workflow_run.conclusion == 'failure' }} steps: - name: Checkout @@ -131,6 +134,10 @@ jobs: run: | echo "## Manifest Generation" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY + if [ "${{ github.event.workflow_run.conclusion }}" = "failure" ]; then + echo "**Note:** [Mirror Sync](${{ github.event.workflow_run.html_url }}) failed. These manifests list what it did mirror; anything it missed waits for a later sync." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + fi if [ "${{ inputs.dry_run }}" = "true" ]; then echo "**Mode:** Dry run (no changes made)" >> $GITHUB_STEP_SUMMARY elif [ "${{ steps.check-changes.outputs.changed }}" = "true" ]; then diff --git a/.github/workflows/mirror-sync.yml b/.github/workflows/mirror-sync.yml index a2ab9b1..8322da1 100644 --- a/.github/workflows/mirror-sync.yml +++ b/.github/workflows/mirror-sync.yml @@ -71,4 +71,4 @@ jobs: echo "" >> $GITHUB_STEP_SUMMARY echo "Synced new binaries from upstream that were not already in R2." >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "To generate updated manifests, run the 'Generate Manifests from R2' workflow." >> $GITHUB_STEP_SUMMARY + echo "'Generate Manifests from R2' runs when this workflow finishes, also after a failure." >> $GITHUB_STEP_SUMMARY diff --git a/CLAUDE.md b/CLAUDE.md index 1c5440d..c7dbe8f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -323,7 +323,7 @@ cd src && go test -cover ./... # With coverage | `integration-test.yml` | Manual dispatch | Full integration test suite (runtimes + migrations) | | `integration-test-runtimes.yml` | Manual dispatch | Runtime install/uninstall tests only | | `integration-test-migrations.yml` | Manual dispatch | Migration tests only (all platforms/managers) | -| `generate-manifests-from-r2.yml` | Manual/scheduled | Generate version manifests from R2 mirror | +| `generate-manifests-from-r2.yml` | Manual, after Mirror Sync (also a failed one) | Generate version manifests from R2 mirror | | `deploy-manifests.yml` | Push to main (manifests/) | Deploy manifest files to R2 | | `mirror-all.yml` | Manual dispatch | Mirror all runtime binaries to R2 | | `mirror-sync.yml` | Scheduled | Sync new versions to R2 mirror (installs msitools for Python) | diff --git a/scripts/mirror-binaries/python.go b/scripts/mirror-binaries/python.go index a1b0f93..fb8337b 100644 --- a/scripts/mirror-binaries/python.go +++ b/scripts/mirror-binaries/python.go @@ -125,18 +125,17 @@ func fetchGitHubReleases(releasesURL string, pageSize, maxReleases int) ([]githu // fetchGitHubReleasesPage fetches a single page of releases and returns it // along with the URL of the next page ("" on the last page). func fetchGitHubReleasesPage(url string) ([]githubRelease, string, error) { - resp, err := httpGetWithRetry(url, 3) + resp, err := fetchWithRetry(url, 3) if err != nil { return nil, "", fmt.Errorf("fetching releases: %w", err) } - defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, "", fmt.Errorf("fetching releases: HTTP %d", resp.StatusCode) } var releases []githubRelease - if err := json.NewDecoder(resp.Body).Decode(&releases); err != nil { + if err := json.Unmarshal(resp.Body, &releases); err != nil { return nil, "", fmt.Errorf("parsing releases: %w", err) } diff --git a/scripts/mirror-binaries/python_test.go b/scripts/mirror-binaries/python_test.go index f7e45d8..16832a9 100644 --- a/scripts/mirror-binaries/python_test.go +++ b/scripts/mirror-binaries/python_test.go @@ -54,9 +54,10 @@ func TestNextPageURL(t *testing.T) { } // newReleasesServer serves total releases tagged "1".."total" (newest first), -// paginated like the GitHub releases API. If failPage is non-zero, that page -// responds with HTTP 403. The returned counter tracks requests served. -func newReleasesServer(t *testing.T, total, failPage int) (*httptest.Server, *atomic.Int32) { +// paginated like the GitHub releases API. If intercept is not nil, it sees +// each request's page first and answers it instead when it returns true. The +// returned counter tracks requests served. +func newReleasesServer(t *testing.T, total int, intercept func(w http.ResponseWriter, page int) bool) (*httptest.Server, *atomic.Int32) { t.Helper() var requests atomic.Int32 @@ -73,8 +74,7 @@ func newReleasesServer(t *testing.T, total, failPage int) (*httptest.Server, *at page, _ = strconv.Atoi(p) } - if page == failPage { - http.Error(w, "rate limited", http.StatusForbidden) + if intercept != nil && intercept(w, page) { return } @@ -151,7 +151,7 @@ func TestFetchGitHubReleases(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - srv, requests := newReleasesServer(t, tt.total, 0) + srv, requests := newReleasesServer(t, tt.total, nil) releases, err := fetchGitHubReleases(srv.URL+"/releases", tt.pageSize, tt.maxReleases) if err != nil { @@ -176,8 +176,40 @@ func TestFetchGitHubReleases(t *testing.T) { } } +// A page cut off part way is fetched again. GitHub does this now and then to +// the large python-build-standalone pages ("stream error: ... CANCEL"). +func TestFetchGitHubReleases_CutOffPage(t *testing.T) { + noRetryDelay(t) + + var cutOff atomic.Bool + srv, requests := newReleasesServer(t, 25, func(w http.ResponseWriter, page int) bool { + if page != 2 || !cutOff.CompareAndSwap(false, true) { + return false + } + writeCutOff(t, w) + return true + }) + + releases, err := fetchGitHubReleases(srv.URL+"/releases", 10, 100) + if err != nil { + t.Fatalf("fetchGitHubReleases() error = %v, want nil", err) + } + if got := len(releases); got != 25 { + t.Errorf("len(releases) = %d, want 25", got) + } + if got := requests.Load(); got != 4 { + t.Errorf("requests = %d, want 4 (page 2 twice)", got) + } +} + func TestFetchGitHubReleases_PageError(t *testing.T) { - srv, _ := newReleasesServer(t, 30, 2) + srv, _ := newReleasesServer(t, 30, func(w http.ResponseWriter, page int) bool { + if page != 2 { + return false + } + http.Error(w, "rate limited", http.StatusForbidden) + return true + }) releases, err := fetchGitHubReleases(srv.URL+"/releases", 10, 100) if err == nil { diff --git a/scripts/mirror-binaries/python_winbuild.go b/scripts/mirror-binaries/python_winbuild.go index 24d317d..93aacdd 100644 --- a/scripts/mirror-binaries/python_winbuild.go +++ b/scripts/mirror-binaries/python_winbuild.go @@ -655,15 +655,14 @@ func (a *pythonArchive) close() ([]byte, error) { // download fetches url and returns its body, failing on any non-200 status. func download(url string) ([]byte, error) { - resp, err := httpGetWithRetry(url, 3) + resp, err := fetchWithRetry(url, 3) if err != nil { return nil, fmt.Errorf("downloading %s: %w", url, err) } - defer func() { _ = resp.Body.Close() }() if resp.StatusCode != 200 { return nil, fmt.Errorf("downloading %s: HTTP %d", url, resp.StatusCode) } - return io.ReadAll(resp.Body) + return resp.Body, nil } func fetchText(url string) (string, error) { diff --git a/scripts/mirror-binaries/ruby.go b/scripts/mirror-binaries/ruby.go index 84c6e98..e837b2f 100644 --- a/scripts/mirror-binaries/ruby.go +++ b/scripts/mirror-binaries/ruby.go @@ -23,18 +23,17 @@ var rubyInstallerPattern = regexp.MustCompile( func (s *RubyInstallerSource) FetchVersions() ([]MirrorJob, error) { // Fetch releases from GitHub API with retries url := "https://api.github.com/repos/oneclick/rubyinstaller2/releases?per_page=100" - resp, err := httpGetWithRetry(url, 3) + resp, err := fetchWithRetry(url, 3) if err != nil { return nil, fmt.Errorf("fetching releases: %w", err) } - defer resp.Body.Close() if resp.StatusCode != 200 { return nil, fmt.Errorf("fetching releases: HTTP %d", resp.StatusCode) } var releases []githubRelease - if err := json.NewDecoder(resp.Body).Decode(&releases); err != nil { + if err := json.Unmarshal(resp.Body, &releases); err != nil { return nil, fmt.Errorf("parsing releases: %w", err) } @@ -111,18 +110,17 @@ var rubyBuilderPattern = regexp.MustCompile( func (s *RubyBuilderSource) FetchVersions() ([]MirrorJob, error) { // Fetch the toolcache release from GitHub API with retries url := "https://api.github.com/repos/ruby/ruby-builder/releases/tags/toolcache" - resp, err := httpGetWithRetry(url, 3) + resp, err := fetchWithRetry(url, 3) if err != nil { return nil, fmt.Errorf("fetching release: %w", err) } - defer resp.Body.Close() if resp.StatusCode != 200 { return nil, fmt.Errorf("fetching release: HTTP %d", resp.StatusCode) } var release githubRelease - if err := json.NewDecoder(resp.Body).Decode(&release); err != nil { + if err := json.Unmarshal(resp.Body, &release); err != nil { return nil, fmt.Errorf("parsing release: %w", err) } diff --git a/scripts/mirror-binaries/sources.go b/scripts/mirror-binaries/sources.go index c8e36ce..7dc4274 100644 --- a/scripts/mirror-binaries/sources.go +++ b/scripts/mirror-binaries/sources.go @@ -3,6 +3,7 @@ package main import ( "errors" "fmt" + "io" "net/http" "os" "time" @@ -41,8 +42,25 @@ func newRequest(url string) (*http.Request, error) { return req, nil } -// httpGetWithRetry performs an HTTP GET with retries for transient failures -func httpGetWithRetry(url string, maxRetries int) (*http.Response, error) { +// fetchedResponse is an HTTP response whose body has been read in full. +type fetchedResponse struct { + StatusCode int + Header http.Header + Body []byte +} + +// retryDelay is the wait before the second attempt of a request. Each later +// attempt waits one retryDelay longer than the one before. +var retryDelay = 2 * time.Second + +// fetchWithRetry performs an HTTP GET and reads the whole body. It tries again +// when the request fails, when the body is cut off part way, or when the +// server answers 5xx; any other status is returned for the caller to check. +// +// Reading the body inside the retry matters for GitHub's releases API, whose +// pages run to tens of megabytes: GitHub sometimes drops one part way through +// ("stream error: stream ID 13; CANCEL; received from peer"). +func fetchWithRetry(url string, maxRetries int) (*fetchedResponse, error) { req, err := newRequest(url) if err != nil { return nil, err @@ -50,28 +68,39 @@ func httpGetWithRetry(url string, maxRetries int) (*http.Response, error) { var lastErr error for attempt := 1; attempt <= maxRetries; attempt++ { - resp, err := httpClient.Do(req) + if attempt > 1 { + time.Sleep(time.Duration(attempt-1) * retryDelay) + } + + resp, err := fetchOnce(req) if err != nil { lastErr = err - if attempt < maxRetries { - time.Sleep(time.Duration(attempt) * 2 * time.Second) - } continue } - - // Retry on server errors (5xx) if resp.StatusCode >= 500 { - resp.Body.Close() lastErr = fmt.Errorf("HTTP %d", resp.StatusCode) - if attempt < maxRetries { - time.Sleep(time.Duration(attempt) * 2 * time.Second) - } continue } return resp, nil } - return nil, lastErr + return nil, fmt.Errorf("after %d attempts: %w", maxRetries, lastErr) +} + +// fetchOnce sends req and reads the whole response body. +func fetchOnce(req *http.Request) (*fetchedResponse, error) { + resp, err := httpClient.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + + return &fetchedResponse{StatusCode: resp.StatusCode, Header: resp.Header, Body: body}, nil } // getUpstreamSources returns all upstream sources for a given runtime diff --git a/scripts/mirror-binaries/sources_test.go b/scripts/mirror-binaries/sources_test.go index 9e296ce..355fc3b 100644 --- a/scripts/mirror-binaries/sources_test.go +++ b/scripts/mirror-binaries/sources_test.go @@ -2,7 +2,10 @@ package main import ( "errors" + "net/http" + "net/http/httptest" "strings" + "sync/atomic" "testing" ) @@ -65,6 +68,142 @@ func TestNewRequest_InvalidURL(t *testing.T) { } } +// noRetryDelay makes retries immediate for the rest of the test. +func noRetryDelay(t *testing.T) { + t.Helper() + + old := retryDelay + retryDelay = 0 + t.Cleanup(func() { retryDelay = old }) +} + +// writeCutOff starts a 200 response that promises more body than it sends and +// then drops the connection, as when GitHub cancels a stream part way. +func writeCutOff(t *testing.T, w http.ResponseWriter) { + t.Helper() + + hijacker, ok := w.(http.Hijacker) + if !ok { + t.Error("response writer cannot hijack the connection") + return + } + conn, buf, err := hijacker.Hijack() + if err != nil { + t.Errorf("hijacking connection: %v", err) + return + } + defer func() { _ = conn.Close() }() + + _, _ = buf.WriteString("HTTP/1.1 200 OK\r\nContent-Length: 1000\r\n\r\n[{\"tag_name\":") + _ = buf.Flush() +} + +func TestFetchWithRetry(t *testing.T) { + const body = `[{"tag_name":"20261113"}]` + writeBody := func(w http.ResponseWriter) { _, _ = w.Write([]byte(body)) } + + tests := []struct { + name string + // respond answers the nth request (counting from 1). + respond func(t *testing.T, w http.ResponseWriter, n int32) + wantStatus int + wantBody string + wantRequests int32 + wantErrText string + }{ + { + name: "complete on the first try", + respond: func(_ *testing.T, w http.ResponseWriter, _ int32) { writeBody(w) }, + wantStatus: http.StatusOK, + wantBody: body, + wantRequests: 1, + }, + { + name: "body cut off, then complete", + respond: func(t *testing.T, w http.ResponseWriter, n int32) { + if n == 1 { + writeCutOff(t, w) + return + } + writeBody(w) + }, + wantStatus: http.StatusOK, + wantBody: body, + wantRequests: 2, + }, + { + name: "server error, then complete", + respond: func(_ *testing.T, w http.ResponseWriter, n int32) { + if n == 1 { + http.Error(w, "bad gateway", http.StatusBadGateway) + return + } + writeBody(w) + }, + wantStatus: http.StatusOK, + wantBody: body, + wantRequests: 2, + }, + { + name: "client error returned without a retry", + respond: func(_ *testing.T, w http.ResponseWriter, _ int32) { + http.Error(w, "not found", http.StatusNotFound) + }, + wantStatus: http.StatusNotFound, + wantBody: "not found\n", + wantRequests: 1, + }, + { + name: "body cut off every time", + respond: func(t *testing.T, w http.ResponseWriter, _ int32) { writeCutOff(t, w) }, + wantRequests: 3, + wantErrText: "after 3 attempts: reading response: ", + }, + { + name: "server error every time", + respond: func(_ *testing.T, w http.ResponseWriter, _ int32) { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + }, + wantRequests: 3, + wantErrText: "after 3 attempts: HTTP 503", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + noRetryDelay(t) + + var requests atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + tt.respond(t, w, requests.Add(1)) + })) + t.Cleanup(srv.Close) + + resp, err := fetchWithRetry(srv.URL, 3) + + if got := requests.Load(); got != tt.wantRequests { + t.Errorf("requests = %d, want %d", got, tt.wantRequests) + } + + if tt.wantErrText != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErrText) { + t.Errorf("fetchWithRetry() error = %v, want error containing %q", err, tt.wantErrText) + } + return + } + if err != nil { + t.Fatalf("fetchWithRetry() error = %v, want nil", err) + } + if resp.StatusCode != tt.wantStatus { + t.Errorf("StatusCode = %d, want %d", resp.StatusCode, tt.wantStatus) + } + if string(resp.Body) != tt.wantBody { + t.Errorf("Body = %q, want %q", resp.Body, tt.wantBody) + } + }) + } +} + // fakeSource is an UpstreamSource returning canned jobs or an error. type fakeSource struct { name string