Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion .github/workflows/generate-manifests-from-r2.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,10 @@ jobs:
generate:
name: Generate Manifests
runs-on: ubuntu-latest
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
# Runs after a failed sync too: one source or download failing must not
# hold back everything the other jobs mirrored. Manifests only list what
# R2 holds, and a build's metadata is uploaded after its archive.
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' || github.event.workflow_run.conclusion == 'failure' }}

steps:
- name: Checkout
Expand Down Expand Up @@ -131,6 +134,10 @@ jobs:
run: |
echo "## Manifest Generation" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
if [ "${{ github.event.workflow_run.conclusion }}" = "failure" ]; then
echo "**Note:** [Mirror Sync](${{ github.event.workflow_run.html_url }}) failed. These manifests list what it did mirror; anything it missed waits for a later sync." >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
fi
if [ "${{ inputs.dry_run }}" = "true" ]; then
echo "**Mode:** Dry run (no changes made)" >> $GITHUB_STEP_SUMMARY
elif [ "${{ steps.check-changes.outputs.changed }}" = "true" ]; then
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/mirror-sync.yml
Original file line number Diff line number Diff line change
Expand Up @@ -71,4 +71,4 @@ jobs:
echo "" >> $GITHUB_STEP_SUMMARY
echo "Synced new binaries from upstream that were not already in R2." >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "To generate updated manifests, run the 'Generate Manifests from R2' workflow." >> $GITHUB_STEP_SUMMARY
echo "'Generate Manifests from R2' runs when this workflow finishes, also after a failure." >> $GITHUB_STEP_SUMMARY
2 changes: 1 addition & 1 deletion CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -323,7 +323,7 @@ cd src && go test -cover ./... # With coverage
| `integration-test.yml` | Manual dispatch | Full integration test suite (runtimes + migrations) |
| `integration-test-runtimes.yml` | Manual dispatch | Runtime install/uninstall tests only |
| `integration-test-migrations.yml` | Manual dispatch | Migration tests only (all platforms/managers) |
| `generate-manifests-from-r2.yml` | Manual/scheduled | Generate version manifests from R2 mirror |
| `generate-manifests-from-r2.yml` | Manual, after Mirror Sync (also a failed one) | Generate version manifests from R2 mirror |
| `deploy-manifests.yml` | Push to main (manifests/) | Deploy manifest files to R2 |
| `mirror-all.yml` | Manual dispatch | Mirror all runtime binaries to R2 |
| `mirror-sync.yml` | Scheduled | Sync new versions to R2 mirror (installs msitools for Python) |
Expand Down
5 changes: 2 additions & 3 deletions scripts/mirror-binaries/python.go
Original file line number Diff line number Diff line change
Expand Up @@ -125,18 +125,17 @@ func fetchGitHubReleases(releasesURL string, pageSize, maxReleases int) ([]githu
// fetchGitHubReleasesPage fetches a single page of releases and returns it
// along with the URL of the next page ("" on the last page).
func fetchGitHubReleasesPage(url string) ([]githubRelease, string, error) {
resp, err := httpGetWithRetry(url, 3)
resp, err := fetchWithRetry(url, 3)
if err != nil {
return nil, "", fmt.Errorf("fetching releases: %w", err)
}
defer resp.Body.Close()

if resp.StatusCode != http.StatusOK {
return nil, "", fmt.Errorf("fetching releases: HTTP %d", resp.StatusCode)
}

var releases []githubRelease
if err := json.NewDecoder(resp.Body).Decode(&releases); err != nil {
if err := json.Unmarshal(resp.Body, &releases); err != nil {
return nil, "", fmt.Errorf("parsing releases: %w", err)
}

Expand Down
46 changes: 39 additions & 7 deletions scripts/mirror-binaries/python_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,10 @@ func TestNextPageURL(t *testing.T) {
}

// newReleasesServer serves total releases tagged "1".."total" (newest first),
// paginated like the GitHub releases API. If failPage is non-zero, that page
// responds with HTTP 403. The returned counter tracks requests served.
func newReleasesServer(t *testing.T, total, failPage int) (*httptest.Server, *atomic.Int32) {
// paginated like the GitHub releases API. If intercept is not nil, it sees
// each request's page first and answers it instead when it returns true. The
// returned counter tracks requests served.
func newReleasesServer(t *testing.T, total int, intercept func(w http.ResponseWriter, page int) bool) (*httptest.Server, *atomic.Int32) {
t.Helper()

var requests atomic.Int32
Expand All @@ -73,8 +74,7 @@ func newReleasesServer(t *testing.T, total, failPage int) (*httptest.Server, *at
page, _ = strconv.Atoi(p)
}

if page == failPage {
http.Error(w, "rate limited", http.StatusForbidden)
if intercept != nil && intercept(w, page) {
return
}

Expand Down Expand Up @@ -151,7 +151,7 @@ func TestFetchGitHubReleases(t *testing.T) {

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
srv, requests := newReleasesServer(t, tt.total, 0)
srv, requests := newReleasesServer(t, tt.total, nil)

releases, err := fetchGitHubReleases(srv.URL+"/releases", tt.pageSize, tt.maxReleases)
if err != nil {
Expand All @@ -176,8 +176,40 @@ func TestFetchGitHubReleases(t *testing.T) {
}
}

// A page cut off part way is fetched again. GitHub does this now and then to
// the large python-build-standalone pages ("stream error: ... CANCEL").
func TestFetchGitHubReleases_CutOffPage(t *testing.T) {
noRetryDelay(t)

var cutOff atomic.Bool
srv, requests := newReleasesServer(t, 25, func(w http.ResponseWriter, page int) bool {
if page != 2 || !cutOff.CompareAndSwap(false, true) {
return false
}
writeCutOff(t, w)
return true
})

releases, err := fetchGitHubReleases(srv.URL+"/releases", 10, 100)
if err != nil {
t.Fatalf("fetchGitHubReleases() error = %v, want nil", err)
}
if got := len(releases); got != 25 {
t.Errorf("len(releases) = %d, want 25", got)
}
if got := requests.Load(); got != 4 {
t.Errorf("requests = %d, want 4 (page 2 twice)", got)
}
}

func TestFetchGitHubReleases_PageError(t *testing.T) {
srv, _ := newReleasesServer(t, 30, 2)
srv, _ := newReleasesServer(t, 30, func(w http.ResponseWriter, page int) bool {
if page != 2 {
return false
}
http.Error(w, "rate limited", http.StatusForbidden)
return true
})

releases, err := fetchGitHubReleases(srv.URL+"/releases", 10, 100)
if err == nil {
Expand Down
5 changes: 2 additions & 3 deletions scripts/mirror-binaries/python_winbuild.go
Original file line number Diff line number Diff line change
Expand Up @@ -655,15 +655,14 @@ func (a *pythonArchive) close() ([]byte, error) {

// download fetches url and returns its body, failing on any non-200 status.
func download(url string) ([]byte, error) {
resp, err := httpGetWithRetry(url, 3)
resp, err := fetchWithRetry(url, 3)
if err != nil {
return nil, fmt.Errorf("downloading %s: %w", url, err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != 200 {
return nil, fmt.Errorf("downloading %s: HTTP %d", url, resp.StatusCode)
}
return io.ReadAll(resp.Body)
return resp.Body, nil
}

func fetchText(url string) (string, error) {
Expand Down
10 changes: 4 additions & 6 deletions scripts/mirror-binaries/ruby.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,18 +23,17 @@ var rubyInstallerPattern = regexp.MustCompile(
func (s *RubyInstallerSource) FetchVersions() ([]MirrorJob, error) {
// Fetch releases from GitHub API with retries
url := "https://api.github.com/repos/oneclick/rubyinstaller2/releases?per_page=100"
resp, err := httpGetWithRetry(url, 3)
resp, err := fetchWithRetry(url, 3)
if err != nil {
return nil, fmt.Errorf("fetching releases: %w", err)
}
defer resp.Body.Close()

if resp.StatusCode != 200 {
return nil, fmt.Errorf("fetching releases: HTTP %d", resp.StatusCode)
}

var releases []githubRelease
if err := json.NewDecoder(resp.Body).Decode(&releases); err != nil {
if err := json.Unmarshal(resp.Body, &releases); err != nil {
return nil, fmt.Errorf("parsing releases: %w", err)
}

Expand Down Expand Up @@ -111,18 +110,17 @@ var rubyBuilderPattern = regexp.MustCompile(
func (s *RubyBuilderSource) FetchVersions() ([]MirrorJob, error) {
// Fetch the toolcache release from GitHub API with retries
url := "https://api.github.com/repos/ruby/ruby-builder/releases/tags/toolcache"
resp, err := httpGetWithRetry(url, 3)
resp, err := fetchWithRetry(url, 3)
if err != nil {
return nil, fmt.Errorf("fetching release: %w", err)
}
defer resp.Body.Close()

if resp.StatusCode != 200 {
return nil, fmt.Errorf("fetching release: HTTP %d", resp.StatusCode)
}

var release githubRelease
if err := json.NewDecoder(resp.Body).Decode(&release); err != nil {
if err := json.Unmarshal(resp.Body, &release); err != nil {
return nil, fmt.Errorf("parsing release: %w", err)
}

Expand Down
55 changes: 42 additions & 13 deletions scripts/mirror-binaries/sources.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package main
import (
"errors"
"fmt"
"io"
"net/http"
"os"
"time"
Expand Down Expand Up @@ -41,37 +42,65 @@ func newRequest(url string) (*http.Request, error) {
return req, nil
}

// httpGetWithRetry performs an HTTP GET with retries for transient failures
func httpGetWithRetry(url string, maxRetries int) (*http.Response, error) {
// fetchedResponse is an HTTP response whose body has been read in full.
type fetchedResponse struct {
StatusCode int
Header http.Header
Body []byte
}

// retryDelay is the wait before the second attempt of a request. Each later
// attempt waits one retryDelay longer than the one before.
var retryDelay = 2 * time.Second

// fetchWithRetry performs an HTTP GET and reads the whole body. It tries again
// when the request fails, when the body is cut off part way, or when the
// server answers 5xx; any other status is returned for the caller to check.
//
// Reading the body inside the retry matters for GitHub's releases API, whose
// pages run to tens of megabytes: GitHub sometimes drops one part way through
// ("stream error: stream ID 13; CANCEL; received from peer").
func fetchWithRetry(url string, maxRetries int) (*fetchedResponse, error) {
req, err := newRequest(url)
if err != nil {
return nil, err
}

var lastErr error
for attempt := 1; attempt <= maxRetries; attempt++ {
resp, err := httpClient.Do(req)
if attempt > 1 {
time.Sleep(time.Duration(attempt-1) * retryDelay)
}

resp, err := fetchOnce(req)
if err != nil {
lastErr = err
if attempt < maxRetries {
time.Sleep(time.Duration(attempt) * 2 * time.Second)
}
continue
}

// Retry on server errors (5xx)
if resp.StatusCode >= 500 {
resp.Body.Close()
lastErr = fmt.Errorf("HTTP %d", resp.StatusCode)
if attempt < maxRetries {
time.Sleep(time.Duration(attempt) * 2 * time.Second)
}
continue
}

return resp, nil
}
return nil, lastErr
return nil, fmt.Errorf("after %d attempts: %w", maxRetries, lastErr)
}

// fetchOnce sends req and reads the whole response body.
func fetchOnce(req *http.Request) (*fetchedResponse, error) {
resp, err := httpClient.Do(req)
if err != nil {
return nil, err
}
defer func() { _ = resp.Body.Close() }()

body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("reading response: %w", err)
}

return &fetchedResponse{StatusCode: resp.StatusCode, Header: resp.Header, Body: body}, nil
}

// getUpstreamSources returns all upstream sources for a given runtime
Expand Down
Loading
Loading