From b1badac258e1d717af144f2b18272abe1cfa24df Mon Sep 17 00:00:00 2001 From: "Calvin A. Allen" Date: Mon, 5 Oct 2026 14:09:47 -0400 Subject: [PATCH] fix(mirror): paginate python-build-standalone release discovery Each python-build-standalone release now lists ~900 assets (~2MB of JSON), so requesting 100 releases in one GitHub API call times out with HTTP 504. The sync logged this as a warning, found zero python jobs, and still exited successfully, so no new Python versions have been mirrored since December 2025. - Fetch releases in pages of 10, following the Link header, still bounded to the newest 100 releases - Authenticate GitHub API calls with GITHUB_TOKEN when set, to avoid the shared unauthenticated rate limit on CI runners - Keep mirroring jobs from healthy sources when one fails, but exit non-zero so a broken source fails the workflow instead of passing --- .github/workflows/mirror-sync.yml | 1 + scripts/mirror-binaries/main.go | 21 ++- scripts/mirror-binaries/python.go | 98 +++++++++-- scripts/mirror-binaries/python_test.go | 206 ++++++++++++++++++++++++ scripts/mirror-binaries/sources.go | 40 ++++- scripts/mirror-binaries/sources_test.go | 140 ++++++++++++++++ 6 files changed, 488 insertions(+), 18 deletions(-) create mode 100644 scripts/mirror-binaries/python_test.go create mode 100644 scripts/mirror-binaries/sources_test.go diff --git a/.github/workflows/mirror-sync.yml b/.github/workflows/mirror-sync.yml index 9ef546a..5487650 100644 --- a/.github/workflows/mirror-sync.yml +++ b/.github/workflows/mirror-sync.yml @@ -48,6 +48,7 @@ jobs: R2_BUCKET: ${{ secrets.CLOUDFLARE_R2_BUILDS_BUCKET }} R2_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }} R2_SECRET_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | ./scripts/mirror-binaries/mirror-binaries \ --runtime=${{ matrix.runtime }} \ diff --git a/scripts/mirror-binaries/main.go b/scripts/mirror-binaries/main.go index 2189154..5fbb297 100644 --- a/scripts/mirror-binaries/main.go +++ b/scripts/mirror-binaries/main.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "flag" "fmt" "io" @@ -120,17 +121,20 @@ func main() { } } - // Collect all jobs from upstream sources + // Collect all jobs from upstream sources. A failing source is reported at + // the end rather than aborting, so jobs from healthy sources still mirror. var jobs []MirrorJob + var fetchErrs []error for _, rt := range runtimes { fmt.Printf("Discovering %s versions from upstream...\n", rt) rtJobs, err := fetchJobsFromUpstream(rt) if err != nil { fmt.Fprintf(os.Stderr, "Error fetching upstream versions for %s: %v\n", rt, err) - os.Exit(1) + fetchErrs = append(fetchErrs, err) } jobs = append(jobs, rtJobs...) } + fetchErr := errors.Join(fetchErrs...) fmt.Printf("Total jobs to process: %d\n", len(jobs)) @@ -150,6 +154,7 @@ func main() { } fmt.Printf("\nTotal: %d files (%d with upstream checksum, %d will generate)\n", len(jobs), withChecksum, withoutChecksum) + exitOnFetchError(fetchErr) return } @@ -172,6 +177,7 @@ func main() { if len(jobs) == 0 { fmt.Println("No files to mirror") + exitOnFetchError(fetchErr) return } @@ -188,11 +194,22 @@ func main() { fmt.Printf("Upstream checksums: %d\n", stats.UpstreamChecksum) fmt.Printf("Generated checksums: %d\n", stats.GeneratedChecksum) + exitOnFetchError(fetchErr) if stats.Failed > 0 { os.Exit(1) } } +// exitOnFetchError exits non-zero when any upstream source failed to fetch. +// Without this a broken source mirrors nothing while the workflow still +// reports success, hiding missing versions indefinitely. +func exitOnFetchError(err error) { + if err != nil { + fmt.Fprintf(os.Stderr, "\nUpstream fetch failed: %v\n", err) + os.Exit(1) + } +} + func createS3Client() (*s3.Client, error) { cfg, err := config.LoadDefaultConfig(context.Background(), config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( diff --git a/scripts/mirror-binaries/python.go b/scripts/mirror-binaries/python.go index 7defcc0..521719e 100644 --- a/scripts/mirror-binaries/python.go +++ b/scripts/mirror-binaries/python.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "io" + "net/http" "regexp" "strings" ) @@ -33,22 +34,22 @@ var pythonStandalonePattern = regexp.MustCompile( `^cpython-(\d+\.\d+\.\d+)\+\d+-([^-]+-[^-]+-[^-]+(?:-[^-]+)?)-install_only\.(tar\.gz|tar\.zst)$`, ) -func (s *PythonStandaloneSource) FetchVersions() ([]MirrorJob, error) { - // Fetch releases from GitHub API with retries - url := "https://api.github.com/repos/astral-sh/python-build-standalone/releases?per_page=100" - resp, err := httpGetWithRetry(url, 3) - if err != nil { - return nil, fmt.Errorf("fetching releases: %w", err) - } - defer resp.Body.Close() +const ( + pythonStandaloneReleasesURL = "https://api.github.com/repos/astral-sh/python-build-standalone/releases" - if resp.StatusCode != 200 { - return nil, fmt.Errorf("fetching releases: HTTP %d", resp.StatusCode) - } + // pythonStandalonePageSize is kept small because each release lists ~900 + // assets (~2MB of JSON). Pages of 50 or more make the GitHub API time out + // with HTTP 504. + pythonStandalonePageSize = 10 - var releases []githubRelease - if err := json.NewDecoder(resp.Body).Decode(&releases); err != nil { - return nil, fmt.Errorf("parsing releases: %w", err) + // pythonStandaloneMaxReleases bounds how far back in release history to look. + pythonStandaloneMaxReleases = 100 +) + +func (s *PythonStandaloneSource) FetchVersions() ([]MirrorJob, error) { + releases, err := fetchGitHubReleases(pythonStandaloneReleasesURL, pythonStandalonePageSize, pythonStandaloneMaxReleases) + if err != nil { + return nil, err } var jobs []MirrorJob @@ -98,6 +99,75 @@ func (s *PythonStandaloneSource) FetchVersions() ([]MirrorJob, error) { return jobs, nil } +// fetchGitHubReleases fetches up to maxReleases releases (newest first) from a +// GitHub releases API URL, following the Link header across pages of pageSize. +func fetchGitHubReleases(releasesURL string, pageSize, maxReleases int) ([]githubRelease, error) { + var releases []githubRelease + url := fmt.Sprintf("%s?per_page=%d", releasesURL, pageSize) + + for page := 1; url != "" && len(releases) < maxReleases; page++ { + pageReleases, next, err := fetchGitHubReleasesPage(url) + if err != nil { + return nil, fmt.Errorf("page %d: %w", page, err) + } + + releases = append(releases, pageReleases...) + url = next + } + + if len(releases) > maxReleases { + releases = releases[:maxReleases] + } + + return releases, nil +} + +// fetchGitHubReleasesPage fetches a single page of releases and returns it +// along with the URL of the next page ("" on the last page). +func fetchGitHubReleasesPage(url string) ([]githubRelease, string, error) { + resp, err := httpGetWithRetry(url, 3) + if err != nil { + return nil, "", fmt.Errorf("fetching releases: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, "", fmt.Errorf("fetching releases: HTTP %d", resp.StatusCode) + } + + var releases []githubRelease + if err := json.NewDecoder(resp.Body).Decode(&releases); err != nil { + return nil, "", fmt.Errorf("parsing releases: %w", err) + } + + return releases, nextPageURL(resp.Header.Get("Link")), nil +} + +// nextPageURL extracts the rel="next" target from a GitHub Link header, e.g. +// `; rel="next", <...>; rel="last"`. +// It returns "" on the last page. +func nextPageURL(linkHeader string) string { + for _, link := range strings.Split(linkHeader, ",") { + target, params, found := strings.Cut(link, ";") + if !found { + continue + } + + target = strings.TrimSpace(target) + if !strings.HasPrefix(target, "<") || !strings.HasSuffix(target, ">") { + continue + } + + for _, param := range strings.Split(params, ";") { + if strings.TrimSpace(param) == `rel="next"` { + return target[1 : len(target)-1] + } + } + } + + return "" +} + func (s *PythonStandaloneSource) fetchShasums(release githubRelease) map[string]string { shasums := make(map[string]string) diff --git a/scripts/mirror-binaries/python_test.go b/scripts/mirror-binaries/python_test.go new file mode 100644 index 0000000..f7e45d8 --- /dev/null +++ b/scripts/mirror-binaries/python_test.go @@ -0,0 +1,206 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync/atomic" + "testing" +) + +func TestNextPageURL(t *testing.T) { + tests := []struct { + name string + header string + want string + }{ + { + name: "next before last", + header: `; rel="next", ; rel="last"`, + want: "https://api.github.com/r?per_page=10&page=2", + }, + { + name: "next after prev", + header: `; rel="prev", ; rel="next"`, + want: "https://api.github.com/r?page=3", + }, + { + name: "last page has no next", + header: `; rel="first", ; rel="prev"`, + want: "", + }, + { + name: "empty header", + header: "", + want: "", + }, + { + name: "target without angle brackets", + header: `https://api.github.com/r?page=2; rel="next"`, + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := nextPageURL(tt.header); got != tt.want { + t.Errorf("nextPageURL(%q) = %q, want %q", tt.header, got, tt.want) + } + }) + } +} + +// newReleasesServer serves total releases tagged "1".."total" (newest first), +// paginated like the GitHub releases API. If failPage is non-zero, that page +// responds with HTTP 403. The returned counter tracks requests served. +func newReleasesServer(t *testing.T, total, failPage int) (*httptest.Server, *atomic.Int32) { + t.Helper() + + var requests atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + + perPage, err := strconv.Atoi(r.URL.Query().Get("per_page")) + if err != nil { + http.Error(w, "missing per_page", http.StatusBadRequest) + return + } + page := 1 + if p := r.URL.Query().Get("page"); p != "" { + page, _ = strconv.Atoi(p) + } + + if page == failPage { + http.Error(w, "rate limited", http.StatusForbidden) + return + } + + start := (page - 1) * perPage + end := min(start+perPage, total) + var releases []githubRelease + for i := start; i < end; i++ { + releases = append(releases, githubRelease{TagName: strconv.Itoa(i + 1)}) + } + + if end < total { + w.Header().Set("Link", fmt.Sprintf(`; rel="next"`, + r.Host, r.URL.Path, perPage, page+1)) + } + if err := json.NewEncoder(w).Encode(releases); err != nil { + t.Errorf("encoding releases: %v", err) + } + })) + t.Cleanup(srv.Close) + + return srv, &requests +} + +func TestFetchGitHubReleases(t *testing.T) { + tests := []struct { + name string + total int + pageSize int + maxReleases int + wantReleases int + wantRequests int32 + }{ + { + name: "follows every page", + total: 25, + pageSize: 10, + maxReleases: 100, + wantReleases: 25, + wantRequests: 3, + }, + { + name: "single page", + total: 4, + pageSize: 10, + maxReleases: 100, + wantReleases: 4, + wantRequests: 1, + }, + { + name: "stops at max releases", + total: 50, + pageSize: 10, + maxReleases: 15, + wantReleases: 15, + wantRequests: 2, + }, + { + name: "max releases on page boundary", + total: 50, + pageSize: 10, + maxReleases: 20, + wantReleases: 20, + wantRequests: 2, + }, + { + name: "no releases", + total: 0, + pageSize: 10, + maxReleases: 100, + wantReleases: 0, + wantRequests: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + srv, requests := newReleasesServer(t, tt.total, 0) + + releases, err := fetchGitHubReleases(srv.URL+"/releases", tt.pageSize, tt.maxReleases) + if err != nil { + t.Fatalf("fetchGitHubReleases() error = %v, want nil", err) + } + + if got := len(releases); got != tt.wantReleases { + t.Errorf("len(releases) = %d, want %d", got, tt.wantReleases) + } + if got := requests.Load(); got != tt.wantRequests { + t.Errorf("requests = %d, want %d", got, tt.wantRequests) + } + + // Releases must stay newest first across page boundaries + for i, release := range releases { + if want := strconv.Itoa(i + 1); release.TagName != want { + t.Errorf("releases[%d].TagName = %q, want %q", i, release.TagName, want) + break + } + } + }) + } +} + +func TestFetchGitHubReleases_PageError(t *testing.T) { + srv, _ := newReleasesServer(t, 30, 2) + + releases, err := fetchGitHubReleases(srv.URL+"/releases", 10, 100) + if err == nil { + t.Fatalf("fetchGitHubReleases() error = nil, want error for failed page 2") + } + if releases != nil { + t.Errorf("fetchGitHubReleases() releases = %v, want nil on error", releases) + } + + want := "page 2: fetching releases: HTTP 403" + if err.Error() != want { + t.Errorf("fetchGitHubReleases() error = %q, want %q", err.Error(), want) + } +} + +func TestFetchGitHubReleases_MalformedPage(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("not json")) + })) + t.Cleanup(srv.Close) + + _, err := fetchGitHubReleases(srv.URL+"/releases", 10, 100) + if err == nil || !strings.HasPrefix(err.Error(), "page 1: parsing releases:") { + t.Errorf("fetchGitHubReleases() error = %v, want page 1 parsing error", err) + } +} diff --git a/scripts/mirror-binaries/sources.go b/scripts/mirror-binaries/sources.go index 8ab5afb..72cb993 100644 --- a/scripts/mirror-binaries/sources.go +++ b/scripts/mirror-binaries/sources.go @@ -1,8 +1,10 @@ package main import ( + "errors" "fmt" "net/http" + "os" "time" ) @@ -19,11 +21,36 @@ var httpClient = &http.Client{ Timeout: 60 * time.Second, } +// newRequest builds a GET request, authenticating GitHub API calls with +// GITHUB_TOKEN when it is set. Unauthenticated calls share a 60 requests/hour +// limit per IP, which CI runners regularly exhaust. +func newRequest(url string) (*http.Request, error) { + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + // Only send the token to the GitHub API, never to arbitrary download hosts + if req.URL.Host == "api.github.com" { + req.Header.Set("Accept", "application/vnd.github+json") + if token := os.Getenv("GITHUB_TOKEN"); token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + } + + return req, nil +} + // httpGetWithRetry performs an HTTP GET with retries for transient failures func httpGetWithRetry(url string, maxRetries int) (*http.Response, error) { + req, err := newRequest(url) + if err != nil { + return nil, err + } + var lastErr error for attempt := 1; attempt <= maxRetries; attempt++ { - resp, err := httpClient.Get(url) + resp, err := httpClient.Do(req) if err != nil { lastErr = err if attempt < maxRetries { @@ -76,7 +103,15 @@ func fetchJobsFromUpstream(runtime string) ([]MirrorJob, error) { return nil, err } + return collectJobs(sources) +} + +// collectJobs fetches mirror jobs from each source in order. A failing source +// does not stop the others: jobs from the sources that succeeded are returned +// together with an error describing every source that failed. +func collectJobs(sources []UpstreamSource) ([]MirrorJob, error) { var allJobs []MirrorJob + var errs []error seen := make(map[string]bool) // Track version+platform to avoid duplicates for _, source := range sources { @@ -84,6 +119,7 @@ func fetchJobsFromUpstream(runtime string) ([]MirrorJob, error) { jobs, err := source.FetchVersions() if err != nil { fmt.Printf(" Warning: failed to fetch from %s: %v\n", source.Name(), err) + errs = append(errs, fmt.Errorf("%s: %w", source.Name(), err)) continue } @@ -100,5 +136,5 @@ func fetchJobsFromUpstream(runtime string) ([]MirrorJob, error) { fmt.Printf(" Found %d versions from %s (%d new)\n", len(jobs), source.Name(), added) } - return allJobs, nil + return allJobs, errors.Join(errs...) } diff --git a/scripts/mirror-binaries/sources_test.go b/scripts/mirror-binaries/sources_test.go new file mode 100644 index 0000000..2ed960a --- /dev/null +++ b/scripts/mirror-binaries/sources_test.go @@ -0,0 +1,140 @@ +package main + +import ( + "errors" + "strings" + "testing" +) + +func TestNewRequest(t *testing.T) { + tests := []struct { + name string + url string + token string + wantAuth string + wantGHAPI bool + }{ + { + name: "GitHub API with token", + url: "https://api.github.com/repos/astral-sh/python-build-standalone/releases", + token: "secret", + wantAuth: "Bearer secret", + wantGHAPI: true, + }, + { + name: "GitHub API without token", + url: "https://api.github.com/repos/astral-sh/python-build-standalone/releases", + wantGHAPI: true, + }, + { + name: "download host never receives token", + url: "https://github.com/astral-sh/python-build-standalone/releases/download/20261003/SHA256SUMS", + token: "secret", + }, + { + name: "unrelated host never receives token", + url: "https://nodejs.org/dist/index.json", + token: "secret", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("GITHUB_TOKEN", tt.token) + + req, err := newRequest(tt.url) + if err != nil { + t.Fatalf("newRequest(%q) error = %v, want nil", tt.url, err) + } + + if got := req.Header.Get("Authorization"); got != tt.wantAuth { + t.Errorf("Authorization header = %q, want %q", got, tt.wantAuth) + } + + gotGHAPI := req.Header.Get("Accept") == "application/vnd.github+json" + if gotGHAPI != tt.wantGHAPI { + t.Errorf("GitHub Accept header set = %v, want %v", gotGHAPI, tt.wantGHAPI) + } + }) + } +} + +func TestNewRequest_InvalidURL(t *testing.T) { + if _, err := newRequest("://bad"); err == nil { + t.Error("newRequest(\"://bad\") error = nil, want error") + } +} + +// fakeSource is an UpstreamSource returning canned jobs or an error. +type fakeSource struct { + name string + jobs []MirrorJob + err error +} + +func (s *fakeSource) Name() string { return s.name } +func (s *fakeSource) FetchVersions() ([]MirrorJob, error) { return s.jobs, s.err } + +func TestCollectJobs(t *testing.T) { + primary := &fakeSource{ + name: "primary", + jobs: []MirrorJob{ + {Version: "3.13.16", Platform: "windows-amd64", URL: "primary"}, + {Version: "3.13.16", Platform: "linux-amd64", URL: "primary"}, + }, + } + fallback := &fakeSource{ + name: "fallback", + jobs: []MirrorJob{ + {Version: "3.13.16", Platform: "windows-amd64", URL: "fallback"}, + {Version: "3.11.17", Platform: "windows-amd64", URL: "fallback"}, + }, + } + broken := &fakeSource{name: "broken", err: errors.New("fetching releases: HTTP 504")} + + tests := []struct { + name string + sources []UpstreamSource + wantURLs []string + wantErrText string + }{ + { + name: "first source wins on duplicates", + sources: []UpstreamSource{primary, fallback}, + wantURLs: []string{"primary", "primary", "fallback"}, + }, + { + name: "failing source keeps jobs from healthy sources", + sources: []UpstreamSource{broken, fallback}, + wantURLs: []string{"fallback", "fallback"}, + wantErrText: "broken: fetching releases: HTTP 504", + }, + { + name: "only source fails", + sources: []UpstreamSource{broken}, + wantErrText: "broken: fetching releases: HTTP 504", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + jobs, err := collectJobs(tt.sources) + + if tt.wantErrText == "" { + if err != nil { + t.Errorf("collectJobs() error = %v, want nil", err) + } + } else if err == nil || !strings.Contains(err.Error(), tt.wantErrText) { + t.Errorf("collectJobs() error = %v, want error containing %q", err, tt.wantErrText) + } + + var gotURLs []string + for _, job := range jobs { + gotURLs = append(gotURLs, job.URL) + } + if strings.Join(gotURLs, ",") != strings.Join(tt.wantURLs, ",") { + t.Errorf("collectJobs() job URLs = %v, want %v", gotURLs, tt.wantURLs) + } + }) + } +}