fix(mirror): paginate python-build-standalone release discovery

.github/workflows/mirror-sync.yml

@@ -48,6 +48,7 @@ jobs:
           R2_BUCKET: ${{ secrets.CLOUDFLARE_R2_BUILDS_BUCKET }}
           R2_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
           R2_SECRET_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           ./scripts/mirror-binaries/mirror-binaries \
             --runtime=${{ matrix.runtime }} \

scripts/mirror-binaries/main.go

@@ -6,6 +6,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -120,17 +121,20 @@ func main() {
 		}
 	}
 
-	// Collect all jobs from upstream sources
+	// Collect all jobs from upstream sources. A failing source is reported at
+	// the end rather than aborting, so jobs from healthy sources still mirror.
 	var jobs []MirrorJob
+	var fetchErrs []error
 	for _, rt := range runtimes {
 		fmt.Printf("Discovering %s versions from upstream...\n", rt)
 		rtJobs, err := fetchJobsFromUpstream(rt)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error fetching upstream versions for %s: %v\n", rt, err)
-			os.Exit(1)
+			fetchErrs = append(fetchErrs, err)
 		}
 		jobs = append(jobs, rtJobs...)
 	}
+	fetchErr := errors.Join(fetchErrs...)
 
 	fmt.Printf("Total jobs to process: %d\n", len(jobs))
 
@@ -150,6 +154,7 @@ func main() {
 		}
 		fmt.Printf("\nTotal: %d files (%d with upstream checksum, %d will generate)\n",
 			len(jobs), withChecksum, withoutChecksum)
+		exitOnFetchError(fetchErr)
 		return
 	}
 
@@ -172,6 +177,7 @@ func main() {
 
 	if len(jobs) == 0 {
 		fmt.Println("No files to mirror")
+		exitOnFetchError(fetchErr)
 		return
 	}
 
@@ -188,11 +194,22 @@ func main() {
 	fmt.Printf("Upstream checksums: %d\n", stats.UpstreamChecksum)
 	fmt.Printf("Generated checksums: %d\n", stats.GeneratedChecksum)
 
+	exitOnFetchError(fetchErr)
 	if stats.Failed > 0 {
 		os.Exit(1)
 	}
 }
 
+// exitOnFetchError exits non-zero when any upstream source failed to fetch.
+// Without this a broken source mirrors nothing while the workflow still
+// reports success, hiding missing versions indefinitely.
+func exitOnFetchError(err error) {
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "\nUpstream fetch failed: %v\n", err)
+		os.Exit(1)
+	}
+}
+
 func createS3Client() (*s3.Client, error) {
 	cfg, err := config.LoadDefaultConfig(context.Background(),
 		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(

scripts/mirror-binaries/python.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"net/http"
 	"regexp"
 	"strings"
 )
@@ -33,22 +34,22 @@ var pythonStandalonePattern = regexp.MustCompile(
 	`^cpython-(\d+\.\d+\.\d+)\+\d+-([^-]+-[^-]+-[^-]+(?:-[^-]+)?)-install_only\.(tar\.gz|tar\.zst)$`,
 )
 
-func (s *PythonStandaloneSource) FetchVersions() ([]MirrorJob, error) {
-	// Fetch releases from GitHub API with retries
-	url := "https://api.github.com/repos/astral-sh/python-build-standalone/releases?per_page=100"
-	resp, err := httpGetWithRetry(url, 3)
-	if err != nil {
-		return nil, fmt.Errorf("fetching releases: %w", err)
-	}
-	defer resp.Body.Close()
+const (
+	pythonStandaloneReleasesURL = "https://api.github.com/repos/astral-sh/python-build-standalone/releases"
 
-	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("fetching releases: HTTP %d", resp.StatusCode)
-	}
+	// pythonStandalonePageSize is kept small because each release lists ~900
+	// assets (~2MB of JSON). Pages of 50 or more make the GitHub API time out
+	// with HTTP 504.
+	pythonStandalonePageSize = 10
 
-	var releases []githubRelease
-	if err := json.NewDecoder(resp.Body).Decode(&releases); err != nil {
-		return nil, fmt.Errorf("parsing releases: %w", err)
+	// pythonStandaloneMaxReleases bounds how far back in release history to look.
+	pythonStandaloneMaxReleases = 100
+)
+
+func (s *PythonStandaloneSource) FetchVersions() ([]MirrorJob, error) {
+	releases, err := fetchGitHubReleases(pythonStandaloneReleasesURL, pythonStandalonePageSize, pythonStandaloneMaxReleases)
+	if err != nil {
+		return nil, err
 	}
 
 	var jobs []MirrorJob
@@ -98,6 +99,75 @@ func (s *PythonStandaloneSource) FetchVersions() ([]MirrorJob, error) {
 	return jobs, nil
 }
 
+// fetchGitHubReleases fetches up to maxReleases releases (newest first) from a
+// GitHub releases API URL, following the Link header across pages of pageSize.
+func fetchGitHubReleases(releasesURL string, pageSize, maxReleases int) ([]githubRelease, error) {
+	var releases []githubRelease
+	url := fmt.Sprintf("%s?per_page=%d", releasesURL, pageSize)
+
+	for page := 1; url != "" && len(releases) < maxReleases; page++ {
+		pageReleases, next, err := fetchGitHubReleasesPage(url)
+		if err != nil {
+			return nil, fmt.Errorf("page %d: %w", page, err)
+		}
+
+		releases = append(releases, pageReleases...)
+		url = next
+	}
+
+	if len(releases) > maxReleases {
+		releases = releases[:maxReleases]
+	}
+
+	return releases, nil
+}
+
+// fetchGitHubReleasesPage fetches a single page of releases and returns it
+// along with the URL of the next page ("" on the last page).
+func fetchGitHubReleasesPage(url string) ([]githubRelease, string, error) {
+	resp, err := httpGetWithRetry(url, 3)
+	if err != nil {
+		return nil, "", fmt.Errorf("fetching releases: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, "", fmt.Errorf("fetching releases: HTTP %d", resp.StatusCode)
+	}
+
+	var releases []githubRelease
+	if err := json.NewDecoder(resp.Body).Decode(&releases); err != nil {
+		return nil, "", fmt.Errorf("parsing releases: %w", err)
+	}
+
+	return releases, nextPageURL(resp.Header.Get("Link")), nil
+}
+
+// nextPageURL extracts the rel="next" target from a GitHub Link header, e.g.
+// `<https://api.github.com/...&page=2>; rel="next", <...>; rel="last"`.
+// It returns "" on the last page.
+func nextPageURL(linkHeader string) string {
+	for _, link := range strings.Split(linkHeader, ",") {
+		target, params, found := strings.Cut(link, ";")
+		if !found {
+			continue
+		}
+
+		target = strings.TrimSpace(target)
+		if !strings.HasPrefix(target, "<") || !strings.HasSuffix(target, ">") {
+			continue
+		}
+
+		for _, param := range strings.Split(params, ";") {
+			if strings.TrimSpace(param) == `rel="next"` {
+				return target[1 : len(target)-1]
+			}
+		}
+	}
+
+	return ""
+}
+
 func (s *PythonStandaloneSource) fetchShasums(release githubRelease) map[string]string {
 	shasums := make(map[string]string)