From a838f13f48075734d7120f8da5a16d9ce4754663 Mon Sep 17 00:00:00 2001 From: Phat Pham Date: Sat, 14 Nov 2026 10:39:22 +0700 Subject: [PATCH] fix(sources): report GitHub rate limits cleanly GitHub's trending page can return a 429 abuse page with a raw JSON body from Cloudflare's shared egress. Store a concise "temporarily rate-limiting this source; showing cached items" message instead, so the health payload and UI banner stay readable. Cached GitHub items remain in place until the next refresh. Add a source-level test that stubs three 429 responses and verifies the clean message (and that GitHub's raw body is not exposed). Co-Authored-By: Claude Code --- README.md | 2 +- core/sources/github.ts | 7 +++++++ core/test/github.spec.ts | 45 ++++++++++++++++++++++++++++++++++++++-- docs/ARCHITECTURE.md | 2 +- docs/DEPLOYMENT.md | 2 +- docs/RUNBOOK.md | 2 +- 6 files changed, 54 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 6278c9b..83e8e86 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,7 @@ Key properties: - fetch failures do not wipe existing data - sources without a native article date fall back to the initial fetch time (`first_seen_at`) for ordering - later refreshes preserve the original published/fetched ordering timestamps for existing items -- source fetches send a descriptive User-Agent, honor `Retry-After` on 429s, and cap error details; Hugging Face reads its daily-papers JSON API instead of scraping the trending HTML page +- source fetches send a descriptive User-Agent, honor `Retry-After` on 429s, and cap error details; GitHub 429s show a concise rate-limit message while cached items remain, and Hugging Face reads its daily-papers JSON API instead of scraping the trending HTML page --- diff --git a/core/sources/github.ts b/core/sources/github.ts index ce51684..686177b 100644 --- a/core/sources/github.ts +++ b/core/sources/github.ts @@ -24,6 +24,13 @@ export const gitHubTrendingSource: Source = { async fetch(): Promise { const response = await getWithRetry(HOMEPAGE_URL); if (!response.ok) { + if (response.status === 429) { + await response.body?.cancel().catch(() => {}); + throw new HttpError( + response.status, + "GitHub is temporarily rate-limiting this source; showing cached items until the next refresh", + ); + } throw new HttpError( response.status, await readHttpErrorDetail(response), diff --git a/core/test/github.spec.ts b/core/test/github.spec.ts index da8cea9..0f4f8bc 100644 --- a/core/test/github.spec.ts +++ b/core/test/github.spec.ts @@ -1,6 +1,14 @@ // Mirrors internal/sources/github_test.go's fixture and assertions. -import { describe, expect, it } from "vitest"; -import { parseGitHubTrending } from "../sources/github.ts"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + gitHubTrendingSource, + parseGitHubTrending, +} from "../sources/github.ts"; + +afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); +}); describe("parseGitHubTrending", () => { it("normalizes the repo path from the href", () => { @@ -35,3 +43,36 @@ describe("parseGitHubTrending", () => { expect(item.sourceRank).toBe(1); }); }); + +describe("gitHubTrendingSource", () => { + it("reports a concise message when GitHub rate-limits the source", async () => { + vi.useFakeTimers(); + let calls = 0; + vi.stubGlobal("fetch", async () => { + calls += 1; + return new Response( + JSON.stringify({ + code: "429", + message: "Access to this site has been restricted.", + }), + { + status: 429, + headers: { "content-type": "application/json" }, + }, + ); + }); + + const promise = gitHubTrendingSource.fetch().catch((error) => error); + await vi.runAllTimersAsync(); + const error = await promise; + + expect(calls).toBe(3); + expect(error).toBeInstanceOf(Error); + if (!(error instanceof Error)) { + throw new Error("expected an error"); + } + expect(error.message).toContain("temporarily rate-limiting"); + expect(error.message).toContain("cached items"); + expect(error.message).not.toContain("Access to this site"); + }); +}); diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 29009f5..5d06a52 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -51,7 +51,7 @@ Browser-only preferences and reading state live in `localStorage`: theme prefere - Items are upserted by `(source, external_id)`. - A refresh never deletes existing rows — a failed fetch records the failure in `sync_state` and leaves prior data in place. - The original `published_at` is preserved across re-fetches via `coalesce(items.published_at, excluded.published_at)` so an upstream source cannot reorder an existing item by later emitting a different date. -- Source requests share a retry helper with a descriptive `User-Agent`/`Accept` header, `Retry-After`-aware 429 backoff, and error details capped at 200 characters (HTML error pages become a fixed message). Hugging Face uses its daily-papers JSON API (the same 50 entries) instead of scraping the ~1.4 MB trending HTML page, which reduces CPU and avoids HTML challenge bodies. +- Source requests share a retry helper with a descriptive `User-Agent`/`Accept` header, `Retry-After`-aware 429 backoff, and error details capped at 200 characters (HTML error pages become a fixed message). GitHub 429s become a concise “temporarily rate-limiting; showing cached items” message instead of GitHub's raw JSON. Hugging Face uses its daily-papers JSON API (the same 50 entries) instead of scraping the ~1.4 MB trending HTML page, which reduces CPU and avoids HTML challenge bodies. - Sorting and pagination intentionally happen in application memory after SQL filtering. Total item count across the three sources is small enough that keeping one shared comparator is simpler and less error-prone than encoding the full fallback ordering logic separately per adapter. ## `FeedRepository` diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index b6177d1..4392871 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -62,7 +62,7 @@ The checked-in `wrangler.toml` config does not hardcode a route. Domain attachme - D1: 5GB storage, 5M row reads/day, 100k row writes/day. - Workers: 100k requests/day, 10ms CPU time per invocation, 50 subrequests per request. -- Source refreshes fetch one page per source per hourly fan-out. Hugging Face reads about 272 KB of JSON from `/api/daily_papers` instead of the ~1.4 MB `/papers/trending` HTML page; source error details are capped at 200 characters, and HTML error pages are reduced to a fixed message before they reach D1 or the UI. +- Source refreshes fetch one page per source per hourly fan-out. Hugging Face reads about 272 KB of JSON from `/api/daily_papers` instead of the ~1.4 MB `/papers/trending` HTML page; source error details are capped at 200 characters, and HTML error pages are reduced to a fixed message before they reach D1 or the UI. GitHub 429s use a concise rate-limit message while cached GitHub items remain available. - The five-minute home-page cache absorbs most `GET /` traffic, so only cache misses run the feed query and render the HTML. The browser response is `private, no-store`; edge caching is the Worker Cache API, not the CDN, so the zone's Browser Cache TTL setting cannot rewrite the client TTL. Each cold-isolate cache miss also reads `style.css` and `theme-init.js` through the `ASSETS` binding (two local subrequests), then reuses them in memory. - The standalone launch veil and skeleton styles are inlined with the existing HTML/CSS, so they add no extra requests. Bump the shell cache version when the HTML shell or `app.js` changes; see [RUNBOOK.md](RUNBOOK.md#pwa-shell-updates). - Article previews add one outbound fetch (and one `node-html-parser` parse) per uncached `/api/preview` request. The 400 KB HTML body cap, 30-minute edge cache (5 minutes for empty results), and a per-IP limiter (20 cache-miss previews per minute; cached previews bypass it) bound that cost; transient upstream `429`/`5xx` responses are retried once. PDFs/Office documents and HTML pages with no readable blocks can additionally call Workers AI's `toMarkdown` (5 MB download cap; document text conversion is free and the fallback disables AI image-description conversion). That call runs only on a cache miss and its result is cached for the same 30 minutes; set `FEEDREADER_AI_PREVIEW_FALLBACK=false` to disable it. HTML pages that still have no readable content after AI conversion can call Browser Run's `markdown` Quick Action; Workers Free includes 10 browser-minutes/day and paces Quick Actions to one request per 10 seconds. Browser Run runs only on a cache miss, uses no browser sessions (so the concurrent-browser limit does not apply), and its result is cached for 30 minutes; set `FEEDREADER_BROWSER_PREVIEW_FALLBACK=false` to disable it. Hugging Face paper pages parse only ``+`
` with one combined noise selector, reducing their extraction CPU. The browser retries a preview `503` once after 500 ms; `429`s are not retried. Browser-facing preview and `/api/items` responses are `private, no-store`; their edge copies live in the Cache API. Watch the CPU-time panel if preview usage is heavy — bursts can hit the Workers Free 10 ms limit, so see [RUNBOOK.md](RUNBOOK.md#article-preview-failures). Richer preview blocks (inline links, captions, tables) increase the cached response payload but not the upstream fetch count; the same byte cap and cache TTL apply. diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md index 610ecfd..ac683f0 100644 --- a/docs/RUNBOOK.md +++ b/docs/RUNBOOK.md @@ -19,7 +19,7 @@ POST /internal/refresh/huggingface ─┘ X-Refresh-Secret header This preserves per-source fault isolation: one source's failure updates only that source's `sync_state` row and never wipes its existing `items` rows. -**Source fetch reliability:** all source requests send a descriptive `User-Agent` and `Accept` header through `core/sources/util.ts`. Retryable responses are attempted twice more; `429` responses honor `Retry-After` (capped at 5 seconds) or use 1s/3s backoff instead of the short 5xx backoff. Error details are capped at 200 characters, and HTML error pages are stored as `upstream returned an HTML error page`, so a large challenge page never lands in `/healthz`, the UI banner, or D1's `last_error` column. +**Source fetch reliability:** all source requests send a descriptive `User-Agent` and `Accept` header through `core/sources/util.ts`. Retryable responses are attempted twice more; `429` responses honor `Retry-After` (capped at 5 seconds) or use 1s/3s backoff instead of the short 5xx backoff. Error details are capped at 200 characters, and HTML error pages are stored as `upstream returned an HTML error page`, so a large challenge page never lands in `/healthz`, the UI banner, or D1's `last_error` column. GitHub's HTML page can return a 429 abuse/rate-limit page from Cloudflare's shared egress; that case is stored as a concise “temporarily rate-limiting this source; showing cached items” message instead of GitHub's raw JSON, and existing GitHub items remain in place until the next refresh. Hugging Face trending papers are fetched from `https://huggingface.co/api/daily_papers` (about 272 KB of JSON, same 50 entries) instead of the ~1.4 MB `/papers/trending` HTML page. The HTML page was the main source of the 429 challenge bodies and cost more Worker CPU to regex-parse; the JSON API keeps the source invocation comfortably inside the Free-tier CPU budget.