diff --git a/README.md b/README.md index debfdd4..afec1e2 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,7 @@ Key properties: - fetch failures do not wipe existing data - sources without a native article date fall back to the initial fetch time (`first_seen_at`) for ordering - later refreshes preserve the original published/fetched ordering timestamps for existing items +- source fetches send a descriptive User-Agent, honor `Retry-After` on 429s, and cap error details; Hugging Face reads its daily-papers JSON API instead of scraping the trending HTML page --- diff --git a/core/sources/github.ts b/core/sources/github.ts index f825a0a..ce51684 100644 --- a/core/sources/github.ts +++ b/core/sources/github.ts @@ -11,7 +11,7 @@ import { getWithRetry, HttpError, parseDigits, - readLimitedText, + readHttpErrorDetail, } from "./util.ts"; const HOMEPAGE_URL = "https://github.com/trending"; @@ -26,7 +26,7 @@ export const gitHubTrendingSource: Source = { if (!response.ok) { throw new HttpError( response.status, - await readLimitedText(response, 1024), + await readHttpErrorDetail(response), ); } const html = await response.text(); diff --git a/core/sources/hackernews.ts b/core/sources/hackernews.ts index 4c6d0d4..5601a6f 100644 --- a/core/sources/hackernews.ts +++ b/core/sources/hackernews.ts @@ -7,7 +7,7 @@ import { firstNonEmpty, getWithRetry, HttpError, - readLimitedText, + readHttpErrorDetail, unescapeHtmlEntities, } from "./util.ts"; @@ -42,7 +42,7 @@ export const hackerNewsSource: Source = { if (!response.ok) { throw new HttpError( response.status, - await readLimitedText(response, 1024), + await readHttpErrorDetail(response), ); } const payload = (await response.json()) as HnFrontPage; diff --git a/core/sources/huggingface.ts b/core/sources/huggingface.ts index 6fb38ad..e8e94e1 100644 --- a/core/sources/huggingface.ts +++ b/core/sources/huggingface.ts @@ -1,5 +1,8 @@ -// Port of internal/sources/huggingface.go. Extracts an embedded JSON blob -// via one regex — no DOM parsing needed. +// Uses Hugging Face's daily-papers JSON API. The public /papers/trending +// page is ~1.4 MB of HTML with the same data embedded in a data-props blob; +// fetching and parsing it wasted Worker CPU and its HTML fallback page is +// also what surfaced large 429 challenge bodies. The API returns the same +// 50 daily-paper entries as JSON. import type { FeedItem } from "../domain.ts"; import type { Source } from "./index.ts"; @@ -8,12 +11,11 @@ import { firstNonEmpty, getWithRetry, HttpError, - readLimitedText, - unescapeHtmlEntities, + readHttpErrorDetail, } from "./util.ts"; const HOMEPAGE_URL = "https://huggingface.co/papers/trending"; -const DAILY_PAPERS_PATTERN = /data-target="DailyPapers"\s+data-props="([^"]+)"/; +const DAILY_PAPERS_API_URL = "https://huggingface.co/api/daily_papers"; interface HfAuthor { name?: string; @@ -34,40 +36,37 @@ interface HfDailyPaperEntry { title?: string; summary?: string; upvotes?: number | null; + numComments?: number | null; publishedAt?: string; submittedBy?: { fullname?: string; name?: string }; paper?: HfPaper; } -interface HfDailyProps { - dailyPapers?: HfDailyPaperEntry[]; -} - export const huggingFacePapersSource: Source = { key: () => "huggingface", label: () => "Hugging Face Trending Papers", homepageUrl: () => HOMEPAGE_URL, async fetch(): Promise { - const response = await getWithRetry(HOMEPAGE_URL); + const response = await getWithRetry(DAILY_PAPERS_API_URL, { + headers: { Accept: "application/json" }, + }); if (!response.ok) { throw new HttpError( response.status, - await readLimitedText(response, 1024), + await readHttpErrorDetail(response), ); } - const body = await response.text(); - return parseHuggingFacePapers(body); + const payload = (await response.json()) as unknown; + if (!Array.isArray(payload)) { + throw new Error("unexpected daily papers payload"); + } + return parseHuggingFaceDailyPapers(payload); }, }; -export function parseHuggingFacePapers(payload: string): FeedItem[] { - const match = DAILY_PAPERS_PATTERN.exec(payload); - if (!match) { - throw new Error("unable to locate DailyPapers payload"); - } - const props = JSON.parse(unescapeHtmlEntities(match[1]!)) as HfDailyProps; - const entries = props.dailyPapers ?? []; - +export function parseHuggingFaceDailyPapers( + entries: HfDailyPaperEntry[], +): FeedItem[] { const items: FeedItem[] = []; entries.forEach((entry, i) => { const paper = entry.paper ?? {}; @@ -79,8 +78,9 @@ export function parseHuggingFacePapers(payload: string): FeedItem[] { .filter((name) => name !== ""); const metadata: Record = {}; - if (paper.numComments != null) { - metadata.comments_count = paper.numComments; + const commentCount = firstDefined(entry.numComments, paper.numComments); + if (commentCount != null) { + metadata.comments_count = commentCount; } if (authors.length > 0) { metadata.authors = authors.length > 6 ? authors.slice(0, 6) : authors; diff --git a/core/sources/util.ts b/core/sources/util.ts index e884dad..ac4f2b3 100644 --- a/core/sources/util.ts +++ b/core/sources/util.ts @@ -1,7 +1,16 @@ // Port of internal/sources/util.go. const RETRY_DELAYS_MS = [250, 750]; +// 429s usually need a longer pause than a transient 5xx; Retry-After wins +// when the upstream provides one. +const RATE_LIMIT_RETRY_DELAYS_MS = [1_000, 3_000]; +const MAX_RETRY_AFTER_MS = 5_000; const RETRYABLE_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]); +const MAX_HTTP_ERROR_DETAIL = 200; +const DEFAULT_SOURCE_USER_AGENT = + "feedreader/0.1 (+https://reader.boringcode.dev)"; +const DEFAULT_SOURCE_ACCEPT = + "application/json, text/html;q=0.9, */*;q=0.8"; export class HttpError extends Error { readonly statusCode: number; @@ -30,35 +39,114 @@ function sleep(ms: number, signal?: AbortSignal): Promise { }); } -/** Mirrors getWithRetry: up to 2 retries (250ms, 750ms backoff) on 408/425/429/5xx. */ +function retryDelayMs(response: Response | null, attemptIndex: number): number { + const retryAfter = response?.headers.get("retry-after"); + if (retryAfter) { + const seconds = Number.parseInt(retryAfter, 10); + if (Number.isFinite(seconds) && seconds >= 0) { + return Math.min(seconds * 1000, MAX_RETRY_AFTER_MS); + } + const at = Date.parse(retryAfter); + if (!Number.isNaN(at)) { + return Math.min(Math.max(at - Date.now(), 0), MAX_RETRY_AFTER_MS); + } + } + + const base = + response?.status === 429 + ? (RATE_LIMIT_RETRY_DELAYS_MS[attemptIndex] ?? 0) + : (RETRY_DELAYS_MS[attemptIndex] ?? 0); + return base + Math.floor(Math.random() * 200); +} + +/** Mirrors getWithRetry: up to 2 retries on 408/425/429/5xx. Source requests + * carry a descriptive user agent instead of the Worker default so upstream + * bot protection can identify the client. */ export async function getWithRetry( url: string, init: RequestInit = {}, ): Promise { + const headers = new Headers(init.headers); + if (!headers.has("user-agent")) { + headers.set("User-Agent", DEFAULT_SOURCE_USER_AGENT); + } + if (!headers.has("accept")) { + headers.set("Accept", DEFAULT_SOURCE_ACCEPT); + } + const requestInit: RequestInit = { ...init, headers }; + const attempts = RETRY_DELAYS_MS.length + 1; let lastError: unknown; for (let attempt = 1; attempt <= attempts; attempt++) { + let delay = 0; try { - const response = await fetch(url, init); + const response = await fetch(url, requestInit); if (!RETRYABLE_STATUSES.has(response.status) || attempt === attempts) { return response; } - await response.body?.cancel(); + await response.body?.cancel().catch(() => {}); + delay = retryDelayMs(response, attempt - 1); } catch (error) { if (init.signal?.aborted || attempt === attempts) throw error; lastError = error; + delay = retryDelayMs(null, attempt - 1); } - await sleep(RETRY_DELAYS_MS[attempt - 1] ?? 0, init.signal ?? undefined); + await sleep(delay, init.signal ?? undefined); } throw lastError ?? new Error(`upstream retry loop exhausted for ${url}`); } +/** Reads at most `limit` bytes from the response body and decodes them as + * UTF-8. Error pages can be large (the old Hugging Face HTML path returned a + * 1.4 MB challenge page), and only a short diagnostic prefix is needed. */ export async function readLimitedText( response: Response, limit: number, ): Promise { - const text = await response.text(); - return text.slice(0, limit); + if (!response.body) { + const text = await response.text(); + return text.slice(0, limit); + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let bytesRead = 0; + let out = ""; + try { + while (bytesRead < limit) { + const { done, value } = await reader.read(); + if (done) break; + if (!value || value.byteLength === 0) continue; + const remaining = limit - bytesRead; + const chunk = + value.byteLength > remaining ? value.subarray(0, remaining) : value; + bytesRead += chunk.byteLength; + out += decoder.decode(chunk, { stream: true }); + if (value.byteLength > remaining) break; + } + out += decoder.decode(); + } finally { + await reader.cancel().catch(() => {}); + } + return out; +} + +/** Returns a short, safe error detail. HTML challenge/error pages are + * replaced with a fixed message so their markup never reaches /healthz, the + * UI error banner, or D1's last_error column. */ +export async function readHttpErrorDetail( + response: Response, + limit = MAX_HTTP_ERROR_DETAIL, +): Promise { + const text = await readLimitedText(response, Math.max(limit, 512)); + const contentType = response.headers.get("content-type")?.toLowerCase() ?? ""; + if ( + contentType.includes("text/html") || + /<(?:!doctype|html|head|body)\b/i.test(text) + ) { + return "upstream returned an HTML error page"; + } + return text.replace(/\s+/g, " ").trim().slice(0, limit); } export function extractInt(value: string, pattern: RegExp): number | undefined { diff --git a/core/test/huggingface.spec.ts b/core/test/huggingface.spec.ts index 29cbf9a..84b12d9 100644 --- a/core/test/huggingface.spec.ts +++ b/core/test/huggingface.spec.ts @@ -1,35 +1,29 @@ -// No huggingface_test.go exists in the Go repo to port from directly — this -// exercises parseHuggingFacePapers against a synthetic payload shaped like -// the real embedded data-props blob described in huggingface.go. import { describe, expect, it } from "vitest"; -import { parseHuggingFacePapers } from "../sources/huggingface.ts"; +import { parseHuggingFaceDailyPapers } from "../sources/huggingface.ts"; -describe("parseHuggingFacePapers", () => { - it("extracts papers from the embedded DailyPapers JSON", () => { - const props = { - dailyPapers: [ - { +describe("parseHuggingFaceDailyPapers", () => { + it("extracts papers from the daily papers API entries", () => { + const entries = [ + { + id: "2606.00001", + title: "A Paper Title", + summary: "A short summary.", + numComments: 4, + publishedAt: "2026-06-20T00:00:00.000Z", + submittedBy: { fullname: "Jane Doe", name: "janedoe" }, + paper: { id: "2606.00001", title: "A Paper Title", summary: "A short summary.", - upvotes: 12, publishedAt: "2026-06-20T00:00:00.000Z", - submittedBy: { fullname: "Jane Doe", name: "janedoe" }, - paper: { - id: "2606.00001", - title: "A Paper Title", - summary: "A short summary.", - publishedAt: "2026-06-20T00:00:00.000Z", - upvotes: 12, - numComments: 4, - authors: [{ name: "Jane Doe" }, { name: "John Roe" }], - }, + upvotes: 12, + numComments: null, + authors: [{ name: "Jane Doe" }, { name: "John Roe" }], }, - ], - }; - const payload = `
`; + }, + ]; - const items = parseHuggingFacePapers(payload); + const items = parseHuggingFaceDailyPapers(entries); expect(items).toHaveLength(1); const item = items[0]!; @@ -46,7 +40,27 @@ describe("parseHuggingFacePapers", () => { expect(item.sourceRank).toBe(1); }); - it("throws when the DailyPapers payload is missing", () => { - expect(() => parseHuggingFacePapers("
no papers here
")).toThrow(); + it("falls back to the paper's comment count and title fields", () => { + const items = parseHuggingFaceDailyPapers([ + { + id: "2606.00002", + paper: { + id: "2606.00002", + title: "Paper-only title", + summary: "Paper-only summary.", + numComments: 7, + authors: [{ name: "Alice Smith" }], + }, + }, + ]); + + expect(items).toHaveLength(1); + expect(items[0]?.title).toBe("Paper-only title"); + expect(items[0]?.summary).toBe("Paper-only summary."); + expect(items[0]?.metadata.comments_count).toBe(7); + }); + + it("skips entries without a paper id", () => { + expect(parseHuggingFaceDailyPapers([{}, { paper: {} }])).toEqual([]); }); }); diff --git a/core/test/util.spec.ts b/core/test/util.spec.ts index d6f99e2..feb1f75 100644 --- a/core/test/util.spec.ts +++ b/core/test/util.spec.ts @@ -1,6 +1,10 @@ // Ports internal/sources/http_test.go's three getWithRetry cases. import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { getWithRetry } from "../sources/util.ts"; +import { + getWithRetry, + readHttpErrorDetail, + readLimitedText, +} from "../sources/util.ts"; describe("getWithRetry", () => { beforeEach(() => { @@ -55,4 +59,70 @@ describe("getWithRetry", () => { expect(attempts).toBe(1); expect(response.status).toBe(404); }); + + it("sends descriptive source headers", async () => { + let headers: Headers | undefined; + vi.stubGlobal( + "fetch", + async (_input: unknown, init?: RequestInit) => { + headers = new Headers(init?.headers); + return new Response("ok", { status: 200 }); + }, + ); + + await getWithRetry("https://example.com"); + + expect(headers?.get("user-agent")).toContain("feedreader/0.1"); + expect(headers?.get("accept")).toContain("application/json"); + }); + + it("honors Retry-After on a 429", async () => { + let attempts = 0; + vi.stubGlobal("fetch", async () => { + attempts += 1; + return attempts === 1 + ? new Response("limited", { + status: 429, + headers: { "retry-after": "1" }, + }) + : new Response("ok", { status: 200 }); + }); + + const promise = getWithRetry("https://example.com"); + await vi.runAllTimersAsync(); + const response = await promise; + + expect(attempts).toBe(2); + expect(response.status).toBe(200); + }); +}); + +describe("readHttpErrorDetail", () => { + it("replaces an HTML error page with a fixed message", async () => { + const detail = await readHttpErrorDetail( + new Response("blocked", { + headers: { "content-type": "text/html; charset=utf-8" }, + }), + ); + + expect(detail).toBe("upstream returned an HTML error page"); + }); + + it("returns a compact short body for non-HTML errors", async () => { + const detail = await readHttpErrorDetail( + new Response('{\n "error": "rate limited"\n}', { + headers: { "content-type": "application/json" }, + }), + ); + + expect(detail).toBe('{ "error": "rate limited" }'); + }); +}); + +describe("readLimitedText", () => { + it("reads only the requested prefix of a body", async () => { + const text = await readLimitedText(new Response("abcdefghij"), 4); + + expect(text).toBe("abcd"); + }); }); diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index d275a1c..eeac55f 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -49,6 +49,7 @@ Browser-only preferences and reading state live in `localStorage`: theme prefere - Items are upserted by `(source, external_id)`. - A refresh never deletes existing rows — a failed fetch records the failure in `sync_state` and leaves prior data in place. - The original `published_at` is preserved across re-fetches via `coalesce(items.published_at, excluded.published_at)` so an upstream source cannot reorder an existing item by later emitting a different date. +- Source requests share a retry helper with a descriptive `User-Agent`/`Accept` header, `Retry-After`-aware 429 backoff, and error details capped at 200 characters (HTML error pages become a fixed message). Hugging Face uses its daily-papers JSON API (the same 50 entries) instead of scraping the ~1.4 MB trending HTML page, which reduces CPU and avoids HTML challenge bodies. - Sorting and pagination intentionally happen in application memory after SQL filtering. Total item count across the three sources is small enough that keeping one shared comparator is simpler and less error-prone than encoding the full fallback ordering logic separately per adapter. ## `FeedRepository` diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index b38fbd3..226af61 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -62,6 +62,7 @@ The checked-in `wrangler.toml` config does not hardcode a route. Domain attachme - D1: 5GB storage, 5M row reads/day, 100k row writes/day. - Workers: 100k requests/day, 10ms CPU time per invocation, 50 subrequests per request. +- Source refreshes fetch one page per source per hourly fan-out. Hugging Face reads about 272 KB of JSON from `/api/daily_papers` instead of the ~1.4 MB `/papers/trending` HTML page; source error details are capped at 200 characters, and HTML error pages are reduced to a fixed message before they reach D1 or the UI. - The five-minute home-page cache absorbs most `GET /` traffic, so only cache misses run the feed query and render the HTML. The browser response is `private, no-store`; edge caching is the Worker Cache API, not the CDN, so the zone's Browser Cache TTL setting cannot rewrite the client TTL. Each cold-isolate cache miss also reads `style.css` and `theme-init.js` through the `ASSETS` binding (two local subrequests), then reuses them in memory. - The standalone launch veil and skeleton styles are inlined with the existing HTML/CSS, so they add no extra requests. Bump the shell cache version when the HTML shell or `app.js` changes; see [RUNBOOK.md](RUNBOOK.md#pwa-shell-updates). - Article previews add one outbound fetch (and one `node-html-parser` parse) per uncached `/api/preview` request. The 400 KB HTML body cap and 30-minute edge cache in `core/preview.ts` bound that cost; transient upstream `429`/`5xx` responses are retried once. PDFs/Office documents and HTML pages with no readable blocks can additionally call Workers AI's `toMarkdown` (5 MB download cap; document text conversion is free and the fallback disables AI image-description conversion). That call runs only on a cache miss and its result is cached for the same 30 minutes; set `FEEDREADER_AI_PREVIEW_FALLBACK=false` to disable it. Browser-facing preview and `/api/items` responses are `private, no-store`; their edge copies live in the Cache API. Watch the CPU-time panel if preview usage is heavy — bursts can hit the Workers Free 10 ms limit, so see [RUNBOOK.md](RUNBOOK.md#article-preview-failures). Richer preview blocks (inline links, captions, tables) increase the cached response payload but not the upstream fetch count; the same byte cap and cache TTL apply. diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md index 5e9dabd..5f05f12 100644 --- a/docs/RUNBOOK.md +++ b/docs/RUNBOOK.md @@ -19,6 +19,10 @@ POST /internal/refresh/huggingface ─┘ X-Refresh-Secret header This preserves per-source fault isolation: one source's failure updates only that source's `sync_state` row and never wipes its existing `items` rows. +**Source fetch reliability:** all source requests send a descriptive `User-Agent` and `Accept` header through `core/sources/util.ts`. Retryable responses are attempted twice more; `429` responses honor `Retry-After` (capped at 5 seconds) or use 1s/3s backoff instead of the short 5xx backoff. Error details are capped at 200 characters, and HTML error pages are stored as `upstream returned an HTML error page`, so a large challenge page never lands in `/healthz`, the UI banner, or D1's `last_error` column. + +Hugging Face trending papers are fetched from `https://huggingface.co/api/daily_papers` (about 272 KB of JSON, same 50 entries) instead of the ~1.4 MB `/papers/trending` HTML page. The HTML page was the main source of the 429 challenge bodies and cost more Worker CPU to regex-parse; the JSON API keeps the source invocation comfortably inside the Free-tier CPU budget. + **If a source consistently nears the CPU ceiling:** - inspect whether the upstream page size changed dramatically