Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,7 @@ Key properties:
- fetch failures do not wipe existing data
- sources without a native article date fall back to the initial fetch time (`first_seen_at`) for ordering
- later refreshes preserve the original published/fetched ordering timestamps for existing items
- source fetches send a descriptive User-Agent, honor `Retry-After` on 429s, and cap error details; Hugging Face reads its daily-papers JSON API instead of scraping the trending HTML page

---

Expand Down
4 changes: 2 additions & 2 deletions core/sources/github.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import {
getWithRetry,
HttpError,
parseDigits,
readLimitedText,
readHttpErrorDetail,
} from "./util.ts";

const HOMEPAGE_URL = "https://github.com/trending";
Expand All @@ -26,7 +26,7 @@ export const gitHubTrendingSource: Source = {
if (!response.ok) {
throw new HttpError(
response.status,
await readLimitedText(response, 1024),
await readHttpErrorDetail(response),
);
}
const html = await response.text();
Expand Down
4 changes: 2 additions & 2 deletions core/sources/hackernews.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import {
firstNonEmpty,
getWithRetry,
HttpError,
readLimitedText,
readHttpErrorDetail,
unescapeHtmlEntities,
} from "./util.ts";

Expand Down Expand Up @@ -42,7 +42,7 @@ export const hackerNewsSource: Source = {
if (!response.ok) {
throw new HttpError(
response.status,
await readLimitedText(response, 1024),
await readHttpErrorDetail(response),
);
}
const payload = (await response.json()) as HnFrontPage;
Expand Down
46 changes: 23 additions & 23 deletions core/sources/huggingface.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
// Port of internal/sources/huggingface.go. Extracts an embedded JSON blob
// via one regex — no DOM parsing needed.
// Uses Hugging Face's daily-papers JSON API. The public /papers/trending
// page is ~1.4 MB of HTML with the same data embedded in a data-props blob;
// fetching and parsing it wasted Worker CPU and its HTML fallback page is
// also what surfaced large 429 challenge bodies. The API returns the same
// 50 daily-paper entries as JSON.

import type { FeedItem } from "../domain.ts";
import type { Source } from "./index.ts";
Expand All @@ -8,12 +11,11 @@ import {
firstNonEmpty,
getWithRetry,
HttpError,
readLimitedText,
unescapeHtmlEntities,
readHttpErrorDetail,
} from "./util.ts";

const HOMEPAGE_URL = "https://huggingface.co/papers/trending";
const DAILY_PAPERS_PATTERN = /data-target="DailyPapers"\s+data-props="([^"]+)"/;
const DAILY_PAPERS_API_URL = "https://huggingface.co/api/daily_papers";

interface HfAuthor {
name?: string;
Expand All @@ -34,40 +36,37 @@ interface HfDailyPaperEntry {
title?: string;
summary?: string;
upvotes?: number | null;
numComments?: number | null;
publishedAt?: string;
submittedBy?: { fullname?: string; name?: string };
paper?: HfPaper;
}

interface HfDailyProps {
dailyPapers?: HfDailyPaperEntry[];
}

export const huggingFacePapersSource: Source = {
key: () => "huggingface",
label: () => "Hugging Face Trending Papers",
homepageUrl: () => HOMEPAGE_URL,
async fetch(): Promise<FeedItem[]> {
const response = await getWithRetry(HOMEPAGE_URL);
const response = await getWithRetry(DAILY_PAPERS_API_URL, {
headers: { Accept: "application/json" },
});
if (!response.ok) {
throw new HttpError(
response.status,
await readLimitedText(response, 1024),
await readHttpErrorDetail(response),
);
}
const body = await response.text();
return parseHuggingFacePapers(body);
const payload = (await response.json()) as unknown;
if (!Array.isArray(payload)) {
throw new Error("unexpected daily papers payload");
}
return parseHuggingFaceDailyPapers(payload);
},
};

export function parseHuggingFacePapers(payload: string): FeedItem[] {
const match = DAILY_PAPERS_PATTERN.exec(payload);
if (!match) {
throw new Error("unable to locate DailyPapers payload");
}
const props = JSON.parse(unescapeHtmlEntities(match[1]!)) as HfDailyProps;
const entries = props.dailyPapers ?? [];

export function parseHuggingFaceDailyPapers(
entries: HfDailyPaperEntry[],
): FeedItem[] {
const items: FeedItem[] = [];
entries.forEach((entry, i) => {
const paper = entry.paper ?? {};
Expand All @@ -79,8 +78,9 @@ export function parseHuggingFacePapers(payload: string): FeedItem[] {
.filter((name) => name !== "");

const metadata: Record<string, unknown> = {};
if (paper.numComments != null) {
metadata.comments_count = paper.numComments;
const commentCount = firstDefined(entry.numComments, paper.numComments);
if (commentCount != null) {
metadata.comments_count = commentCount;
}
if (authors.length > 0) {
metadata.authors = authors.length > 6 ? authors.slice(0, 6) : authors;
Expand Down
100 changes: 94 additions & 6 deletions core/sources/util.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,16 @@
// Port of internal/sources/util.go.

const RETRY_DELAYS_MS = [250, 750];
// 429s usually need a longer pause than a transient 5xx; Retry-After wins
// when the upstream provides one.
const RATE_LIMIT_RETRY_DELAYS_MS = [1_000, 3_000];
const MAX_RETRY_AFTER_MS = 5_000;
const RETRYABLE_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]);
const MAX_HTTP_ERROR_DETAIL = 200;
const DEFAULT_SOURCE_USER_AGENT =
"feedreader/0.1 (+https://reader.boringcode.dev)";
const DEFAULT_SOURCE_ACCEPT =
"application/json, text/html;q=0.9, */*;q=0.8";

export class HttpError extends Error {
readonly statusCode: number;
Expand Down Expand Up @@ -30,35 +39,114 @@ function sleep(ms: number, signal?: AbortSignal): Promise<void> {
});
}

/** Mirrors getWithRetry: up to 2 retries (250ms, 750ms backoff) on 408/425/429/5xx. */
function retryDelayMs(response: Response | null, attemptIndex: number): number {
const retryAfter = response?.headers.get("retry-after");
if (retryAfter) {
const seconds = Number.parseInt(retryAfter, 10);
if (Number.isFinite(seconds) && seconds >= 0) {
return Math.min(seconds * 1000, MAX_RETRY_AFTER_MS);
}
const at = Date.parse(retryAfter);
if (!Number.isNaN(at)) {
return Math.min(Math.max(at - Date.now(), 0), MAX_RETRY_AFTER_MS);
}
}

const base =
response?.status === 429
? (RATE_LIMIT_RETRY_DELAYS_MS[attemptIndex] ?? 0)
: (RETRY_DELAYS_MS[attemptIndex] ?? 0);
return base + Math.floor(Math.random() * 200);
}

/** Mirrors getWithRetry: up to 2 retries on 408/425/429/5xx. Source requests
* carry a descriptive user agent instead of the Worker default so upstream
* bot protection can identify the client. */
export async function getWithRetry(
url: string,
init: RequestInit = {},
): Promise<Response> {
const headers = new Headers(init.headers);
if (!headers.has("user-agent")) {
headers.set("User-Agent", DEFAULT_SOURCE_USER_AGENT);
}
if (!headers.has("accept")) {
headers.set("Accept", DEFAULT_SOURCE_ACCEPT);
}
const requestInit: RequestInit = { ...init, headers };

const attempts = RETRY_DELAYS_MS.length + 1;
let lastError: unknown;
for (let attempt = 1; attempt <= attempts; attempt++) {
let delay = 0;
try {
const response = await fetch(url, init);
const response = await fetch(url, requestInit);
if (!RETRYABLE_STATUSES.has(response.status) || attempt === attempts) {
return response;
}
await response.body?.cancel();
await response.body?.cancel().catch(() => {});
delay = retryDelayMs(response, attempt - 1);
} catch (error) {
if (init.signal?.aborted || attempt === attempts) throw error;
lastError = error;
delay = retryDelayMs(null, attempt - 1);
}
await sleep(RETRY_DELAYS_MS[attempt - 1] ?? 0, init.signal ?? undefined);
await sleep(delay, init.signal ?? undefined);
}
throw lastError ?? new Error(`upstream retry loop exhausted for ${url}`);
}

/** Reads at most `limit` bytes from the response body and decodes them as
* UTF-8. Error pages can be large (the old Hugging Face HTML path returned a
* 1.4 MB challenge page), and only a short diagnostic prefix is needed. */
export async function readLimitedText(
response: Response,
limit: number,
): Promise<string> {
const text = await response.text();
return text.slice(0, limit);
if (!response.body) {
const text = await response.text();
return text.slice(0, limit);
}

const reader = response.body.getReader();
const decoder = new TextDecoder();
let bytesRead = 0;
let out = "";
try {
while (bytesRead < limit) {
const { done, value } = await reader.read();
if (done) break;
if (!value || value.byteLength === 0) continue;
const remaining = limit - bytesRead;
const chunk =
value.byteLength > remaining ? value.subarray(0, remaining) : value;
bytesRead += chunk.byteLength;
out += decoder.decode(chunk, { stream: true });
if (value.byteLength > remaining) break;
}
out += decoder.decode();
} finally {
await reader.cancel().catch(() => {});
}
return out;
}

/** Returns a short, safe error detail. HTML challenge/error pages are
* replaced with a fixed message so their markup never reaches /healthz, the
* UI error banner, or D1's last_error column. */
export async function readHttpErrorDetail(
response: Response,
limit = MAX_HTTP_ERROR_DETAIL,
): Promise<string> {
const text = await readLimitedText(response, Math.max(limit, 512));
const contentType = response.headers.get("content-type")?.toLowerCase() ?? "";
if (
contentType.includes("text/html") ||
/<(?:!doctype|html|head|body)\b/i.test(text)
) {
return "upstream returned an HTML error page";
}
return text.replace(/\s+/g, " ").trim().slice(0, limit);
}

export function extractInt(value: string, pattern: RegExp): number | undefined {
Expand Down
66 changes: 40 additions & 26 deletions core/test/huggingface.spec.ts
Original file line number Diff line number Diff line change
@@ -1,35 +1,29 @@
// No huggingface_test.go exists in the Go repo to port from directly — this
// exercises parseHuggingFacePapers against a synthetic payload shaped like
// the real embedded data-props blob described in huggingface.go.
import { describe, expect, it } from "vitest";
import { parseHuggingFacePapers } from "../sources/huggingface.ts";
import { parseHuggingFaceDailyPapers } from "../sources/huggingface.ts";

describe("parseHuggingFacePapers", () => {
it("extracts papers from the embedded DailyPapers JSON", () => {
const props = {
dailyPapers: [
{
describe("parseHuggingFaceDailyPapers", () => {
it("extracts papers from the daily papers API entries", () => {
const entries = [
{
id: "2606.00001",
title: "A Paper Title",
summary: "A short summary.",
numComments: 4,
publishedAt: "2026-06-20T00:00:00.000Z",
submittedBy: { fullname: "Jane Doe", name: "janedoe" },
paper: {
id: "2606.00001",
title: "A Paper Title",
summary: "A short summary.",
upvotes: 12,
publishedAt: "2026-06-20T00:00:00.000Z",
submittedBy: { fullname: "Jane Doe", name: "janedoe" },
paper: {
id: "2606.00001",
title: "A Paper Title",
summary: "A short summary.",
publishedAt: "2026-06-20T00:00:00.000Z",
upvotes: 12,
numComments: 4,
authors: [{ name: "Jane Doe" }, { name: "John Roe" }],
},
upvotes: 12,
numComments: null,
authors: [{ name: "Jane Doe" }, { name: "John Roe" }],
},
],
};
const payload = `<div data-target="DailyPapers" data-props="${JSON.stringify(props).replace(/"/g, "&quot;")}"></div>`;
},
];

const items = parseHuggingFacePapers(payload);
const items = parseHuggingFaceDailyPapers(entries);
expect(items).toHaveLength(1);

const item = items[0]!;
Expand All @@ -46,7 +40,27 @@ describe("parseHuggingFacePapers", () => {
expect(item.sourceRank).toBe(1);
});

it("throws when the DailyPapers payload is missing", () => {
expect(() => parseHuggingFacePapers("<div>no papers here</div>")).toThrow();
it("falls back to the paper's comment count and title fields", () => {
const items = parseHuggingFaceDailyPapers([
{
id: "2606.00002",
paper: {
id: "2606.00002",
title: "Paper-only title",
summary: "Paper-only summary.",
numComments: 7,
authors: [{ name: "Alice Smith" }],
},
},
]);

expect(items).toHaveLength(1);
expect(items[0]?.title).toBe("Paper-only title");
expect(items[0]?.summary).toBe("Paper-only summary.");
expect(items[0]?.metadata.comments_count).toBe(7);
});

it("skips entries without a paper id", () => {
expect(parseHuggingFaceDailyPapers([{}, { paper: {} }])).toEqual([]);
});
});
Loading
Loading