From bb271273acb24b4fa2154933ae134441674641fb Mon Sep 17 00:00:00 2001 From: Phat Pham Date: Fri, 13 Nov 2026 22:30:21 +0700 Subject: [PATCH] perf(preview): focus Hugging Face paper extraction on main Hugging Face paper pages are ~150-220 KB of HTML with large header, footer, and hydration chrome. Parse only +
before extraction, preserving title/byline/date metadata and all paper, link, and citation blocks while cutting DOM work. Combine the 16 noise-removal selectors into one comma-separated selector so node-html-parser traverses the tree once instead of once per tag. Local benchmarks on the ten HF URLs that hit the Workers Free exceededCpu limit show ~20% lower extraction time with identical block counts; a regression test covers the main-only boundary. Co-Authored-By: Claude Code --- core/preview.ts | 54 ++++++++++++++++++++++----------------- core/test/preview.spec.ts | 26 +++++++++++++++++++ docs/ARCHITECTURE.md | 1 + docs/DEPLOYMENT.md | 2 +- docs/RUNBOOK.md | 2 +- 5 files changed, 59 insertions(+), 26 deletions(-) diff --git a/core/preview.ts b/core/preview.ts index 0198e22..1f19314 100644 --- a/core/preview.ts +++ b/core/preview.ts @@ -47,24 +47,11 @@ const MIN_PARAGRAPH_CHARS = 25; // GitHub don't leak repository chrome into the reader view. const MIN_ARTICLE_SCORE = 300; -const NOISE_SELECTORS = [ - "script", - "style", - "noscript", - "template", - "svg", - "canvas", - "iframe", - "object", - "embed", - "form", - "nav", - "header", - "footer", - "aside", - "[hidden]", - '[aria-hidden="true"]', -]; +// One comma-separated selector lets node-html-parser traverse the tree once +// instead of once per tag. Removing a parent also removes nested matches, so +// the deduplicated result is equivalent to the old per-selector loop. +const NOISE_SELECTOR = + 'script,style,noscript,template,svg,canvas,iframe,object,embed,form,nav,header,footer,aside,[hidden],[aria-hidden="true"]'; export type PreviewTextBlockType = "heading" | "paragraph" | "list" | "quote"; @@ -396,7 +383,7 @@ export function extractArticlePreview( html: string, pageUrl: string, ): ArticlePreview { - const document = parseHtmlDocument(html); + const document = parseHtmlDocument(focusPreviewHtml(html, pageUrl)); const rawTitle = firstNonEmpty( metaContent(document, [ @@ -432,11 +419,9 @@ export function extractArticlePreview( ? extractHuggingFacePaperCitations(document, pageUrl) : []; - for (const selector of NOISE_SELECTORS) { - document.querySelectorAll(selector).forEach((element) => { - element.remove(); - }); - } + document.querySelectorAll(NOISE_SELECTOR).forEach((element) => { + element.remove(); + }); const root = isPaperPage ? (pickHuggingFacePaperRoot(document) ?? pickContentRoot(document)) @@ -954,6 +939,27 @@ function sanitizePreviewTitle(rawTitle: string, pageUrl: string): string { return cleaned.trim() || title; } +/** + * Hugging Face paper pages put the article inside
, surrounded by a + * large header/footer and client-hydration chrome. Parsing only and + *
keeps title/byline/date metadata and the paper/citation blocks + * while cutting the DOM by roughly a quarter on the pages that have hit the + * Workers Free CPU limit. + */ +function focusPreviewHtml(html: string, pageUrl: string): string { + if (!isHuggingFacePaperPage(pageUrl)) return html; + + const mainStart = /)/i.exec(html); + if (!mainStart) return html; + const mainHtml = html.slice(mainStart.index); + const mainClose = /<\/main\s*>/i.exec(mainHtml); + if (!mainClose) return html; + + const focusedMain = mainHtml.slice(0, mainClose.index + mainClose[0].length); + const head = /]*>[\s\S]*?<\/head\s*>/i.exec(html)?.[0] ?? ""; + return `${head}${focusedMain}`; +} + function isHuggingFacePaperPage(pageUrl: string): boolean { try { const url = new URL(pageUrl); diff --git a/core/test/preview.spec.ts b/core/test/preview.spec.ts index 415d9fa..658b51c 100644 --- a/core/test/preview.spec.ts +++ b/core/test/preview.spec.ts @@ -368,6 +368,32 @@ describe("extractArticlePreview", () => { ]); }); + it("keeps Hugging Face paper extraction inside main", () => { + const inside = "Inside main article content that should be previewed. ".repeat( + 20, + ).trim(); + const outside = "Outside chrome content that must not be previewed. ".repeat( + 20, + ).trim(); + const html = ` + + +

${outside}

+

Main Only

${inside}

+

${outside}

+ `; + + const preview = extractArticlePreview( + html, + "https://huggingface.co/papers/2611.00010", + ); + + expect(preview.title).toBe("Main Only"); + const text = preview.blocks.map(blockText).join(" "); + expect(text).toContain("Inside main article content"); + expect(text).not.toContain("Outside chrome content"); + }); + it("uses the Hugging Face paper author list as the byline", () => { const preview = extractArticlePreview( ` diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index af6bce0..9fdff3d 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -102,6 +102,7 @@ The optional article preview is served by `GET /api/preview?url=…`. - `core/preview.ts` additionally rejects non-http(s) schemes, credentials, non-standard ports, and loopback/private/link-local hosts, and re-checks the final URL after redirects. - Article fetches time out after 10 seconds and read at most 400 KB of HTML (5 MB when a document is handed to the optional fallback). Transient upstream `429`/`5xx` responses are retried once with jitter. `PreviewUnavailableError` carries a structured reason (`blocked`, `rate_limited`, `upstream_error`, `timeout`, `unsupported_content_type`, `too_large`, `empty_body`, `redirect`, `invalid_url`) plus the optional upstream status; the endpoint logs `preview_unavailable`/`preview_empty` events and returns the reason so the client can show a specific message. The extractor removes scripts/navigation/footer noise and returns structured text segments, image blocks, and table blocks only — upstream HTML is never returned or injected. Text renders with `textContent`/DOM nodes; inline anchors are limited to URLs that pass the same public http(s) validation, and image URLs are re-validated before the browser loads them directly with `loading="lazy"` and `referrerpolicy="no-referrer"`. - If the built-in extractor finds no blocks (`no_content`), truncation left no blocks (`too_large`), or the upstream is a non-HTML document (`unsupported_content_type`), `fetchArticlePreview` may call an injected fallback. Core stays platform-free: the Worker supplies a chain (`platforms/cloudflare/src/previewFallbackChain.ts`). Workers AI (`previewFallback.ts`) calls `env.AI.toMarkdown`; Browser Run (`browserPreviewFallback.ts`) calls `env.BROWSER.quickAction("markdown", …)` only for `no_content` after AI fails. Both map the converted Markdown through `core/previewMarkdown.ts`, which re-checks every link and image with `normalizePreviewUrl`, strips YAML frontmatter and trailing JSON-LD, and drops PDF filename/page-marker headings. Browser Run waits for `networkidle2` with 8-second navigation/action and 9-second outer timeouts, rejects images/media/fonts and literal private-host requests, and validates the rendered page status and `meta.finalUrl`. Blocked, timeout, rate-limited, and empty-body failures never reach the fallback. `FEEDREADER_AI_PREVIEW_FALLBACK=false` and `FEEDREADER_BROWSER_PREVIEW_FALLBACK=false` disable each adapter. +- Hugging Face paper pages are focused to ``+`
` before DOM parsing, and noise removal uses one combined selector instead of one traversal per tag. This preserves title/byline/date metadata and the paper/citation blocks while cutting parse work by roughly 20% locally on the pages that have hit the Workers Free CPU limit. - Image blocks may include a `figcaption`-derived caption and intrinsic width/height to reserve layout space. Known avatar/profile-image hosts and paths, SVG wordmarks, and common badge hosts are dropped before the image cap so author-list and README chrome doesn't displace article figures. Table blocks carry a header flag and capped rows/cells. The response also includes byline, publication date, and an estimated reading time. For Hugging Face paper URLs the extractor starts at the Abstract section, then appends the paper's resource links and citing model, dataset, space, and collection lists while excluding community comment chrome; GitHub titles drop the site prefix. The preview and card `…` dialogs expose a stored Hacker News comments URL for HN items; other sources omit it. - The client highlights the card matching the open preview and provides previous/next navigation over the currently rendered card list, skipping duplicate or unusable cards. When the last rendered card is previewed, Next remains enabled if another page exists and appends it before advancing. On mobile the open panel pins the body scroll position (and reapplies it after rotation), the glass header/footer auto-hide on scroll down and return on scroll up, and the `theme-color` tint switches to the panel surface while the panel covers the viewport. A left-edge rightward drag follows the finger and reveals the preserved feed position underneath; releasing past a distance/velocity threshold slides the panel away and closes it, while a shorter drag springs back. Vertical edge drags remain native scrolling and the gesture is inactive while the `…` dialog is open. - Card host rows keep the optional bookmark toggle and a more-actions dialog trigger. The dialog exposes Share, the stored Hacker News comments URL when present, and Open source; choosing an action does not open the preview. diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 796149a..9984cf9 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -65,6 +65,6 @@ The checked-in `wrangler.toml` config does not hardcode a route. Domain attachme - Source refreshes fetch one page per source per hourly fan-out. Hugging Face reads about 272 KB of JSON from `/api/daily_papers` instead of the ~1.4 MB `/papers/trending` HTML page; source error details are capped at 200 characters, and HTML error pages are reduced to a fixed message before they reach D1 or the UI. - The five-minute home-page cache absorbs most `GET /` traffic, so only cache misses run the feed query and render the HTML. The browser response is `private, no-store`; edge caching is the Worker Cache API, not the CDN, so the zone's Browser Cache TTL setting cannot rewrite the client TTL. Each cold-isolate cache miss also reads `style.css` and `theme-init.js` through the `ASSETS` binding (two local subrequests), then reuses them in memory. - The standalone launch veil and skeleton styles are inlined with the existing HTML/CSS, so they add no extra requests. Bump the shell cache version when the HTML shell or `app.js` changes; see [RUNBOOK.md](RUNBOOK.md#pwa-shell-updates). -- Article previews add one outbound fetch (and one `node-html-parser` parse) per uncached `/api/preview` request. The 400 KB HTML body cap, 30-minute edge cache (5 minutes for empty results), and a per-IP limiter (20 cache-miss previews per minute; cached previews bypass it) bound that cost; transient upstream `429`/`5xx` responses are retried once. PDFs/Office documents and HTML pages with no readable blocks can additionally call Workers AI's `toMarkdown` (5 MB download cap; document text conversion is free and the fallback disables AI image-description conversion). That call runs only on a cache miss and its result is cached for the same 30 minutes; set `FEEDREADER_AI_PREVIEW_FALLBACK=false` to disable it. HTML pages that still have no readable content after AI conversion can call Browser Run's `markdown` Quick Action; Workers Free includes 10 browser-minutes/day and paces Quick Actions to one request per 10 seconds. Browser Run runs only on a cache miss, uses no browser sessions (so the concurrent-browser limit does not apply), and its result is cached for 30 minutes; set `FEEDREADER_BROWSER_PREVIEW_FALLBACK=false` to disable it. Browser-facing preview and `/api/items` responses are `private, no-store`; their edge copies live in the Cache API. Watch the CPU-time panel if preview usage is heavy — bursts can hit the Workers Free 10 ms limit, so see [RUNBOOK.md](RUNBOOK.md#article-preview-failures). Richer preview blocks (inline links, captions, tables) increase the cached response payload but not the upstream fetch count; the same byte cap and cache TTL apply. +- Article previews add one outbound fetch (and one `node-html-parser` parse) per uncached `/api/preview` request. The 400 KB HTML body cap, 30-minute edge cache (5 minutes for empty results), and a per-IP limiter (20 cache-miss previews per minute; cached previews bypass it) bound that cost; transient upstream `429`/`5xx` responses are retried once. PDFs/Office documents and HTML pages with no readable blocks can additionally call Workers AI's `toMarkdown` (5 MB download cap; document text conversion is free and the fallback disables AI image-description conversion). That call runs only on a cache miss and its result is cached for the same 30 minutes; set `FEEDREADER_AI_PREVIEW_FALLBACK=false` to disable it. HTML pages that still have no readable content after AI conversion can call Browser Run's `markdown` Quick Action; Workers Free includes 10 browser-minutes/day and paces Quick Actions to one request per 10 seconds. Browser Run runs only on a cache miss, uses no browser sessions (so the concurrent-browser limit does not apply), and its result is cached for 30 minutes; set `FEEDREADER_BROWSER_PREVIEW_FALLBACK=false` to disable it. Hugging Face paper pages parse only ``+`
` with one combined noise selector, reducing their extraction CPU. Browser-facing preview and `/api/items` responses are `private, no-store`; their edge copies live in the Cache API. Watch the CPU-time panel if preview usage is heavy — bursts can hit the Workers Free 10 ms limit, so see [RUNBOOK.md](RUNBOOK.md#article-preview-failures). Richer preview blocks (inline links, captions, tables) increase the cached response payload but not the upstream fetch count; the same byte cap and cache TTL apply. Check the Cloudflare dashboard's D1 usage and Workers CPU-time panels periodically — see [RUNBOOK.md](RUNBOOK.md) for what to do if a source consistently nears the CPU ceiling. diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md index 4f7c2cb..dd7517c 100644 --- a/docs/RUNBOOK.md +++ b/docs/RUNBOOK.md @@ -91,7 +91,7 @@ Before returning `unsupported_content_type` for a non-HTML response, `too_large` If the AI conversion returns nothing for an HTML page, the chain calls Browser Run's `markdown` Quick Action (`platforms/cloudflare/src/browserPreviewFallback.ts`) before returning `no_content`. It is limited to `no_content`; PDFs/Office documents, `too_large`, `blocked`, `timeout`, and rate-limited failures never reach it. Browser Run waits for `networkidle2` with 8-second navigation/action and 9-second outer timeouts, blocks images/media/fonts and literal private-host requests, and validates the rendered page status and final URL. Workers Free includes 10 browser-minutes/day and paces Quick Actions to one request per 10 seconds; a `429` is logged and treated as no preview. Set `FEEDREADER_BROWSER_PREVIEW_FALLBACK=false` to disable it. Browser Run successes are cached for 30 minutes; the preview cache key is `preview:v11`. -The Worker logs `preview_unavailable` and `preview_empty` events with the URL, reason, and upstream status; `preview_empty` also records `cache_ttl_seconds`. It logs a `preview_ai_fallback` event for each AI attempt (`kind`, `outcome` = `ok`/`empty`/`error`, `ms`, `blocks`, `tokens`, and a capped `error`), a `preview_browser_fallback` event for each Browser Run attempt (`outcome`, `status`, `page_status`, `browser_ms`, `blocks`, and a capped `error`), and `preview_rate_limited` when the local per-IP limiter rejects a cache miss. Successful previews, including AI- and Browser Run-converted ones, are edge-cached for 30 minutes; empty `no_content` results use a 5-minute TTL so a transient fallback failure can retry. Cache-miss previews pass a per-IP fixed-window limiter (Cache API, 20 per minute, per-colo, fail-open); cache hits bypass it. Because the in-Worker limiter is per-colo and fail-open, add a zone-level Cloudflare Rate Limiting rule for `/api/preview` if you need a global limit. Browser-facing responses are `private, no-store`. Transient upstream `429`/`5xx` responses are retried once with jitter. If bursts produce transient `503`s, check Workers Logs/CPU: the extractor runs in the request invocation, and sequential retries succeeding usually points at the Workers Free 10 ms CPU limit rather than a lasting upstream block. Lowering `DEFAULT_MAX_BYTES` in `core/preview.ts` (400 KB today) reduces the parse cost. +The Worker logs `preview_unavailable` and `preview_empty` events with the URL, reason, and upstream status; `preview_empty` also records `cache_ttl_seconds`. It logs a `preview_ai_fallback` event for each AI attempt (`kind`, `outcome` = `ok`/`empty`/`error`, `ms`, `blocks`, `tokens`, and a capped `error`), a `preview_browser_fallback` event for each Browser Run attempt (`outcome`, `status`, `page_status`, `browser_ms`, `blocks`, and a capped `error`), and `preview_rate_limited` when the local per-IP limiter rejects a cache miss. Successful previews, including AI- and Browser Run-converted ones, are edge-cached for 30 minutes; empty `no_content` results use a 5-minute TTL so a transient fallback failure can retry. Cache-miss previews pass a per-IP fixed-window limiter (Cache API, 20 per minute, per-colo, fail-open); cache hits bypass it. Because the in-Worker limiter is per-colo and fail-open, add a zone-level Cloudflare Rate Limiting rule for `/api/preview` if you need a global limit. Browser-facing responses are `private, no-store`. Transient upstream `429`/`5xx` responses are retried once with jitter. If bursts produce transient `503`s, check Workers Logs/CPU: the extractor runs in the request invocation, and sequential retries succeeding usually points at the Workers Free 10 ms CPU limit rather than a lasting upstream block. Hugging Face paper pages are parsed as ``+`
` with one combined noise selector to keep them closer to that budget; if `exceededCpu` 503s recur for HF URLs, the next lever is the paper Markdown-content endpoint or a stored-summary fallback. Lowering `DEFAULT_MAX_BYTES` in `core/preview.ts` (400 KB today) reduces the parse cost. ## Local SSR smoke procedure