From 2c784bd375d12c27dae43a1ec6ce9cf6bd42a687 Mon Sep 17 00:00:00 2001 From: Phat Pham Date: Fri, 13 Nov 2026 11:54:23 +0700 Subject: [PATCH 1/2] feat(preview): add Workers AI Markdown fallback PDFs, Office documents, and HTML pages the built-in extractor cannot read now fall back to Workers AI's toMarkdown conversion. The converted Markdown is mapped into the same validated PreviewBlock shape, with unsafe links and images dropped; core stays platform-free behind an injected fallback, and FEEDREADER_AI_PREVIEW_FALLBACK=false disables it. Bump the preview cache key to v10 so previously cached empty previews get a fallback attempt. Bound non-HTML downloads at 5 MB, skip oversized documents using Content-Length, and race the AI call against 10 seconds. Strip PDF filename/page-marker headings and trailing JSON-LD. Co-Authored-By: Claude Code --- README.md | 5 +- core/preview.ts | 168 ++++- core/previewMarkdown.ts | 673 ++++++++++++++++++ core/test/preview.spec.ts | 215 ++++++ core/test/previewMarkdown.spec.ts | 318 +++++++++ docs/ARCHITECTURE.md | 7 +- docs/DEPLOYMENT.md | 2 +- docs/RUNBOOK.md | 8 +- platforms/cloudflare/src/env.d.ts | 3 + platforms/cloudflare/src/index.ts | 17 +- platforms/cloudflare/src/previewFallback.ts | 302 ++++++++ .../cloudflare/test/previewFallback.spec.ts | 173 +++++ platforms/cloudflare/wrangler.toml | 4 + 13 files changed, 1875 insertions(+), 20 deletions(-) create mode 100644 core/previewMarkdown.ts create mode 100644 core/test/previewMarkdown.spec.ts create mode 100644 platforms/cloudflare/src/previewFallback.ts create mode 100644 platforms/cloudflare/test/previewFallback.spec.ts diff --git a/README.md b/README.md index a203b66..debfdd4 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ npx lighthouse https://reader.boringcode.dev --view - **Scheduled refresh** every 1 hour on wall-clock boundaries - **Manual refresh** from the header re-fetches the current feed view from backend stored items only; it does **not** re-fetch upstream sources - **Persisted visited-link dimming** for feed card titles across reload/reopen using local storage -- **Optional article preview** (off by default, toggled in Reader settings) that opens a reader view beside the feed on wide screens or full-screen on phones, with inline links/emphasis, image captions, tables, publication date/reading time, source-specific Hacker News comments, and an **Open source** button +- **Optional article preview** (off by default, toggled in Reader settings) that opens a reader view beside the feed on wide screens or full-screen on phones, with inline links/emphasis, image captions, tables, publication date/reading time, a Workers AI Markdown fallback for PDFs, Office documents, and HTML pages with no readable blocks, source-specific Hacker News comments, and an **Open source** button - **Saved articles** with an optional per-card bookmark toggle, a `Saved` filter, and a preview-panel toggle; bookmark controls are disabled by default and can be enabled in Reader settings, with saved items stored locally under `feedreader.bookmarks` - **PWA-ready assets and offline caching** including manifest, service worker, touch icons, cached shell assets, cached `/api/items` responses for previously visited views, a standalone launch veil, and skeleton loading states - **Reconnect list refresh** re-fetches the current view from backend stored items only; it does **not** refresh upstream sources @@ -220,6 +220,7 @@ Runtime configuration: | ----------------------------- | ----------------: | -------------------------------------------------------------- | | `FEEDREADER_ITEMS_PER_SOURCE` | `20` | Per-source item count used in source dashboard/health contexts | | `FEEDREADER_USER_AGENT` | `feedreader/0.1` | Outbound fetch user agent | +| `FEEDREADER_AI_PREVIEW_FALLBACK` | `true` | Workers AI Markdown fallback for PDFs/Office documents and empty previews; `false` disables it | | `REFRESH_SECRET` | _required secret_ | Shared secret for internal per-source refresh fan-out routes | --- @@ -265,7 +266,7 @@ Returns a reader view for a stored feed item: rich text segments, image blocks w - `url` — required; must exactly match the URL of an item currently stored in D1 -The Worker fetches the article with a timeout and 400 KB cap, rejects non-HTML responses and non-public hosts, and never returns upstream markup. Inline targets are restricted to validated public http(s) URLs. Responses are edge-cached for 30 minutes. Returns `404` for unknown URLs and `502` when the upstream page cannot be previewed. +The Worker fetches the article with a 10-second timeout and a 400 KB HTML cap, rejects non-public hosts, and never returns upstream markup. When the built-in extractor finds no readable blocks or the response is a document (PDF/Office), the Worker can convert it through Workers AI's Markdown conversion with a 5 MB cap; set `FEEDREADER_AI_PREVIEW_FALLBACK=false` to disable that fallback. Inline targets and images are restricted to validated public http(s) URLs. Responses are edge-cached for 30 minutes. Returns `404` for unknown URLs and `502` when the upstream page cannot be previewed. ### `POST /api/refresh` diff --git a/core/preview.ts b/core/preview.ts index 03e42b6..0684868 100644 --- a/core/preview.ts +++ b/core/preview.ts @@ -25,6 +25,10 @@ import { } from "./sources/dom.ts"; const DEFAULT_MAX_BYTES = 400_000; +// Non-HTML bodies are handed to the optional platform fallback (for example +// Workers AI's toMarkdown) instead of being parsed here. Keep the cap well +// below the Worker's memory budget while still covering typical papers. +const DEFAULT_MAX_FALLBACK_BYTES = 5_000_000; const DEFAULT_TIMEOUT_MS = 10_000; const MAX_REDIRECTS = 5; const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]); @@ -118,6 +122,33 @@ export interface ArticlePreview { blocks: PreviewBlock[]; } +/** Content the primary extractor cannot turn into blocks. A platform adapter + * may convert it another way (for example Workers AI's Markdown conversion); + * core only passes bytes/text and receives already-validated blocks back. */ +export type PreviewFallbackKind = + | "no_content" + | "too_large" + | "unsupported_content_type"; + +export interface PreviewFallbackInput { + kind: PreviewFallbackKind; + url: string; + contentType: string; + /** Present for HTML fallbacks. */ + html?: string; + /** Present for non-HTML fallbacks, capped by `maxFallbackBytes`. */ + bytes?: Uint8Array; +} + +/** + * Optional platform hook for previews the built-in extractor can't produce. + * It must return plain `PreviewBlock` data (never upstream HTML) or null to + * preserve the original failure. Core never imports a platform SDK. + */ +export type PreviewFallback = ( + input: PreviewFallbackInput, +) => Promise; + export type PreviewFailureReason = | "invalid_url" | "fetch_failed" @@ -170,6 +201,11 @@ export interface FetchArticlePreviewOptions { retryDelayMs?: number; /** Random jitter added to the retry delay. */ retryJitterMs?: number; + /** Attempted only when the built-in extractor finds no readable blocks or + * the response is not HTML. */ + fallback?: PreviewFallback; + /** Cap for non-HTML response bodies handed to `fallback`. */ + maxFallbackBytes?: number; } /** @@ -218,7 +254,11 @@ export async function fetchArticlePreview( try { lastResult = await fetchWithSafeRedirects(fetchImpl, target, { headers: { - Accept: "text/html, application/xhtml+xml", + // PDF/octet-stream let document URLs negotiate their document + // instead of an HTML wrapper; HTML stays the preferred type for + // ordinary article URLs. + Accept: + "text/html, application/xhtml+xml, application/pdf;q=0.9, application/octet-stream;q=0.5", "User-Agent": options.userAgent, }, signal: timeoutSignal, @@ -272,11 +312,44 @@ export async function fetchArticlePreview( } const contentType = (response.headers.get("content-type") ?? "").toLowerCase(); - if ( - contentType !== "" && - !contentType.includes("text/html") && - !contentType.includes("application/xhtml+xml") - ) { + const isHtml = + contentType === "" || + contentType.includes("text/html") || + contentType.includes("application/xhtml+xml"); + const fallback = options.fallback; + + if (!isHtml) { + const maxFallbackBytes = + options.maxFallbackBytes ?? DEFAULT_MAX_FALLBACK_BYTES; + const declaredLength = Number.parseInt( + response.headers.get("content-length") ?? "", + 10, + ); + // Do not download a document that already declares itself too large: the + // fallback would reject the partial file anyway. + const declaredTooLarge = + Number.isFinite(declaredLength) && declaredLength > maxFallbackBytes; + if (fallback && !declaredTooLarge) { + try { + const { bytes, truncated } = await readBytesLimited( + response, + maxFallbackBytes, + ); + // A partial PDF/Office file is unlikely to convert; preserve the + // original unsupported-content reason instead of spending an AI call. + if (!truncated) { + const converted = await tryPreviewFallback(fallback, { + kind: "unsupported_content_type", + url: finalUrl, + contentType, + bytes, + }); + if (converted) return converted; + } + } catch { + // The fallback is best-effort; report the original reason below. + } + } throw new PreviewUnavailableError( "article is not HTML", "unsupported_content_type", @@ -292,6 +365,15 @@ export async function fetchArticlePreview( } const preview = extractArticlePreview(html, finalUrl); + if (preview.blocks.length === 0 && fallback) { + const converted = await tryPreviewFallback(fallback, { + kind: truncated ? "too_large" : "no_content", + url: finalUrl, + contentType, + html, + }); + if (converted) return converted; + } if (truncated && preview.blocks.length === 0) { throw new PreviewUnavailableError( "article body exceeded the preview limit before readable content", @@ -689,8 +771,7 @@ function extractImageBlock( } } if (resolved === "") return null; - if (isAvatarImageSrc(resolved)) return null; - if (isDecorativeImageSrc(resolved)) return null; + if (isDecorativePreviewImageSrc(resolved)) return null; const caption = normalizeText( image.closest("figure")?.querySelector("figcaption")?.textContent, @@ -746,6 +827,15 @@ function resolveImageUrl( } } +/** + * True for interface chrome that should never become a reader-view figure. + * Shared with the Markdown fallback mapper so both extractors apply the same + * avatar/wordmark/badge policy. + */ +export function isDecorativePreviewImageSrc(rawUrl: string): boolean { + return isAvatarImageSrc(rawUrl) || isDecorativeImageSrc(rawUrl); +} + /** * Avatar and profile images are interface chrome, not article figures. They * often appear before the content and would otherwise consume the image cap @@ -1058,7 +1148,7 @@ function extractPublishedAt(document: HtmlDocument): string | undefined { return undefined; } -function estimateReadingMinutes( +export function estimateReadingMinutes( blocks: PreviewBlock[], ): number | undefined { let words = 0; @@ -1199,6 +1289,18 @@ function isPrivateIpv6(hostname: string): boolean { return firstGroup.startsWith("ff"); } +async function tryPreviewFallback( + fallback: PreviewFallback, + input: PreviewFallbackInput, +): Promise { + try { + const preview = await fallback(input); + return preview && preview.blocks.length > 0 ? preview : null; + } catch { + return null; + } +} + /** * Follows redirects manually so every hop's Location is checked by * normalizePreviewUrl before the request is issued. `redirect: "follow"` @@ -1298,3 +1400,51 @@ async function readBodyLimited( } return { text: out, truncated }; } + +async function readBytesLimited( + response: Response, + maxBytes: number, +): Promise<{ bytes: Uint8Array; truncated: boolean }> { + if (!response.body) { + const bytes = new Uint8Array(await response.arrayBuffer()); + return { + bytes: bytes.slice(0, maxBytes), + truncated: bytes.byteLength > maxBytes, + }; + } + + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let bytesRead = 0; + let truncated = false; + try { + while (bytesRead < maxBytes) { + const { done, value } = await reader.read(); + if (done) break; + if (!value || value.byteLength === 0) continue; + const remaining = maxBytes - bytesRead; + if (value.byteLength > remaining) { + chunks.push(value.subarray(0, remaining)); + bytesRead += remaining; + truncated = true; + break; + } + chunks.push(value); + bytesRead += value.byteLength; + } + if (!truncated && bytesRead >= maxBytes) { + const extra = await reader.read(); + truncated = !extra.done; + } + } finally { + await reader.cancel().catch(() => {}); + } + + const bytes = new Uint8Array(bytesRead); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + return { bytes, truncated }; +} diff --git a/core/previewMarkdown.ts b/core/previewMarkdown.ts new file mode 100644 index 0000000..14a81d8 --- /dev/null +++ b/core/previewMarkdown.ts @@ -0,0 +1,673 @@ +// Markdown-to-PreviewBlock mapping for the optional platform fallback (for +// example Workers AI's toMarkdown). The output is the same validated block +// shape as core/preview.ts: no upstream HTML, and every link/image target is +// re-checked by normalizePreviewUrl before it reaches the client. + +import { + isDecorativePreviewImageSrc, + normalizePreviewUrl, + type PreviewBlock, + type PreviewSegment, + type PreviewTextBlockType, +} from "./preview.ts"; + +// Keep these bounds in the same ballpark as core/preview.ts. The Markdown +// source is model output, so cap it before it becomes a response payload. +const MAX_BLOCKS = 120; +const MAX_IMAGES = 20; +const MAX_TABLES = 5; +const MAX_TABLE_ROWS = 15; +const MAX_TABLE_COLUMNS = 6; +const MAX_TABLE_CELL_CHARS = 150; +const MAX_BLOCK_CHARS = 4_000; +const MAX_TOTAL_CHARS = 40_000; +const MIN_PARAGRAPH_CHARS = 25; +const MIN_QUOTE_CHARS = 10; +const MIN_LIST_CHARS = 2; + +const HTML_ENTITIES: Record = { + amp: "&", + apos: "'", + bull: "•", + copy: "©", + gt: ">", + hellip: "…", + ldquo: "“", + lsquo: "‘", + lt: "<", + mdash: "—", + middot: "·", + nbsp: " ", + ndash: "–", + quot: '"', + rdquo: "”", + reg: "®", + rsquo: "’", + trade: "™", +}; + +const INLINE_TOKEN = + /(?\s]+>)/gu; + +interface InlineMarks { + href?: string; + strong?: boolean; + em?: boolean; + code?: boolean; +} + +interface FenceStart { + marker: string; +} + +interface StandaloneImage { + alt: string; + src: string; +} + +export interface MarkdownPreviewOptions { + /** Article URL, used to resolve relative links and images. */ + pageUrl: string; + /** Skip a heading that repeats the stored article title. */ + title?: string; +} + +export function markdownToPreviewBlocks( + markdown: string, + options: MarkdownPreviewOptions, +): PreviewBlock[] { + const source = stripTrailingJsonLd(markdown.replace(/\r\n?/g, "\n")); + const lines = source.split("\n"); + const blocks: PreviewBlock[] = []; + const seenImages = new Set(); + const titleKey = normalizeText(options.title); + let totalChars = 0; + let imageCount = 0; + let tableCount = 0; + + const addTextBlock = ( + type: PreviewTextBlockType, + segments: PreviewSegment[], + minChars: number, + ): void => { + const text = segments.map((segment) => segment.text).join(""); + if (text.length < minChars) return; + if (blocks.length >= MAX_BLOCKS || totalChars >= MAX_TOTAL_CHARS) return; + + const cappedSegments = capSegments(segments, MAX_BLOCK_CHARS); + const cappedText = cappedSegments + .map((segment) => segment.text) + .join(""); + if (cappedText === "") return; + + const last = blocks[blocks.length - 1]; + if (last && textBlockPlainText(last) === cappedText) return; + + if (hasInlineMarks(cappedSegments)) { + blocks.push({ type, segments: cappedSegments }); + } else { + blocks.push({ type, text: cappedText }); + } + totalChars += cappedText.length; + }; + + const addImage = (image: StandaloneImage): void => { + if (imageCount >= MAX_IMAGES || blocks.length >= MAX_BLOCKS) return; + const src = resolvePublicUrl(image.src, options.pageUrl); + if (src === "") return; + if (seenImages.has(src) || isDecorativePreviewImageSrc(src)) return; + seenImages.add(src); + blocks.push({ + type: "image", + src, + alt: normalizeText(plainInlineText(image.alt)).slice(0, 300), + }); + imageCount += 1; + }; + + let index = 0; + while ( + index < lines.length && + blocks.length < MAX_BLOCKS && + totalChars < MAX_TOTAL_CHARS + ) { + const line = lines[index]!; + if (line.trim() === "") { + index += 1; + continue; + } + + const fence = parseFenceStart(line); + if (fence) { + const code = readCodeFence(lines, index, fence); + index = code.nextIndex; + const text = code.text.slice(0, MAX_BLOCK_CHARS); + if (text !== "" && blocks.length < MAX_BLOCKS) { + blocks.push({ type: "code", text }); + totalChars += text.length; + } + continue; + } + + const heading = /^ {0,3}#{1,6}\s+(.+?)\s*#*\s*$/.exec(line); + if (heading) { + const text = plainInlineText(heading[1] ?? ""); + if (text !== "" && normalizeText(text) !== titleKey) { + addTextBlock("heading", [{ text }], 1); + } + index += 1; + continue; + } + + if (isHorizontalRule(line)) { + index += 1; + continue; + } + + if (isTableStart(lines, index)) { + if (tableCount < MAX_TABLES) { + const table = buildTable(lines, index); + if (table) { + blocks.push(table.block); + tableCount += 1; + totalChars += table.chars; + } + } + index = consumeTable(lines, index); + continue; + } + + if (/^ {0,3}>/.test(line)) { + let text = ""; + while (index < lines.length) { + const quoteMatch = /^ {0,3}>\s?(.*)$/.exec(lines[index]!); + if (!quoteMatch) break; + text += `${quoteMatch[1] ?? ""} `; + index += 1; + } + addTextBlock( + "quote", + parseInlineSegments(text, options.pageUrl), + MIN_QUOTE_CHARS, + ); + continue; + } + + if (isListItemLine(line)) { + const item = parseListItem(lines, index); + if (item) { + const text = item.text.replace(/^\[[ xX]\]\s+/, ""); + addTextBlock( + "list", + parseInlineSegments(text, options.pageUrl), + MIN_LIST_CHARS, + ); + } + index = item?.nextIndex ?? index + 1; + continue; + } + + const standaloneImage = parseStandaloneImage(line); + if (standaloneImage) { + addImage(standaloneImage); + index += 1; + continue; + } + + const paragraphLines: string[] = []; + while (index < lines.length) { + const current = lines[index]!; + if (current.trim() === "") break; + if (paragraphLines.length > 0 && isBlockStart(lines, index)) break; + paragraphLines.push(current.trim()); + index += 1; + } + const paragraph = extractInlineImages( + paragraphLines.join(" "), + options.pageUrl, + ); + for (const image of paragraph.images) addImage(image); + addTextBlock( + "paragraph", + parseInlineSegments(paragraph.text, options.pageUrl), + MIN_PARAGRAPH_CHARS, + ); + } + + return blocks; +} + +function readCodeFence( + lines: string[], + start: number, + fence: FenceStart, +): { text: string; nextIndex: number } { + const content: string[] = []; + let index = start + 1; + while (index < lines.length && !isFenceClose(lines[index]!, fence.marker)) { + content.push(lines[index]!); + index += 1; + } + if (index < lines.length) index += 1; + return { + text: content.join("\n").replace(/\s+$/, ""), + nextIndex: index, + }; +} + +function parseFenceStart(line: string): FenceStart | null { + const match = /^ {0,3}(`{3,}|~{3,})/.exec(line); + return match ? { marker: match[1]! } : null; +} + +function isFenceClose(line: string, marker: string): boolean { + const trimmed = line.trim(); + if (trimmed.length < marker.length) return false; + const char = marker[0]!; + for (const candidate of trimmed) { + if (candidate !== char) return false; + } + return true; +} + +function isHorizontalRule(line: string): boolean { + const compact = line.trim().replace(/[ \t]/g, ""); + return compact.length >= 3 && /^(?:\*+|_+|-+)$/.test(compact); +} + +function isListItemLine(line: string): boolean { + return /^ {0,3}(?:[-*+]|\d{1,9}[.)])\s+/.test(line); +} + +function parseListItem( + lines: string[], + start: number, +): { text: string; nextIndex: number } | null { + const match = /^ {0,3}(?:[-*+]|\d{1,9}[.)])\s+(.*)$/.exec(lines[start]!); + if (!match) return null; + + let text = match[1] ?? ""; + let index = start + 1; + while (index < lines.length) { + const line = lines[index]!; + if (line.trim() === "") break; + if (isListItemLine(line) || isBlockStart(lines, index)) break; + if (!/^\s{2,}\S/.test(line)) break; + text += ` ${line.trim()}`; + index += 1; + } + return { text, nextIndex: index }; +} + +function isTableStart(lines: string[], index: number): boolean { + const line = lines[index]; + const next = lines[index + 1]; + if (!line || !next || !line.includes("|")) return false; + return isTableSeparator(next); +} + +function isTableSeparator(line: string): boolean { + if (!line.includes("-")) return false; + const cells = splitTableRow(line); + return ( + cells.length > 0 && + cells.every((cell) => /^:?-{3,}:?$/.test(cell.trim())) + ); +} + +function consumeTable(lines: string[], start: number): number { + let index = start + 2; + while (index < lines.length) { + const line = lines[index]!; + if (line.trim() === "" || !line.includes("|")) break; + index += 1; + } + return index; +} + +function buildTable( + lines: string[], + start: number, +): { block: PreviewBlock; chars: number } | null { + const rows: string[][] = []; + let chars = 0; + for (let index = start; index < lines.length; index += 1) { + const line = lines[index]!; + if (line.trim() === "" || !line.includes("|")) break; + if (index === start + 1) continue; + const row = splitTableRow(line) + .slice(0, MAX_TABLE_COLUMNS) + .map((cell) => + plainInlineText(cell).slice(0, MAX_TABLE_CELL_CHARS), + ); + if (row.length === 0 || row.every((cell) => cell === "")) continue; + rows.push(row); + chars += row.reduce((sum, cell) => sum + cell.length, 0); + if (rows.length >= MAX_TABLE_ROWS) break; + } + if (rows.length === 0) return null; + return { block: { type: "table", header: true, rows }, chars }; +} + +function splitTableRow(line: string): string[] { + let raw = line.trim(); + if (raw.startsWith("|")) raw = raw.slice(1); + if (raw.endsWith("|") && !raw.endsWith("\\|")) { + raw = raw.slice(0, -1); + } + const cells: string[] = []; + let current = ""; + for (let index = 0; index < raw.length; index += 1) { + const char = raw[index]!; + if (char === "\\" && raw[index + 1] === "|") { + current += "|"; + index += 1; + continue; + } + if (char === "|") { + cells.push(current); + current = ""; + continue; + } + current += char; + } + cells.push(current); + return cells; +} + +function parseStandaloneImage(line: string): StandaloneImage | null { + const match = + /^ {0,3}!\[([^\]]*)\]\(\s*([^)\s]+)(?:\s+"[^"]*")?\s*\)\s*$/.exec(line); + if (!match) return null; + return { alt: match[1] ?? "", src: match[2] ?? "" }; +} + +function extractInlineImages( + raw: string, + pageUrl: string, +): { text: string; images: StandaloneImage[] } { + const images: StandaloneImage[] = []; + const text = raw.replace( + /!\[([^\]]*)\]\(\s*([^)\s]+)(?:\s+"[^"]*")?\s*\)/g, + (_match, alt: string, src: string) => { + images.push({ alt, src }); + return " "; + }, + ); + // Resolve here only to drop obviously unusable sources early; addImage does + // the authoritative validation and deduplication. + return { + text, + images: images.filter((image) => resolvePublicUrl(image.src, pageUrl) !== ""), + }; +} + +function isBlockStart(lines: string[], index: number): boolean { + const line = lines[index]!; + if (line.trim() === "") return true; + if (parseFenceStart(line)) return true; + if (/^ {0,3}#{1,6}\s+/.test(line)) return true; + if (/^ {0,3}>/.test(line)) return true; + if (isListItemLine(line)) return true; + if (isHorizontalRule(line)) return true; + if (isTableStart(lines, index)) return true; + return parseStandaloneImage(line) !== null; +} + +function parseInlineSegments( + raw: string, + pageUrl: string, + marks: InlineMarks = {}, +): PreviewSegment[] { + const out: PreviewSegment[] = []; + let lastIndex = 0; + for (const match of raw.matchAll(INLINE_TOKEN)) { + const index = match.index ?? 0; + if (index > lastIndex) { + appendText(out, raw.slice(lastIndex, index), marks); + } + const token = match[0]; + if (token.startsWith("**") && token.endsWith("**")) { + out.push( + ...parseInlineSegments(token.slice(2, -2), pageUrl, { + ...marks, + strong: true, + }), + ); + } else if (token.startsWith("__") && token.endsWith("__")) { + out.push( + ...parseInlineSegments(token.slice(2, -2), pageUrl, { + ...marks, + strong: true, + }), + ); + } else if (token.startsWith("*") && token.endsWith("*")) { + out.push( + ...parseInlineSegments(token.slice(1, -1), pageUrl, { + ...marks, + em: true, + }), + ); + } else if (token.startsWith("`") && token.endsWith("`")) { + appendSegment(out, { ...marks, text: token.slice(1, -1), code: true }); + } else { + const link = /^\[([^\]]*)\]\(([^)\s]+)(?:\s+"[^"]*")?\)$/.exec(token); + if (link) { + const href = resolvePublicUrl(link[2] ?? "", pageUrl); + out.push( + ...parseInlineSegments( + link[1] ?? "", + pageUrl, + href === "" ? marks : { ...marks, href }, + ), + ); + } else { + const autolink = /^<(https?:\/\/[^>\s]+)>$/.exec(token); + if (autolink) { + const href = resolvePublicUrl(autolink[1] ?? "", pageUrl); + appendText( + out, + autolink[1] ?? "", + href === "" ? marks : { ...marks, href }, + ); + } else { + appendText(out, token, marks); + } + } + } + lastIndex = index + token.length; + } + if (lastIndex < raw.length) appendText(out, raw.slice(lastIndex), marks); + trimSegments(out); + return out; +} + +function appendText( + out: PreviewSegment[], + raw: string, + marks: InlineMarks, +): void { + const text = unescapeMarkdown(decodeHtmlEntities(raw)).replace(/\s+/g, " "); + if (text === "") return; + appendSegment(out, { ...marks, text }); +} + +function appendSegment(out: PreviewSegment[], segment: PreviewSegment): void { + if (segment.text === "") return; + const last = out[out.length - 1]; + if (last && sameInlineMarks(last, segment)) { + last.text += segment.text; + return; + } + const clean: PreviewSegment = { text: segment.text }; + if (segment.href !== undefined) clean.href = segment.href; + if (segment.strong) clean.strong = true; + if (segment.em) clean.em = true; + if (segment.code) clean.code = true; + out.push(clean); +} + +function sameInlineMarks(a: PreviewSegment, b: PreviewSegment): boolean { + return ( + a.href === b.href && + Boolean(a.strong) === Boolean(b.strong) && + Boolean(a.em) === Boolean(b.em) && + Boolean(a.code) === Boolean(b.code) + ); +} + +function trimSegments(segments: PreviewSegment[]): void { + while (segments.length > 0) { + segments[0]!.text = segments[0]!.text.replace(/^ +/, ""); + if (segments[0]!.text !== "") break; + segments.shift(); + } + while (segments.length > 0) { + const last = segments[segments.length - 1]!; + last.text = last.text.replace(/ +$/, ""); + if (last.text !== "") break; + segments.pop(); + } +} + +function capSegments( + segments: PreviewSegment[], + maxChars: number, +): PreviewSegment[] { + const out: PreviewSegment[] = []; + let length = 0; + for (const segment of segments) { + if (length >= maxChars) break; + const text = segment.text.slice(0, maxChars - length); + if (text === "") continue; + out.push({ ...segment, text }); + length += text.length; + } + return out; +} + +function hasInlineMarks(segments: PreviewSegment[]): boolean { + return segments.some( + (segment) => + segment.href !== undefined || + segment.strong === true || + segment.em === true || + segment.code === true, + ); +} + +function textBlockPlainText(block: PreviewBlock): string | null { + if ( + block.type !== "heading" && + block.type !== "paragraph" && + block.type !== "list" && + block.type !== "quote" + ) { + return null; + } + return "segments" in block + ? block.segments.map((segment) => segment.text).join("") + : block.text; +} + +function plainInlineText(raw: string): string { + let text = raw; + text = text.replace(/!\[([^\]]*)\]\([^)]*\)/g, "$1"); + text = text.replace(/\[([^\]]*)\]\([^)]*\)/g, "$1"); + text = text.replace(/<(https?:\/\/[^>\s]+)>/g, "$1"); + text = text.replace(/\*\*([^*]+)\*\*/g, "$1"); + text = text.replace(/__([^_]+)__/g, "$1"); + text = text.replace(/\*([^*]+)\*/g, "$1"); + text = text.replace(/`([^`]+)`/g, "$1"); + text = text.replace(/~~([^~]+)~~/g, "$1"); + return normalizeText(unescapeMarkdown(decodeHtmlEntities(text))); +} + +function unescapeMarkdown(value: string): string { + return value.replace(/\\([\\`*_{}\[\]()#+\-.!>~|])/g, "$1"); +} + +function resolvePublicUrl(raw: string, pageUrl: string): string { + const trimmed = decodeHtmlEntities(raw) + .trim() + .replace(/^$/, ""); + if (trimmed === "") return ""; + try { + return normalizePreviewUrl(new URL(trimmed, pageUrl).toString()) ?? ""; + } catch { + return ""; + } +} + +function normalizeText(value: string | null | undefined): string { + return (value ?? "").replace(/\s+/g, " ").trim(); +} + +function decodeHtmlEntities(value: string): string { + return value.replace( + /&(#x?[0-9a-f]+|[a-z]+);/gi, + (match, entity: string) => { + const key = entity.toLowerCase(); + if (key.startsWith("#x")) { + return codePoint(match, Number.parseInt(key.slice(2), 16)); + } + if (key.startsWith("#")) { + return codePoint(match, Number.parseInt(key.slice(1), 10)); + } + return HTML_ENTITIES[key] ?? match; + }, + ); +} + +function codePoint(fallback: string, value: number): string { + if (!Number.isFinite(value) || value < 0 || value > 0x10ffff) { + return fallback; + } + return String.fromCodePoint(value); +} + +/** + * HTML conversion appends the page's JSON-LD at the end of the Markdown. + * Remove only a trailing JSON-LD-shaped object/array so a legitimate JSON + * code sample elsewhere in the article is left alone. Limit how many opening + * braces are tried so a long PDF with many braces cannot make this quadratic. + */ +function stripTrailingJsonLd(markdown: string): string { + const fenced = /```(?:json)?[ \t]*\n([\s\S]*?)\n```\s*$/i.exec(markdown); + if (fenced && isJsonLd(fenced[1] ?? "")) { + return markdown.slice(0, fenced.index).trimEnd(); + } + + const leading = markdown.trimStart(); + if (leading.startsWith("{") && isJsonLd(leading.trim())) { + return ""; + } + + let start = markdown.lastIndexOf("\n{"); + for (let attempts = 0; start !== -1 && attempts < 5; attempts += 1) { + const candidate = markdown.slice(start + 1).trim(); + if (isJsonLd(candidate)) return markdown.slice(0, start).trimEnd(); + start = markdown.lastIndexOf("\n{", start - 1); + } + return markdown; +} + +function isJsonLd(text: string): boolean { + let parsed: unknown; + try { + parsed = JSON.parse(text) as unknown; + } catch { + return false; + } + return hasJsonLdShape(parsed); +} + +function hasJsonLdShape(value: unknown): boolean { + if (Array.isArray(value)) return value.some((entry) => hasJsonLdShape(entry)); + if (!value || typeof value !== "object") return false; + const record = value as Record; + return ( + record["@context"] !== undefined || typeof record["@type"] === "string" + ); +} diff --git a/core/test/preview.spec.ts b/core/test/preview.spec.ts index 51340a2..8d1f901 100644 --- a/core/test/preview.spec.ts +++ b/core/test/preview.spec.ts @@ -5,6 +5,7 @@ import { normalizePreviewUrl, PreviewUnavailableError, type PreviewBlock, + type PreviewFallbackInput, } from "../preview.ts"; function blockText(block: PreviewBlock): string { @@ -877,4 +878,218 @@ describe("fetchArticlePreview", () => { }), ).rejects.toMatchObject({ reason: "too_large" }); }); + + it("converts non-HTML content through the optional fallback", async () => { + const inputs: PreviewFallbackInput[] = []; + const bytes = new Uint8Array([0x25, 0x50, 0x44, 0x46, 0x2d]); + + const preview = await fetchArticlePreview("https://example.com/paper.pdf", { + userAgent: "feedreader-test", + fetchImpl: async () => + new Response(bytes, { + headers: { "content-type": "application/pdf; charset=binary" }, + }), + fallback: async (input) => { + inputs.push(input); + return { + title: "Paper", + siteName: "example.com", + blocks: [{ type: "paragraph", text: "Converted PDF text." }], + }; + }, + }); + + expect(inputs).toHaveLength(1); + expect(inputs[0]).toMatchObject({ + kind: "unsupported_content_type", + url: "https://example.com/paper.pdf", + contentType: "application/pdf; charset=binary", + }); + expect(inputs[0]?.bytes).toEqual(bytes); + expect(inputs[0]?.html).toBeUndefined(); + expect(preview.blocks).toEqual([ + { type: "paragraph", text: "Converted PDF text." }, + ]); + }); + + it("preserves the unsupported-content reason when the fallback gives up", async () => { + const response = () => + new Response("%PDF-1.7", { + headers: { "content-type": "application/pdf" }, + }); + + await expect( + fetchArticlePreview("https://example.com/paper.pdf", { + userAgent: "feedreader-test", + fetchImpl: async () => response(), + fallback: async () => null, + }), + ).rejects.toMatchObject({ reason: "unsupported_content_type" }); + + await expect( + fetchArticlePreview("https://example.com/paper.pdf", { + userAgent: "feedreader-test", + fetchImpl: async () => response(), + fallback: async () => { + throw new Error("AI unavailable"); + }, + }), + ).rejects.toMatchObject({ reason: "unsupported_content_type" }); + }); + + it("does not hand a truncated non-HTML body to the fallback", async () => { + let called = false; + + await expect( + fetchArticlePreview("https://example.com/large.pdf", { + userAgent: "feedreader-test", + maxFallbackBytes: 4, + fetchImpl: async () => + new Response(new Uint8Array([1, 2, 3, 4, 5]), { + headers: { "content-type": "application/pdf" }, + }), + fallback: async () => { + called = true; + return null; + }, + }), + ).rejects.toMatchObject({ reason: "unsupported_content_type" }); + + expect(called).toBe(false); + }); + + it("does not read a document whose content-length is over the fallback cap", async () => { + let called = false; + + await expect( + fetchArticlePreview("https://example.com/large.pdf", { + userAgent: "feedreader-test", + maxFallbackBytes: 4, + fetchImpl: async () => + new Response(new Uint8Array([1, 2, 3, 4, 5]), { + headers: { + "content-type": "application/pdf", + "content-length": "5", + }, + }), + fallback: async () => { + called = true; + return null; + }, + }), + ).rejects.toMatchObject({ reason: "unsupported_content_type" }); + + expect(called).toBe(false); + }); + + it("uses the fallback when an HTML page has no readable blocks", async () => { + const inputs: PreviewFallbackInput[] = []; + + const preview = await fetchArticlePreview("https://example.com/shell", { + userAgent: "feedreader-test", + fetchImpl: async () => + htmlResponse("
tiny
"), + fallback: async (input) => { + inputs.push(input); + return { + title: "Shell", + siteName: "example.com", + blocks: [{ type: "paragraph", text: "Recovered article text." }], + }; + }, + }); + + expect(inputs).toHaveLength(1); + expect(inputs[0]).toMatchObject({ + kind: "no_content", + url: "https://example.com/shell", + contentType: "text/html; charset=utf-8", + }); + expect(inputs[0]?.html).toBe( + "
tiny
", + ); + expect(inputs[0]?.bytes).toBeUndefined(); + expect(preview.blocks).toEqual([ + { type: "paragraph", text: "Recovered article text." }, + ]); + }); + + it("uses the fallback when truncation left no readable blocks", async () => { + const inputs: PreviewFallbackInput[] = []; + const html = `
${"a".repeat(400)}
`; + + const preview = await fetchArticlePreview("https://example.com/huge", { + userAgent: "feedreader-test", + maxBytes: 120, + fetchImpl: async () => htmlResponse(html), + fallback: async (input) => { + inputs.push(input); + return { + title: "Huge", + siteName: "example.com", + blocks: [{ type: "paragraph", text: "Recovered from the prefix." }], + }; + }, + }); + + expect(inputs[0]?.kind).toBe("too_large"); + expect(preview.blocks).toEqual([ + { type: "paragraph", text: "Recovered from the prefix." }, + ]); + }); + + it("keeps the too_large reason when the fallback cannot recover", async () => { + const html = `
${"a".repeat(400)}
`; + + await expect( + fetchArticlePreview("https://example.com/huge", { + userAgent: "feedreader-test", + maxBytes: 120, + fetchImpl: async () => htmlResponse(html), + fallback: async () => null, + }), + ).rejects.toMatchObject({ reason: "too_large" }); + }); + + it("does not invoke the fallback when the primary extractor succeeds", async () => { + let called = false; + + const preview = await fetchArticlePreview("https://example.com/edge", { + userAgent: "feedreader-test", + fetchImpl: async () => htmlResponse(ARTICLE_HTML), + fallback: async () => { + called = true; + return null; + }, + }); + + expect(called).toBe(false); + expect(preview.blocks.length).toBeGreaterThan(0); + }); + + it("does not invoke the fallback for blocked responses or empty bodies", async () => { + let called = false; + const fallback = async (): Promise => { + called = true; + return null; + }; + + await expect( + fetchArticlePreview("https://example.com/blocked", { + userAgent: "feedreader-test", + fetchImpl: async () => new Response("blocked", { status: 403 }), + fallback, + }), + ).rejects.toMatchObject({ reason: "blocked" }); + + await expect( + fetchArticlePreview("https://example.com/empty", { + userAgent: "feedreader-test", + fetchImpl: async () => htmlResponse(""), + fallback, + }), + ).rejects.toMatchObject({ reason: "empty_body" }); + + expect(called).toBe(false); + }); }); diff --git a/core/test/previewMarkdown.spec.ts b/core/test/previewMarkdown.spec.ts new file mode 100644 index 0000000..b9681a3 --- /dev/null +++ b/core/test/previewMarkdown.spec.ts @@ -0,0 +1,318 @@ +import { describe, expect, it } from "vitest"; +import type { PreviewBlock } from "../preview.ts"; +import { markdownToPreviewBlocks } from "../previewMarkdown.ts"; + +function blockText(block: PreviewBlock): string { + if (block.type === "image") return block.alt; + if (block.type === "table") return block.rows.flat().join(" "); + if (block.type === "code") return block.text; + return "segments" in block + ? block.segments.map((segment) => segment.text).join("") + : block.text; +} + +describe("markdownToPreviewBlocks", () => { + it("maps headings, prose, lists, quotes, code, images, and tables", () => { + const markdown = `# Edge Caching Deep Dive + +Edge caching keeps responses close to users, cutting latency and origin load. + +## How it works + +The **cache key** includes the [request URL](https://example.com/docs). A hit returns a stored response; a miss runs the origin handler. + +- Lower latency +- Fewer origin reads +- [x] Verified + +> Cache the edges, not the decisions. + +\`\`\`js +const cached = await cache.match(request); +\`\`\` + +![Latency chart](https://example.com/chart.png) + +| Region | Latency | +| --- | --- | +| US | 20ms | +`; + + const blocks = markdownToPreviewBlocks(markdown, { + pageUrl: "https://example.com/posts/edge", + title: "Edge Caching Deep Dive", + }); + + expect(blocks.map((block) => block.type)).toEqual([ + "paragraph", + "heading", + "paragraph", + "list", + "list", + "list", + "quote", + "code", + "image", + "table", + ]); + expect(blocks[0]).toEqual({ + type: "paragraph", + text: "Edge caching keeps responses close to users, cutting latency and origin load.", + }); + expect(blocks[1]).toEqual({ type: "heading", text: "How it works" }); + expect(blocks[2]).toEqual({ + type: "paragraph", + segments: [ + { text: "The " }, + { text: "cache key", strong: true }, + { + text: " includes the ", + }, + { text: "request URL", href: "https://example.com/docs" }, + { + text: ". A hit returns a stored response; a miss runs the origin handler.", + }, + ], + }); + expect(blocks.filter((block) => block.type === "list")).toEqual([ + { type: "list", text: "Lower latency" }, + { type: "list", text: "Fewer origin reads" }, + { type: "list", text: "Verified" }, + ]); + expect(blocks.find((block) => block.type === "quote")).toEqual({ + type: "quote", + text: "Cache the edges, not the decisions.", + }); + expect(blocks.find((block) => block.type === "code")).toEqual({ + type: "code", + text: "const cached = await cache.match(request);", + }); + expect(blocks.find((block) => block.type === "image")).toEqual({ + type: "image", + src: "https://example.com/chart.png", + alt: "Latency chart", + }); + expect(blocks.find((block) => block.type === "table")).toEqual({ + type: "table", + header: true, + rows: [ + ["Region", "Latency"], + ["US", "20ms"], + ], + }); + }); + + it("drops unsafe links and images while keeping readable text", () => { + const markdown = `This paragraph has a [dangerous link](javascript:alert(1)) and a [private link](http://127.0.0.1/admin) plus a [safe link](https://example.com/ok) for context. + +![Private image](http://127.0.0.1/private.png) +![Badge](https://img.shields.io/badge/build-passing.svg) +![Relative chart](/images/chart.png) +`; + + const blocks = markdownToPreviewBlocks(markdown, { + pageUrl: "https://example.com/posts/unsafe", + }); + + const paragraph = blocks.find((block) => block.type === "paragraph"); + expect(paragraph).toBeDefined(); + if ( + paragraph?.type !== "paragraph" || + !("segments" in paragraph) + ) { + throw new Error("expected a rich paragraph"); + } + expect( + paragraph.segments + .map((segment) => segment.href) + .filter((href) => href !== undefined), + ).toEqual(["https://example.com/ok"]); + expect(paragraph.segments.map((segment) => segment.text).join("")).toContain( + "dangerous link", + ); + + expect(blocks.filter((block) => block.type === "image")).toEqual([ + { + type: "image", + src: "https://example.com/images/chart.png", + alt: "Relative chart", + }, + ]); + }); + + it("extracts inline images as figure blocks", () => { + const markdown = `This paragraph is long enough to pass the minimum length check and contains ![Latency chart](https://example.com/chart.png) an inline figure plus enough trailing words.`; + + const blocks = markdownToPreviewBlocks(markdown, { + pageUrl: "https://example.com/posts/inline-image", + }); + + expect(blocks.map((block) => block.type)).toEqual(["image", "paragraph"]); + expect(blocks[0]).toEqual({ + type: "image", + src: "https://example.com/chart.png", + alt: "Latency chart", + }); + expect(blockText(blocks[1]!)).not.toContain("chart.png"); + expect(blockText(blocks[1]!)).not.toContain("Latency chart"); + }); + + it("strips JSON-LD appended by the HTML converter", () => { + const markdown = `A long enough article paragraph that should remain in the preview. + +{"@context":"https://schema.org","@type":"Article","headline":"A story"}`; + + const blocks = markdownToPreviewBlocks(markdown, { + pageUrl: "https://example.com/posts/json-ld", + }); + + expect(blocks).toEqual([ + { + type: "paragraph", + text: "A long enough article paragraph that should remain in the preview.", + }, + ]); + }); + + it("strips a fenced JSON-LD block but keeps ordinary JSON code", () => { + const markdown = `A long enough article paragraph that should remain in the preview. + +\`\`\`json +{"@context":"https://schema.org","@type":"Article"} +\`\`\` +`; + + expect( + markdownToPreviewBlocks(markdown, { + pageUrl: "https://example.com/posts/json-ld-fenced", + }).map((block) => block.type), + ).toEqual(["paragraph"]); + + const ordinary = `\`\`\`json +{"enabled":true} +\`\`\``; + expect( + markdownToPreviewBlocks(ordinary, { + pageUrl: "https://example.com/posts/json-code", + }), + ).toEqual([{ type: "code", text: '{"enabled":true}' }]); + }); + + it("skips a heading that repeats the stored title", () => { + const markdown = `# Same Title + +A long enough paragraph that should remain in the preview.`; + + expect( + markdownToPreviewBlocks(markdown, { + pageUrl: "https://example.com/posts/title", + title: "Same Title", + }).map((block) => block.type), + ).toEqual(["paragraph"]); + + expect( + markdownToPreviewBlocks(markdown, { + pageUrl: "https://example.com/posts/title", + })[0], + ).toEqual({ type: "heading", text: "Same Title" }); + }); + + it("does not interpret Markdown inside fenced code", () => { + const markdown = `\`\`\`md +# Not a heading +**not bold** +\`\`\``; + + expect( + markdownToPreviewBlocks(markdown, { + pageUrl: "https://example.com/posts/code", + }), + ).toEqual([ + { type: "code", text: "# Not a heading\n**not bold**" }, + ]); + }); + + it("supports autolinks and avoids emphasis inside snake_case", () => { + const markdown = + "Read and use snake_case_name in this long enough paragraph."; + + const blocks = markdownToPreviewBlocks(markdown, { + pageUrl: "https://example.com/posts/autolink", + }); + + expect(blocks).toHaveLength(1); + const paragraph = blocks[0]; + if ( + paragraph?.type !== "paragraph" || + !("segments" in paragraph) + ) { + throw new Error("expected a rich paragraph"); + } + expect(paragraph.segments).toContainEqual({ + text: "https://example.com/docs", + href: "https://example.com/docs", + }); + expect(paragraph.segments.some((segment) => segment.strong)).toBe(false); + expect(paragraph.segments.some((segment) => segment.em)).toBe(false); + }); + + it("handles escaped pipes in tables", () => { + const markdown = `| Name | Value | +| --- | --- | +| a \\| b | 1 |`; + + expect( + markdownToPreviewBlocks(markdown, { + pageUrl: "https://example.com/posts/table", + }), + ).toEqual([ + { + type: "table", + header: true, + rows: [ + ["Name", "Value"], + ["a | b", "1"], + ], + }, + ]); + }); + + it("unescapes Markdown punctuation and decodes entities", () => { + const markdown = + "Use \\[brackets\\] and \\*stars\\* … in this long enough paragraph."; + + expect( + markdownToPreviewBlocks(markdown, { + pageUrl: "https://example.com/posts/escaped", + }), + ).toEqual([ + { + type: "paragraph", + text: "Use [brackets] and *stars* … in this long enough paragraph.", + }, + ]); + }); + + it("does not mistake other trailing JSON for JSON-LD", () => { + const markdown = '{"Content-Type":["text/html"],"X-Test":["ignore"]}'; + + expect( + markdownToPreviewBlocks(markdown, { + pageUrl: "https://example.com/posts/plain-json", + }), + ).toEqual([{ type: "paragraph", text: markdown }]); + }); + + it("returns no blocks for empty or JSON-LD-only input", () => { + expect( + markdownToPreviewBlocks("", { + pageUrl: "https://example.com/posts/empty", + }), + ).toEqual([]); + expect( + markdownToPreviewBlocks('{"@context":"https://schema.org"}', { + pageUrl: "https://example.com/posts/json-only", + }), + ).toEqual([]); + }); +}); diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 7f6366b..403b119 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -9,9 +9,11 @@ | [`core/service.ts`](../core/service.ts) | Refresh orchestration, feed pagination, health payloads, card-building, and error projection. | | [`core/render.ts`](../core/render.ts) | Server-side HTML rendering for the feed UI. | | [`core/preview.ts`](../core/preview.ts) | Reader-view URL validation, guarded article fetch, and rich text/image/table extraction. | +| [`core/previewMarkdown.ts`](../core/previewMarkdown.ts) | Validated Markdown-to-`PreviewBlock` mapping for the optional platform fallback. | | [`core/sources/`](../core/sources) | Upstream source adapters plus shared HTTP/HTML parsing helpers. | | [`platforms/cloudflare/src/index.ts`](../platforms/cloudflare/src/index.ts) | Worker request router and scheduled entrypoint. | | [`platforms/cloudflare/src/repository.ts`](../platforms/cloudflare/src/repository.ts) | D1-backed `FeedRepository` implementation. | +| [`platforms/cloudflare/src/previewFallback.ts`](../platforms/cloudflare/src/previewFallback.ts) | Workers AI `toMarkdown` adapter for document/empty previews. | | [`platforms/cloudflare/migrations/`](../platforms/cloudflare/migrations) | D1 schema migrations. | | [`web-static/`](../web-static) | Browser assets: CSS, JS, icons, manifest, service worker. | @@ -74,7 +76,7 @@ This keeps refresh logic, rendering, and feed shaping independent of D1-specific ## Caching - `GET /` is cached per URL (including the raw query string) for five minutes through the Cache API. `APP_VERSION` is part of the key and changes on every deploy, so a release invalidates old HTML; local `wrangler dev` keeps `APP_VERSION` at `dev` and skips the home cache so template/CSS edits are never hidden. The browser-facing response is `private, no-store`: the five-minute copy is internal to the Cache API, which keeps Cloudflare's Browser Cache TTL setting from rewriting the client TTL. -- `GET /api/items` is edge-cached until the underlying sources next refresh (the key includes `sync_state.last_success_at`). Successful article previews are cached for 30 minutes. Both endpoints use the same split as the home page: the Cache API holds the edge copy, while clients receive `private, no-store`, so the CDN/Browser Cache TTL cannot extend the client TTL. +- `GET /api/items` is edge-cached until the underlying sources next refresh (the key includes `sync_state.last_success_at`). Successful article previews, including AI-converted ones, are cached for 30 minutes. Both endpoints use the same split as the home page: the Cache API holds the edge copy, while clients receive `private, no-store`, so the CDN/Browser Cache TTL cannot extend the client TTL. - `style.css` and `theme-init.js` are read from the `ASSETS` binding once per isolate and inlined into the server-rendered HTML, so first paint has no render-blocking network requests. The files remain the single source of truth and are still served directly for cached shells. - [`web-static/_headers`](../web-static/_headers) marks versioned static assets as immutable for a year; `service-worker.js` keeps `must-revalidate` so updates land. @@ -95,7 +97,8 @@ The optional article preview is served by `GET /api/preview?url=…`. - The requested URL must exactly match a stored `items.url`. This check, not the fetch layer, is what keeps the endpoint from acting as an open proxy; `feedreader.articlePreviewEnabled` only controls the client behavior. - `core/preview.ts` additionally rejects non-http(s) schemes, credentials, non-standard ports, and loopback/private/link-local hosts, and re-checks the final URL after redirects. -- Article fetches time out after 10 seconds and read at most 400 KB. Transient upstream `429`/`5xx` responses are retried once with jitter. `PreviewUnavailableError` carries a structured reason (`blocked`, `rate_limited`, `upstream_error`, `timeout`, `unsupported_content_type`, `empty_body`, `redirect`, `invalid_url`) plus the optional upstream status; the endpoint logs `preview_unavailable`/`preview_empty` events and returns the reason so the client can show a specific message. The extractor removes scripts/navigation/footer noise and returns structured text segments, image blocks, and table blocks only — upstream HTML is never returned or injected. Text renders with `textContent`/DOM nodes; inline anchors are limited to URLs that pass the same public http(s) validation, and image URLs are re-validated before the browser loads them directly with `loading="lazy"` and `referrerpolicy="no-referrer"`. +- Article fetches time out after 10 seconds and read at most 400 KB of HTML (5 MB when a document is handed to the optional fallback). Transient upstream `429`/`5xx` responses are retried once with jitter. `PreviewUnavailableError` carries a structured reason (`blocked`, `rate_limited`, `upstream_error`, `timeout`, `unsupported_content_type`, `empty_body`, `redirect`, `invalid_url`) plus the optional upstream status; the endpoint logs `preview_unavailable`/`preview_empty` events and returns the reason so the client can show a specific message. The extractor removes scripts/navigation/footer noise and returns structured text segments, image blocks, and table blocks only — upstream HTML is never returned or injected. Text renders with `textContent`/DOM nodes; inline anchors are limited to URLs that pass the same public http(s) validation, and image URLs are re-validated before the browser loads them directly with `loading="lazy"` and `referrerpolicy="no-referrer"`. +- If the built-in extractor finds no blocks (`no_content`), truncation left no blocks (`too_large`), or the upstream is a non-HTML document (`unsupported_content_type`), `fetchArticlePreview` may call an injected fallback. Core stays AI-free: the Worker supplies `platforms/cloudflare/src/previewFallback.ts`, which calls `env.AI.toMarkdown`, maps the converted Markdown through `core/previewMarkdown.ts`, and returns the same validated `PreviewBlock` shape. The mapper re-checks every link and image with `normalizePreviewUrl`, strips trailing JSON-LD, and drops PDF filename/page-marker headings. Blocked, timeout, rate-limited, and empty-body failures never reach the fallback. `FEEDREADER_AI_PREVIEW_FALLBACK=false` disables it. - Image blocks may include a `figcaption`-derived caption and intrinsic width/height to reserve layout space. Known avatar/profile-image hosts and paths, SVG wordmarks, and common badge hosts are dropped before the image cap so author-list and README chrome doesn't displace article figures. Table blocks carry a header flag and capped rows/cells. The response also includes byline, publication date, and an estimated reading time. For Hugging Face paper URLs the extractor starts at the Abstract section, then appends the paper's resource links and citing model, dataset, space, and collection lists while excluding community comment chrome; GitHub titles drop the site prefix. The preview and card `…` dialogs expose a stored Hacker News comments URL for HN items; other sources omit it. - The client highlights the card matching the open preview and provides previous/next navigation over the currently rendered card list, skipping duplicate or unusable cards. When the last rendered card is previewed, Next remains enabled if another page exists and appends it before advancing. On mobile the open panel pins the body scroll position (and reapplies it after rotation), the glass header/footer auto-hide on scroll down and return on scroll up, and the `theme-color` tint switches to the panel surface while the panel covers the viewport. A left-edge rightward drag follows the finger and reveals the preserved feed position underneath; releasing past a distance/velocity threshold slides the panel away and closes it, while a shorter drag springs back. Vertical edge drags remain native scrolling and the gesture is inactive while the `…` dialog is open. - Card host rows keep the optional bookmark toggle and a more-actions dialog trigger. The dialog exposes Share, the stored Hacker News comments URL when present, and Open source; choosing an action does not open the preview. diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 911de9d..b38fbd3 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -64,6 +64,6 @@ The checked-in `wrangler.toml` config does not hardcode a route. Domain attachme - Workers: 100k requests/day, 10ms CPU time per invocation, 50 subrequests per request. - The five-minute home-page cache absorbs most `GET /` traffic, so only cache misses run the feed query and render the HTML. The browser response is `private, no-store`; edge caching is the Worker Cache API, not the CDN, so the zone's Browser Cache TTL setting cannot rewrite the client TTL. Each cold-isolate cache miss also reads `style.css` and `theme-init.js` through the `ASSETS` binding (two local subrequests), then reuses them in memory. - The standalone launch veil and skeleton styles are inlined with the existing HTML/CSS, so they add no extra requests. Bump the shell cache version when the HTML shell or `app.js` changes; see [RUNBOOK.md](RUNBOOK.md#pwa-shell-updates). -- Article previews add one outbound fetch (and one `node-html-parser` parse) per uncached `/api/preview` request. The 400 KB body cap and 30-minute edge cache in `core/preview.ts` bound that cost; transient upstream `429`/`5xx` responses are retried once. Browser-facing preview and `/api/items` responses are `private, no-store`; their edge copies live in the Cache API. Watch the CPU-time panel if preview usage is heavy — bursts can hit the Workers Free 10 ms limit, so see [RUNBOOK.md](RUNBOOK.md#article-preview-failures). Richer preview blocks (inline links, captions, tables) increase the cached response payload but not the upstream fetch count; the same byte cap and cache TTL apply. +- Article previews add one outbound fetch (and one `node-html-parser` parse) per uncached `/api/preview` request. The 400 KB HTML body cap and 30-minute edge cache in `core/preview.ts` bound that cost; transient upstream `429`/`5xx` responses are retried once. PDFs/Office documents and HTML pages with no readable blocks can additionally call Workers AI's `toMarkdown` (5 MB download cap; document text conversion is free and the fallback disables AI image-description conversion). That call runs only on a cache miss and its result is cached for the same 30 minutes; set `FEEDREADER_AI_PREVIEW_FALLBACK=false` to disable it. Browser-facing preview and `/api/items` responses are `private, no-store`; their edge copies live in the Cache API. Watch the CPU-time panel if preview usage is heavy — bursts can hit the Workers Free 10 ms limit, so see [RUNBOOK.md](RUNBOOK.md#article-preview-failures). Richer preview blocks (inline links, captions, tables) increase the cached response payload but not the upstream fetch count; the same byte cap and cache TTL apply. Check the Cloudflare dashboard's D1 usage and Workers CPU-time panels periodically — see [RUNBOOK.md](RUNBOOK.md) for what to do if a source consistently nears the CPU ceiling. diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md index a47641a..84e9af3 100644 --- a/docs/RUNBOOK.md +++ b/docs/RUNBOOK.md @@ -78,10 +78,12 @@ Use Cloudflare dashboard → Workers & Pages → `feedreader` → Triggers → C - `502 blocked` — the upstream returned `401`/`403` (for example DataDome/paywall protection) - `502 rate_limited` / `502 upstream_error` — the upstream returned `429`/`5xx` after one retry - `504 timeout` — the fetch exceeded the 10-second timeout -- `502 unsupported_content_type` / `empty_body` / `redirect` / `fetch_failed` — other fetch or extraction failures -- `200` with `reason: "no_content"` — the page was fetched but no readable blocks were found +- `502 unsupported_content_type` / `empty_body` / `redirect` / `fetch_failed` — other fetch or extraction failures. `unsupported_content_type` also covers a document the AI fallback could not convert (disabled, over 5 MB, timed out, or unsupported by `toMarkdown`). +- `200` with `reason: "no_content"` — the page was fetched and had no readable blocks, and the AI fallback was disabled or returned nothing. -The Worker logs `preview_unavailable` and `preview_empty` events with the URL, reason, and upstream status. Successful previews are edge-cached for 30 minutes via the Cache API; browser-facing responses are `private, no-store`. Transient upstream `429`/`5xx` responses are retried once with jitter. If bursts produce transient `503`s, check Workers Logs/CPU: the extractor runs in the request invocation, and sequential retries succeeding usually points at the Workers Free 10 ms CPU limit rather than a lasting upstream block. Lowering `DEFAULT_MAX_BYTES` in `core/preview.ts` (400 KB today) reduces the parse cost. +Before returning `unsupported_content_type` for a non-HTML response, `too_large` for a truncated page, or `no_content` for an empty HTML page, `fetchArticlePreview` calls the optional Workers AI `toMarkdown` fallback (`platforms/cloudflare/src/previewFallback.ts`). It downloads at most 5 MB (`DEFAULT_MAX_FALLBACK_BYTES`), races the conversion against a 10-second timeout, strips the filename/page-marker headings from PDF output, and maps the Markdown back into validated preview blocks. Standalone images are skipped; the fallback disables AI image-description conversion. Set `FEEDREADER_AI_PREVIEW_FALLBACK=false` to disable it. Document text conversion is free in Workers AI. + +The Worker logs `preview_unavailable` and `preview_empty` events with the URL, reason, and upstream status, plus a `preview_ai_fallback` event for each attempt (`kind`, `outcome` = `ok`/`empty`/`error`, `ms`, `blocks`, `tokens`, and a capped `error`). Successful previews, including AI-converted ones, are edge-cached for 30 minutes via the Cache API; browser-facing responses are `private, no-store`. Transient upstream `429`/`5xx` responses are retried once with jitter. If bursts produce transient `503`s, check Workers Logs/CPU: the extractor runs in the request invocation, and sequential retries succeeding usually points at the Workers Free 10 ms CPU limit rather than a lasting upstream block. Lowering `DEFAULT_MAX_BYTES` in `core/preview.ts` (400 KB today) reduces the parse cost. ## Local SSR smoke procedure diff --git a/platforms/cloudflare/src/env.d.ts b/platforms/cloudflare/src/env.d.ts index 2097c72..5dee823 100644 --- a/platforms/cloudflare/src/env.d.ts +++ b/platforms/cloudflare/src/env.d.ts @@ -18,4 +18,7 @@ export interface Env { * embedding-similarity order on its own. See src/index.ts's * handlePersonalize. */ FEEDREADER_PERSONALIZE_POLISH_POOL_SIZE?: string; + /** Set to "false" to disable the Workers AI toMarkdown fallback for + * non-HTML and empty/truncated article previews. */ + FEEDREADER_AI_PREVIEW_FALLBACK?: string; } diff --git a/platforms/cloudflare/src/index.ts b/platforms/cloudflare/src/index.ts index 67c7b3f..45c9c80 100644 --- a/platforms/cloudflare/src/index.ts +++ b/platforms/cloudflare/src/index.ts @@ -32,6 +32,7 @@ import { paginate } from "../../../core/sources/listInMemory.ts"; import { D1Repository } from "./repository.ts"; import { CloudflareLlmRanker } from "./llmRanker.ts"; import { CloudflareEmbedder } from "./embedder.ts"; +import { aiPreviewFallback } from "./previewFallback.ts"; import { loadConfig } from "./config.ts"; import type { Env } from "./env.d.ts"; @@ -291,8 +292,17 @@ async function handlePreview( if (cached) return clientCacheResponse(cached); try { + // PDFs/Office documents and pages with no readable blocks may still be + // previewable through Workers AI's Markdown conversion. The fallback is + // best-effort: a null result keeps the original failure reason. + const aiFallbackEnabled = + (env.FEEDREADER_AI_PREVIEW_FALLBACK ?? "true").trim().toLowerCase() !== + "false"; const preview = await fetchArticlePreview(target, { userAgent: loadConfig(env).userAgent, + ...(aiFallbackEnabled + ? { fallback: (input) => aiPreviewFallback(env, input, item.title) } + : {}), }); const payload = { url: item.url, @@ -374,9 +384,10 @@ async function handlePreview( * `.invalid` is reserved by RFC 2606 for exactly this kind of placeholder * use, so it's guaranteed not to collide with a real host. */ function previewCacheRequest(target: string): Request { - // v9 adds paper resource/citation sections; v8 added arXiv links and v7 - // covered author bylines. - const raw = `preview:v9:${target}`; + // v10 adds the Workers AI Markdown fallback, so previously cached empty + // previews get another extraction attempt; v9 added paper resource/citation + // sections, v8 added arXiv links, and v7 covered author bylines. + const raw = `preview:v10:${target}`; const url = `https://feedreader-internal.invalid/preview-cache?k=${encodeURIComponent(raw)}`; return new Request(url, { method: "GET" }); } diff --git a/platforms/cloudflare/src/previewFallback.ts b/platforms/cloudflare/src/previewFallback.ts new file mode 100644 index 0000000..1ae62e9 --- /dev/null +++ b/platforms/cloudflare/src/previewFallback.ts @@ -0,0 +1,302 @@ +// Workers AI Markdown Conversion fallback for previews the built-in +// extractor cannot read: PDFs/Office documents and HTML pages with no +// readable blocks (often a JavaScript shell) or a body truncated before the +// content. Core passes only validated URLs and bounded bytes; this adapter +// turns the converted Markdown back into the same PreviewBlock shape. +// +// The AI binding has no request-level timeout, so race the conversion. +// Failures are logged and return null, which preserves the original preview +// failure reason instead of turning an optional enhancement into an error. + +import { + estimateReadingMinutes, + type ArticlePreview, + type PreviewBlock, + type PreviewFallbackInput, +} from "../../../core/preview.ts"; +import { markdownToPreviewBlocks } from "../../../core/previewMarkdown.ts"; +import type { Env } from "./env.d.ts"; + +const AI_TIMEOUT_MS = 10_000; +const MAX_LOG_ERROR_CHARS = 300; +const TITLE_MAX_CHARS = 200; + +// toMarkdown also accepts images, but turning a standalone image into an AI +// description is not a useful article preview and can consume the Workers AI +// allocation. Keep the fallback on document formats. +const AI_DOCUMENT_MIME_TYPES = new Set([ + "application/pdf", + "application/vnd.apple.numbers", + "application/vnd.ms-excel", + "application/vnd.ms-excel.sheet.binary.macroenabled.12", + "application/vnd.ms-excel.sheet.macroenabled.12", + "application/vnd.oasis.opendocument.spreadsheet", + "application/vnd.oasis.opendocument.text", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/xml", + "text/csv", +]); + +const EXTENSION_MIME_TYPES: Record = { + bmp: "image/bmp", + csv: "text/csv", + docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + gif: "image/gif", + jpeg: "image/jpeg", + jpg: "image/jpeg", + ods: "application/vnd.oasis.opendocument.spreadsheet", + odt: "application/vnd.oasis.opendocument.text", + pdf: "application/pdf", + png: "image/png", + svg: "image/svg+xml", + webp: "image/webp", + xls: "application/vnd.ms-excel", + xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", +}; + +const MIME_TYPE_EXTENSIONS: Record = Object.fromEntries( + Object.entries(EXTENSION_MIME_TYPES).map(([extension, mimeType]) => [ + mimeType, + extension, + ]), +); + +/** + * Returns null when the AI conversion fails, times out, or produces no + * previewable blocks. `titleHint` is the stored feed-item title. + */ +export async function aiPreviewFallback( + env: Env, + input: PreviewFallbackInput, + titleHint: string, +): Promise { + const fallbackFlag = (env.FEEDREADER_AI_PREVIEW_FALLBACK ?? "true") + .trim() + .toLowerCase(); + if (fallbackFlag === "false") return null; + + const startedAt = Date.now(); + const mimeType = normalizeMimeType(input.contentType, input.url); + if (input.bytes && !AI_DOCUMENT_MIME_TYPES.has(mimeType)) { + return null; + } + try { + const document = buildMarkdownDocument(input, mimeType); + if (!document) return null; + + const result = await withTimeout( + env.AI.toMarkdown(document, { + conversionOptions: { + html: { + hostname: input.url, + images: { convert: false }, + }, + pdf: { metadata: false, images: { convert: false } }, + }, + }), + AI_TIMEOUT_MS, + ); + + if (result.format === "error") { + logFallback(input, mimeType, startedAt, { + outcome: "error", + error: result.error, + }); + return null; + } + + const blocks = cleanConvertedBlocks( + markdownToPreviewBlocks(result.data, { + pageUrl: input.url, + title: titleHint, + }), + document.name, + mimeType === "application/pdf", + ); + if (blocks.length === 0) { + logFallback(input, mimeType, startedAt, { + outcome: "empty", + tokens: result.tokens, + }); + return null; + } + + const readingMinutes = estimateReadingMinutes(blocks); + const preview: ArticlePreview = { + title: + titleHint.trim().slice(0, TITLE_MAX_CHARS) || + firstHeading(blocks) || + "Article preview", + siteName: hostLabel(input.url), + blocks, + }; + if (readingMinutes !== undefined) preview.readingMinutes = readingMinutes; + logFallback(input, mimeType, startedAt, { + outcome: "ok", + blocks: blocks.length, + tokens: result.tokens, + }); + return preview; + } catch (error) { + logFallback(input, mimeType, startedAt, { + outcome: "error", + error: errorMessage(error), + }); + return null; + } +} + +function buildMarkdownDocument( + input: PreviewFallbackInput, + mimeType: string, +): MarkdownDocument | null { + if (input.bytes) { + return { + name: documentName(mimeType), + blob: new Blob([input.bytes], { type: mimeType }), + }; + } + if (input.html !== undefined) { + return { + name: "article.html", + blob: new Blob([input.html], { type: "text/html" }), + }; + } + return null; +} + +function normalizeMimeType(contentType: string, pageUrl: string): string { + const base = contentType.split(";")[0]?.trim().toLowerCase() ?? ""; + if ( + base !== "" && + base !== "application/octet-stream" && + base !== "binary/octet-stream" + ) { + return base; + } + const extension = urlExtension(pageUrl); + const inferred = extension ? EXTENSION_MIME_TYPES[extension] : undefined; + return inferred ?? (base !== "" ? base : "application/octet-stream"); +} + +function documentName(mimeType: string): string { + if (mimeType === "text/html") return "article.html"; + const extension = MIME_TYPE_EXTENSIONS[mimeType]; + return extension ? `article.${extension}` : "article.bin"; +} + +function urlExtension(rawUrl: string): string { + try { + return /\.([a-z0-9]+)$/i.exec(new URL(rawUrl).pathname)?.[1]?.toLowerCase() ?? ""; + } catch { + return ""; + } +} + +function firstHeading(blocks: PreviewBlock[]): string { + for (const block of blocks) { + if (block.type !== "heading") continue; + const text = + "segments" in block + ? block.segments.map((segment) => segment.text).join("") + : block.text; + const trimmed = text.trim(); + if (trimmed !== "") return trimmed; + } + return ""; +} + +/** + * PDF conversion emits a heading for the uploaded filename and for each page + * ("Page 1", "Page 2", ...). Neither is article content; drop them so the + * reader view starts with the paper's text. + */ +function cleanConvertedBlocks( + blocks: PreviewBlock[], + documentName: string, + isPdf: boolean, +): PreviewBlock[] { + const nameKey = documentName.trim().toLowerCase(); + return blocks.filter((block) => { + const text = blockPlainText(block); + if (text === null) return true; + const trimmed = text.trim(); + if (nameKey !== "" && trimmed.toLowerCase() === nameKey) return false; + if (isPdf && /^page\s+\d+(?:\s+of\s+\d+)?$/i.test(trimmed)) { + return false; + } + return true; + }); +} + +function blockPlainText(block: PreviewBlock): string | null { + if ( + block.type !== "heading" && + block.type !== "paragraph" && + block.type !== "list" && + block.type !== "quote" + ) { + return null; + } + return "segments" in block + ? block.segments.map((segment) => segment.text).join("") + : block.text; +} + +function hostLabel(rawUrl: string): string { + try { + return new URL(rawUrl).hostname.toLowerCase().replace(/^www\./, ""); + } catch { + return ""; + } +} + +function errorMessage(error: unknown): string { + const message = error instanceof Error ? error.message : String(error); + return message.slice(0, MAX_LOG_ERROR_CHARS); +} + +async function withTimeout(promise: Promise, timeoutMs: number): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error("AI Markdown conversion timed out")), + timeoutMs, + ); + }), + ]); + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} + +function logFallback( + input: PreviewFallbackInput, + mimeType: string, + startedAt: number, + result: { + outcome: "ok" | "empty" | "error"; + blocks?: number; + tokens?: number; + error?: string; + }, +): void { + const entry: Record = { + event: "preview_ai_fallback", + kind: input.kind, + url: input.url, + content_type: mimeType, + outcome: result.outcome, + ms: Date.now() - startedAt, + }; + if (result.blocks !== undefined) entry.blocks = result.blocks; + if (result.tokens !== undefined) entry.tokens = result.tokens; + if (result.error !== undefined) entry.error = result.error; + const line = JSON.stringify(entry); + if (result.outcome === "ok") console.log(line); + else console.warn(line); +} diff --git a/platforms/cloudflare/test/previewFallback.spec.ts b/platforms/cloudflare/test/previewFallback.spec.ts new file mode 100644 index 0000000..9e1de5c --- /dev/null +++ b/platforms/cloudflare/test/previewFallback.spec.ts @@ -0,0 +1,173 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { PreviewFallbackInput } from "../../../core/preview.ts"; +import type { Env } from "../src/env.d.ts"; +import { aiPreviewFallback } from "../src/previewFallback.ts"; + +const HTML_INPUT: PreviewFallbackInput = { + kind: "no_content", + url: "https://example.com/posts/shell", + contentType: "text/html; charset=utf-8", + html: "
", +}; + +const PDF_INPUT: PreviewFallbackInput = { + kind: "unsupported_content_type", + url: "https://example.com/paper.pdf", + contentType: "application/pdf", + bytes: new Uint8Array([0x25, 0x50, 0x44, 0x46]), +}; + +function envWith( + value: string, + toMarkdown: (...args: unknown[]) => Promise, +): Env { + return { + FEEDREADER_AI_PREVIEW_FALLBACK: value, + AI: { toMarkdown }, + } as unknown as Env; +} + +function conversion(data: string, tokens = 10): Record { + return { + id: "conversion-1", + name: "article.html", + mimeType: "text/html", + format: "markdown", + tokens, + data, + }; +} + +beforeEach(() => { + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "warn").mockImplementation(() => {}); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("aiPreviewFallback", () => { + it("does nothing when the env flag is false", async () => { + const toMarkdown = vi.fn(async () => conversion("Unused content.")); + const result = await aiPreviewFallback( + envWith("false", toMarkdown), + HTML_INPUT, + "Stored title", + ); + + expect(result).toBeNull(); + expect(toMarkdown).not.toHaveBeenCalled(); + }); + + it("maps HTML conversion output and strips trailing JSON-LD", async () => { + const toMarkdown = vi.fn(async () => + conversion( + "A long enough paragraph recovered from the converted page.\n\n" + + '{"@context":"https://schema.org","@type":"Article"}', + 42, + ), + ); + + const result = await aiPreviewFallback( + envWith("true", toMarkdown), + HTML_INPUT, + "Stored title", + ); + + expect(toMarkdown).toHaveBeenCalledTimes(1); + expect(result).toMatchObject({ + title: "Stored title", + siteName: "example.com", + readingMinutes: 1, + blocks: [ + { + type: "paragraph", + text: "A long enough paragraph recovered from the converted page.", + }, + ], + }); + }); + + it("drops PDF filename and page-marker headings", async () => { + const toMarkdown = vi.fn(async () => + conversion( + [ + "# article.pdf", + "# Page 1", + "A long enough paragraph from the first page of the paper.", + "# Page 2", + "Another long enough paragraph from the second page of the paper.", + ].join("\n\n"), + 120, + ), + ); + + const result = await aiPreviewFallback( + envWith("true", toMarkdown), + PDF_INPUT, + "A Paper", + ); + + expect(toMarkdown).toHaveBeenCalledTimes(1); + expect(result?.blocks).toEqual([ + { + type: "paragraph", + text: "A long enough paragraph from the first page of the paper.", + }, + { + type: "paragraph", + text: "Another long enough paragraph from the second page of the paper.", + }, + ]); + }); + + it("skips standalone images without calling the AI service", async () => { + const toMarkdown = vi.fn(async () => conversion("Unused content.")); + const result = await aiPreviewFallback( + envWith("true", toMarkdown), + { + kind: "unsupported_content_type", + url: "https://example.com/figure.png", + contentType: "image/png", + bytes: new Uint8Array([0x89, 0x50, 0x4e, 0x47]), + }, + "A figure", + ); + + expect(result).toBeNull(); + expect(toMarkdown).not.toHaveBeenCalled(); + }); + + it("returns null when the AI conversion reports an error", async () => { + const toMarkdown = vi.fn(async () => ({ + id: "conversion-1", + name: "article.bin", + mimeType: "application/octet-stream", + format: "error", + error: "unsupported file type", + })); + + const result = await aiPreviewFallback( + envWith("true", toMarkdown), + PDF_INPUT, + "A Paper", + ); + + expect(result).toBeNull(); + }); + + it("returns null when the AI call throws", async () => { + const toMarkdown = vi.fn(async () => { + throw new Error("AI unavailable"); + }); + + const result = await aiPreviewFallback( + envWith("true", toMarkdown), + HTML_INPUT, + "Stored title", + ); + + expect(result).toBeNull(); + }); +}); diff --git a/platforms/cloudflare/wrangler.toml b/platforms/cloudflare/wrangler.toml index 02ffd7b..092e2e7 100644 --- a/platforms/cloudflare/wrangler.toml +++ b/platforms/cloudflare/wrangler.toml @@ -69,6 +69,10 @@ FEEDREADER_MAX_ITEMS_PER_SOURCE = "1000" # the remaining slots are filled from the top similarity hits. See # src/index.ts's handlePersonalize and src/embedder.ts. FEEDREADER_PERSONALIZE_POLISH_POOL_SIZE = "30" +# Workers AI Markdown Conversion fallback for previews the built-in extractor +# can't read (PDFs/Office documents, JavaScript shells, truncated bodies). +# Set to "false" to disable it without a code change. +FEEDREADER_AI_PREVIEW_FALLBACK = "true" # REFRESH_SECRET is a secret, not a var — set it with: # wrangler secret put REFRESH_SECRET --config platforms/cloudflare/wrangler.toml From 06e6e4f209785d1f15fb54169ebc83c9ce747590 Mon Sep 17 00:00:00 2001 From: Phat Pham Date: Fri, 13 Nov 2026 11:56:53 +0700 Subject: [PATCH 2/2] fix(preview): report oversized documents as too_large When the AI fallback is enabled but the document exceeds the 5 MB cap (declared by Content-Length or discovered while reading), return too_large instead of unsupported_content_type so the client shows the size message. No fallback, unsupported formats, and failed conversions still report unsupported_content_type. Co-Authored-By: Claude Code --- core/preview.ts | 49 +++++++++++++++++++++++---------------- core/test/preview.spec.ts | 4 ++-- docs/ARCHITECTURE.md | 2 +- docs/RUNBOOK.md | 3 ++- 4 files changed, 34 insertions(+), 24 deletions(-) diff --git a/core/preview.ts b/core/preview.ts index 0684868..0198e22 100644 --- a/core/preview.ts +++ b/core/preview.ts @@ -329,30 +329,39 @@ export async function fetchArticlePreview( // fallback would reject the partial file anyway. const declaredTooLarge = Number.isFinite(declaredLength) && declaredLength > maxFallbackBytes; - if (fallback && !declaredTooLarge) { - try { - const { bytes, truncated } = await readBytesLimited( - response, - maxFallbackBytes, - ); - // A partial PDF/Office file is unlikely to convert; preserve the - // original unsupported-content reason instead of spending an AI call. - if (!truncated) { - const converted = await tryPreviewFallback(fallback, { - kind: "unsupported_content_type", - url: finalUrl, - contentType, - bytes, - }); - if (converted) return converted; + let tooLarge = false; + if (fallback) { + if (declaredTooLarge) { + tooLarge = true; + } else { + try { + const { bytes, truncated } = await readBytesLimited( + response, + maxFallbackBytes, + ); + // A partial PDF/Office file is unlikely to convert; report the size + // limit rather than spending an AI call on corrupt input. + if (truncated) { + tooLarge = true; + } else { + const converted = await tryPreviewFallback(fallback, { + kind: "unsupported_content_type", + url: finalUrl, + contentType, + bytes, + }); + if (converted) return converted; + } + } catch { + // The fallback is best-effort; report the original reason below. } - } catch { - // The fallback is best-effort; report the original reason below. } } throw new PreviewUnavailableError( - "article is not HTML", - "unsupported_content_type", + tooLarge + ? "article document is too large to preview" + : "article is not HTML", + tooLarge ? "too_large" : "unsupported_content_type", ); } diff --git a/core/test/preview.spec.ts b/core/test/preview.spec.ts index 8d1f901..415d9fa 100644 --- a/core/test/preview.spec.ts +++ b/core/test/preview.spec.ts @@ -953,7 +953,7 @@ describe("fetchArticlePreview", () => { return null; }, }), - ).rejects.toMatchObject({ reason: "unsupported_content_type" }); + ).rejects.toMatchObject({ reason: "too_large" }); expect(called).toBe(false); }); @@ -977,7 +977,7 @@ describe("fetchArticlePreview", () => { return null; }, }), - ).rejects.toMatchObject({ reason: "unsupported_content_type" }); + ).rejects.toMatchObject({ reason: "too_large" }); expect(called).toBe(false); }); diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 403b119..d275a1c 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -97,7 +97,7 @@ The optional article preview is served by `GET /api/preview?url=…`. - The requested URL must exactly match a stored `items.url`. This check, not the fetch layer, is what keeps the endpoint from acting as an open proxy; `feedreader.articlePreviewEnabled` only controls the client behavior. - `core/preview.ts` additionally rejects non-http(s) schemes, credentials, non-standard ports, and loopback/private/link-local hosts, and re-checks the final URL after redirects. -- Article fetches time out after 10 seconds and read at most 400 KB of HTML (5 MB when a document is handed to the optional fallback). Transient upstream `429`/`5xx` responses are retried once with jitter. `PreviewUnavailableError` carries a structured reason (`blocked`, `rate_limited`, `upstream_error`, `timeout`, `unsupported_content_type`, `empty_body`, `redirect`, `invalid_url`) plus the optional upstream status; the endpoint logs `preview_unavailable`/`preview_empty` events and returns the reason so the client can show a specific message. The extractor removes scripts/navigation/footer noise and returns structured text segments, image blocks, and table blocks only — upstream HTML is never returned or injected. Text renders with `textContent`/DOM nodes; inline anchors are limited to URLs that pass the same public http(s) validation, and image URLs are re-validated before the browser loads them directly with `loading="lazy"` and `referrerpolicy="no-referrer"`. +- Article fetches time out after 10 seconds and read at most 400 KB of HTML (5 MB when a document is handed to the optional fallback). Transient upstream `429`/`5xx` responses are retried once with jitter. `PreviewUnavailableError` carries a structured reason (`blocked`, `rate_limited`, `upstream_error`, `timeout`, `unsupported_content_type`, `too_large`, `empty_body`, `redirect`, `invalid_url`) plus the optional upstream status; the endpoint logs `preview_unavailable`/`preview_empty` events and returns the reason so the client can show a specific message. The extractor removes scripts/navigation/footer noise and returns structured text segments, image blocks, and table blocks only — upstream HTML is never returned or injected. Text renders with `textContent`/DOM nodes; inline anchors are limited to URLs that pass the same public http(s) validation, and image URLs are re-validated before the browser loads them directly with `loading="lazy"` and `referrerpolicy="no-referrer"`. - If the built-in extractor finds no blocks (`no_content`), truncation left no blocks (`too_large`), or the upstream is a non-HTML document (`unsupported_content_type`), `fetchArticlePreview` may call an injected fallback. Core stays AI-free: the Worker supplies `platforms/cloudflare/src/previewFallback.ts`, which calls `env.AI.toMarkdown`, maps the converted Markdown through `core/previewMarkdown.ts`, and returns the same validated `PreviewBlock` shape. The mapper re-checks every link and image with `normalizePreviewUrl`, strips trailing JSON-LD, and drops PDF filename/page-marker headings. Blocked, timeout, rate-limited, and empty-body failures never reach the fallback. `FEEDREADER_AI_PREVIEW_FALLBACK=false` disables it. - Image blocks may include a `figcaption`-derived caption and intrinsic width/height to reserve layout space. Known avatar/profile-image hosts and paths, SVG wordmarks, and common badge hosts are dropped before the image cap so author-list and README chrome doesn't displace article figures. Table blocks carry a header flag and capped rows/cells. The response also includes byline, publication date, and an estimated reading time. For Hugging Face paper URLs the extractor starts at the Abstract section, then appends the paper's resource links and citing model, dataset, space, and collection lists while excluding community comment chrome; GitHub titles drop the site prefix. The preview and card `…` dialogs expose a stored Hacker News comments URL for HN items; other sources omit it. - The client highlights the card matching the open preview and provides previous/next navigation over the currently rendered card list, skipping duplicate or unusable cards. When the last rendered card is previewed, Next remains enabled if another page exists and appends it before advancing. On mobile the open panel pins the body scroll position (and reapplies it after rotation), the glass header/footer auto-hide on scroll down and return on scroll up, and the `theme-color` tint switches to the panel surface while the panel covers the viewport. A left-edge rightward drag follows the finger and reveals the preserved feed position underneath; releasing past a distance/velocity threshold slides the panel away and closes it, while a shorter drag springs back. Vertical edge drags remain native scrolling and the gesture is inactive while the `…` dialog is open. diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md index 84e9af3..5e9dabd 100644 --- a/docs/RUNBOOK.md +++ b/docs/RUNBOOK.md @@ -78,7 +78,8 @@ Use Cloudflare dashboard → Workers & Pages → `feedreader` → Triggers → C - `502 blocked` — the upstream returned `401`/`403` (for example DataDome/paywall protection) - `502 rate_limited` / `502 upstream_error` — the upstream returned `429`/`5xx` after one retry - `504 timeout` — the fetch exceeded the 10-second timeout -- `502 unsupported_content_type` / `empty_body` / `redirect` / `fetch_failed` — other fetch or extraction failures. `unsupported_content_type` also covers a document the AI fallback could not convert (disabled, over 5 MB, timed out, or unsupported by `toMarkdown`). +- `502 too_large` — the HTML exceeded the 400 KB cap before readable content, or a document exceeded the 5 MB fallback cap +- `502 unsupported_content_type` / `empty_body` / `redirect` / `fetch_failed` — other fetch or extraction failures. `unsupported_content_type` also covers a document the AI fallback could not convert (disabled, timed out, or unsupported by `toMarkdown`). - `200` with `reason: "no_content"` — the page was fetched and had no readable blocks, and the AI fallback was disabled or returned nothing. Before returning `unsupported_content_type` for a non-HTML response, `too_large` for a truncated page, or `no_content` for an empty HTML page, `fetchArticlePreview` calls the optional Workers AI `toMarkdown` fallback (`platforms/cloudflare/src/previewFallback.ts`). It downloads at most 5 MB (`DEFAULT_MAX_FALLBACK_BYTES`), races the conversion against a 10-second timeout, strips the filename/page-marker headings from PDF output, and maps the Markdown back into validated preview blocks. Standalone images are skipped; the fallback disables AI image-description conversion. Set `FEEDREADER_AI_PREVIEW_FALLBACK=false` to disable it. Document text conversion is free in Workers AI.