From 07224446f4fb99bbf87f7602c7d900e2e73ce8a0 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Tue, 18 Aug 2026 09:31:14 +0000 Subject: [PATCH 1/2] Reader: a 500 is a hiccup, not a refusal, and should not cost a day MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A post on c0mpute.com would not load in the reader. Nothing was wrong with it: the page answered 200 to every request I made. What had happened is that the reader probed it once, that one request got a 500, and the failure was cached for twenty-four hours — so the post stayed broken long after the origin was fine again. Measured against the live site: one 500 in ninety-nine requests. Not a cold start (the first request after 60, 120 and 180 seconds idle was fine) and not load (forty parallel requests were fine). Just an occasional error, of the kind every origin has. A one-per-cent event was costing a full day of downtime for that post, which is the wrong order of magnitude by about three. The cache itself is right — without it every view of a paywalled post is another request to the publisher — but it had one number for two different situations. A paywall will still be a paywall tomorrow. A 500 is the server telling us it failed, and a network error is us never having heard from it at all; both are worth asking again shortly. So transient failures now retry after ten minutes and everything else keeps the day. The distinction is drawn on the response rather than on any opinion about the site: 5xx and fetch errors are transient, 4xx is an answer, `blocked-host` is a refusal we made ourselves, and `empty` is a paywall or a JavaScript-only page that will parse to nothing at teatime too. Ten minutes is long enough that a genuinely broken origin is not hammered once per viewer, and short enough that nobody files a bug about it. Co-Authored-By: Claude Opus 5 (1M context) --- packages/db/src/extracts.js | 41 ++++++++++++++++++++++++++++++- packages/db/test/extracts.test.js | 41 +++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/packages/db/src/extracts.js b/packages/db/src/extracts.js index 0c0dbb5..77324a7 100644 --- a/packages/db/src/extracts.js +++ b/packages/db/src/extracts.js @@ -36,6 +36,45 @@ import { nowIso } from './client.js'; */ export const RETRY_AFTER_MS = 24 * 60 * 60 * 1000; +/** + * How long a *transient* failure stands, which is a different question. + * + * The comment above is right about a paywall and wrong about a hiccup, and it + * had them share a number. An origin that answered 500 once is not refusing us; + * it dropped one request. Measured on a real post: c0mpute.com served a single + * 500 in ninety-nine requests — no cold start, no load trigger, just an + * occasional error — and the reader's one probe happened to land on it. That + * cost the post its reader page for a full day, which is a thousandfold + * over-reaction to a one-per-cent event. + * + * Ten minutes is long enough that a genuinely broken origin is not hammered by + * every viewer, and short enough that nobody files a bug about it. + */ +export const TRANSIENT_RETRY_MS = 10 * 60 * 1000; + +/** + * Whether a stored failure is the kind that clears up by itself. + * + * The distinction is the response, not our opinion of the site: a 5xx is the + * server saying it failed, and a network error is us never having heard from + * it. Both are worth asking again shortly. A 4xx — unauthorised, forbidden, + * gone — is an answer, and asking again in ten minutes would only be rude. + * `empty` is an answer too: a paywall or a JavaScript-only page parses to + * nothing today and will parse to nothing at teatime. + * + * @param {Extract} stored + * @returns {boolean} + */ +export function isTransient(stored) { + if (stored.status === 'error') return true; + if (stored.status !== 'blocked') return false; + + // Recorded by the reader as `http-`; anything else under `blocked` + // (a private address, say) is a refusal we made ourselves and means it. + const match = /^http-(\d{3})$/.exec(stored.reason ?? ''); + return match ? Number(match[1]) >= 500 : false; +} + /** * A successful extraction is kept, and not refreshed. * @@ -99,7 +138,7 @@ export function shouldFetch(stored, now = Date.now()) { // An unparseable timestamp is a row we cannot reason about; treat it as due // rather than as permanently fresh. if (!Number.isFinite(age)) return true; - return age >= RETRY_AFTER_MS; + return age >= (isTransient(stored) ? TRANSIENT_RETRY_MS : RETRY_AFTER_MS); } /** diff --git a/packages/db/test/extracts.test.js b/packages/db/test/extracts.test.js index 46d9514..9099b4d 100644 --- a/packages/db/test/extracts.test.js +++ b/packages/db/test/extracts.test.js @@ -111,3 +111,44 @@ test('a failure goes stale and is tried again', async () => { test('a row with an unreadable timestamp is treated as due, not as permanent', () => { assert.equal(e.shouldFetch({ status: 'empty', fetchedAt: 'not a date' }), true); }); + +test('a 5xx is a hiccup and is retried in minutes, not tomorrow', () => { + // The case this exists for: c0mpute.com answered 500 once in ninety-nine + // requests, the reader's single probe landed on it, and the post lost its + // reader page for a day. A server saying it failed is not a server refusing. + const at = Date.parse('2026-08-18T08:54:06.232Z'); + const stored = { + status: /** @type {const} */ ('blocked'), + reason: 'http-500', + fetchedAt: '2026-08-18T08:54:06.232Z', + }; + + assert.equal(e.isTransient(stored), true); + assert.equal(e.shouldFetch(stored, at + e.TRANSIENT_RETRY_MS - 1000), false); + assert.equal(e.shouldFetch(stored, at + e.TRANSIENT_RETRY_MS), true); + // And emphatically sooner than the day a refusal gets. + assert.ok(e.TRANSIENT_RETRY_MS < e.RETRY_AFTER_MS); +}); + +test('a network error is transient too — we never heard an answer', () => { + const stored = { status: 'error', reason: 'fetch-failed', fetchedAt: '2026-08-18T08:54:06.232Z' }; + assert.equal(e.isTransient(stored), true); +}); + +test('a refusal is an answer, and keeps the full day', () => { + const at = Date.parse('2026-08-18T08:54:06.232Z'); + for (const reason of ['http-401', 'http-403', 'http-404', 'http-451', 'blocked-host']) { + const stored = { status: 'blocked', reason, fetchedAt: '2026-08-18T08:54:06.232Z' }; + assert.equal(e.isTransient(stored), false, reason); + assert.equal(e.shouldFetch(stored, at + e.TRANSIENT_RETRY_MS), false, reason); + assert.equal(e.shouldFetch(stored, at + e.RETRY_AFTER_MS), true, reason); + } +}); + +test('a paywall parses to nothing today and tomorrow, so it waits the day', () => { + const at = Date.parse('2026-08-18T08:54:06.232Z'); + const stored = { status: 'empty', reason: 'no-article', fetchedAt: '2026-08-18T08:54:06.232Z' }; + + assert.equal(e.isTransient(stored), false); + assert.equal(e.shouldFetch(stored, at + e.TRANSIENT_RETRY_MS), false); +}); From 33238bad93dcff3acb4f3f5b5491997d320a589e Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sat, 29 Aug 2026 08:15:59 +0000 Subject: [PATCH 2/2] feat(dock): route enclosures through the shared engine picker The dock set `src={track.src}` on its