diff --git a/apps/web/src/lib/corpus.js b/apps/web/src/lib/corpus.js index f609b0e..61dc6ab 100644 --- a/apps/web/src/lib/corpus.js +++ b/apps/web/src/lib/corpus.js @@ -37,8 +37,21 @@ const TTL_MS = 60 * 60 * 1000; */ const MAX_STALE_MS = 30 * 24 * 60 * 60 * 1000; -/** Shorter than the client's own deadline, so a hung read gives the page back. */ -const TIMEOUT_MS = 8 * 1000; +/** + * Shorter than the client's own deadline, so a hung read gives the page back. + * + * Fifteen seconds against roughly three of measured work, and the headroom is + * the lesson rather than the number. This shipped at eight against a read that + * took twenty-three, and the failure was not "the page was slow once" — it was + * that `remember` stores nothing when the computation fails, so the entry never + * filled and the two figures it feeds were missing from the sales page for good. + * A timeout under the cost of the work is not a safety margin, it is an off + * switch with a delay on it. + * + * Paid only on a genuinely cold cache: one success, by anyone, serves every + * later reader for an hour and is refreshed behind them after that. + */ +const TIMEOUT_MS = 15 * 1000; /** * Corpus-specific counts: full-text articles, authors, and publishers opted out. diff --git a/packages/db/src/dataset.js b/packages/db/src/dataset.js index f8a2b34..09a52bd 100644 --- a/packages/db/src/dataset.js +++ b/packages/db/src/dataset.js @@ -680,11 +680,33 @@ export async function optedOutCount(db) { * it snippets?") for a cost the site can pay, and calling it a sample costs * nothing but a word. * + * ## Why the sample is 2,000 and not 20,000 + * + * It shipped at 20,000 and that number silently emptied two rows off the sales + * page. Every row in the sample is a row lookup for a column no index covers, so + * the cost is linear in the sample and nothing else. Measured against production + * on 2026-08-29: + * + * count(*) where status='ok' 1,946ms 274,885 + * avg over 20,000 22,662ms 7,810 + * avg over 2,000 584ms 7,929 + * + * The 20,000 blew through `corpusFigures`' timeout, so the whole read failed — + * and a read-through cache stores nothing on failure, which is the exact trap + * `cache.js` was written to describe: it does not fail once and recover, it + * fails for ever, and the page renders its graceful fallback (no article count, + * no author count) permanently. + * + * Ten times the sample bought a difference of 1.5% in a figure already labelled + * "sampled", against 39 times the cost and a page that could not show it at all. + * If a defensible exact total is ever wanted, the answer is a `sum(text_length)` + * warmed by the poller on its patient connection, not a bigger sample here. + * * @param {Client} db * @returns {Promise<{ articles: number, sampledAvgChars: number, sampleSize: number }>} */ export async function articleFigures(db) { - const sampleSize = 20_000; + const sampleSize = 2_000; const [count, sample] = await Promise.all([ db.execute(`select count(*) as n from item_extracts where status = 'ok'`), diff --git a/packages/db/test/dataset.test.js b/packages/db/test/dataset.test.js index 25a9961..c2add61 100644 --- a/packages/db/test/dataset.test.js +++ b/packages/db/test/dataset.test.js @@ -374,6 +374,32 @@ test('an author arrives with their links parsed apart, and never with their addr assert.ok(!('email' in author), 'authors.email must never leave in the corpus'); }); +// ---------------------------------------------------------------- figures + +test('the article sample stays small enough to actually complete', async () => { + // A bound rather than an equality, because the number itself is a judgement + // call and may reasonably move. What must not move is the order of magnitude. + // + // `text_length` is covered by no index, so this read costs one row lookup per + // sampled row and nothing else: 2,000 measured at 584ms against production + // where 20,000 measured at 22,662ms. The second number exceeded the caller's + // timeout, and because a read-through cache stores nothing on failure, the two + // figures this feeds were absent from /sales permanently rather than briefly. + // + // So this guards the failure that actually happened. Raising the sample past + // this bound does not make the page slower — it makes it empty. + const figures = await dataset.articleFigures(db); + + assert.ok( + figures.sampleSize <= 5_000, + `sample of ${figures.sampleSize} risks exceeding the caller's timeout; see the note on articleFigures`, + ); + // Reported alongside the average so the page can say "sampled over N" rather + // than presenting an estimate as a total. + assert.equal(typeof figures.sampledAvgChars, 'number'); + assert.equal(figures.articles, 4, 'one ok extract per seeded feed'); +}); + // ---------------------------------------------------------------- enquiries test('an enquiry is stored, and repeats from one address are countable', async () => {