Make the corpus figures cheap enough to ever appear

apps/web/src/lib/corpus.js

@@ -37,8 +37,21 @@ const TTL_MS = 60 * 60 * 1000;
  */
 const MAX_STALE_MS = 30 * 24 * 60 * 60 * 1000;
 
-/** Shorter than the client's own deadline, so a hung read gives the page back. */
-const TIMEOUT_MS = 8 * 1000;
+/**
+ * Shorter than the client's own deadline, so a hung read gives the page back.
+ *
+ * Fifteen seconds against roughly three of measured work, and the headroom is
+ * the lesson rather than the number. This shipped at eight against a read that
+ * took twenty-three, and the failure was not "the page was slow once" — it was
+ * that `remember` stores nothing when the computation fails, so the entry never
+ * filled and the two figures it feeds were missing from the sales page for good.
+ * A timeout under the cost of the work is not a safety margin, it is an off
+ * switch with a delay on it.
+ *
+ * Paid only on a genuinely cold cache: one success, by anyone, serves every
+ * later reader for an hour and is refreshed behind them after that.
+ */
+const TIMEOUT_MS = 15 * 1000;
 
 /**
  * Corpus-specific counts: full-text articles, authors, and publishers opted out.

packages/db/src/dataset.js

@@ -680,11 +680,33 @@ export async function optedOutCount(db) {
  * it snippets?") for a cost the site can pay, and calling it a sample costs
  * nothing but a word.
  *
+ * ## Why the sample is 2,000 and not 20,000
+ *
+ * It shipped at 20,000 and that number silently emptied two rows off the sales
+ * page. Every row in the sample is a row lookup for a column no index covers, so
+ * the cost is linear in the sample and nothing else. Measured against production
+ * on 2026-08-29:
+ *
+ *     count(*) where status='ok'      1,946ms   274,885
+ *     avg over 20,000                22,662ms     7,810
+ *     avg over 2,000                    584ms     7,929
+ *
+ * The 20,000 blew through `corpusFigures`' timeout, so the whole read failed —
+ * and a read-through cache stores nothing on failure, which is the exact trap
+ * `cache.js` was written to describe: it does not fail once and recover, it
+ * fails for ever, and the page renders its graceful fallback (no article count,
+ * no author count) permanently.
+ *
+ * Ten times the sample bought a difference of 1.5% in a figure already labelled
+ * "sampled", against 39 times the cost and a page that could not show it at all.
+ * If a defensible exact total is ever wanted, the answer is a `sum(text_length)`
+ * warmed by the poller on its patient connection, not a bigger sample here.
+ *
  * @param {Client} db
  * @returns {Promise<{ articles: number, sampledAvgChars: number, sampleSize: number }>}
  */
 export async function articleFigures(db) {
-  const sampleSize = 20_000;
+  const sampleSize = 2_000;
 
   const [count, sample] = await Promise.all([
     db.execute(`select count(*) as n from item_extracts where status = 'ok'`),

packages/db/test/dataset.test.js

@@ -374,6 +374,32 @@ test('an author arrives with their links parsed apart, and never with their addr
   assert.ok(!('email' in author), 'authors.email must never leave in the corpus');
 });
 
+// ---------------------------------------------------------------- figures
+
+test('the article sample stays small enough to actually complete', async () => {
+  // A bound rather than an equality, because the number itself is a judgement
+  // call and may reasonably move. What must not move is the order of magnitude.
+  //
+  // `text_length` is covered by no index, so this read costs one row lookup per
+  // sampled row and nothing else: 2,000 measured at 584ms against production
+  // where 20,000 measured at 22,662ms. The second number exceeded the caller's
+  // timeout, and because a read-through cache stores nothing on failure, the two
+  // figures this feeds were absent from /sales permanently rather than briefly.
+  //
+  // So this guards the failure that actually happened. Raising the sample past
+  // this bound does not make the page slower — it makes it empty.
+  const figures = await dataset.articleFigures(db);
+
+  assert.ok(
+    figures.sampleSize <= 5_000,
+    `sample of ${figures.sampleSize} risks exceeding the caller's timeout; see the note on articleFigures`,
+  );
+  // Reported alongside the average so the page can say "sampled over N" rather
+  // than presenting an estimate as a total.
+  assert.equal(typeof figures.sampledAvgChars, 'number');
+  assert.equal(figures.articles, 4, 'one ok extract per seeded feed');
+});
+
 // ---------------------------------------------------------------- enquiries
 
 test('an enquiry is stored, and repeats from one address are countable', async () => {