diff --git a/packages/social/src/x/fetch.js b/packages/social/src/x/fetch.js index 16e1887..0b653ae 100644 --- a/packages/social/src/x/fetch.js +++ b/packages/social/src/x/fetch.js @@ -109,16 +109,34 @@ export async function fetchXSource(feed, opts) { { sessions, onEvent, signal: opts.signal }, ); } catch (error) { - // A rate limit is a schedule instruction, not evidence about the account. - // Returning it as `throttled` routes it to `markThrottled`, which moves - // `next_fetch_at` and leaves every health column alone — the same treatment - // an ordinary publisher's 429 gets, and for the same reason (§16). - if (error?.name === 'XRateLimited') { - return { ok: false, throttled: true, retryAfter: error.retryAfter ?? null, error: 'rate-limited' }; + // **Exactly one kind of failure is about the source.** A deleted, suspended + // or protected account is a fact about the account; everything else — a rate + // limit, a provider outage, a dead session, no provider configured at all — + // is a fact about us or about the upstream, and recording it against the + // account is how a directory deletes itself. + // + // The arithmetic, because it is what makes this urgent rather than tidy: + // `markCrawlFailure` retires a feed at ten consecutive failures, and an X + // source polls on a five-minute floor. Ten strikes is **fifty minutes**. So + // switching X on before a provider is reachable — which is the ordinary + // order of operations, since the flag is how you find out — would quietly + // mark every X source dead within the hour, and nothing in the logs would + // say "no provider configured" rather than "these accounts are broken". + // + // Returning `throttled` routes to `markThrottled`, which moves + // `next_fetch_at` and leaves `status`, `error_count`, `last_error` and + // `last_success_at` exactly as they were — the same treatment an ordinary + // publisher's 429 gets, and for the same reason (§16, §40). + if (error?.name === 'XNoSuchSource') { + return { ok: false, error: String(error?.message ?? 'no-such-source').slice(0, 200) }; } - // A deleted, suspended or protected account is the one failure that is - // genuinely about the source, so it is the one that counts against it. - return { ok: false, error: String(error?.message ?? 'x-fetch-failed').slice(0, 200) }; + + return { + ok: false, + throttled: true, + retryAfter: retryAfterFor(error), + error: String(error?.message ?? 'x-fetch-failed').slice(0, 200), + }; } const posts = result.posts ?? []; @@ -149,6 +167,30 @@ export async function fetchXSource(feed, opts) { }; } +/** + * How long to wait before trying this source again, by what went wrong. + * + * The three intervals are three different guesses about when the situation + * changes. A rate limit usually names its own; a provider outage is minutes; + * and "no provider is configured" is a deployment that has not happened yet, so + * asking again in ten minutes is a thousand pointless wake-ups a day across the + * directory and answers no sooner than asking in an hour. + * + * @param {Error & { retryAfter?: number|null }} error + * @returns {number} seconds + */ +function retryAfterFor(error) { + if (error?.name === 'XRateLimited') { + return Number(error.retryAfter) > 0 ? Number(error.retryAfter) : ANOMALY_MINUTES * 60; + } + // Nothing is set up to collect with. Distinguished by message rather than by + // type because `XUnavailable` covers both this and a provider that is merely + // down, and the two deserve very different patience. + if (/no X provider is configured/i.test(String(error?.message ?? ''))) return 3600; + + return ANOMALY_MINUTES * 60; +} + /** * The per-source toggles of §6.3, with the PRD's defaults. * diff --git a/packages/social/test/providers.test.js b/packages/social/test/providers.test.js index f15fa3a..d14a2db 100644 --- a/packages/social/test/providers.test.js +++ b/packages/social/test/providers.test.js @@ -445,3 +445,74 @@ test('a source whose ref we cannot read fails loudly rather than crawling nothin assert.equal(result.ok, false); assert.equal(result.error, 'invalid-x-ref'); }); + +/* --------------------------------------------- what a failure may cost a source */ + +/* + * `markCrawlFailure` retires a feed at ten consecutive failures and an X source + * polls on a five-minute floor, so ten strikes is fifty minutes. That arithmetic + * is why exactly one of the four errors may reach it. + */ + +/** @param {Error} error */ +async function failWith(error) { + const registry = new XRegistry({ + env: { X_PRIMARY_PROVIDER: 'rsshub', X_FALLBACK_PROVIDERS: '' }, + providers: { rsshub: fake('rsshub', [error]) }, + }); + + return fetchXSource( + { social_ref: 'x:user:openai', feed_url: 'https://x.com/OpenAI', item_count: 5 }, + { runtime: { registry, sessions: null, onEvent: () => {} } }, + ); +} + +test('only a missing account counts against the source', async () => { + const gone = await failWith(new XNoSuchSource('no such account')); + assert.equal(gone.ok, false); + assert.equal(gone.throttled, undefined, 'a deleted account is a fact about the account'); +}); + +test('a provider outage reschedules rather than blaming the account', async () => { + const down = await failWith(new XUnavailable('rsshub: upstream-502')); + assert.equal(down.ok, false); + assert.equal(down.throttled, true); + assert.ok(down.retryAfter > 0); +}); + +test('a dead session reschedules — it is our credential, not their account', async () => { + const dead = await failWith(new XAuthFailed('auth-failed-401')); + assert.equal(dead.throttled, true); +}); + +test('X switched on before a provider exists must not retire the directory', async () => { + // The ordinary order of operations: the flag is how you find out whether the + // provider is reachable. With no provider configured the registry refuses + // outright, and that refusal must never look like a broken account. + const registry = new XRegistry({ + env: { X_PRIMARY_PROVIDER: 'rsshub', X_FALLBACK_PROVIDERS: '' }, + providers: { + rsshub: { ...fake('rsshub', [[]]), configured: () => false }, + }, + }); + + const result = await fetchXSource( + { social_ref: 'x:user:openai', feed_url: 'https://x.com/OpenAI', item_count: 5 }, + { runtime: { registry, sessions: null, onEvent: () => {} } }, + ); + + assert.equal(result.ok, false); + assert.equal(result.throttled, true, 'no provider configured is a deployment state, not a dead account'); + // An hour, not ten minutes: nothing changes until somebody deploys something. + assert.equal(result.retryAfter, 3600); +}); + +test('a rate limit still carries the interval the server named', async () => { + const limited = await failWith(new XRateLimited('429', { retryAfter: 90 })); + assert.equal(limited.throttled, true); + assert.equal(limited.retryAfter, 90); + + // And falls back to something sane when it named none. + const bare = await failWith(new XRateLimited('429')); + assert.ok(bare.retryAfter > 0); +});