diff --git a/src/app/codes/[code]/route.ts b/src/app/codes/[code]/route.ts index 777cfa1..6ef7db7 100644 --- a/src/app/codes/[code]/route.ts +++ b/src/app/codes/[code]/route.ts @@ -1,16 +1,26 @@ import { NextRequest } from 'next/server'; +import { codeTarget } from '@/lib/proxy/codes'; import { forward } from '@/lib/proxy/forward'; import { searchApiBase } from '@/lib/searchApi'; -// The DFO APIs' problem-details type registry: an error body's -// `type: "/codes/"` resolves here, mirrored from dfo-apis's root (see -// src/lib/proxy/forward). The code is re-encoded so it stays one segment. -export const GET = (req: NextRequest, { params }: { params: Promise<{ code: string }> }) => - params.then(({ code }) => - forward(req, { - surface: 'api', - target: `${searchApiBase()}/codes/${encodeURIComponent(code)}`, - unreachable: 'The DFO APIs are unreachable.', - }), - ); +// The API's refusal-code registry (GET /codes/{code}), served at the same +// root path on the site's domain. A problem document's `type` is that +// root-relative path, so the link in it resolves here (see src/lib/proxy/codes). +export const GET = async (req: NextRequest, { params }: { params: Promise<{ code: string }> }) => { + const { code } = await params; + const target = codeTarget(searchApiBase(), code, req.nextUrl.search); + + if (!target) { + return Response.json( + { error: 'not_found', retry: 'do_not_retry', detail: 'There is no such refusal code.' }, + { status: 404 }, + ); + } + + return forward(req, { + surface: 'api', + target, + unreachable: 'The DFO APIs are unreachable.', + }); +}; diff --git a/src/lib/proxy/codes.ts b/src/lib/proxy/codes.ts new file mode 100644 index 0000000..53b1335 --- /dev/null +++ b/src/lib/proxy/codes.ts @@ -0,0 +1,15 @@ +/** + * The upstream address of the refusal-code registry. The API's problem + * documents carry a root-relative `type` (`/codes/request_invalid`), which + * resolves against the site's own host, so `src/app/codes/[code]/route.ts` + * forwards that path to the API. Kept apart from forward.ts, which imports + * server-only, so `node --test` can load it. + */ + +// Every registry code is snake_case, so a code of this shape stays in the +// one path segment after /codes/. `.` and `..` would resolve out of it. +const CODE = /^[a-z][a-z0-9_]*$/; + +/** The registry address for `code`, or null for a value no code can be. */ +export const codeTarget = (apiBase: string, code: string, search: string): string | null => + CODE.test(code) ? `${apiBase}/codes/${code}${search}` : null; diff --git a/src/lib/proxy/forward.ts b/src/lib/proxy/forward.ts index 60c19b3..f67eb38 100644 --- a/src/lib/proxy/forward.ts +++ b/src/lib/proxy/forward.ts @@ -7,16 +7,18 @@ import { apisFetch } from '@/lib/apisAuth'; import { limiterKeyFor, setCaller, trustedHops } from './caller'; import { relayBody } from './compress'; import { clientResponseHeaders, dropEncodedLength, pick, upstreamRequestHeaders } from './headers'; -import { hit, rateHeaders, type Surface } from './rateLimit'; +import { rateHeaders } from './rateHeaders'; +import { hit, type Surface } from './rateLimit'; /** * The platform proxy: the DFO APIs and the MCP servers mirrored 1:1 under * the site's own domain, so one public host serves the browser and outside * callers alike (the engines ship no CORS headers, and the harness token - * must not reach a client). An /api/* or /mcp call is rate-limited per IP - * (src/lib/proxy/rateLimit). A call is forwarded - * with its method, query, body, content headers and every mcp-* header - * (src/lib/proxy/headers), and the upstream's response streamed back: SSE + * must not reach a client). An /api/* or /mcp call, and a /codes call the + * route forwards, is rate-limited per IP (src/lib/proxy/rateLimit; the + * X-RateLimit-* headers go out only on a counted call). A call is forwarded + * with its method, query, body, content headers, user-agent and every mcp-* + * header (src/lib/proxy/headers), and the upstream's response streamed back: SSE * (search resolve, studio turns, MCP) and file downloads both depend on * nothing being buffered here. A text, JSON, XML or CSV body of 1,024 bytes * or more is gzipped on the way for a client that accepts it @@ -25,7 +27,7 @@ import { hit, rateHeaders, type Surface } from './rateLimit'; * * Upstreams: * /api/v1/search|datasets|graph/* → SEARCH_API_BASE (dfo-apis, src/lib/searchApi) - * /openapi.json, /codes/* → SEARCH_API_BASE, the same paths + * /openapi.json, /codes/{code} → SEARCH_API_BASE, the same paths * /api/v1/studio/turn|interrupt|cancel|composition * → STUDIO_API_BASE (dfo-ai), STUDIO_TOKEN * diff --git a/src/lib/proxy/headers.ts b/src/lib/proxy/headers.ts index a640cd0..ea411dd 100644 --- a/src/lib/proxy/headers.ts +++ b/src/lib/proxy/headers.ts @@ -2,26 +2,27 @@ * Which headers cross the platform proxy (src/lib/proxy/forward). Kept apart * from forward.ts, which imports server-only, so `node --test` can load it. * - * A request carries its content headers and every `mcp-*` header (the MCP + * A request carries its content headers, the caller's user-agent (the API + * logs it to tell client families apart) and every `mcp-*` header (the MCP * transport defines mcp-method, mcp-name, mcp-session-id and * mcp-protocol-version, and validates the first two against the body). A - * response carries its content headers, location (a redirect, as the - * Datasets API's file links answer), cache-control (the `no-store` on such a - * redirect to a signed URL, which forward.ts's default `no-cache` would - * weaken), retry-after (on a busy refusal), www-authenticate, server-timing, - * every `mcp-*` header and every `x-dfo-*` header (the APIs' x-dfo-trace-id - * and x-dfo-commit). The content-length is left behind when the upstream - * encoded the body, since fetch has decoded it by then. Matching by prefix - * means a protocol header added later needs no change here. No `x-dfo-*` - * header goes upstream from the client: authorization and x-dfo-caller are - * set by forward.ts. + * response carries its content headers, allow (the methods of a 405), + * location (a redirect, as the Datasets API's file links answer), + * cache-control (the `no-store` on such a redirect to a signed URL, which + * forward.ts's default `no-cache` would weaken), retry-after (on a busy + * refusal), www-authenticate, server-timing, every `mcp-*` header and every + * `x-dfo-*` header (the APIs' x-dfo-trace-id and x-dfo-commit). The + * content-length is left behind when the upstream encoded the body, since + * fetch has decoded it by then. Matching by prefix means a protocol header + * added later needs no change here. No `x-dfo-*` header goes upstream from + * the client: authorization and x-dfo-caller are set by forward.ts. */ const REQUEST_PREFIXES = ['mcp-']; const RESPONSE_PREFIXES = ['mcp-', 'x-dfo-']; // Request headers worth carrying upstream; everything else (cookies, the // site's Basic auth, hop-by-hop) stays here. -const REQUEST_HEADERS = ['content-type', 'accept']; +const REQUEST_HEADERS = ['content-type', 'accept', 'user-agent']; // Response headers worth carrying back; the rest is the upstream's own // business (server, date, transfer-encoding…). const RESPONSE_HEADERS = [ @@ -33,6 +34,7 @@ const RESPONSE_HEADERS = [ 'retry-after', 'www-authenticate', 'server-timing', + 'allow', ]; // Copy the named headers present on `source` onto `into`. diff --git a/src/lib/proxy/rateHeaders.ts b/src/lib/proxy/rateHeaders.ts new file mode 100644 index 0000000..986db11 --- /dev/null +++ b/src/lib/proxy/rateHeaders.ts @@ -0,0 +1,25 @@ +/** + * What a rate-limit check decided, and the headers that tell a client so. No + * `server-only` import, so node:test loads it. + */ +import type { Verdict } from './limitCounter'; + +/** A verdict, and whether a count ran to reach it. */ +export type RateVerdict = Verdict & { + // False for the verdict of the fail-open path (a database error or + // timeout): the call passed without a count, so no limit applies. + counted: boolean; +}; + +/** + * The standard rate-limit headers for a verdict. An uncounted verdict gets + * none, since no limit applies to describe. + */ +export const rateHeaders = (verdict: RateVerdict): Record => + verdict.counted + ? { + 'X-RateLimit-Limit': String(verdict.limit), + 'X-RateLimit-Remaining': String(verdict.remaining), + 'X-RateLimit-Reset': String(verdict.reset), + } + : {}; diff --git a/src/lib/proxy/rateLimit.ts b/src/lib/proxy/rateLimit.ts index 5366a10..6fdc91f 100644 --- a/src/lib/proxy/rateLimit.ts +++ b/src/lib/proxy/rateLimit.ts @@ -3,10 +3,11 @@ import 'server-only'; import { counterDb } from '@/lib/db'; import { limiterBucket } from './caller'; -import { countHit, pruneCounters, type Verdict } from './limitCounter'; +import { countHit, pruneCounters } from './limitCounter'; +import type { RateVerdict } from './rateHeaders'; /** - * Per-IP rate limit for the platform proxy's /api/* and /mcp routes + * Per-IP rate limit for the platform proxy's /api/*, /codes and /mcp calls * (src/lib/proxy/forward): a fixed window counted in Postgres * (./limitCounter), shared by every instance. An IPv6 client is counted by * its /64 (see limiterBucket). A count runs on its own small pool @@ -15,9 +16,14 @@ import { countHit, pruneCounters, type Verdict } from './limitCounter'; * timed-out count fails open: the * platform's own doors are the backstop, and the limit is a courtesy. * + * A fail-open call gets a verdict with `counted: false`, and the response + * carries no X-RateLimit-* headers (see rateHeaders), so the proxy advertises + * a limit only while one is counted. + * * The window and the allowance per surface are env-tunable: * RATE_LIMIT_WINDOW_S (default 60) - * RATE_LIMIT_API calls per window per IP on /api/* (default 120) + * RATE_LIMIT_API calls per window per IP on /api/* and /codes + * (default 120) * RATE_LIMIT_PIN calls per window per IP on /api/download-pin * (default 10) * RATE_LIMIT_MCP requests per window per IP on /mcp (default 300) @@ -44,12 +50,12 @@ let lastPrune = 0; /** * Count one call for `ip` on `surface` and say whether it is within the - * allowance. A database that fails or is slow lets the call through. Each + * allowance. A database that fails or is slow lets the call through, uncounted. Each * instance also deletes dead counters, at most once an hour, the Studio's * included (every Studio call passes here too). The delete uses the same * bounded counter pool as the counts. */ -export const hit = async (surface: Surface, ip: string): Promise => { +export const hit = async (surface: Surface, ip: string): Promise => { const limit = LIMITS[surface]; if (Date.now() - lastPrune > PRUNE_EVERY_MS) { @@ -58,17 +64,17 @@ export const hit = async (surface: Surface, ip: string): Promise => { } try { - return await countHit(counterDb, `${surface}:${limiterBucket(ip)}`, limit, WINDOW_S); + const verdict = await countHit( + counterDb, + `${surface}:${limiterBucket(ip)}`, + limit, + WINDOW_S, + ); + + return { ...verdict, counted: true }; } catch (error) { console.warn('rate limit unavailable, passing', error); - return { allowed: true, limit, remaining: limit, reset: WINDOW_S }; + return { allowed: true, counted: false, limit, remaining: limit, reset: WINDOW_S }; } }; - -/** The standard rate-limit headers for a verdict. */ -export const rateHeaders = (verdict: Verdict): Record => ({ - 'X-RateLimit-Limit': String(verdict.limit), - 'X-RateLimit-Remaining': String(verdict.remaining), - 'X-RateLimit-Reset': String(verdict.reset), -}); diff --git a/src/lib/proxy/test/codes.test.ts b/src/lib/proxy/test/codes.test.ts new file mode 100644 index 0000000..a7edfcf --- /dev/null +++ b/src/lib/proxy/test/codes.test.ts @@ -0,0 +1,20 @@ +/** + * `codes.ts`: the upstream address a `/codes/{code}` request forwards to. + */ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { codeTarget } from '../codes.ts'; + +test('codeTarget: puts the code at the API root, keeping the query', () => { + assert.equal( + codeTarget('https://api.example', 'request_invalid', '?lang=fr'), + 'https://api.example/codes/request_invalid?lang=fr', + ); +}); + +for (const code of ['..', '.', '../openapi.json', 'request_invalid/..', 'Request_Invalid', '']) { + test(`codeTarget: no address for ${JSON.stringify(code)}, which no code can be`, () => { + assert.equal(codeTarget('https://api.example', code, '?q'), null); + }); +} diff --git a/src/lib/proxy/test/headers.test.ts b/src/lib/proxy/test/headers.test.ts index b633d33..f697c05 100644 --- a/src/lib/proxy/test/headers.test.ts +++ b/src/lib/proxy/test/headers.test.ts @@ -42,6 +42,12 @@ test('upstreamRequestHeaders: keeps the content headers and the session and prot }); }); +test("upstreamRequestHeaders: carries the caller's user-agent", () => { + const out = upstreamRequestHeaders(new Headers({ 'User-Agent': 'dfo-python/0.4.1' })); + + assert.deepEqual(Object.fromEntries(out), { 'user-agent': 'dfo-python/0.4.1' }); +}); + test('upstreamRequestHeaders: drops credentials, x-dfo-* and other headers', () => { const out = upstreamRequestHeaders( new Headers({ @@ -165,6 +171,12 @@ test('clientResponseHeaders: carries the upstream Server-Timing and every x-dfo- }); }); +test('clientResponseHeaders: carries the Allow of a 405', () => { + const out = clientResponseHeaders(new Headers({ Allow: 'GET, HEAD' })); + + assert.deepEqual(Object.fromEntries(out), { allow: 'GET, HEAD' }); +}); + test('clientResponseHeaders: leaves the content-length of an encoded body behind', () => { const encoded = { 'content-type': 'application/json', 'content-length': '512' }; diff --git a/src/lib/proxy/test/rateHeaders.test.ts b/src/lib/proxy/test/rateHeaders.test.ts new file mode 100644 index 0000000..06b01ce --- /dev/null +++ b/src/lib/proxy/test/rateHeaders.test.ts @@ -0,0 +1,22 @@ +/** + * `rateHeaders.ts`: a counted call advertises its limit, an uncounted one + * advertises none. + */ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { rateHeaders, type RateVerdict } from '../rateHeaders.ts'; + +const counted: RateVerdict = { allowed: true, counted: true, limit: 120, remaining: 7, reset: 41 }; + +test('a counted call carries the three X-RateLimit headers', () => { + assert.deepEqual(rateHeaders(counted), { + 'X-RateLimit-Limit': '120', + 'X-RateLimit-Remaining': '7', + 'X-RateLimit-Reset': '41', + }); +}); + +test('a call passed without a count carries no X-RateLimit header', () => { + assert.deepEqual(rateHeaders({ ...counted, counted: false, remaining: 120, reset: 60 }), {}); +}); diff --git a/src/lib/search/storedFile.ts b/src/lib/search/storedFile.ts index 15108b8..ea806cb 100644 --- a/src/lib/search/storedFile.ts +++ b/src/lib/search/storedFile.ts @@ -3,7 +3,8 @@ import 'server-only'; import type { NextRequest } from 'next/server'; import { clientIp } from '@/lib/proxy/forward'; -import { hit, rateHeaders } from '@/lib/proxy/rateLimit'; +import { rateHeaders } from '@/lib/proxy/rateHeaders'; +import { hit } from '@/lib/proxy/rateLimit'; import { datasetsFetch } from '@/lib/search/datasetsFetch'; import { artifactHref } from '@/lib/search/searchClient'; import { storedDownload, type VersionLinkDocument } from '@/lib/search/versionLink'; @@ -21,8 +22,9 @@ const VERSION_READ_TIMEOUT_MS = 20_000; /** * Counts the call against the proxy's `api` allowance, the one every - * Datasets API call through the site shares. The headers go on the - * response either way, with `Retry-After` added when the call is over it. + * Datasets API call through the site shares. The X-RateLimit-* headers go + * on the response whenever the call was counted, with `Retry-After` added + * when the call is over it. */ export const downloadAllowance = async ( req: NextRequest,