fix(proxy): forward user-agent, pass back Allow, validate /codes, and drop rate headers on uncounted calls

src/app/codes/[code]/route.ts

@@ -1,16 +1,26 @@
 import { NextRequest } from 'next/server';
 
+import { codeTarget } from '@/lib/proxy/codes';
 import { forward } from '@/lib/proxy/forward';
 import { searchApiBase } from '@/lib/searchApi';
 
-// The DFO APIs' problem-details type registry: an error body's
-// `type: "/codes/<code>"` resolves here, mirrored from dfo-apis's root (see
-// src/lib/proxy/forward). The code is re-encoded so it stays one segment.
-export const GET = (req: NextRequest, { params }: { params: Promise<{ code: string }> }) =>
-    params.then(({ code }) =>
-        forward(req, {
-            surface: 'api',
-            target: `${searchApiBase()}/codes/${encodeURIComponent(code)}`,
-            unreachable: 'The DFO APIs are unreachable.',
-        }),
-    );
+// The API's refusal-code registry (GET /codes/{code}), served at the same
+// root path on the site's domain. A problem document's `type` is that
+// root-relative path, so the link in it resolves here (see src/lib/proxy/codes).
+export const GET = async (req: NextRequest, { params }: { params: Promise<{ code: string }> }) => {
+    const { code } = await params;
+    const target = codeTarget(searchApiBase(), code, req.nextUrl.search);
+
+    if (!target) {
+        return Response.json(
+            { error: 'not_found', retry: 'do_not_retry', detail: 'There is no such refusal code.' },
+            { status: 404 },
+        );
+    }
+
+    return forward(req, {
+        surface: 'api',
+        target,
+        unreachable: 'The DFO APIs are unreachable.',
+    });
+};

src/lib/proxy/codes.ts

@@ -0,0 +1,15 @@
+/**
+ * The upstream address of the refusal-code registry. The API's problem
+ * documents carry a root-relative `type` (`/codes/request_invalid`), which
+ * resolves against the site's own host, so `src/app/codes/[code]/route.ts`
+ * forwards that path to the API. Kept apart from forward.ts, which imports
+ * server-only, so `node --test` can load it.
+ */
+
+// Every registry code is snake_case, so a code of this shape stays in the
+// one path segment after /codes/. `.` and `..` would resolve out of it.
+const CODE = /^[a-z][a-z0-9_]*$/;
+
+/** The registry address for `code`, or null for a value no code can be. */
+export const codeTarget = (apiBase: string, code: string, search: string): string | null =>
+    CODE.test(code) ? `${apiBase}/codes/${code}${search}` : null;

src/lib/proxy/forward.ts

@@ -7,16 +7,18 @@ import { apisFetch } from '@/lib/apisAuth';
 import { limiterKeyFor, setCaller, trustedHops } from './caller';
 import { relayBody } from './compress';
 import { clientResponseHeaders, dropEncodedLength, pick, upstreamRequestHeaders } from './headers';
-import { hit, rateHeaders, type Surface } from './rateLimit';
+import { rateHeaders } from './rateHeaders';
+import { hit, type Surface } from './rateLimit';
 
 /**
  * The platform proxy: the DFO APIs and the MCP servers mirrored 1:1 under
  * the site's own domain, so one public host serves the browser and outside
  * callers alike (the engines ship no CORS headers, and the harness token
- * must not reach a client). An /api/* or /mcp call is rate-limited per IP
- * (src/lib/proxy/rateLimit). A call is forwarded
- * with its method, query, body, content headers and every mcp-* header
- * (src/lib/proxy/headers), and the upstream's response streamed back: SSE
+ * must not reach a client). An /api/* or /mcp call, and a /codes call the
+ * route forwards, is rate-limited per IP (src/lib/proxy/rateLimit; the
+ * X-RateLimit-* headers go out only on a counted call). A call is forwarded
+ * with its method, query, body, content headers, user-agent and every mcp-*
+ * header (src/lib/proxy/headers), and the upstream's response streamed back: SSE
  * (search resolve, studio turns, MCP) and file downloads both depend on
  * nothing being buffered here. A text, JSON, XML or CSV body of 1,024 bytes
  * or more is gzipped on the way for a client that accepts it
@@ -25,7 +27,7 @@ import { hit, rateHeaders, type Surface } from './rateLimit';
  *
  * Upstreams:
  *   /api/v1/search|datasets|graph/*  → SEARCH_API_BASE (dfo-apis, src/lib/searchApi)
- *   /openapi.json, /codes/*          → SEARCH_API_BASE, the same paths
+ *   /openapi.json, /codes/{code}     → SEARCH_API_BASE, the same paths
  *   /api/v1/studio/turn|interrupt|cancel|composition
  *                                    → STUDIO_API_BASE (dfo-ai), STUDIO_TOKEN
  *

src/lib/proxy/headers.ts

@@ -2,26 +2,27 @@
  * Which headers cross the platform proxy (src/lib/proxy/forward). Kept apart
  * from forward.ts, which imports server-only, so `node --test` can load it.
  *
- * A request carries its content headers and every `mcp-*` header (the MCP
+ * A request carries its content headers, the caller's user-agent (the API
+ * logs it to tell client families apart) and every `mcp-*` header (the MCP
  * transport defines mcp-method, mcp-name, mcp-session-id and
  * mcp-protocol-version, and validates the first two against the body). A
- * response carries its content headers, location (a redirect, as the
- * Datasets API's file links answer), cache-control (the `no-store` on such a
- * redirect to a signed URL, which forward.ts's default `no-cache` would
- * weaken), retry-after (on a busy refusal), www-authenticate, server-timing,
- * every `mcp-*` header and every `x-dfo-*` header (the APIs' x-dfo-trace-id
- * and x-dfo-commit). The content-length is left behind when the upstream
- * encoded the body, since fetch has decoded it by then. Matching by prefix
- * means a protocol header added later needs no change here. No `x-dfo-*`
- * header goes upstream from the client: authorization and x-dfo-caller are
- * set by forward.ts.
+ * response carries its content headers, allow (the methods of a 405),
+ * location (a redirect, as the Datasets API's file links answer),
+ * cache-control (the `no-store` on such a redirect to a signed URL, which
+ * forward.ts's default `no-cache` would weaken), retry-after (on a busy
+ * refusal), www-authenticate, server-timing, every `mcp-*` header and every
+ * `x-dfo-*` header (the APIs' x-dfo-trace-id and x-dfo-commit). The
+ * content-length is left behind when the upstream encoded the body, since
+ * fetch has decoded it by then. Matching by prefix means a protocol header
+ * added later needs no change here. No `x-dfo-*` header goes upstream from
+ * the client: authorization and x-dfo-caller are set by forward.ts.
  */
 const REQUEST_PREFIXES = ['mcp-'];
 const RESPONSE_PREFIXES = ['mcp-', 'x-dfo-'];
 
 // Request headers worth carrying upstream; everything else (cookies, the
 // site's Basic auth, hop-by-hop) stays here.
-const REQUEST_HEADERS = ['content-type', 'accept'];
+const REQUEST_HEADERS = ['content-type', 'accept', 'user-agent'];
 // Response headers worth carrying back; the rest is the upstream's own
 // business (server, date, transfer-encoding…).
 const RESPONSE_HEADERS = [
@@ -33,6 +34,7 @@ const RESPONSE_HEADERS = [
     'retry-after',
     'www-authenticate',
     'server-timing',
+    'allow',
 ];
 
 // Copy the named headers present on `source` onto `into`.

src/lib/proxy/rateHeaders.ts

@@ -0,0 +1,25 @@
+/**
+ * What a rate-limit check decided, and the headers that tell a client so. No
+ * `server-only` import, so node:test loads it.
+ */
+import type { Verdict } from './limitCounter';
+
+/** A verdict, and whether a count ran to reach it. */
+export type RateVerdict = Verdict & {
+    // False for the verdict of the fail-open path (a database error or
+    // timeout): the call passed without a count, so no limit applies.
+    counted: boolean;
+};
+
+/**
+ * The standard rate-limit headers for a verdict. An uncounted verdict gets
+ * none, since no limit applies to describe.
+ */
+export const rateHeaders = (verdict: RateVerdict): Record<string, string> =>
+    verdict.counted
+        ? {
+              'X-RateLimit-Limit': String(verdict.limit),
+              'X-RateLimit-Remaining': String(verdict.remaining),
+              'X-RateLimit-Reset': String(verdict.reset),
+          }
+        : {};

src/lib/proxy/rateLimit.ts

@@ -3,10 +3,11 @@ import 'server-only';
 import { counterDb } from '@/lib/db';
 
 import { limiterBucket } from './caller';
-import { countHit, pruneCounters, type Verdict } from './limitCounter';
+import { countHit, pruneCounters } from './limitCounter';
+import type { RateVerdict } from './rateHeaders';
 
 /**
- * Per-IP rate limit for the platform proxy's /api/* and /mcp routes
+ * Per-IP rate limit for the platform proxy's /api/*, /codes and /mcp calls
  * (src/lib/proxy/forward): a fixed window counted in Postgres
  * (./limitCounter), shared by every instance. An IPv6 client is counted by
  * its /64 (see limiterBucket). A count runs on its own small pool
@@ -15,9 +16,14 @@ import { countHit, pruneCounters, type Verdict } from './limitCounter';
  * timed-out count fails open: the
  * platform's own doors are the backstop, and the limit is a courtesy.
  *
+ * A fail-open call gets a verdict with `counted: false`, and the response
+ * carries no X-RateLimit-* headers (see rateHeaders), so the proxy advertises
+ * a limit only while one is counted.
+ *
  * The window and the allowance per surface are env-tunable:
  *   RATE_LIMIT_WINDOW_S   (default 60)
- *   RATE_LIMIT_API        calls per window per IP on /api/* (default 120)
+ *   RATE_LIMIT_API        calls per window per IP on /api/* and /codes
+ *                         (default 120)
  *   RATE_LIMIT_PIN        calls per window per IP on /api/download-pin
  *                         (default 10)
  *   RATE_LIMIT_MCP        requests per window per IP on /mcp (default 300)
@@ -44,12 +50,12 @@ let lastPrune = 0;
 
 /**
  * Count one call for `ip` on `surface` and say whether it is within the
- * allowance. A database that fails or is slow lets the call through. Each
+ * allowance. A database that fails or is slow lets the call through, uncounted. Each
  * instance also deletes dead counters, at most once an hour, the Studio's
  * included (every Studio call passes here too). The delete uses the same
  * bounded counter pool as the counts.
  */
-export const hit = async (surface: Surface, ip: string): Promise<Verdict> => {
+export const hit = async (surface: Surface, ip: string): Promise<RateVerdict> => {
     const limit = LIMITS[surface];
 
     if (Date.now() - lastPrune > PRUNE_EVERY_MS) {
@@ -58,17 +64,17 @@ export const hit = async (surface: Surface, ip: string): Promise<Verdict> => {
     }
 
     try {
-        return await countHit(counterDb, `${surface}:${limiterBucket(ip)}`, limit, WINDOW_S);
+        const verdict = await countHit(
+            counterDb,
+            `${surface}:${limiterBucket(ip)}`,
+            limit,
+            WINDOW_S,
+        );
+
+        return { ...verdict, counted: true };
     } catch (error) {
         console.warn('rate limit unavailable, passing', error);
 
-        return { allowed: true, limit, remaining: limit, reset: WINDOW_S };
+        return { allowed: true, counted: false, limit, remaining: limit, reset: WINDOW_S };
     }
 };
-
-/** The standard rate-limit headers for a verdict. */
-export const rateHeaders = (verdict: Verdict): Record<string, string> => ({
-    'X-RateLimit-Limit': String(verdict.limit),
-    'X-RateLimit-Remaining': String(verdict.remaining),
-    'X-RateLimit-Reset': String(verdict.reset),
-});

src/lib/proxy/test/codes.test.ts

@@ -0,0 +1,20 @@
+/**
+ * `codes.ts`: the upstream address a `/codes/{code}` request forwards to.
+ */
+import assert from 'node:assert/strict';
+import test from 'node:test';
+
+import { codeTarget } from '../codes.ts';
+
+test('codeTarget: puts the code at the API root, keeping the query', () => {
+    assert.equal(
+        codeTarget('https://api.example', 'request_invalid', '?lang=fr'),
+        'https://api.example/codes/request_invalid?lang=fr',
+    );
+});
+
+for (const code of ['..', '.', '../openapi.json', 'request_invalid/..', 'Request_Invalid', '']) {
+    test(`codeTarget: no address for ${JSON.stringify(code)}, which no code can be`, () => {
+        assert.equal(codeTarget('https://api.example', code, '?q'), null);
+    });
+}

src/lib/proxy/test/headers.test.ts

@@ -42,6 +42,12 @@ test('upstreamRequestHeaders: keeps the content headers and the session and prot
     });
 });
 
+test("upstreamRequestHeaders: carries the caller's user-agent", () => {
+    const out = upstreamRequestHeaders(new Headers({ 'User-Agent': 'dfo-python/0.4.1' }));
+
+    assert.deepEqual(Object.fromEntries(out), { 'user-agent': 'dfo-python/0.4.1' });
+});
+
 test('upstreamRequestHeaders: drops credentials, x-dfo-* and other headers', () => {
     const out = upstreamRequestHeaders(
         new Headers({
@@ -165,6 +171,12 @@ test('clientResponseHeaders: carries the upstream Server-Timing and every x-dfo-
     });
 });
 
+test('clientResponseHeaders: carries the Allow of a 405', () => {
+    const out = clientResponseHeaders(new Headers({ Allow: 'GET, HEAD' }));
+
+    assert.deepEqual(Object.fromEntries(out), { allow: 'GET, HEAD' });
+});
+
 test('clientResponseHeaders: leaves the content-length of an encoded body behind', () => {
     const encoded = { 'content-type': 'application/json', 'content-length': '512' };
 

src/lib/proxy/test/rateHeaders.test.ts

@@ -0,0 +1,22 @@
+/**
+ * `rateHeaders.ts`: a counted call advertises its limit, an uncounted one
+ * advertises none.
+ */
+import assert from 'node:assert/strict';
+import test from 'node:test';
+
+import { rateHeaders, type RateVerdict } from '../rateHeaders.ts';
+
+const counted: RateVerdict = { allowed: true, counted: true, limit: 120, remaining: 7, reset: 41 };
+
+test('a counted call carries the three X-RateLimit headers', () => {
+    assert.deepEqual(rateHeaders(counted), {
+        'X-RateLimit-Limit': '120',
+        'X-RateLimit-Remaining': '7',
+        'X-RateLimit-Reset': '41',
+    });
+});
+
+test('a call passed without a count carries no X-RateLimit header', () => {
+    assert.deepEqual(rateHeaders({ ...counted, counted: false, remaining: 120, reset: 60 }), {});
+});

src/lib/search/storedFile.ts

@@ -3,7 +3,8 @@ import 'server-only';
 import type { NextRequest } from 'next/server';
 
 import { clientIp } from '@/lib/proxy/forward';
-import { hit, rateHeaders } from '@/lib/proxy/rateLimit';
+import { rateHeaders } from '@/lib/proxy/rateHeaders';
+import { hit } from '@/lib/proxy/rateLimit';
 import { datasetsFetch } from '@/lib/search/datasetsFetch';
 import { artifactHref } from '@/lib/search/searchClient';
 import { storedDownload, type VersionLinkDocument } from '@/lib/search/versionLink';
@@ -21,8 +22,9 @@ const VERSION_READ_TIMEOUT_MS = 20_000;
 
 /**
  * Counts the call against the proxy's `api` allowance, the one every
- * Datasets API call through the site shares. The headers go on the
- * response either way, with `Retry-After` added when the call is over it.
+ * Datasets API call through the site shares. The X-RateLimit-* headers go
+ * on the response whenever the call was counted, with `Retry-After` added
+ * when the call is over it.
  */
 export const downloadAllowance = async (
     req: NextRequest,