diff --git a/CHANGELOG.md b/CHANGELOG.md index 4598894..3d6d74f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### `instrumentLlmClient` keeps the SDK's promise + +- `runtime-node`: `instrumentLlmClient` returned a plain `Promise` for every call, so `.withResponse()` and `.asResponse()` on an instrumented OpenAI or Anthropic client threw a `TypeError` in the app, which broke code that reads rate-limit headers or request ids. The SDK's own `APIPromise` is now returned, and the span ends when its body is parsed, whether through `await`, `.then()` or `.withResponse()`. A call read only through `.asResponse()` is not recorded, because its body is left for the caller. + ## [1.21.0] - 2026-11-17 Packages: `runtime-node` 1.17.0, and the Python adapter (`adapters/python`, copied, not published). The other packages are unchanged; the `otlp-ingester` README is updated without a release. diff --git a/packages/runtime-node/src/llm-instrument.ts b/packages/runtime-node/src/llm-instrument.ts index 5fe62b0..23ce6a0 100644 --- a/packages/runtime-node/src/llm-instrument.ts +++ b/packages/runtime-node/src/llm-instrument.ts @@ -281,6 +281,12 @@ function followEventStream( return stream; } +type ApiPromise = Promise & { parse(): Promise }; + +function isApiPromise(value: Promise): value is ApiPromise { + return typeof (value as Partial).parse === "function"; +} + function isAsyncIterable(value: unknown): value is object { return !!value && typeof value === "object" && Symbol.asyncIterator in value; } @@ -357,13 +363,23 @@ export function instrumentLlmClient( // synchronously. return settle(result, span, model, bedrock); } - return result.then( - (resolved) => settle(resolved, span, model, bedrock), - (err) => { - endWithError(span, err); - throw err; - }, - ); + const observe = (settled: Promise) => + settled.then( + (resolved) => settle(resolved, span, model, bedrock), + (err) => { + endWithError(span, err); + throw err; + }, + ); + if (!isApiPromise(result)) return observe(result); + // The SDKs' APIPromise parses the body only when `then`, `catch`, + // `finally` or `withResponse()` ask for it, all through `parse()`. + // Observing that call keeps the promise itself, its `withResponse()` + // and `asResponse()`, and leaves the body of `asResponse()` unread. + const parse = result.parse; + let observed: Promise | undefined; + result.parse = () => (observed ??= observe(parse.call(result))); + return result; } /** The model a call is for, or undefined when it isn't a model call. */ diff --git a/packages/runtime-node/test/llm-instrument.test.mjs b/packages/runtime-node/test/llm-instrument.test.mjs index d96d57e..bd409e5 100644 --- a/packages/runtime-node/test/llm-instrument.test.mjs +++ b/packages/runtime-node/test/llm-instrument.test.mjs @@ -6,6 +6,7 @@ import { InMemorySpanExporter, SimpleSpanProcessor, } from "@opentelemetry/sdk-trace-base"; +import { quietPeriod } from "./support/wait.mjs"; import { instrumentLlmClient, trackLlmCall, @@ -561,6 +562,164 @@ test("send() with any other command is not traced", async () => { assert.equal(exporter.getFinishedSpans().length, 0); }); +// ------------------------------------------------ SDK promise methods --- + +/** + * Mirrors the APIPromise of the openai and @anthropic-ai/sdk packages: a + * Promise subclass that resolves to nothing by itself, parses the response + * body when `then`, `catch`, `finally`, `parse` or `withResponse` ask for it, + * keeps the unparsed `Response` for `asResponse()`, and holds private state + * that only works with the real instance as `this`. + */ +class FakeApiPromise extends Promise { + #client; + constructor(client, responsePromise, parseResponse) { + super((resolve) => resolve(null)); + this.responsePromise = responsePromise; + this.parseResponse = parseResponse; + this.#client = client; + } + asResponse() { + return this.responsePromise.then((props) => props.response); + } + async withResponse() { + const [data, response] = await Promise.all([this.parse(), this.asResponse()]); + return { data, response, request_id: response.headers.get("x-request-id") }; + } + parse() { + this.parsedPromise ??= this.responsePromise.then((props) => this.parseResponse(this.#client, props)); + return this.parsedPromise; + } + then(onfulfilled, onrejected) { + return this.parse().then(onfulfilled, onrejected); + } + catch(onrejected) { + return this.parse().catch(onrejected); + } + finally(onfinally) { + return this.parse().finally(onfinally); + } +} + +const COMPLETION = { + model: "gpt-4o-2024-08-06", + usage: { prompt_tokens: 10000, completion_tokens: 500, total_tokens: 10500 }, +}; + +async function parseApiResponse(_client, { response, stream }) { + if (stream) return gen([{ model: COMPLETION.model, choices: [], usage: COMPLETION.usage }]); + const data = await response.json(); + return Object.defineProperty(data, "_request_id", { + value: response.headers.get("x-request-id"), + enumerable: false, + }); +} + +class ApiPromiseOpenAI { + baseURL = "https://api.openai.com/v1"; + chat = { + completions: { + create: (req) => + new FakeApiPromise( + this, + req.failWith + ? Promise.reject(req.failWith) + : Promise.resolve({ + stream: req.stream, + response: new Response(JSON.stringify(COMPLETION), { + headers: { "x-request-id": "req_1" }, + }), + }), + parseApiResponse, + ), + }, + }; +} + +const apiCall = { model: "gpt-4o", messages: [] }; + +test("the SDK's promise comes back from an instrumented call with its own methods", async () => { + const openai = instrumentLlmClient(new ApiPromiseOpenAI()); + const { attrs, ai } = await capture("api-promise-identity", async () => { + const call = openai.chat.completions.create(apiCall); + assert.ok(call instanceof FakeApiPromise); + assert.equal(typeof call.withResponse, "function"); + assert.equal(typeof call.asResponse, "function"); + const data = await call; + assert.equal(data._request_id, "req_1"); + }); + assert.equal(attrs["gen_ai.usage.input_tokens"], 10000); + assert.equal(ai.calls, 1); +}); + +test("withResponse() returns the parsed data, the response and the request id, and records the usage", async () => { + const openai = instrumentLlmClient(new ApiPromiseOpenAI()); + const { attrs, ai, span } = await capture("api-promise-with-response", async () => { + const { data, response, request_id } = await openai.chat.completions.create(apiCall).withResponse(); + assert.equal(data.model, COMPLETION.model); + assert.equal(request_id, "req_1"); + assert.equal(response.headers.get("x-request-id"), "req_1"); + }); + assert.equal(span.status.code, 1); + assert.equal(attrs["gen_ai.usage.input_tokens"], 10000); + assert.equal(attrs["gen_ai.usage.output_tokens"], 500); + assert.equal(ai.calls, 1); +}); + +test("asResponse() hands over the response with its body unread", async () => { + const openai = instrumentLlmClient(new ApiPromiseOpenAI()); + const response = await openai.chat.completions.create(apiCall).asResponse(); + assert.equal(response.bodyUsed, false); + assert.deepEqual(await response.json(), COMPLETION); +}); + +test("awaiting and withResponse() on the same call share one parse and end the span once", async () => { + const openai = instrumentLlmClient(new ApiPromiseOpenAI()); + const { ai } = await capture("api-promise-shared", async () => { + const call = openai.chat.completions.create(apiCall); + const awaited = await call; + const { data } = await call.withResponse(); + assert.equal(data, awaited); + await call.then((again) => assert.equal(again, awaited)); + }); + assert.equal(ai.calls, 1); +}); + +test("a stream from withResponse() still ends the span with its usage", async () => { + const openai = instrumentLlmClient(new ApiPromiseOpenAI()); + const { attrs, ai } = await capture("api-promise-stream", async () => { + const { data } = await openai.chat.completions.create({ ...apiCall, stream: true }).withResponse(); + for await (const _ of data) { + /* consume */ + } + }); + assert.equal(attrs["gen_ai.usage.input_tokens"], 10000); + assert.equal(attrs["autter.llm.streamed"], true); + assert.equal(ai.calls, 1); +}); + +test("a failed call rejects withResponse() and then() alike, ends the span once as an error and leaves no unhandled rejection", async () => { + const openai = instrumentLlmClient(new ApiPromiseOpenAI()); + const unhandled = []; + const record = (reason) => unhandled.push(reason); + process.on("unhandledRejection", record); + try { + const { span } = await capture("api-promise-failure", async () => { + await assert.rejects( + openai.chat.completions.create({ ...apiCall, failWith: new Error("rate limited") }).withResponse(), + /rate limited/, + ); + }); + assert.equal(span.status.code, 2); + const call = openai.chat.completions.create({ ...apiCall, failWith: new Error("overloaded") }); + await assert.rejects(call, /overloaded/); + await quietPeriod(); + assert.deepEqual(unhandled, []); + } finally { + process.off("unhandledRejection", record); + } +}); + // ------------------------------------------------------- manual helpers --- test("withLlmCall: the new fields emit the contract and the convention marker", async () => {