From 7844ff34132622aa5174e035c31ba07edb02541d Mon Sep 17 00:00:00 2001 From: Kunal Roy Choudhury <71250874+kunalgoesbyken@users.noreply.github.com> Date: Tue, 17 Nov 2026 16:53:51 +0530 Subject: [PATCH] runtime-node: end the LLM span for streams read with toReadableStream() or tee() wrapStream proxied the stream and only intercepted Symbol.asyncIterator. Every other method was bound to the raw stream, so toReadableStream() and tee() read chunks the finisher never saw, and the span of a call served with new Response(stream.toReadableStream()) never ended or exported. The openai and @anthropic-ai/sdk streams read a public iterator field from [Symbol.asyncIterator], tee() and toReadableStream(), and use private fields that only work with the stream as this, so a proxy receiver is not an option. wrapStream now wraps that source on the stream itself (or Symbol.asyncIterator for other async iterables) and returns the stream unchanged. One finisher covers iteration, both branches of a tee, cancel and error. A stream that cannot be modified still works and its span ends without usage. --- CHANGELOG.md | 4 + packages/runtime-node/src/llm-instrument.ts | 97 ++++--- .../runtime-node/test/llm-instrument.test.mjs | 257 ++++++++++++++++++ 3 files changed, 317 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4598894..64fd48a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Streams read with `toReadableStream()` or `tee()` are recorded + +- `runtime-node`: `instrumentLlmClient` ended the span of a streamed OpenAI or Anthropic call only when the stream was iterated. `new Response(stream.toReadableStream())` (the usual Next.js route handler) and `stream.tee()` read the stream without iterating it, so the span never ended and the call, its tokens and its cost were never exported. The stream is now instrumented at its own `iterator` source, so every way of reading it, and every branch of a tee, ends the span once, on completion, error or cancel. The result is the SDK's own stream object instead of a proxy of it. + ## [1.21.0] - 2026-11-17 Packages: `runtime-node` 1.17.0, and the Python adapter (`adapters/python`, copied, not published). The other packages are unchanged; the `otlp-ingester` README is updated without a release. diff --git a/packages/runtime-node/src/llm-instrument.ts b/packages/runtime-node/src/llm-instrument.ts index 5fe62b0..2c3d03e 100644 --- a/packages/runtime-node/src/llm-instrument.ts +++ b/packages/runtime-node/src/llm-instrument.ts @@ -153,11 +153,18 @@ function streamFinisher(span: Span, model: string, seen: RawUsage) { } /** - * Wrap a streaming result so the span ends when the stream finishes (or - * errors), accumulating usage from the chunks that carry it: OpenAI's final - * chunk (with `stream_options: { include_usage: true }`), OpenAI Responses' + * Make a streaming result end the span when the stream finishes (or errors), + * accumulating usage from the chunks that carry it: OpenAI's final chunk + * (with `stream_options: { include_usage: true }`), OpenAI Responses' * `response.completed`, Anthropic's message_start/message_delta events, * Google's per-chunk usageMetadata, Bedrock's ConverseStream `metadata`. + * + * The stream itself is instrumented, not proxied. The openai and + * @anthropic-ai/sdk streams read their own `iterator` field from + * `[Symbol.asyncIterator]`, `tee()` and `toReadableStream()`, and use private + * fields that only work with the stream as `this`, so wrapping that one + * source covers every way of reading it and every branch of a tee. Other + * async iterables are wrapped at `[Symbol.asyncIterator]`. */ function wrapStream( stream: T, @@ -167,46 +174,54 @@ function wrapStream( ): T { const seen: RawUsage = {}; const finish = streamFinisher(span, model, seen); - return new Proxy(stream, { - get(target, prop, receiver) { - if (prop === Symbol.asyncIterator) { - return () => { - const inner = ( - target as unknown as AsyncIterable - )[Symbol.asyncIterator](); - return { - async next(): Promise> { - try { - const step = await inner.next(); - if (!step.done) { - mergeRawUsage(seen, extractRawUsage(step.value, provider)); - } else { - finish(); - } - return step; - } catch (err) { - finish({ error: err }); - throw err; - } - }, - async return(value?: unknown): Promise> { - finish(); - return inner.return - ? inner.return(value) - : { done: true, value: undefined }; - }, - async throw(err?: unknown): Promise> { - finish({ error: err }); - if (inner.throw) return inner.throw(err); - throw err; - }, - }; - }; + const entry = + typeof (stream as { iterator?: unknown }).iterator === "function" + ? "iterator" + : Symbol.asyncIterator; + const open = (stream as Record AsyncIterator>)[entry]; + try { + Object.defineProperty(stream, entry, { + value: () => trackIterator(open.call(stream), finish, seen, provider), + configurable: true, + writable: true, + }); + } catch { + finish(); + } + return stream; +} + +function trackIterator( + inner: AsyncIterator, + finish: ReturnType, + seen: RawUsage, + provider: string, +): AsyncIterator { + return { + async next(): Promise> { + try { + const step = await inner.next(); + if (!step.done) { + mergeRawUsage(seen, extractRawUsage(step.value, provider)); + } else { + finish(); + } + return step; + } catch (err) { + finish({ error: err }); + throw err; } - const value = Reflect.get(target, prop, receiver); - return typeof value === "function" ? value.bind(target) : value; }, - }); + async return(value?: unknown): Promise> { + finish(); + return inner.return ? inner.return(value) : { done: true, value: undefined }; + }, + async throw(err?: unknown): Promise> { + finish({ error: err }); + if (inner.throw) return inner.throw(err); + throw err; + }, + }; } interface EventStreamHelper { diff --git a/packages/runtime-node/test/llm-instrument.test.mjs b/packages/runtime-node/test/llm-instrument.test.mjs index d96d57e..aec4f76 100644 --- a/packages/runtime-node/test/llm-instrument.test.mjs +++ b/packages/runtime-node/test/llm-instrument.test.mjs @@ -1,5 +1,6 @@ import assert from "node:assert/strict"; import { test } from "node:test"; +import { types } from "node:util"; import { trace } from "@opentelemetry/api"; import { BasicTracerProvider, @@ -13,6 +14,7 @@ import { withRuntimeOperation, } from "../dist/index.js"; import { captureRuntime, expectOperation } from "../dist/testing.js"; +import { until } from "./support/wait.mjs"; // No initAutterServer: autterLlmTracer() falls back to the global provider, // so an in-memory exporter sees every LLM span with its raw attributes. @@ -180,6 +182,261 @@ test("OpenAI Responses stream: usage from response.completed", async () => { assert.equal(attrs["gen_ai.response.model"], "gpt-5.6-terra"); }); +// ------------------------------------------- SDK stream entry points --- + +/** + * Mirrors the Stream class of the openai and @anthropic-ai/sdk packages: the + * source is a public `iterator` field, `[Symbol.asyncIterator]` and + * `toReadableStream()` read it through `this`, `tee()` splits it, and a + * native private field is touched on cancel, so the stream only works when + * `this` is the stream itself and not a proxy of it. + */ +class FakeStream { + #teeBranch = false; + constructor(iterator, controller = new AbortController()) { + this.iterator = iterator; + this.controller = controller; + } + [Symbol.asyncIterator]() { + return this.iterator(); + } + openForTee() { + return this.iterator(); + } + tee() { + const source = this.openForTee(); + const queues = [[], []]; + const open = [true, true]; + const branch = (index) => { + const stream = new FakeStream(() => ({ + next() { + if (!open[index]) return Promise.resolve({ value: undefined, done: true }); + if (queues[index].length === 0) { + const result = source.next(); + queues[0].push(result); + queues[1].push(result); + } + return queues[index].shift(); + }, + async return() { + open[index] = false; + if (!open[0] && !open[1]) await source.return?.(); + return { value: undefined, done: true }; + }, + })); + stream.#teeBranch = true; + return stream; + }; + return [branch(0), branch(1)]; + } + toReadableStream() { + let iter; + let cancellation; + const release = () => (cancellation ??= this.#release(iter)); + return new ReadableStream({ + start: async () => { + iter = this[Symbol.asyncIterator](); + }, + pull: async (controller) => { + try { + const { value, done } = await iter.next(); + if (done) return controller.close(); + controller.enqueue(new TextEncoder().encode(`${JSON.stringify(value)}\n`)); + } catch (err) { + controller.error(err); + void release().catch(() => undefined); + } + }, + cancel: release, + }); + } + async #release(iter) { + if (!this.#teeBranch) this.controller.abort(); + await iter.return?.(); + } +} + +class AnthropicShapedStream extends FakeStream { + openForTee() { + return this[Symbol.asyncIterator](); + } +} + +const STREAM_CHUNKS = [ + { model: "gpt-4o-2024-08-06", choices: [{ delta: { content: "hi" } }], usage: null }, + { + model: "gpt-4o-2024-08-06", + choices: [], + usage: { prompt_tokens: 10000, completion_tokens: 500, total_tokens: 10500 }, + }, +]; + +function streamingClient(Stream, { failWith, seal } = {}) { + const probe = { released: false }; + async function* source() { + try { + yield STREAM_CHUNKS[0]; + if (failWith) throw failWith; + yield STREAM_CHUNKS[1]; + } finally { + probe.released = true; + } + } + class OpenAI { + baseURL = "https://api.openai.com/v1"; + chat = { + completions: { + create: async () => { + const stream = new Stream(source); + return seal ? Object.freeze(stream) : stream; + }, + }, + }; + } + return { client: instrumentLlmClient(new OpenAI()), probe }; +} + +const callBody = { model: "gpt-4o", messages: [], stream: true, stream_options: { include_usage: true } }; + +async function readLines(readable) { + const reader = readable.getReader(); + const decoder = new TextDecoder(); + let text = ""; + for (;;) { + const { done, value } = await reader.read(); + if (done) return text.split("\n").filter(Boolean).map((line) => JSON.parse(line)); + text += decoder.decode(value); + } +} + +async function drain(stream) { + const items = []; + for await (const item of stream) items.push(item); + return items; +} + +test("OpenAI stream read through toReadableStream(): one span with usage, the bytes unchanged", async () => { + const { client } = streamingClient(FakeStream); + const { attrs, ai, span } = await capture("stream-readable", async () => { + const stream = await client.chat.completions.create(callBody); + assert.deepEqual(await readLines(stream.toReadableStream()), STREAM_CHUNKS); + }); + assert.equal(span.status.code, 1); + assert.equal(attrs["gen_ai.usage.input_tokens"], 10000); + assert.equal(attrs["gen_ai.usage.output_tokens"], 500); + assert.equal(attrs["autter.llm.streamed"], true); + assert.equal(ai.calls, 1); + assert.equal(ai.input_tokens, 10000); +}); + +for (const [shape, Stream] of [ + ["tee() through the iterator field", FakeStream], + ["tee() through Symbol.asyncIterator", AnthropicShapedStream], +]) { + test(`stream ${shape}: both branches read to the end give one span with usage`, async () => { + const { client } = streamingClient(Stream); + const { attrs, ai, span } = await capture(`stream-tee-${shape}`, async () => { + const stream = await client.chat.completions.create(callBody); + const [left, right] = stream.tee(); + const [fromLeft, fromRight] = await Promise.all([drain(left), drain(right)]); + assert.deepEqual(fromLeft, STREAM_CHUNKS); + assert.deepEqual(fromRight, STREAM_CHUNKS); + }); + assert.equal(span.status.code, 1); + assert.equal(attrs["gen_ai.usage.input_tokens"], 10000); + assert.equal(ai.calls, 1); + }); + + test(`stream ${shape}: the span stays open until the last branch is done`, async () => { + const { client, probe } = streamingClient(Stream); + const { attrs, ai } = await capture(`stream-tee-close-${shape}`, async () => { + const stream = await client.chat.completions.create(callBody); + const [left, right] = stream.tee(); + const leftIterator = left[Symbol.asyncIterator](); + await leftIterator.next(); + await leftIterator.return(); + assert.equal(exporter.getFinishedSpans().length, 0); + assert.deepEqual(await drain(right), STREAM_CHUNKS); + }); + assert.equal(attrs["gen_ai.usage.input_tokens"], 10000); + assert.equal(ai.calls, 1); + assert.equal(probe.released, true); + }); +} + +test("tee() with both branches closed ends the span once and releases the source", async () => { + const { client, probe } = streamingClient(FakeStream); + const { span, ai } = await capture("stream-tee-both-closed", async () => { + const stream = await client.chat.completions.create(callBody); + const [left, right] = stream.tee(); + for (const branch of [left, right]) { + const iterator = branch[Symbol.asyncIterator](); + await iterator.next(); + await iterator.return(); + } + }); + assert.equal(span.status.code, 1); + assert.equal(ai.calls, 1); + assert.equal(probe.released, true); +}); + +test("toReadableStream() cancelled mid-stream ends the span once with the usage seen so far", async () => { + const { client, probe } = streamingClient(FakeStream); + const { attrs, ai, span } = await capture("stream-readable-cancel", async () => { + const stream = await client.chat.completions.create(callBody); + const reader = stream.toReadableStream().getReader(); + await reader.read(); + await reader.cancel(); + }); + assert.equal(span.status.code, 1); + assert.equal(attrs["gen_ai.usage.input_tokens"], undefined); + assert.equal(attrs["autter.llm.streamed"], true); + assert.equal(ai.calls, 1); + assert.equal(probe.released, true); +}); + +test("toReadableStream() failing mid-stream ends the span once as an error", async () => { + const { client, probe } = streamingClient(FakeStream, { failWith: new Error("upstream reset") }); + const { span, ai } = await capture("stream-readable-error", async () => { + const stream = await client.chat.completions.create(callBody); + await assert.rejects(readLines(stream.toReadableStream()), /upstream reset/); + await until(() => probe.released); + }); + assert.equal(span.status.code, 2); + assert.equal(ai.calls, 1); +}); + +test("iterating still ends the span, on a break too, and the result is the SDK's own stream", async () => { + const { client } = streamingClient(FakeStream); + const { attrs, ai, span } = await capture("stream-iterate", async () => { + const stream = await client.chat.completions.create(callBody); + assert.ok(stream instanceof FakeStream); + assert.equal(types.isProxy(stream), false); + assert.deepEqual(await drain(stream), STREAM_CHUNKS); + }); + assert.equal(span.status.code, 1); + assert.equal(attrs["gen_ai.usage.input_tokens"], 10000); + assert.equal(ai.calls, 1); + + const broken = await capture("stream-break", async () => { + const stream = await client.chat.completions.create(callBody); + for await (const _ of stream) break; + }); + assert.equal(broken.span.status.code, 1); + assert.equal(broken.attrs["gen_ai.usage.input_tokens"], undefined); + assert.equal(broken.ai.calls, 1); +}); + +test("a stream that cannot be instrumented in place still works, and its span ends", async () => { + const { client } = streamingClient(FakeStream, { seal: true }); + const { span, ai } = await capture("stream-sealed", async () => { + const stream = await client.chat.completions.create(callBody); + assert.deepEqual(await readLines(stream.toReadableStream()), STREAM_CHUNKS); + }); + assert.equal(span.status.code, 1); + assert.equal(ai.calls, 1); +}); + // ------------------------------------------------------------- Anthropic --- const READ_USAGE = {