test(e2e): Move OpenAI and Anthropic send-to-sentry cases into gen-ai-libraries

dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gen-ai-libraries",
-  "description": "Real gen_ai spans for every instrumented AI library (OpenAI, Anthropic, Mistral, Together, Vercel AI), each driven through OpenRouter with a chat query and a tool call, on Node and on Cloudflare",
+  "description": "Real gen_ai spans for every instrumented AI library (OpenAI, Anthropic, Mistral, Together, Vercel AI), each driven through OpenRouter with a chat query and a tool call, plus provider-specific calls (streaming, the OpenAI Responses and embeddings APIs, the Anthropic stream helper), on Node and on Cloudflare",
   "version": "1.0.0",
   "private": true,
   "type": "module",
@@ -13,7 +13,8 @@
     "test:build": "pnpm install",
     "test:build:cloudflare": "pnpm install && vite build",
     "test:assert": "pnpm test",
-    "test:assert:cloudflare": "RUNTIME=cloudflare pnpm test"
+    "test:assert:cloudflare": "RUNTIME=cloudflare pnpm test",
+    "test:build-latest": "pnpm install && pnpm add openai@latest && pnpm update '@anthropic-ai/sdk@<1'"
   },
   "dependencies": {
     "@anthropic-ai/sdk": "0.63.0",
@@ -46,6 +47,10 @@
         "build-command": "pnpm test:build:cloudflare",
         "assert-command": "pnpm test:assert:cloudflare",
         "label": "gen-ai-libraries (cloudflare)"
+      },
+      {
+        "build-command": "pnpm test:build-latest",
+        "label": "gen-ai-libraries (latest)"
       }
     ]
   }

dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts

@@ -17,8 +17,9 @@ export default Sentry.withSentry(
       const url = new URL(request.url);
       const [, id, action] = url.pathname.split('/');
       const library = id ? byId.get(id) : undefined;
+      const extra = library && action ? library.extras?.[action] : undefined;
 
-      if (!library || (action !== 'chat' && action !== 'tools')) {
+      if (!library || !action || (action !== 'chat' && action !== 'tools' && !extra)) {
         return new Response('Not found', { status: 404 });
       }
 
@@ -28,8 +29,9 @@ export default Sentry.withSentry(
       }
 
       try {
-        const spanName = action === 'tools' ? 'ai-tool-workflow' : 'ai-workflow';
-        const result = await Sentry.startSpan({ name: spanName, op: 'function' }, () => library[action](apiKey));
+        const spanName = extra ? `${library.id}/${action}` : action === 'tools' ? 'ai-tool-workflow' : 'ai-workflow';
+        const call = extra ?? library[action as 'chat' | 'tools'];
+        const result = await Sentry.startSpan({ name: spanName, op: 'function' }, () => call(apiKey));
         return Response.json({ result });
       } catch (error) {
         return Response.json({ message: (error as Error).message }, { status: 500 });

dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.node.ts

@@ -33,6 +33,17 @@ for (const library of libraries) {
       next(error);
     }
   });
+
+  for (const [name, call] of Object.entries(library.extras ?? {})) {
+    app.get(`/${library.id}/${name}`, async (_req, res, next) => {
+      try {
+        const result = await Sentry.startSpan({ name: `${library.id}/${name}`, op: 'function' }, () => call(apiKey));
+        res.send({ result });
+      } catch (error) {
+        next(error);
+      }
+    });
+  }
 }
 
 Sentry.setupExpressErrorHandler(app);

dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts

@@ -60,6 +60,53 @@ export interface Library {
   origin?: string;
   chat: (apiKey: string) => Promise<unknown>;
   tools: (apiKey: string) => Promise<unknown>;
+  /** Calls only this library's SDK has (streaming, other endpoints), served as `/:lib/:name`. */
+  extras?: Record<string, (apiKey: string) => Promise<unknown>>;
+}
+
+const EMBEDDINGS_MODEL = 'openai/text-embedding-3-small';
+
+const openAiClient = (apiKey: string): OpenAI => new OpenAI({ apiKey, baseURL: OPENROUTER_V1 });
+
+// OpenRouter's Anthropic skin authenticates with a bearer token, so the key goes in `authToken`
+// (Authorization: Bearer) rather than `apiKey` (x-api-key).
+const anthropicClient = (apiKey: string): Anthropic =>
+  new Anthropic({ authToken: apiKey, baseURL: OPENROUTER_BASE, fetch: fetchWithoutDone });
+
+/**
+ * OpenRouter ends each stream with an OpenAI-style `data: [DONE]` frame, which is not part of
+ * Anthropic's protocol. Since 0.136.0 the SDK JSON-parses every named event, so this frame
+ * makes it throw. Drop the frame before the SDK sees it.
+ */
+async function fetchWithoutDone(input: string | URL | Request, init?: RequestInit): Promise<Response> {
+  const response = await fetch(input, init);
+  if (!response.body || !response.headers.get('content-type')?.includes('text/event-stream')) {
+    return response;
+  }
+
+  let buffer = '';
+  const keepFrame = (frame: string): boolean => !/^data: \[DONE\]$/m.test(frame);
+  const body = response.body.pipeThrough(new TextDecoderStream()).pipeThrough(
+    new TransformStream<string, string>({
+      transform(chunk, controller) {
+        buffer += chunk;
+        const frames = buffer.split(/(?<=\r?\n\r?\n)/);
+        buffer = /\r?\n\r?\n$/.test(frames[frames.length - 1] ?? '') ? '' : (frames.pop() ?? '');
+        controller.enqueue(frames.filter(keepFrame).join(''));
+      },
+      flush(controller) {
+        if (keepFrame(buffer)) {
+          controller.enqueue(buffer);
+        }
+      },
+    }),
+  );
+
+  return new Response(body.pipeThrough(new TextEncoderStream()), {
+    status: response.status,
+    statusText: response.statusText,
+    headers: response.headers,
+  });
 }
 
 /** Chat + forced tool call for an OpenAI-compatible chat-completions client (OpenAI, Together). */
@@ -100,7 +147,47 @@ function openAiCompatible(
 }
 
 export const libraries: Library[] = [
-  openAiCompatible('openai', 'openai', 'auto.ai.openai', apiKey => new OpenAI({ apiKey, baseURL: OPENROUTER_V1 })),
+  {
+    ...openAiCompatible('openai', 'openai', 'auto.ai.openai', openAiClient),
+    extras: {
+      'chat-stream': async apiKey => {
+        const stream = await openAiClient(apiKey).chat.completions.create({
+          model: MODEL,
+          messages: [
+            { role: 'system', content: SYSTEM },
+            { role: 'user', content: CHAT_PROMPT },
+          ],
+          temperature: 0,
+          max_tokens: 32,
+          stream: true,
+          // Ask for the usage in the final chunk, so the streamed span carries token counts.
+          stream_options: { include_usage: true },
+        });
+        let answer = '';
+        for await (const chunk of stream) {
+          answer += chunk.choices[0]?.delta.content ?? '';
+        }
+        return answer;
+      },
+      responses: async apiKey => {
+        const response = await openAiClient(apiKey).responses.create({
+          model: MODEL,
+          instructions: SYSTEM,
+          input: CHAT_PROMPT,
+          temperature: 0,
+          max_output_tokens: 32,
+        });
+        return response.output_text;
+      },
+      embeddings: async apiKey => {
+        const embedding = await openAiClient(apiKey).embeddings.create({
+          model: EMBEDDINGS_MODEL,
+          input: 'The capital of France is Paris.',
+        });
+        return embedding.data[0]?.embedding.length ?? 0;
+      },
+    },
+  },
   openAiCompatible(
     'together',
     'together_ai',
@@ -146,10 +233,7 @@ export const libraries: Library[] = [
     provider: 'anthropic',
     origin: 'auto.ai.anthropic',
     chat: async apiKey => {
-      // OpenRouter's Anthropic skin authenticates with a bearer token, so the key goes in `authToken`
-      // (Authorization: Bearer) rather than `apiKey` (x-api-key).
-      const client = new Anthropic({ authToken: apiKey, baseURL: OPENROUTER_BASE });
-      const message = await client.messages.create({
+      const message = await anthropicClient(apiKey).messages.create({
         model: MODEL,
         max_tokens: 32,
         temperature: 0,
@@ -160,8 +244,7 @@ export const libraries: Library[] = [
       return first && first.type === 'text' ? first.text : '';
     },
     tools: async apiKey => {
-      const client = new Anthropic({ authToken: apiKey, baseURL: OPENROUTER_BASE });
-      const message = await client.messages.create({
+      const message = await anthropicClient(apiKey).messages.create({
         model: MODEL,
         max_tokens: 64,
         messages: [{ role: 'user', content: WEATHER_PROMPT }],
@@ -180,6 +263,38 @@ export const libraries: Library[] = [
       });
       return (message.content ?? []).filter(block => block.type === 'tool_use');
     },
+    extras: {
+      // `messages.create({ stream: true })` returns an async iterable of events the caller drains.
+      'chat-stream': async apiKey => {
+        const stream = await anthropicClient(apiKey).messages.create({
+          model: MODEL,
+          max_tokens: 32,
+          system: SYSTEM,
+          messages: [{ role: 'user', content: CHAT_PROMPT }],
+          stream: true,
+        });
+        let answer = '';
+        for await (const event of stream) {
+          if (event.type === 'content_block_delta' && event.delta.type === 'text_delta') {
+            answer += event.delta.text;
+          }
+        }
+        return answer;
+      },
+      // The SDK's streaming helper, which the integration instruments separately from `create`.
+      'stream-helper': async apiKey => {
+        const message = await anthropicClient(apiKey)
+          .messages.stream({
+            model: MODEL,
+            max_tokens: 32,
+            system: SYSTEM,
+            messages: [{ role: 'user', content: CHAT_PROMPT }],
+          })
+          .finalMessage();
+        const first = message.content?.[0];
+        return first && first.type === 'text' ? first.text : '';
+      },
+    },
   },
 
   {

dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/extras.test.ts

@@ -0,0 +1,96 @@
+import { expect, test } from '@playwright/test';
+import type { SerializedStreamedSpan } from '@sentry-internal/test-utils';
+import { collectStreamedSpans, getSpanOp } from '@sentry-internal/test-utils';
+import { APP } from './constants';
+import { attr, describeTree, expectCommonChatAttributes, LIBRARIES } from './utils';
+
+const OPENAI = LIBRARIES.find(library => library.id === 'openai')!;
+const ANTHROPIC = LIBRARIES.find(library => library.id === 'anthropic')!;
+
+/**
+ * Calls `/:lib/:extra` and returns its trace once the request span (the segment) has arrived. The
+ * segment ends last, so every model-call span of the request is in by then.
+ */
+async function requestExtra(baseURL: string | undefined, path: string): Promise<SerializedStreamedSpan[]> {
+  const spansPromise = collectStreamedSpans(
+    APP,
+    spansOfTrace => spansOfTrace.some(span => span.name === path) && spansOfTrace.some(span => span.is_segment),
+  );
+
+  const response = await fetch(`${baseURL}/${path}`);
+  expect(response.status, await response.text()).toBe(200);
+
+  return spansPromise;
+}
+
+/** The only span of the trace with `op`; finding exactly one also catches a call instrumented twice. */
+function onlySpanWithOp(spans: SerializedStreamedSpan[], op: string): SerializedStreamedSpan {
+  const matching = spans.filter(span => getSpanOp(span) === op);
+  expect(matching, `expected exactly one ${op} span in:\n${describeTree(spans)}`).toHaveLength(1);
+  return matching[0]!;
+}
+
+/** `gen_ai.response.finish_reasons` is a JSON array; a streamed answer can repeat the reason per chunk. */
+function finishReasons(span: SerializedStreamedSpan): string[] {
+  const raw = attr(span, 'gen_ai.response.finish_reasons');
+  expect(typeof raw, 'gen_ai.response.finish_reasons').toBe('string');
+  return JSON.parse(raw as string);
+}
+
+test('openai: a streamed chat completion records its token usage', async ({ baseURL }) => {
+  const spans = await requestExtra(baseURL, 'openai/chat-stream');
+  const span = onlySpanWithOp(spans, 'gen_ai.chat');
+
+  // The usage only arrives in the final chunk, which the streaming instrumentation has to pick up.
+  expectCommonChatAttributes(span, OPENAI);
+  expect(attr(span, 'gen_ai.response.streaming')).toBe(true);
+  expect(finishReasons(span)).toContain('stop');
+});
+
+test('openai: a Responses API call emits a gen_ai.chat span', async ({ baseURL }) => {
+  const spans = await requestExtra(baseURL, 'openai/responses');
+  const span = onlySpanWithOp(spans, 'gen_ai.chat');
+
+  expectCommonChatAttributes(span, OPENAI);
+  expect(finishReasons(span)).toEqual(['completed']);
+  expect(attr(span, 'gen_ai.input.messages')).toContain('capital of France');
+});
+
+test('openai: an embeddings call emits a gen_ai.embeddings span', async ({ baseURL }) => {
+  const spans = await requestExtra(baseURL, 'openai/embeddings');
+  const span = onlySpanWithOp(spans, 'gen_ai.embeddings');
+
+  expect(span.name).toBe('embeddings openai/text-embedding-3-small');
+  expect(span.status).toBe('ok');
+  expect(attr(span, 'sentry.origin')).toBe('auto.ai.openai');
+  expect(attr(span, 'gen_ai.operation.name')).toBe('embeddings');
+  expect(attr(span, 'gen_ai.provider.name')).toBe('openai');
+  expect(attr(span, 'gen_ai.embeddings.input')).toContain('Paris');
+  expect(attr(span, 'gen_ai.usage.input_tokens')).toBeGreaterThan(0);
+});
+
+test('anthropic: a streamed message records its output usage', async ({ baseURL }) => {
+  const spans = await requestExtra(baseURL, 'anthropic/chat-stream');
+  const span = onlySpanWithOp(spans, 'gen_ai.chat');
+
+  expect(attr(span, 'gen_ai.response.streaming')).toBe(true);
+  expect(finishReasons(span)).toContain('end_turn');
+  expect(attr(span, 'gen_ai.usage.output_tokens')).toBeGreaterThan(0);
+  // The input usage is not asserted: the instrumentation takes it from `message_start`, where
+  // OpenRouter reports 0, and ignores the real count OpenRouter (and the Anthropic API) put in
+  // `message_delta`.
+  expect(attr(span, 'gen_ai.provider.name')).toBe(ANTHROPIC.provider);
+  expect(attr(span, 'sentry.origin')).toBe(ANTHROPIC.origin);
+});
+
+test('anthropic: the stream helper emits a single gen_ai.chat span', async ({ baseURL }) => {
+  const spans = await requestExtra(baseURL, 'anthropic/stream-helper');
+  // `messages.stream()` calls `create` underneath, which must not produce a second span.
+  const span = onlySpanWithOp(spans, 'gen_ai.chat');
+
+  expect(attr(span, 'gen_ai.response.streaming')).toBe(true);
+  expect(finishReasons(span)).toContain('end_turn');
+  expect(attr(span, 'gen_ai.usage.output_tokens')).toBeGreaterThan(0);
+  expect(attr(span, 'gen_ai.provider.name')).toBe(ANTHROPIC.provider);
+  expect(attr(span, 'sentry.origin')).toBe(ANTHROPIC.origin);
+});