From cc0a3a15e9c826b6c3657ee40310368f5a5da9cf Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 28 Oct 2026 17:14:22 +0100 Subject: [PATCH 1/9] test(e2e): Move OpenAI and Anthropic send-to-sentry cases into gen-ai-libraries gen-ai-libraries already runs chat and tool calls for both SDKs against real OpenRouter through the event proxy. Moves the cases only the send-to-sentry apps had (streaming usage, Responses API, embeddings, the Anthropic stream helper) into it, so they also run on Cloudflare, adds a (latest) variant for both SDKs, and removes node-openai-send-to-sentry and node-anthropic-send-to-sentry. Co-Authored-By: Claude Opus 5.5 --- .../gen-ai-libraries/package.json | 9 +- .../gen-ai-libraries/src/entry.cloudflare.ts | 8 +- .../gen-ai-libraries/src/entry.node.ts | 11 ++ .../gen-ai-libraries/src/libraries.ts | 129 ++++++++++++- .../gen-ai-libraries/tests/extras.test.ts | 96 ++++++++++ .../node-anthropic-send-to-sentry/.gitignore | 5 - .../package.json | 36 ---- .../playwright.config.ts | 28 --- .../pnpm-workspace.yaml | 1 - .../node-anthropic-send-to-sentry/src/app.ts | 165 ----------------- .../src/instrument.ts | 10 - .../tests/send-to-sentry.test.ts | 151 --------------- .../tests/utils/sentry-api.ts | 131 ------------- .../tsconfig.json | 13 -- .../node-openai-send-to-sentry/.gitignore | 5 - .../node-openai-send-to-sentry/package.json | 38 ---- .../playwright.config.ts | 24 --- .../pnpm-workspace.yaml | 1 - .../node-openai-send-to-sentry/src/app.ts | 151 --------------- .../src/instrument.ts | 10 - .../tests/send-to-sentry.test.ts | 172 ------------------ .../node-openai-send-to-sentry/tsconfig.json | 13 -- 22 files changed, 241 insertions(+), 966 deletions(-) create mode 100644 dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/extras.test.ts delete mode 100644 dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/.gitignore delete mode 100644 dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/package.json delete mode 100644 dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/playwright.config.ts delete mode 100644 dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/pnpm-workspace.yaml delete mode 100644 dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/src/app.ts delete mode 100644 dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/src/instrument.ts delete mode 100644 dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/tests/send-to-sentry.test.ts delete mode 100644 dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/tests/utils/sentry-api.ts delete mode 100644 dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/tsconfig.json delete mode 100644 dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/.gitignore delete mode 100644 dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/package.json delete mode 100644 dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/playwright.config.ts delete mode 100644 dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/pnpm-workspace.yaml delete mode 100644 dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/src/app.ts delete mode 100644 dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/src/instrument.ts delete mode 100644 dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/tests/send-to-sentry.test.ts delete mode 100644 dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/tsconfig.json diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json index 35f5df09e3e3..10dfd01d6656 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json @@ -1,6 +1,6 @@ { "name": "gen-ai-libraries", - "description": "Real gen_ai spans for every instrumented AI library (OpenAI, Anthropic, Mistral, Together, Vercel AI), each driven through OpenRouter with a chat query and a tool call, on Node and on Cloudflare", + "description": "Real gen_ai spans for every instrumented AI library (OpenAI, Anthropic, Mistral, Together, Vercel AI), each driven through OpenRouter with a chat query and a tool call, plus provider-specific calls (streaming, the OpenAI Responses and embeddings APIs, the Anthropic stream helper), on Node and on Cloudflare", "version": "1.0.0", "private": true, "type": "module", @@ -13,7 +13,8 @@ "test:build": "pnpm install", "test:build:cloudflare": "pnpm install && vite build", "test:assert": "pnpm test", - "test:assert:cloudflare": "RUNTIME=cloudflare pnpm test" + "test:assert:cloudflare": "RUNTIME=cloudflare pnpm test", + "test:build-latest": "pnpm install && pnpm add openai@latest && pnpm update '@anthropic-ai/sdk@<1'" }, "dependencies": { "@anthropic-ai/sdk": "0.63.0", @@ -46,6 +47,10 @@ "build-command": "pnpm test:build:cloudflare", "assert-command": "pnpm test:assert:cloudflare", "label": "gen-ai-libraries (cloudflare)" + }, + { + "build-command": "pnpm test:build-latest", + "label": "gen-ai-libraries (latest)" } ] } diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts index 265b8122f0df..c928ffd5d635 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts @@ -17,8 +17,9 @@ export default Sentry.withSentry( const url = new URL(request.url); const [, id, action] = url.pathname.split('/'); const library = id ? byId.get(id) : undefined; + const extra = library && action ? library.extras?.[action] : undefined; - if (!library || (action !== 'chat' && action !== 'tools')) { + if (!library || !action || (action !== 'chat' && action !== 'tools' && !extra)) { return new Response('Not found', { status: 404 }); } @@ -28,8 +29,9 @@ export default Sentry.withSentry( } try { - const spanName = action === 'tools' ? 'ai-tool-workflow' : 'ai-workflow'; - const result = await Sentry.startSpan({ name: spanName, op: 'function' }, () => library[action](apiKey)); + const spanName = extra ? `${library.id}/${action}` : action === 'tools' ? 'ai-tool-workflow' : 'ai-workflow'; + const call = extra ?? library[action as 'chat' | 'tools']; + const result = await Sentry.startSpan({ name: spanName, op: 'function' }, () => call(apiKey)); return Response.json({ result }); } catch (error) { return Response.json({ message: (error as Error).message }, { status: 500 }); diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.node.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.node.ts index 726924dbd69d..b67803a4da58 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.node.ts +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.node.ts @@ -33,6 +33,17 @@ for (const library of libraries) { next(error); } }); + + for (const [name, call] of Object.entries(library.extras ?? {})) { + app.get(`/${library.id}/${name}`, async (_req, res, next) => { + try { + const result = await Sentry.startSpan({ name: `${library.id}/${name}`, op: 'function' }, () => call(apiKey)); + res.send({ result }); + } catch (error) { + next(error); + } + }); + } } Sentry.setupExpressErrorHandler(app); diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts index 1b1490e7e9b4..a9fdfb5f3f98 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts @@ -60,6 +60,53 @@ export interface Library { origin?: string; chat: (apiKey: string) => Promise; tools: (apiKey: string) => Promise; + /** Calls only this library's SDK has (streaming, other endpoints), served as `/:lib/:name`. */ + extras?: Record Promise>; +} + +const EMBEDDINGS_MODEL = 'openai/text-embedding-3-small'; + +const openAiClient = (apiKey: string): OpenAI => new OpenAI({ apiKey, baseURL: OPENROUTER_V1 }); + +// OpenRouter's Anthropic skin authenticates with a bearer token, so the key goes in `authToken` +// (Authorization: Bearer) rather than `apiKey` (x-api-key). +const anthropicClient = (apiKey: string): Anthropic => + new Anthropic({ authToken: apiKey, baseURL: OPENROUTER_BASE, fetch: fetchWithoutDone }); + +/** + * OpenRouter ends each stream with an OpenAI-style `data: [DONE]` frame, which is not part of + * Anthropic's protocol. Since 0.136.0 the SDK JSON-parses every named event, so this frame + * makes it throw. Drop the frame before the SDK sees it. + */ +async function fetchWithoutDone(input: string | URL | Request, init?: RequestInit): Promise { + const response = await fetch(input, init); + if (!response.body || !response.headers.get('content-type')?.includes('text/event-stream')) { + return response; + } + + let buffer = ''; + const keepFrame = (frame: string): boolean => !/^data: \[DONE\]$/m.test(frame); + const body = response.body.pipeThrough(new TextDecoderStream()).pipeThrough( + new TransformStream({ + transform(chunk, controller) { + buffer += chunk; + const frames = buffer.split(/(?<=\r?\n\r?\n)/); + buffer = /\r?\n\r?\n$/.test(frames[frames.length - 1] ?? '') ? '' : (frames.pop() ?? ''); + controller.enqueue(frames.filter(keepFrame).join('')); + }, + flush(controller) { + if (keepFrame(buffer)) { + controller.enqueue(buffer); + } + }, + }), + ); + + return new Response(body.pipeThrough(new TextEncoderStream()), { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }); } /** Chat + forced tool call for an OpenAI-compatible chat-completions client (OpenAI, Together). */ @@ -100,7 +147,47 @@ function openAiCompatible( } export const libraries: Library[] = [ - openAiCompatible('openai', 'openai', 'auto.ai.openai', apiKey => new OpenAI({ apiKey, baseURL: OPENROUTER_V1 })), + { + ...openAiCompatible('openai', 'openai', 'auto.ai.openai', openAiClient), + extras: { + 'chat-stream': async apiKey => { + const stream = await openAiClient(apiKey).chat.completions.create({ + model: MODEL, + messages: [ + { role: 'system', content: SYSTEM }, + { role: 'user', content: CHAT_PROMPT }, + ], + temperature: 0, + max_tokens: 32, + stream: true, + // Ask for the usage in the final chunk, so the streamed span carries token counts. + stream_options: { include_usage: true }, + }); + let answer = ''; + for await (const chunk of stream) { + answer += chunk.choices[0]?.delta.content ?? ''; + } + return answer; + }, + responses: async apiKey => { + const response = await openAiClient(apiKey).responses.create({ + model: MODEL, + instructions: SYSTEM, + input: CHAT_PROMPT, + temperature: 0, + max_output_tokens: 32, + }); + return response.output_text; + }, + embeddings: async apiKey => { + const embedding = await openAiClient(apiKey).embeddings.create({ + model: EMBEDDINGS_MODEL, + input: 'The capital of France is Paris.', + }); + return embedding.data[0]?.embedding.length ?? 0; + }, + }, + }, openAiCompatible( 'together', 'together_ai', @@ -146,10 +233,7 @@ export const libraries: Library[] = [ provider: 'anthropic', origin: 'auto.ai.anthropic', chat: async apiKey => { - // OpenRouter's Anthropic skin authenticates with a bearer token, so the key goes in `authToken` - // (Authorization: Bearer) rather than `apiKey` (x-api-key). - const client = new Anthropic({ authToken: apiKey, baseURL: OPENROUTER_BASE }); - const message = await client.messages.create({ + const message = await anthropicClient(apiKey).messages.create({ model: MODEL, max_tokens: 32, temperature: 0, @@ -160,8 +244,7 @@ export const libraries: Library[] = [ return first && first.type === 'text' ? first.text : ''; }, tools: async apiKey => { - const client = new Anthropic({ authToken: apiKey, baseURL: OPENROUTER_BASE }); - const message = await client.messages.create({ + const message = await anthropicClient(apiKey).messages.create({ model: MODEL, max_tokens: 64, messages: [{ role: 'user', content: WEATHER_PROMPT }], @@ -180,6 +263,38 @@ export const libraries: Library[] = [ }); return (message.content ?? []).filter(block => block.type === 'tool_use'); }, + extras: { + // `messages.create({ stream: true })` returns an async iterable of events the caller drains. + 'chat-stream': async apiKey => { + const stream = await anthropicClient(apiKey).messages.create({ + model: MODEL, + max_tokens: 32, + system: SYSTEM, + messages: [{ role: 'user', content: CHAT_PROMPT }], + stream: true, + }); + let answer = ''; + for await (const event of stream) { + if (event.type === 'content_block_delta' && event.delta.type === 'text_delta') { + answer += event.delta.text; + } + } + return answer; + }, + // The SDK's streaming helper, which the integration instruments separately from `create`. + 'stream-helper': async apiKey => { + const message = await anthropicClient(apiKey) + .messages.stream({ + model: MODEL, + max_tokens: 32, + system: SYSTEM, + messages: [{ role: 'user', content: CHAT_PROMPT }], + }) + .finalMessage(); + const first = message.content?.[0]; + return first && first.type === 'text' ? first.text : ''; + }, + }, }, { diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/extras.test.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/extras.test.ts new file mode 100644 index 000000000000..315cf0df19dd --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/extras.test.ts @@ -0,0 +1,96 @@ +import { expect, test } from '@playwright/test'; +import type { SerializedStreamedSpan } from '@sentry-internal/test-utils'; +import { collectStreamedSpans, getSpanOp } from '@sentry-internal/test-utils'; +import { APP } from './constants'; +import { attr, describeTree, expectCommonChatAttributes, LIBRARIES } from './utils'; + +const OPENAI = LIBRARIES.find(library => library.id === 'openai')!; +const ANTHROPIC = LIBRARIES.find(library => library.id === 'anthropic')!; + +/** + * Calls `/:lib/:extra` and returns its trace once the request span (the segment) has arrived. The + * segment ends last, so every model-call span of the request is in by then. + */ +async function requestExtra(baseURL: string | undefined, path: string): Promise { + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => spansOfTrace.some(span => span.name === path) && spansOfTrace.some(span => span.is_segment), + ); + + const response = await fetch(`${baseURL}/${path}`); + expect(response.status, await response.text()).toBe(200); + + return spansPromise; +} + +/** The only span of the trace with `op`; finding exactly one also catches a call instrumented twice. */ +function onlySpanWithOp(spans: SerializedStreamedSpan[], op: string): SerializedStreamedSpan { + const matching = spans.filter(span => getSpanOp(span) === op); + expect(matching, `expected exactly one ${op} span in:\n${describeTree(spans)}`).toHaveLength(1); + return matching[0]!; +} + +/** `gen_ai.response.finish_reasons` is a JSON array; a streamed answer can repeat the reason per chunk. */ +function finishReasons(span: SerializedStreamedSpan): string[] { + const raw = attr(span, 'gen_ai.response.finish_reasons'); + expect(typeof raw, 'gen_ai.response.finish_reasons').toBe('string'); + return JSON.parse(raw as string); +} + +test('openai: a streamed chat completion records its token usage', async ({ baseURL }) => { + const spans = await requestExtra(baseURL, 'openai/chat-stream'); + const span = onlySpanWithOp(spans, 'gen_ai.chat'); + + // The usage only arrives in the final chunk, which the streaming instrumentation has to pick up. + expectCommonChatAttributes(span, OPENAI); + expect(attr(span, 'gen_ai.response.streaming')).toBe(true); + expect(finishReasons(span)).toContain('stop'); +}); + +test('openai: a Responses API call emits a gen_ai.chat span', async ({ baseURL }) => { + const spans = await requestExtra(baseURL, 'openai/responses'); + const span = onlySpanWithOp(spans, 'gen_ai.chat'); + + expectCommonChatAttributes(span, OPENAI); + expect(finishReasons(span)).toEqual(['completed']); + expect(attr(span, 'gen_ai.input.messages')).toContain('capital of France'); +}); + +test('openai: an embeddings call emits a gen_ai.embeddings span', async ({ baseURL }) => { + const spans = await requestExtra(baseURL, 'openai/embeddings'); + const span = onlySpanWithOp(spans, 'gen_ai.embeddings'); + + expect(span.name).toBe('embeddings openai/text-embedding-3-small'); + expect(span.status).toBe('ok'); + expect(attr(span, 'sentry.origin')).toBe('auto.ai.openai'); + expect(attr(span, 'gen_ai.operation.name')).toBe('embeddings'); + expect(attr(span, 'gen_ai.provider.name')).toBe('openai'); + expect(attr(span, 'gen_ai.embeddings.input')).toContain('Paris'); + expect(attr(span, 'gen_ai.usage.input_tokens')).toBeGreaterThan(0); +}); + +test('anthropic: a streamed message records its output usage', async ({ baseURL }) => { + const spans = await requestExtra(baseURL, 'anthropic/chat-stream'); + const span = onlySpanWithOp(spans, 'gen_ai.chat'); + + expect(attr(span, 'gen_ai.response.streaming')).toBe(true); + expect(finishReasons(span)).toContain('end_turn'); + expect(attr(span, 'gen_ai.usage.output_tokens')).toBeGreaterThan(0); + // The input usage is not asserted: the instrumentation takes it from `message_start`, where + // OpenRouter reports 0, and ignores the real count OpenRouter (and the Anthropic API) put in + // `message_delta`. + expect(attr(span, 'gen_ai.provider.name')).toBe(ANTHROPIC.provider); + expect(attr(span, 'sentry.origin')).toBe(ANTHROPIC.origin); +}); + +test('anthropic: the stream helper emits a single gen_ai.chat span', async ({ baseURL }) => { + const spans = await requestExtra(baseURL, 'anthropic/stream-helper'); + // `messages.stream()` calls `create` underneath, which must not produce a second span. + const span = onlySpanWithOp(spans, 'gen_ai.chat'); + + expect(attr(span, 'gen_ai.response.streaming')).toBe(true); + expect(finishReasons(span)).toContain('end_turn'); + expect(attr(span, 'gen_ai.usage.output_tokens')).toBeGreaterThan(0); + expect(attr(span, 'gen_ai.provider.name')).toBe(ANTHROPIC.provider); + expect(attr(span, 'sentry.origin')).toBe(ANTHROPIC.origin); +}); diff --git a/dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/.gitignore b/dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/.gitignore deleted file mode 100644 index 795f6b6d40d1..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -node_modules -pnpm-lock.yaml -dist -test-results -playwright-report diff --git a/dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/package.json b/dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/package.json deleted file mode 100644 index 9f7d8ae4fa75..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/package.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "name": "node-anthropic-send-to-sentry", - "description": "The Anthropic integration as a user runs it: a plain Sentry.init on an express app, real chat, streaming, stream-helper and tool-call requests through OpenRouter, and the gen_ai spans read back from a real Sentry project", - "version": "1.0.0", - "private": true, - "type": "module", - "scripts": { - "build": "tsc", - "start": "node --import ./dist/instrument.js dist/app.js", - "test": "playwright test", - "clean": "npx rimraf node_modules dist pnpm-lock.yaml", - "test:build": "pnpm install && pnpm build", - "test:build-latest": "pnpm install && pnpm add @anthropic-ai/sdk@latest && pnpm build", - "test:assert": "pnpm test" - }, - "dependencies": { - "@anthropic-ai/sdk": "0.63.0", - "@sentry/node": "file:../../packed/sentry-node-packed.tgz", - "express": "^4.21.2" - }, - "devDependencies": { - "@playwright/test": "1.64.1", - "@types/express": "^4.17.21", - "@types/node": "^24.0.0", - "typescript": "~5.9.0" - }, - "sentryTest": { - "optional": true, - "optionalVariants": [ - { - "build-command": "pnpm test:build-latest", - "label": "node-anthropic-send-to-sentry (latest)" - } - ] - } -} diff --git a/dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/playwright.config.ts b/dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/playwright.config.ts deleted file mode 100644 index ef4764ce8d73..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/playwright.config.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { defineConfig } from '@playwright/test'; - -const port = 3030; - -export default defineConfig({ - testDir: './tests', - /* - * Spans take ~2min to become queryable via the trace endpoint, and each poll has its own 180s - * budget. The first test polls twice in a row (the model span, then its parent), so the ceiling - * has to hold two polls back to back. - */ - timeout: 400_000, - fullyParallel: true, - forbidOnly: !!process.env.CI, - retries: 0, - // Every test spends most of its time polling Sentry, so run them all at once. - workers: '100%', - reporter: process.env.CI ? [['list'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list', - use: { - baseURL: `http://localhost:${port}`, - }, - webServer: { - command: 'pnpm start', - port, - stdout: 'pipe', - stderr: 'pipe', - }, -}); diff --git a/dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/pnpm-workspace.yaml b/dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/pnpm-workspace.yaml deleted file mode 100644 index a1c031fc3310..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/pnpm-workspace.yaml +++ /dev/null @@ -1 +0,0 @@ -minimumReleaseAge: 0 diff --git a/dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/src/app.ts b/dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/src/app.ts deleted file mode 100644 index 693f1ef4f3b3..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/src/app.ts +++ /dev/null @@ -1,165 +0,0 @@ -// `instrument.ts` is preloaded via `node --import`, so Sentry is already initialised here. -import * as Sentry from '@sentry/node'; -import Anthropic from '@anthropic-ai/sdk'; -import express from 'express'; - -const apiKey = process.env.E2E_OPENROUTER_API_KEY; -if (!apiKey) { - throw new Error('E2E_OPENROUTER_API_KEY is not set'); -} - -// OpenRouter serves an Anthropic-compatible `/api/v1/messages`, so the stock client only needs a -// different base URL. It authenticates with a bearer token, so the key goes in `authToken` -// (Authorization: Bearer) rather than `apiKey` (x-api-key). The model is incidental: what is under -// test is the SDK's own request/response code path, which is what Sentry instruments. -const client = new Anthropic({ authToken: apiKey, baseURL: 'https://openrouter.ai/api', fetch: fetchWithoutDone }); - -/** - * OpenRouter ends each stream with an OpenAI-style `data: [DONE]` frame, which is not part of - * Anthropic's protocol. Since 0.136.0 the SDK JSON-parses every named event, so this frame - * makes it throw. Drop the frame before the SDK sees it. - */ -async function fetchWithoutDone(input: string | URL | Request, init?: RequestInit): Promise { - const response = await fetch(input, init); - if (!response.body || !response.headers.get('content-type')?.includes('text/event-stream')) { - return response; - } - - let buffer = ''; - const keepFrame = (frame: string): boolean => !/^data: \[DONE\]$/m.test(frame); - const body = response.body.pipeThrough(new TextDecoderStream()).pipeThrough( - new TransformStream({ - transform(chunk, controller) { - buffer += chunk; - const frames = buffer.split(/(?<=\r?\n\r?\n)/); - buffer = /\r?\n\r?\n$/.test(frames[frames.length - 1] ?? '') ? '' : (frames.pop() ?? ''); - controller.enqueue(frames.filter(keepFrame).join('')); - }, - flush(controller) { - if (keepFrame(buffer)) { - controller.enqueue(buffer); - } - }, - }), - ); - - return new Response(body.pipeThrough(new TextEncoderStream()), { - status: response.status, - statusText: response.statusText, - headers: response.headers, - }); -} - -const MODEL = 'openai/gpt-4o-mini'; - -const SHORT_ANSWER = 'Answer in at most five words.'; -const CHAT_PROMPT = `What is the capital of France? ${SHORT_ANSWER}`; -// Deliberately does not name the tool: `tool_choice` forces the call. -const WEATHER_PROMPT = `What is the weather in Paris? ${SHORT_ANSWER}`; -const SYSTEM = 'You are a helpful assistant used by an automated test.'; - -const WEATHER_TOOL = { - name: 'get_weather', - description: 'Get the current weather for a city.', - input_schema: { - type: 'object' as const, - properties: { city: { type: 'string', description: 'The city name' } }, - required: ['city'], - }, -}; - -const app = express(); - -/** The trace this request is recorded under, so the test can look its spans up in Sentry. */ -function currentTraceId(): string | undefined { - return Sentry.getActiveSpan()?.spanContext().traceId; -} - -/** The text of a message's first text block. */ -function textOf(message: Anthropic.Message): string { - const first = message.content[0]; - return first?.type === 'text' ? first.text : ''; -} - -// Each route makes one call through the `@anthropic-ai/sdk` client and answers with the trace id, so -// the test can find the request's spans in Sentry. - -app.get('/chat', async (_req, res, next) => { - try { - const message = await client.messages.create({ - model: MODEL, - max_tokens: 32, - system: SYSTEM, - messages: [{ role: 'user', content: CHAT_PROMPT }], - }); - res.send({ traceId: currentTraceId(), answer: textOf(message) }); - } catch (error) { - next(error); - } -}); - -// `messages.create({ stream: true })` returns an async iterable of events the caller drains. -app.get('/chat-stream', async (_req, res, next) => { - try { - const stream = await client.messages.create({ - model: MODEL, - max_tokens: 32, - system: SYSTEM, - messages: [{ role: 'user', content: CHAT_PROMPT }], - stream: true, - }); - - let answer = ''; - for await (const event of stream) { - if (event.type === 'content_block_delta' && event.delta.type === 'text_delta') { - answer += event.delta.text; - } - } - res.send({ traceId: currentTraceId(), answer }); - } catch (error) { - next(error); - } -}); - -// `messages.stream()` is the SDK's streaming helper: it accumulates the events into the final -// message, and the integration instruments it separately from `create`. -app.get('/stream-helper', async (_req, res, next) => { - try { - const message = await client.messages - .stream({ - model: MODEL, - max_tokens: 32, - system: SYSTEM, - messages: [{ role: 'user', content: CHAT_PROMPT }], - }) - .finalMessage(); - res.send({ traceId: currentTraceId(), answer: textOf(message) }); - } catch (error) { - next(error); - } -}); - -app.get('/tools', async (_req, res, next) => { - try { - const message = await client.messages.create({ - model: MODEL, - max_tokens: 64, - messages: [{ role: 'user', content: WEATHER_PROMPT }], - tools: [WEATHER_TOOL], - tool_choice: { type: 'tool', name: 'get_weather' }, - }); - res.send({ traceId: currentTraceId(), toolUse: message.content.filter(block => block.type === 'tool_use') }); - } catch (error) { - next(error); - } -}); - -app.use((error: Error, _req: express.Request, res: express.Response, _next: express.NextFunction) => { - res.status(500).send({ message: error.message }); -}); - -const port = Number(process.env.PORT ?? 3030); -app.listen(port, () => { - // eslint-disable-next-line no-console - console.log(`node-anthropic-send-to-sentry listening on port ${port}`); -}); diff --git a/dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/src/instrument.ts b/dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/src/instrument.ts deleted file mode 100644 index 7d031b0e6d8e..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/src/instrument.ts +++ /dev/null @@ -1,10 +0,0 @@ -import * as Sentry from '@sentry/node'; - -// The setup from the docs, nothing more: preloaded with `node --import`, no tunnel, so the spans go to -// the real Sentry project behind `E2E_TEST_DSN`, where the tests read them back. The Anthropic integration -// is on by default; its runtime channel injection is what picks up the `@anthropic-ai/sdk` client in app.ts. -Sentry.init({ - dsn: process.env.E2E_TEST_DSN, - environment: 'qa', // dynamic sampling bias to keep transactions - tracesSampleRate: 1, -}); diff --git a/dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/tests/send-to-sentry.test.ts b/dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/tests/send-to-sentry.test.ts deleted file mode 100644 index e7a135c2368c..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/tests/send-to-sentry.test.ts +++ /dev/null @@ -1,151 +0,0 @@ -import { expect, test } from '@playwright/test'; -import type { TraceItem } from './utils/sentry-api'; -import { - EVENT_POLLING_OPTIONS, - fetchSpanAttributes, - fetchTrace, - findSpanInTrace, - flattenTrace, -} from './utils/sentry-api'; - -// Mirrors src/app.ts. -const MODEL = 'openai/gpt-4o-mini'; - -type Attributes = Record; - -/** Sends one request to the app and returns the trace its request span was recorded under. */ -async function requestTrace(baseURL: string | undefined, path: string): Promise { - const response = await fetch(`${baseURL}${path}`); - const body = await response.text(); - expect(response.status, `${path} answered ${response.status}: ${body}`).toBe(200); - - const { traceId } = JSON.parse(body) as { traceId?: string }; - expect(traceId, `${path} did not report a trace id: ${body}`).toMatch(/^[0-9a-f]{32}$/); - - console.log(`${path}: https://${process.env.E2E_TEST_SENTRY_ORG_SLUG}.sentry.io/explore/traces/trace/${traceId}/`); - return traceId!; -} - -/** - * Polls Sentry until the trace holds a span with `op`, then returns that span and its attributes. The - * trace endpoint only carries the span's op, name and place in the tree; the attributes need a second - * lookup, which is polled as well since the two can land apart. - */ -async function waitForSpan(traceId: string, op: string): Promise<{ span: TraceItem; attributes: Attributes }> { - let found: { span: TraceItem; attributes: Attributes } | undefined; - - await expect - .poll(async () => { - const span = await findSpanInTrace(traceId, op); - const attributes = span?.event_id ? await fetchSpanAttributes(traceId, span.event_id) : undefined; - found = span && attributes ? { span, attributes } : undefined; - return found; - }, EVENT_POLLING_OPTIONS) - .toBeDefined(); - - return found!; -} - -/** Whether the model-call span sits somewhere below the request span (`http.server`) of the trace. */ -async function isModelSpanUnderRequestSpan(traceId: string, modelSpanId: string): Promise { - const requestSpan = flattenTrace(await fetchTrace(traceId)).find(item => item.op === 'http.server'); - return flattenTrace(requestSpan?.children ?? []).some(item => item.event_id === modelSpanId); -} - -/** - * The attributes every successful model-call span carries once Sentry has stored it, whatever the - * model answers. Model-dependent values (token counts, response id and model) are checked for shape, - * not content. Note the names are the stored ones: `origin` and `span.status` rather than the - * `sentry.*` attributes the SDK sends. - */ -function expectModelCallAttributes(span: TraceItem, attributes: Attributes): void { - expect(span.description).toBe(`chat ${MODEL}`); - expect(attributes).toMatchObject({ - 'span.op': 'gen_ai.chat', - 'span.status': 'ok', - origin: 'auto.ai.anthropic', - 'gen_ai.operation.name': 'chat', - 'gen_ai.provider.name': 'anthropic', - 'gen_ai.request.model': MODEL, - }); - expect(typeof attributes['gen_ai.response.id']).toBe('string'); - expect(typeof attributes['gen_ai.response.model']).toBe('string'); - expect(attributes['gen_ai.usage.output_tokens']).toBeGreaterThan(0); - expect(attributes['gen_ai.usage.total_tokens']).toBeGreaterThan(0); -} - -/** The prompt and the answer, which the SDK records by default and which have to survive the trip. */ -function expectRecordedConversation(attributes: Attributes): void { - // The answer is sent as `gen_ai.response.text` and stored as `gen_ai.output.messages`. - expect(attributes['gen_ai.system_instructions']).toContain('automated test'); - expect(attributes['gen_ai.input.messages']).toContain('capital of France'); - expect(typeof attributes['gen_ai.output.messages']).toBe('string'); -} - -/** `gen_ai.response.finish_reasons` is a JSON array, only recorded for streamed calls. */ -function finishReasons(attributes: Attributes): string[] { - const raw = attributes['gen_ai.response.finish_reasons']; - expect(typeof raw, 'gen_ai.response.finish_reasons').toBe('string'); - return JSON.parse(raw as string); -} - -test('Sends a message to Sentry as a gen_ai.chat span under the request span', async ({ baseURL }) => { - const traceId = await requestTrace(baseURL, '/chat'); - - const { span, attributes } = await waitForSpan(traceId, 'gen_ai.chat'); - - expectModelCallAttributes(span, attributes); - expect(attributes['gen_ai.usage.input_tokens']).toBeGreaterThan(0); - expect(attributes['gen_ai.response.streaming']).toBeUndefined(); - expectRecordedConversation(attributes); - - // The request span is the segment; it ends last, so it can land after its children. - await expect.poll(() => isModelSpanUnderRequestSpan(traceId, span.event_id!), EVENT_POLLING_OPTIONS).toBe(true); -}); - -test('Sends a streamed message to Sentry with its token usage', async ({ baseURL }) => { - const traceId = await requestTrace(baseURL, '/chat-stream'); - - const { span, attributes } = await waitForSpan(traceId, 'gen_ai.chat'); - - expectModelCallAttributes(span, attributes); - expect(attributes['gen_ai.response.streaming']).toBe(true); - expect(finishReasons(attributes)).toContain('end_turn'); - expectRecordedConversation(attributes); - // The input usage is not asserted: the instrumentation takes it from `message_start`, where - // OpenRouter reports 0, and ignores the real count OpenRouter (and the Anthropic API) put in - // `message_delta`. Output usage comes from `message_delta` and is covered by the shared checks. -}); - -test('Sends a message made with the stream helper to Sentry', async ({ baseURL }) => { - const traceId = await requestTrace(baseURL, '/stream-helper'); - - const { span, attributes } = await waitForSpan(traceId, 'gen_ai.chat'); - - // `messages.stream()` calls `create` underneath, which must not produce a second span. The parent - // lands last, so count after it; the short poll only rides out a dropped connection. - await expect.poll(() => isModelSpanUnderRequestSpan(traceId, span.event_id!), EVENT_POLLING_OPTIONS).toBe(true); - await expect - .poll(async () => flattenTrace(await fetchTrace(traceId)).filter(item => item.op === 'gen_ai.chat').length, { - timeout: 30_000, - intervals: [5_000], - }) - .toBe(1); - expectModelCallAttributes(span, attributes); - expect(attributes['gen_ai.response.streaming']).toBe(true); - expect(finishReasons(attributes)).toContain('end_turn'); - expectRecordedConversation(attributes); -}); - -test('Sends a message that returned a tool use to Sentry', async ({ baseURL }) => { - const traceId = await requestTrace(baseURL, '/tools'); - - const { span, attributes } = await waitForSpan(traceId, 'gen_ai.chat'); - - expectModelCallAttributes(span, attributes); - expect(attributes['gen_ai.usage.input_tokens']).toBeGreaterThan(0); - expect(attributes['gen_ai.tool.definitions']).toContain('get_weather'); - expect(attributes['gen_ai.input.messages']).toContain('weather in Paris'); - // The SDK also sends the returned tool use as `gen_ai.response.tool_calls`, but that attribute is - // not stored under that name, and stop reasons are only recorded for streamed calls. -}); diff --git a/dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/tests/utils/sentry-api.ts b/dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/tests/utils/sentry-api.ts deleted file mode 100644 index 521b830eefcb..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/tests/utils/sentry-api.ts +++ /dev/null @@ -1,131 +0,0 @@ -const authToken = process.env.E2E_TEST_AUTH_TOKEN; -const sentryTestOrgSlug = process.env.E2E_TEST_SENTRY_ORG_SLUG; -const sentryTestProject = process.env.E2E_TEST_SENTRY_PROJECT; - -/** - * Spans only become queryable once they have made it through to EAP, which takes - * noticeably longer than the error pipeline (~2min vs ~20s when this was measured). - */ -export const EVENT_POLLING_OPTIONS = { timeout: 180_000, intervals: [5_000] }; - -/** - * A node of the span tree returned by the organization trace endpoint. Spans, errors and - * occurrences all share this shape and are discriminated by `event_type`. - */ -export interface TraceItem { - /** On a span this is the span id. */ - event_id?: string; - event_type?: 'span' | 'error' | 'occurrence' | 'uptime_check'; - op?: string; - /** On a span this is the span name. */ - description?: string; - children?: TraceItem[]; - errors?: TraceItem[]; - occurrences?: TraceItem[]; -} - -/** - * One request to the Sentry API with the E2E token. Returns `undefined` when the connection - * fails or drops mid-response: over the minutes a test polls, the API closes the odd - * connection ("TypeError: terminated"), and an error thrown inside `expect.poll` fails the - * test instead of letting it retry, so a network error is treated like a "not there yet". - */ -async function fetchSentryApi(what: string, path: string): Promise<{ status: number; body: string } | undefined> { - try { - const response = await fetch(`https://sentry.io/api/0/${path}`, { - headers: { Authorization: `Bearer ${authToken}` }, - }); - return { status: response.status, body: await response.text() }; - } catch (error) { - console.log(`${what} failed: ${error}`); - return undefined; - } -} - -export async function fetchTrace(traceId: string): Promise { - const what = `Trace lookup for ${traceId}`; - const response = await fetchSentryApi(what, `organizations/${sentryTestOrgSlug}/trace/${traceId}/?statsPeriod=1h`); - if (!response) { - return []; - } - - // The trace endpoint is org scoped, so the auth token needs `org:read` on top of the - // project scopes the other assertions rely on. That never resolves by waiting, so fail - // loudly instead of polling until the timeout and reporting it as a missing event. - if (response.status === 401 || response.status === 403) { - throw new Error( - `${what} was rejected with ${response.status}: ${response.body}. ` + - 'E2E_TEST_AUTH_TOKEN needs the `org:read` scope.', - ); - } - - // Empty traces and the occasional rate limit are expected while polling, so treat anything - // else that is not a success as "not there yet" -- but log it, since a rejected request and - // a trace that has not landed are otherwise indistinguishable. - if (response.status !== 200) { - console.log(`${what} returned ${response.status}: ${response.body}`); - return []; - } - - return JSON.parse(response.body); -} - -/** - * Errors attach to whichever span was active when they were captured, and relocate from the - * top level into that span once it lands, so a given event can surface at any depth. - */ -export function flattenTrace(items: TraceItem[]): TraceItem[] { - return items.flatMap(item => [ - item, - ...flattenTrace(item.children ?? []), - ...flattenTrace(item.errors ?? []), - ...flattenTrace(item.occurrences ?? []), - ]); -} - -export async function findSpanInTrace(traceId: string, op: string): Promise { - return flattenTrace(await fetchTrace(traceId)).find(item => item.op === op); -} - -/** - * The attributes of one span, keyed by name, as Sentry stores them. The trace endpoint only carries - * a span's op, name and place in the tree; the attributes come from the project trace-items - * endpoint. Returns `undefined` while the span has not landed. - * - * Ingest normalises what the SDK sent: `sentry.origin` comes back as `origin`, `sentry.op` as - * `span.op`, `gen_ai.response.text` as `gen_ai.output.messages`, and some numbers are indexed as - * tags (`tags[,number]`). Integers are returned as strings and are converted here; everything - * else is passed through as is. - */ -export async function fetchSpanAttributes( - traceId: string, - spanId: string, -): Promise | undefined> { - const what = `Span lookup for ${spanId}`; - const response = await fetchSentryApi( - what, - `projects/${sentryTestOrgSlug}/${sentryTestProject}/trace-items/${spanId}/?trace_id=${traceId}&item_type=spans`, - ); - if (!response) { - return undefined; - } - - if (response.status === 401 || response.status === 403) { - throw new Error( - `${what} was rejected with ${response.status}: ${response.body}. ` + - 'E2E_TEST_AUTH_TOKEN needs the `org:read` scope.', - ); - } - - if (response.status !== 200) { - if (response.status !== 404) { - console.log(`${what} returned ${response.status}: ${response.body}`); - } - return undefined; - } - - const { attributes } = JSON.parse(response.body) as { attributes?: { name: string; type: string; value: unknown }[] }; - return Object.fromEntries( - (attributes ?? []).map(({ name, type, value }) => [name, type === 'int' ? Number(value) : value]), - ); -} diff --git a/dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/tsconfig.json b/dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/tsconfig.json deleted file mode 100644 index 795574fd9df2..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/tsconfig.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "compilerOptions": { - "target": "es2022", - "module": "nodenext", - "moduleResolution": "nodenext", - "outDir": "dist", - "strict": true, - "esModuleInterop": true, - "skipLibCheck": true, - "types": ["node"] - }, - "include": ["src/**/*.ts"] -} diff --git a/dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/.gitignore b/dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/.gitignore deleted file mode 100644 index 795f6b6d40d1..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -node_modules -pnpm-lock.yaml -dist -test-results -playwright-report diff --git a/dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/package.json b/dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/package.json deleted file mode 100644 index f94e4d8cf2ba..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/package.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "name": "node-openai-send-to-sentry", - "description": "The OpenAI integration as a user runs it: a plain Sentry.init on an express app, real chat, streaming, tool-call, Responses API and embeddings requests through OpenRouter, and the gen_ai spans read back from a real Sentry project", - "version": "1.0.0", - "private": true, - "type": "module", - "scripts": { - "build": "tsc", - "start": "node --import ./dist/instrument.js dist/app.js", - "test": "playwright test", - "clean": "npx rimraf node_modules dist pnpm-lock.yaml", - "test:build": "pnpm install && pnpm build", - "test:build-latest": "pnpm install && pnpm add openai@latest && pnpm build", - "test:assert": "pnpm test" - }, - "dependencies": { - "@sentry/node": "file:../../packed/sentry-node-packed.tgz", - "express": "^4.21.2", - "openai": "7.5.0" - }, - "devDependencies": { - "@playwright/test": "1.64.1", - "@sentry-internal/test-utils": "file:../../packed/sentry-internal-test-utils-packed.tgz", - "@types/express": "^4.17.21", - "@types/node": "^24.0.0", - "sentry": "~0.44.1", - "typescript": "~5.9.0" - }, - "sentryTest": { - "optional": true, - "optionalVariants": [ - { - "build-command": "pnpm test:build-latest", - "label": "node-openai-send-to-sentry (latest)" - } - ] - } -} diff --git a/dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/playwright.config.ts b/dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/playwright.config.ts deleted file mode 100644 index e7621a23cc8b..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/playwright.config.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { defineConfig } from '@playwright/test'; - -const port = 3030; - -export default defineConfig({ - testDir: './tests', - /* Each test runs two Sentry polls of up to 180s each, after a live model call. */ - timeout: 400_000, - fullyParallel: true, - forbidOnly: !!process.env.CI, - retries: 0, - // Every test spends most of its time polling Sentry, so run them all at once. - workers: '100%', - reporter: process.env.CI ? [['list'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list', - use: { - baseURL: `http://localhost:${port}`, - }, - webServer: { - command: 'pnpm start', - port, - stdout: 'pipe', - stderr: 'pipe', - }, -}); diff --git a/dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/pnpm-workspace.yaml b/dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/pnpm-workspace.yaml deleted file mode 100644 index a1c031fc3310..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/pnpm-workspace.yaml +++ /dev/null @@ -1 +0,0 @@ -minimumReleaseAge: 0 diff --git a/dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/src/app.ts b/dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/src/app.ts deleted file mode 100644 index ac99c6d18eeb..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/src/app.ts +++ /dev/null @@ -1,151 +0,0 @@ -// `instrument.ts` is preloaded via `node --import`, so Sentry is already initialised here. -import * as Sentry from '@sentry/node'; -import express from 'express'; -import OpenAI from 'openai'; - -const apiKey = process.env.E2E_OPENROUTER_API_KEY; -if (!apiKey) { - throw new Error('E2E_OPENROUTER_API_KEY is not set'); -} - -// OpenRouter serves the OpenAI API shape (chat completions, responses, embeddings), so the stock -// client only needs a different base URL. The model is incidental: what is under test is the SDK's own -// request/response code path, which is what Sentry instruments. -const client = new OpenAI({ apiKey, baseURL: 'https://openrouter.ai/api/v1' }); - -const CHAT_MODEL = 'openai/gpt-4o-mini'; -const EMBEDDINGS_MODEL = 'openai/text-embedding-3-small'; - -const SHORT_ANSWER = 'Answer in at most five words.'; -const CHAT_PROMPT = `What is the capital of France? ${SHORT_ANSWER}`; -// Deliberately does not name the tool: `tool_choice: 'required'` forces the call. -const WEATHER_PROMPT = `What is the weather in Paris? ${SHORT_ANSWER}`; -const SYSTEM = 'You are a helpful assistant used by an automated test.'; - -const WEATHER_TOOL = { - type: 'function' as const, - function: { - name: 'get_weather', - description: 'Get the current weather for a city.', - parameters: { - type: 'object', - properties: { city: { type: 'string', description: 'The city name' } }, - required: ['city'], - }, - }, -}; - -const app = express(); - -/** The trace this request is recorded under, so the test can look its spans up in Sentry. */ -function currentTraceId(): string | undefined { - return Sentry.getActiveSpan()?.spanContext().traceId; -} - -// Each route makes one call through the `openai` client and answers with the trace id, so the test can -// find the request's spans in Sentry. - -app.get('/chat', async (_req, res, next) => { - try { - const completion = await client.chat.completions.create({ - model: CHAT_MODEL, - messages: [ - { role: 'system', content: SYSTEM }, - { role: 'user', content: CHAT_PROMPT }, - ], - temperature: 0, - max_tokens: 32, - }); - res.send({ - traceId: currentTraceId(), - answer: completion.choices[0]?.message.content ?? '', - }); - } catch (error) { - next(error); - } -}); - -app.get('/chat-stream', async (_req, res, next) => { - try { - const stream = await client.chat.completions.create({ - model: CHAT_MODEL, - messages: [ - { role: 'system', content: SYSTEM }, - { role: 'user', content: CHAT_PROMPT }, - ], - temperature: 0, - max_tokens: 32, - stream: true, - // Ask for the usage in the final chunk, so the streamed span carries token counts. - stream_options: { include_usage: true }, - }); - - let answer = ''; - for await (const chunk of stream) { - answer += chunk.choices[0]?.delta.content ?? ''; - } - res.send({ traceId: currentTraceId(), answer }); - } catch (error) { - next(error); - } -}); - -app.get('/tools', async (_req, res, next) => { - try { - const completion = await client.chat.completions.create({ - model: CHAT_MODEL, - messages: [{ role: 'user', content: WEATHER_PROMPT }], - tools: [WEATHER_TOOL], - tool_choice: 'required', - max_tokens: 64, - }); - res.send({ - traceId: currentTraceId(), - toolCalls: completion.choices[0]?.message.tool_calls ?? [], - }); - } catch (error) { - next(error); - } -}); - -app.get('/responses', async (_req, res, next) => { - try { - const response = await client.responses.create({ - model: CHAT_MODEL, - instructions: SYSTEM, - input: CHAT_PROMPT, - temperature: 0, - max_output_tokens: 32, - }); - res.send({ traceId: currentTraceId(), answer: response.output_text }); - } catch (error) { - next(error); - } -}); - -app.get('/embeddings', async (_req, res, next) => { - try { - const embedding = await client.embeddings.create({ - model: EMBEDDINGS_MODEL, - input: 'The capital of France is Paris.', - }); - res.send({ - traceId: currentTraceId(), - dimensions: embedding.data[0]?.embedding.length ?? 0, - }); - } catch (error) { - next(error); - } -}); - -Sentry.setupExpressErrorHandler(app); - -app.use((error: Error, _req: express.Request, res: express.Response, _next: express.NextFunction) => { - res.status(500).send({ message: error.message }); -}); - -const port = Number(process.env.PORT ?? 3030); -app.listen(port, () => { - // eslint-disable-next-line no-console - console.log(`node-openai-send-to-sentry listening on port ${port}`); -}); diff --git a/dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/src/instrument.ts b/dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/src/instrument.ts deleted file mode 100644 index c6c15df960cb..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/src/instrument.ts +++ /dev/null @@ -1,10 +0,0 @@ -import * as Sentry from '@sentry/node'; - -// The setup from the docs, nothing more: preloaded with `node --import`, no tunnel, so the spans go to -// the real Sentry project behind `E2E_TEST_DSN`, where the tests read them back. The OpenAI integration -// is on by default; its runtime channel injection is what picks up the `openai` client in app.ts. -Sentry.init({ - dsn: process.env.E2E_TEST_DSN, - environment: 'qa', // dynamic sampling bias to keep transactions - tracesSampleRate: 1, -}); diff --git a/dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/tests/send-to-sentry.test.ts b/dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/tests/send-to-sentry.test.ts deleted file mode 100644 index eee3e75aeafb..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/tests/send-to-sentry.test.ts +++ /dev/null @@ -1,172 +0,0 @@ -import { expect, test } from '@playwright/test'; -import type { TraceItem } from '@sentry-internal/test-utils/cli'; -import { - EVENT_POLLING_OPTIONS, - fetchSpanAttributes, - fetchTrace, - findSpanInTrace, - flattenTrace, -} from '@sentry-internal/test-utils/cli'; - -// Mirrors src/app.ts. -const CHAT_MODEL = 'openai/gpt-4o-mini'; -const EMBEDDINGS_MODEL = 'openai/text-embedding-3-small'; - -type Attributes = Record; - -/** Sends one request to the app and returns the trace its request span was recorded under. */ -async function requestTrace(baseURL: string | undefined, path: string): Promise { - const response = await fetch(`${baseURL}${path}`); - const body = await response.text(); - expect(response.status, `${path} answered ${response.status}: ${body}`).toBe(200); - - const { traceId } = JSON.parse(body) as { traceId?: string }; - expect(traceId, `${path} did not report a trace id: ${body}`).toMatch(/^[0-9a-f]{32}$/); - - console.log(`${path}: https://${process.env.E2E_TEST_SENTRY_ORG_SLUG}.sentry.io/explore/traces/trace/${traceId}/`); - return traceId!; -} - -/** - * Polls Sentry until the trace holds a span with `op`, then returns that span and its attributes. The - * trace endpoint only carries the span's op, name and place in the tree; the attributes need a second - * lookup, which is polled as well since the two can land apart. - */ -async function waitForSpan(traceId: string, op: string): Promise<{ span: TraceItem; attributes: Attributes }> { - let found: { span: TraceItem; attributes: Attributes } | undefined; - - await expect - .poll(() => { - const span = findSpanInTrace(traceId, op); - const attributes = span?.event_id ? fetchSpanAttributes(traceId, span.event_id) : undefined; - found = span && attributes ? { span, attributes } : undefined; - return found; - }, EVENT_POLLING_OPTIONS) - .toBeDefined(); - - return found!; -} - -/** The ids of the spans with `op` below the request span (`http.server`) of the trace. */ -function spanIdsUnderRequestSpan(traceId: string, op: string): string[] { - const requestSpan = flattenTrace(fetchTrace(traceId)).find(item => item.op === 'http.server'); - return flattenTrace(requestSpan?.children ?? []) - .filter(item => item.op === op) - .map(item => item.event_id!); -} - -/** - * The request span is the segment; it ends last, so it can land after its children. Expecting exactly - * one span also catches the call being instrumented twice. - */ -async function expectOnlySpanUnderRequestSpan(traceId: string, op: string, span: TraceItem): Promise { - await expect.poll(() => spanIdsUnderRequestSpan(traceId, op), EVENT_POLLING_OPTIONS).toEqual([span.event_id]); -} - -/** - * The attributes every successful model-call span carries once Sentry has stored it, whatever the - * model answers. Model-dependent values (token counts, response id and model) are checked for shape, - * not content. Note the names are the stored ones: `origin` and `span.status` rather than the - * `sentry.*` attributes the SDK sends. - */ -function expectModelCallAttributes(attributes: Attributes, operation: 'chat' | 'embeddings', model: string): void { - expect(attributes).toMatchObject({ - 'span.op': `gen_ai.${operation}`, - 'span.status': 'ok', - origin: 'auto.ai.openai', - 'gen_ai.operation.name': operation, - 'gen_ai.provider.name': 'openai', - 'gen_ai.request.model': model, - }); - // OpenAI embeddings responses carry no id, so only chat calls are guaranteed one. - if (operation === 'chat') { - expect(typeof attributes['gen_ai.response.id']).toBe('string'); - } - expect(typeof attributes['gen_ai.response.model']).toBe('string'); - // The Sentry API returns int attributes as strings. - expect(Number(attributes['gen_ai.usage.input_tokens'])).toBeGreaterThan(0); - expect(Number(attributes['gen_ai.usage.total_tokens'])).toBeGreaterThan(0); -} - -/** `gen_ai.response.finish_reasons` is a JSON array; a streamed answer can repeat the reason per chunk. */ -function finishReasons(attributes: Attributes): string[] { - return JSON.parse(attributes['gen_ai.response.finish_reasons'] as string); -} - -test('Sends a chat completion to Sentry as a gen_ai.chat span under the request span', async ({ baseURL }) => { - const traceId = await requestTrace(baseURL, '/chat'); - - const { span, attributes } = await waitForSpan(traceId, 'gen_ai.chat'); - - expect(span.description).toBe(`chat ${CHAT_MODEL}`); - expectModelCallAttributes(attributes, 'chat', CHAT_MODEL); - expect(Number(attributes['gen_ai.usage.output_tokens'])).toBeGreaterThan(0); - expect(finishReasons(attributes)).toEqual(['stop']); - expect(attributes['gen_ai.response.streaming']).toBeUndefined(); - - // The SDK records prompts and answers by default, and they have to survive the trip to Sentry. The - // answer is sent as `gen_ai.response.text` and stored as `gen_ai.output.messages`. - expect(attributes['gen_ai.system_instructions']).toContain('automated test'); - expect(attributes['gen_ai.input.messages']).toContain('capital of France'); - expect(attributes['gen_ai.output.messages']).toMatch(/paris/i); - - await expectOnlySpanUnderRequestSpan(traceId, 'gen_ai.chat', span); -}); - -test('Sends a streamed chat completion to Sentry with its token usage', async ({ baseURL }) => { - const traceId = await requestTrace(baseURL, '/chat-stream'); - - const { span, attributes } = await waitForSpan(traceId, 'gen_ai.chat'); - - expect(span.description).toBe(`chat ${CHAT_MODEL}`); - expectModelCallAttributes(attributes, 'chat', CHAT_MODEL); - expect(attributes['gen_ai.response.streaming']).toBe(true); - // The usage only arrives in the final chunk, which the streaming instrumentation has to pick up. - expect(Number(attributes['gen_ai.usage.output_tokens'])).toBeGreaterThan(0); - expect(finishReasons(attributes)).toContain('stop'); - expect(attributes['gen_ai.output.messages']).toMatch(/paris/i); - - await expectOnlySpanUnderRequestSpan(traceId, 'gen_ai.chat', span); -}); - -test('Sends a chat completion that returned a tool call to Sentry', async ({ baseURL }) => { - const traceId = await requestTrace(baseURL, '/tools'); - - const { span, attributes } = await waitForSpan(traceId, 'gen_ai.chat'); - - expectModelCallAttributes(attributes, 'chat', CHAT_MODEL); - expect(finishReasons(attributes)).toEqual(['tool_calls']); - expect(attributes['gen_ai.tool.definitions']).toContain('get_weather'); - // The SDK also sends the returned call as `gen_ai.response.tool_calls`, but that attribute is not - // stored as such, so the finish reason is what proves the tool call made it. - // TODO: Assert the call in `gen_ai.output.messages` once the OpenAI integration records it there. - - await expectOnlySpanUnderRequestSpan(traceId, 'gen_ai.chat', span); -}); - -test('Sends a Responses API call to Sentry as a gen_ai.chat span', async ({ baseURL }) => { - const traceId = await requestTrace(baseURL, '/responses'); - - const { span, attributes } = await waitForSpan(traceId, 'gen_ai.chat'); - - expect(span.description).toBe(`chat ${CHAT_MODEL}`); - expectModelCallAttributes(attributes, 'chat', CHAT_MODEL); - expect(Number(attributes['gen_ai.usage.output_tokens'])).toBeGreaterThan(0); - expect(finishReasons(attributes)).toEqual(['completed']); - expect(attributes['gen_ai.input.messages']).toContain('capital of France'); - expect(attributes['gen_ai.output.messages']).toMatch(/paris/i); - - await expectOnlySpanUnderRequestSpan(traceId, 'gen_ai.chat', span); -}); - -test('Sends an embeddings call to Sentry as a gen_ai.embeddings span', async ({ baseURL }) => { - const traceId = await requestTrace(baseURL, '/embeddings'); - - const { span, attributes } = await waitForSpan(traceId, 'gen_ai.embeddings'); - - expect(span.description).toBe(`embeddings ${EMBEDDINGS_MODEL}`); - expectModelCallAttributes(attributes, 'embeddings', EMBEDDINGS_MODEL); - expect(attributes['gen_ai.embeddings.input']).toContain('Paris'); - - await expectOnlySpanUnderRequestSpan(traceId, 'gen_ai.embeddings', span); -}); diff --git a/dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/tsconfig.json b/dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/tsconfig.json deleted file mode 100644 index 795574fd9df2..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-openai-send-to-sentry/tsconfig.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "compilerOptions": { - "target": "es2022", - "module": "nodenext", - "moduleResolution": "nodenext", - "outDir": "dist", - "strict": true, - "esModuleInterop": true, - "skipLibCheck": true, - "types": ["node"] - }, - "include": ["src/**/*.ts"] -} From 84dffa64c9f7ef85c82be36f5e377083ca290d21 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Fri, 30 Oct 2026 10:37:55 +0100 Subject: [PATCH 2/9] test(e2e): Only keep the streaming extras in gen-ai-libraries Responses, embeddings and the Anthropic stream helper are covered by node-integration-tests already; only streaming benefits from a real wire format. Co-Authored-By: Claude Opus 5.5 --- .../gen-ai-libraries/package.json | 2 +- .../gen-ai-libraries/src/entry.cloudflare.ts | 2 +- .../gen-ai-libraries/src/libraries.ts | 35 +----------------- .../gen-ai-libraries/tests/extras.test.ts | 36 +------------------ 4 files changed, 4 insertions(+), 71 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json index 10dfd01d6656..175d26f749e3 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json @@ -1,6 +1,6 @@ { "name": "gen-ai-libraries", - "description": "Real gen_ai spans for every instrumented AI library (OpenAI, Anthropic, Mistral, Together, Vercel AI), each driven through OpenRouter with a chat query and a tool call, plus provider-specific calls (streaming, the OpenAI Responses and embeddings APIs, the Anthropic stream helper), on Node and on Cloudflare", + "description": "Real gen_ai spans for every instrumented AI library (OpenAI, Anthropic, Mistral, Together, Vercel AI), each driven through OpenRouter with a chat query and a tool call, plus streamed OpenAI and Anthropic chats, on Node and on Cloudflare", "version": "1.0.0", "private": true, "type": "module", diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts index c928ffd5d635..9503a400189e 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts @@ -19,7 +19,7 @@ export default Sentry.withSentry( const library = id ? byId.get(id) : undefined; const extra = library && action ? library.extras?.[action] : undefined; - if (!library || !action || (action !== 'chat' && action !== 'tools' && !extra)) { + if (!library || (action !== 'chat' && action !== 'tools' && !extra)) { return new Response('Not found', { status: 404 }); } diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts index a9fdfb5f3f98..898aa9e51582 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts @@ -60,12 +60,10 @@ export interface Library { origin?: string; chat: (apiKey: string) => Promise; tools: (apiKey: string) => Promise; - /** Calls only this library's SDK has (streaming, other endpoints), served as `/:lib/:name`. */ + /** Library-specific calls, served as `/:lib/:name`. */ extras?: Record Promise>; } -const EMBEDDINGS_MODEL = 'openai/text-embedding-3-small'; - const openAiClient = (apiKey: string): OpenAI => new OpenAI({ apiKey, baseURL: OPENROUTER_V1 }); // OpenRouter's Anthropic skin authenticates with a bearer token, so the key goes in `authToken` @@ -169,23 +167,6 @@ export const libraries: Library[] = [ } return answer; }, - responses: async apiKey => { - const response = await openAiClient(apiKey).responses.create({ - model: MODEL, - instructions: SYSTEM, - input: CHAT_PROMPT, - temperature: 0, - max_output_tokens: 32, - }); - return response.output_text; - }, - embeddings: async apiKey => { - const embedding = await openAiClient(apiKey).embeddings.create({ - model: EMBEDDINGS_MODEL, - input: 'The capital of France is Paris.', - }); - return embedding.data[0]?.embedding.length ?? 0; - }, }, }, openAiCompatible( @@ -264,7 +245,6 @@ export const libraries: Library[] = [ return (message.content ?? []).filter(block => block.type === 'tool_use'); }, extras: { - // `messages.create({ stream: true })` returns an async iterable of events the caller drains. 'chat-stream': async apiKey => { const stream = await anthropicClient(apiKey).messages.create({ model: MODEL, @@ -281,19 +261,6 @@ export const libraries: Library[] = [ } return answer; }, - // The SDK's streaming helper, which the integration instruments separately from `create`. - 'stream-helper': async apiKey => { - const message = await anthropicClient(apiKey) - .messages.stream({ - model: MODEL, - max_tokens: 32, - system: SYSTEM, - messages: [{ role: 'user', content: CHAT_PROMPT }], - }) - .finalMessage(); - const first = message.content?.[0]; - return first && first.type === 'text' ? first.text : ''; - }, }, }, diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/extras.test.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/extras.test.ts index 315cf0df19dd..77ab5caf0a63 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/extras.test.ts +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/extras.test.ts @@ -23,7 +23,7 @@ async function requestExtra(baseURL: string | undefined, path: string): Promise< return spansPromise; } -/** The only span of the trace with `op`; finding exactly one also catches a call instrumented twice. */ +/** The only span of the trace with `op`, so a call instrumented twice fails. */ function onlySpanWithOp(spans: SerializedStreamedSpan[], op: string): SerializedStreamedSpan { const matching = spans.filter(span => getSpanOp(span) === op); expect(matching, `expected exactly one ${op} span in:\n${describeTree(spans)}`).toHaveLength(1); @@ -47,28 +47,6 @@ test('openai: a streamed chat completion records its token usage', async ({ base expect(finishReasons(span)).toContain('stop'); }); -test('openai: a Responses API call emits a gen_ai.chat span', async ({ baseURL }) => { - const spans = await requestExtra(baseURL, 'openai/responses'); - const span = onlySpanWithOp(spans, 'gen_ai.chat'); - - expectCommonChatAttributes(span, OPENAI); - expect(finishReasons(span)).toEqual(['completed']); - expect(attr(span, 'gen_ai.input.messages')).toContain('capital of France'); -}); - -test('openai: an embeddings call emits a gen_ai.embeddings span', async ({ baseURL }) => { - const spans = await requestExtra(baseURL, 'openai/embeddings'); - const span = onlySpanWithOp(spans, 'gen_ai.embeddings'); - - expect(span.name).toBe('embeddings openai/text-embedding-3-small'); - expect(span.status).toBe('ok'); - expect(attr(span, 'sentry.origin')).toBe('auto.ai.openai'); - expect(attr(span, 'gen_ai.operation.name')).toBe('embeddings'); - expect(attr(span, 'gen_ai.provider.name')).toBe('openai'); - expect(attr(span, 'gen_ai.embeddings.input')).toContain('Paris'); - expect(attr(span, 'gen_ai.usage.input_tokens')).toBeGreaterThan(0); -}); - test('anthropic: a streamed message records its output usage', async ({ baseURL }) => { const spans = await requestExtra(baseURL, 'anthropic/chat-stream'); const span = onlySpanWithOp(spans, 'gen_ai.chat'); @@ -82,15 +60,3 @@ test('anthropic: a streamed message records its output usage', async ({ baseURL expect(attr(span, 'gen_ai.provider.name')).toBe(ANTHROPIC.provider); expect(attr(span, 'sentry.origin')).toBe(ANTHROPIC.origin); }); - -test('anthropic: the stream helper emits a single gen_ai.chat span', async ({ baseURL }) => { - const spans = await requestExtra(baseURL, 'anthropic/stream-helper'); - // `messages.stream()` calls `create` underneath, which must not produce a second span. - const span = onlySpanWithOp(spans, 'gen_ai.chat'); - - expect(attr(span, 'gen_ai.response.streaming')).toBe(true); - expect(finishReasons(span)).toContain('end_turn'); - expect(attr(span, 'gen_ai.usage.output_tokens')).toBeGreaterThan(0); - expect(attr(span, 'gen_ai.provider.name')).toBe(ANTHROPIC.provider); - expect(attr(span, 'sentry.origin')).toBe(ANTHROPIC.origin); -}); From 77c2614b2f074ec8f94c030ef946ffbbe8f37222 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Fri, 30 Oct 2026 10:42:50 +0100 Subject: [PATCH 3/9] test(e2e): Describe OpenRouter's [DONE] frame accurately Co-Authored-By: Claude Opus 5.5 --- .../test-applications/gen-ai-libraries/src/libraries.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts index 898aa9e51582..bf6af3697bf9 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts @@ -72,7 +72,7 @@ const anthropicClient = (apiKey: string): Anthropic => new Anthropic({ authToken: apiKey, baseURL: OPENROUTER_BASE, fetch: fetchWithoutDone }); /** - * OpenRouter ends each stream with an OpenAI-style `data: [DONE]` frame, which is not part of + * OpenRouter ends each stream with an `event: data` / `data: [DONE]` frame, which is not part of * Anthropic's protocol. Since 0.136.0 the SDK JSON-parses every named event, so this frame * makes it throw. Drop the frame before the SDK sees it. */ From b5a0998ed96b2c3f46681e9425bab1b2a159a001 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Fri, 30 Oct 2026 10:45:10 +0100 Subject: [PATCH 4/9] test(e2e): Replace the extras map with a chatStream call in gen-ai-libraries Co-Authored-By: Claude Opus 5.5 --- .../gen-ai-libraries/src/entry.cloudflare.ts | 14 ++-- .../gen-ai-libraries/src/entry.node.ts | 9 +-- .../gen-ai-libraries/src/libraries.ts | 72 +++++++++---------- .../tests/chat-stream.test.ts | 52 ++++++++++++++ .../gen-ai-libraries/tests/extras.test.ts | 62 ---------------- 5 files changed, 101 insertions(+), 108 deletions(-) create mode 100644 dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/chat-stream.test.ts delete mode 100644 dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/extras.test.ts diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts index 9503a400189e..9b568302052e 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts @@ -17,9 +17,16 @@ export default Sentry.withSentry( const url = new URL(request.url); const [, id, action] = url.pathname.split('/'); const library = id ? byId.get(id) : undefined; - const extra = library && action ? library.extras?.[action] : undefined; + const call = + action === 'chat' + ? library?.chat + : action === 'tools' + ? library?.tools + : action === 'chat-stream' + ? library?.chatStream + : undefined; - if (!library || (action !== 'chat' && action !== 'tools' && !extra)) { + if (!call) { return new Response('Not found', { status: 404 }); } @@ -29,8 +36,7 @@ export default Sentry.withSentry( } try { - const spanName = extra ? `${library.id}/${action}` : action === 'tools' ? 'ai-tool-workflow' : 'ai-workflow'; - const call = extra ?? library[action as 'chat' | 'tools']; + const spanName = action === 'tools' ? 'ai-tool-workflow' : 'ai-workflow'; const result = await Sentry.startSpan({ name: spanName, op: 'function' }, () => call(apiKey)); return Response.json({ result }); } catch (error) { diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.node.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.node.ts index b67803a4da58..95a405dfa4d0 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.node.ts +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.node.ts @@ -34,11 +34,12 @@ for (const library of libraries) { } }); - for (const [name, call] of Object.entries(library.extras ?? {})) { - app.get(`/${library.id}/${name}`, async (_req, res, next) => { + const chatStream = library.chatStream; + if (chatStream) { + app.get(`/${library.id}/chat-stream`, async (_req, res, next) => { try { - const result = await Sentry.startSpan({ name: `${library.id}/${name}`, op: 'function' }, () => call(apiKey)); - res.send({ result }); + const answer = await Sentry.startSpan({ name: 'ai-workflow', op: 'function' }, () => chatStream(apiKey)); + res.send({ answer }); } catch (error) { next(error); } diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts index bf6af3697bf9..dd002c976962 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts @@ -60,8 +60,8 @@ export interface Library { origin?: string; chat: (apiKey: string) => Promise; tools: (apiKey: string) => Promise; - /** Library-specific calls, served as `/:lib/:name`. */ - extras?: Record Promise>; + /** A streamed chat query; only set for the libraries whose streaming is tested. */ + chatStream?: (apiKey: string) => Promise; } const openAiClient = (apiKey: string): OpenAI => new OpenAI({ apiKey, baseURL: OPENROUTER_V1 }); @@ -147,26 +147,24 @@ function openAiCompatible( export const libraries: Library[] = [ { ...openAiCompatible('openai', 'openai', 'auto.ai.openai', openAiClient), - extras: { - 'chat-stream': async apiKey => { - const stream = await openAiClient(apiKey).chat.completions.create({ - model: MODEL, - messages: [ - { role: 'system', content: SYSTEM }, - { role: 'user', content: CHAT_PROMPT }, - ], - temperature: 0, - max_tokens: 32, - stream: true, - // Ask for the usage in the final chunk, so the streamed span carries token counts. - stream_options: { include_usage: true }, - }); - let answer = ''; - for await (const chunk of stream) { - answer += chunk.choices[0]?.delta.content ?? ''; - } - return answer; - }, + chatStream: async apiKey => { + const stream = await openAiClient(apiKey).chat.completions.create({ + model: MODEL, + messages: [ + { role: 'system', content: SYSTEM }, + { role: 'user', content: CHAT_PROMPT }, + ], + temperature: 0, + max_tokens: 32, + stream: true, + // Ask for the usage in the final chunk, so the streamed span carries token counts. + stream_options: { include_usage: true }, + }); + let answer = ''; + for await (const chunk of stream) { + answer += chunk.choices[0]?.delta.content ?? ''; + } + return answer; }, }, openAiCompatible( @@ -244,23 +242,21 @@ export const libraries: Library[] = [ }); return (message.content ?? []).filter(block => block.type === 'tool_use'); }, - extras: { - 'chat-stream': async apiKey => { - const stream = await anthropicClient(apiKey).messages.create({ - model: MODEL, - max_tokens: 32, - system: SYSTEM, - messages: [{ role: 'user', content: CHAT_PROMPT }], - stream: true, - }); - let answer = ''; - for await (const event of stream) { - if (event.type === 'content_block_delta' && event.delta.type === 'text_delta') { - answer += event.delta.text; - } + chatStream: async apiKey => { + const stream = await anthropicClient(apiKey).messages.create({ + model: MODEL, + max_tokens: 32, + system: SYSTEM, + messages: [{ role: 'user', content: CHAT_PROMPT }], + stream: true, + }); + let answer = ''; + for await (const event of stream) { + if (event.type === 'content_block_delta' && event.delta.type === 'text_delta') { + answer += event.delta.text; } - return answer; - }, + } + return answer; }, }, diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/chat-stream.test.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/chat-stream.test.ts new file mode 100644 index 000000000000..297a3b701465 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/chat-stream.test.ts @@ -0,0 +1,52 @@ +import { expect, test } from '@playwright/test'; +import type { SerializedStreamedSpan } from '@sentry-internal/test-utils'; +import { collectStreamedSpans } from '@sentry-internal/test-utils'; +import { APP } from './constants'; +import type { LibraryUnderTest } from './utils'; +import { attr, describeTree, expectCommonChatAttributes, isModelCallSpan, LIBRARIES } from './utils'; + +// Streaming is where OpenRouter's real wire format can diverge from the mocked responses in +// node-integration-tests, so it is only covered here. + +const OPENAI = LIBRARIES.find(library => library.id === 'openai')!; +const ANTHROPIC = LIBRARIES.find(library => library.id === 'anthropic')!; + +const isStreamedModelCallSpan = (span: SerializedStreamedSpan, library: LibraryUnderTest): boolean => + isModelCallSpan(span, library) && attr(span, 'gen_ai.response.streaming') === true; + +async function requestChatStream( + baseURL: string | undefined, + library: LibraryUnderTest, +): Promise { + const spansPromise = collectStreamedSpans(APP, spansOfTrace => + spansOfTrace.some(span => isStreamedModelCallSpan(span, library)), + ); + + const response = await fetch(`${baseURL}/${library.id}/chat-stream`); + expect(response.status, await response.text()).toBe(200); + + const spans = await spansPromise; + const modelSpan = spans.find(span => isStreamedModelCallSpan(span, library)); + expect(modelSpan, `expected a streamed ${library.op} span in:\n${describeTree(spans)}`).toBeDefined(); + return modelSpan!; +} + +test('openai: a streamed chat completion records its token usage', async ({ baseURL }) => { + const span = await requestChatStream(baseURL, OPENAI); + + // The usage only arrives in the final chunk, which the streaming instrumentation has to pick up. + expectCommonChatAttributes(span, OPENAI); + expect(attr(span, 'gen_ai.response.finish_reasons')).toContain('stop'); +}); + +test('anthropic: a streamed message records its output usage', async ({ baseURL }) => { + const span = await requestChatStream(baseURL, ANTHROPIC); + + // Not `expectCommonChatAttributes`: the instrumentation takes the input usage from `message_start`, + // where OpenRouter reports 0, and ignores the real count in `message_delta`. + expect(span.status).toBe('ok'); + expect(attr(span, 'gen_ai.usage.output_tokens')).toBeGreaterThan(0); + expect(attr(span, 'gen_ai.response.finish_reasons')).toContain('end_turn'); + expect(attr(span, 'gen_ai.provider.name')).toBe(ANTHROPIC.provider); + expect(attr(span, 'sentry.origin')).toBe(ANTHROPIC.origin); +}); diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/extras.test.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/extras.test.ts deleted file mode 100644 index 77ab5caf0a63..000000000000 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/extras.test.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { expect, test } from '@playwright/test'; -import type { SerializedStreamedSpan } from '@sentry-internal/test-utils'; -import { collectStreamedSpans, getSpanOp } from '@sentry-internal/test-utils'; -import { APP } from './constants'; -import { attr, describeTree, expectCommonChatAttributes, LIBRARIES } from './utils'; - -const OPENAI = LIBRARIES.find(library => library.id === 'openai')!; -const ANTHROPIC = LIBRARIES.find(library => library.id === 'anthropic')!; - -/** - * Calls `/:lib/:extra` and returns its trace once the request span (the segment) has arrived. The - * segment ends last, so every model-call span of the request is in by then. - */ -async function requestExtra(baseURL: string | undefined, path: string): Promise { - const spansPromise = collectStreamedSpans( - APP, - spansOfTrace => spansOfTrace.some(span => span.name === path) && spansOfTrace.some(span => span.is_segment), - ); - - const response = await fetch(`${baseURL}/${path}`); - expect(response.status, await response.text()).toBe(200); - - return spansPromise; -} - -/** The only span of the trace with `op`, so a call instrumented twice fails. */ -function onlySpanWithOp(spans: SerializedStreamedSpan[], op: string): SerializedStreamedSpan { - const matching = spans.filter(span => getSpanOp(span) === op); - expect(matching, `expected exactly one ${op} span in:\n${describeTree(spans)}`).toHaveLength(1); - return matching[0]!; -} - -/** `gen_ai.response.finish_reasons` is a JSON array; a streamed answer can repeat the reason per chunk. */ -function finishReasons(span: SerializedStreamedSpan): string[] { - const raw = attr(span, 'gen_ai.response.finish_reasons'); - expect(typeof raw, 'gen_ai.response.finish_reasons').toBe('string'); - return JSON.parse(raw as string); -} - -test('openai: a streamed chat completion records its token usage', async ({ baseURL }) => { - const spans = await requestExtra(baseURL, 'openai/chat-stream'); - const span = onlySpanWithOp(spans, 'gen_ai.chat'); - - // The usage only arrives in the final chunk, which the streaming instrumentation has to pick up. - expectCommonChatAttributes(span, OPENAI); - expect(attr(span, 'gen_ai.response.streaming')).toBe(true); - expect(finishReasons(span)).toContain('stop'); -}); - -test('anthropic: a streamed message records its output usage', async ({ baseURL }) => { - const spans = await requestExtra(baseURL, 'anthropic/chat-stream'); - const span = onlySpanWithOp(spans, 'gen_ai.chat'); - - expect(attr(span, 'gen_ai.response.streaming')).toBe(true); - expect(finishReasons(span)).toContain('end_turn'); - expect(attr(span, 'gen_ai.usage.output_tokens')).toBeGreaterThan(0); - // The input usage is not asserted: the instrumentation takes it from `message_start`, where - // OpenRouter reports 0, and ignores the real count OpenRouter (and the Anthropic API) put in - // `message_delta`. - expect(attr(span, 'gen_ai.provider.name')).toBe(ANTHROPIC.provider); - expect(attr(span, 'sentry.origin')).toBe(ANTHROPIC.origin); -}); From d97e00205f04728e2efe5d4d2da8b19657f4fd04 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Fri, 30 Oct 2026 10:47:10 +0100 Subject: [PATCH 5/9] test(e2e): Update every AI SDK in the gen-ai-libraries latest variant Co-Authored-By: Claude Opus 5.5 --- .../e2e-tests/test-applications/gen-ai-libraries/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json index 175d26f749e3..5df587e8e0c1 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json @@ -14,7 +14,7 @@ "test:build:cloudflare": "pnpm install && vite build", "test:assert": "pnpm test", "test:assert:cloudflare": "RUNTIME=cloudflare pnpm test", - "test:build-latest": "pnpm install && pnpm add openai@latest && pnpm update '@anthropic-ai/sdk@<1'" + "test:build-latest": "pnpm install && pnpm add 'openai@<8' '@anthropic-ai/sdk@<1' '@mistralai/mistralai@<3' 'together-ai@<2' ai@latest @openrouter/ai-sdk-provider@latest" }, "dependencies": { "@anthropic-ai/sdk": "0.63.0", From ead56a7fe14dde416552215397419fc62b502732 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Fri, 30 Oct 2026 10:48:04 +0100 Subject: [PATCH 6/9] test(e2e): Use @latest for every AI SDK not known to be unsupported Co-Authored-By: Claude Opus 5.5 --- .../e2e-tests/test-applications/gen-ai-libraries/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json index 5df587e8e0c1..55b214bedd8b 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json @@ -14,7 +14,7 @@ "test:build:cloudflare": "pnpm install && vite build", "test:assert": "pnpm test", "test:assert:cloudflare": "RUNTIME=cloudflare pnpm test", - "test:build-latest": "pnpm install && pnpm add 'openai@<8' '@anthropic-ai/sdk@<1' '@mistralai/mistralai@<3' 'together-ai@<2' ai@latest @openrouter/ai-sdk-provider@latest" + "test:build-latest": "pnpm install && pnpm add openai@latest together-ai@latest ai@latest @openrouter/ai-sdk-provider@latest '@anthropic-ai/sdk@<1' '@mistralai/mistralai@<3'" }, "dependencies": { "@anthropic-ai/sdk": "0.63.0", From 6ddf5476b78b74eac259d5cfb52bcf1ccc73c8e9 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Fri, 30 Oct 2026 10:51:07 +0100 Subject: [PATCH 7/9] test(e2e): Pick the Cloudflare route's call with a switch Co-Authored-By: Claude Opus 5.5 --- .../gen-ai-libraries/src/entry.cloudflare.ts | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts index 9b568302052e..10ddd8d13a03 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts @@ -1,10 +1,24 @@ // The Cloudflare variant: the same libraries and routes as the Node entry, but instrumented by the // `@sentry/cloudflare/vite` bundler plugin (build-time channel injection) and run on workerd. import * as Sentry from '@sentry/cloudflare'; +import type { Library } from './libraries'; import { libraries } from './libraries'; const byId = new Map(libraries.map(library => [library.id, library])); +function getCall(library: Library, action: string | undefined): ((apiKey: string) => Promise) | undefined { + switch (action) { + case 'chat': + return library.chat; + case 'tools': + return library.tools; + case 'chat-stream': + return library.chatStream; + default: + return undefined; + } +} + export default Sentry.withSentry( (env: Env) => ({ dsn: env.E2E_TEST_DSN, @@ -17,14 +31,7 @@ export default Sentry.withSentry( const url = new URL(request.url); const [, id, action] = url.pathname.split('/'); const library = id ? byId.get(id) : undefined; - const call = - action === 'chat' - ? library?.chat - : action === 'tools' - ? library?.tools - : action === 'chat-stream' - ? library?.chatStream - : undefined; + const call = library && getCall(library, action); if (!call) { return new Response('Not found', { status: 404 }); From 0a8b7d57f88c4dcf58bf82f315f03a2fe6fe6498 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Fri, 30 Oct 2026 10:52:51 +0100 Subject: [PATCH 8/9] test(e2e): Use semantic convention constants in the gen-ai-libraries stream test Co-Authored-By: Claude Opus 5.5 --- .../gen-ai-libraries/package.json | 1 + .../tests/chat-stream.test.ts | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json index 55b214bedd8b..4ee5be3ac607 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json @@ -33,6 +33,7 @@ "@cloudflare/workers-types": "^4.20260426.0", "@playwright/test": "1.64.1", "@sentry-internal/test-utils": "file:../../packed/sentry-internal-test-utils-packed.tgz", + "@sentry/conventions": "0.30.0", "@types/express": "^4.17.21", "@types/node": "^18.19.1", "tsx": "4.21.0", diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/chat-stream.test.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/chat-stream.test.ts index 297a3b701465..2a68b3f6e5f4 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/chat-stream.test.ts +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/chat-stream.test.ts @@ -1,4 +1,11 @@ import { expect, test } from '@playwright/test'; +import { + GEN_AI_PROVIDER_NAME, + GEN_AI_RESPONSE_FINISH_REASONS, + GEN_AI_RESPONSE_STREAMING, + GEN_AI_USAGE_OUTPUT_TOKENS, + SENTRY_ORIGIN, +} from '@sentry/conventions/attributes'; import type { SerializedStreamedSpan } from '@sentry-internal/test-utils'; import { collectStreamedSpans } from '@sentry-internal/test-utils'; import { APP } from './constants'; @@ -12,7 +19,7 @@ const OPENAI = LIBRARIES.find(library => library.id === 'openai')!; const ANTHROPIC = LIBRARIES.find(library => library.id === 'anthropic')!; const isStreamedModelCallSpan = (span: SerializedStreamedSpan, library: LibraryUnderTest): boolean => - isModelCallSpan(span, library) && attr(span, 'gen_ai.response.streaming') === true; + isModelCallSpan(span, library) && attr(span, GEN_AI_RESPONSE_STREAMING) === true; async function requestChatStream( baseURL: string | undefined, @@ -36,7 +43,7 @@ test('openai: a streamed chat completion records its token usage', async ({ base // The usage only arrives in the final chunk, which the streaming instrumentation has to pick up. expectCommonChatAttributes(span, OPENAI); - expect(attr(span, 'gen_ai.response.finish_reasons')).toContain('stop'); + expect(attr(span, GEN_AI_RESPONSE_FINISH_REASONS)).toContain('stop'); }); test('anthropic: a streamed message records its output usage', async ({ baseURL }) => { @@ -45,8 +52,8 @@ test('anthropic: a streamed message records its output usage', async ({ baseURL // Not `expectCommonChatAttributes`: the instrumentation takes the input usage from `message_start`, // where OpenRouter reports 0, and ignores the real count in `message_delta`. expect(span.status).toBe('ok'); - expect(attr(span, 'gen_ai.usage.output_tokens')).toBeGreaterThan(0); - expect(attr(span, 'gen_ai.response.finish_reasons')).toContain('end_turn'); - expect(attr(span, 'gen_ai.provider.name')).toBe(ANTHROPIC.provider); - expect(attr(span, 'sentry.origin')).toBe(ANTHROPIC.origin); + expect(attr(span, GEN_AI_USAGE_OUTPUT_TOKENS)).toBeGreaterThan(0); + expect(attr(span, GEN_AI_RESPONSE_FINISH_REASONS)).toContain('end_turn'); + expect(attr(span, GEN_AI_PROVIDER_NAME)).toBe(ANTHROPIC.provider); + expect(attr(span, SENTRY_ORIGIN)).toBe(ANTHROPIC.origin); }); From 242a9b1c673f7585ab32960fbc5a04ee0528b322 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Fri, 30 Oct 2026 10:53:41 +0100 Subject: [PATCH 9/9] test(e2e): Test the latest Mistral SDK in gen-ai-libraries Mistral 3.x is instrumented since #25827. Co-Authored-By: Claude Opus 5.5 --- .../e2e-tests/test-applications/gen-ai-libraries/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json index 4ee5be3ac607..3d0ff3e11d38 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json @@ -14,7 +14,7 @@ "test:build:cloudflare": "pnpm install && vite build", "test:assert": "pnpm test", "test:assert:cloudflare": "RUNTIME=cloudflare pnpm test", - "test:build-latest": "pnpm install && pnpm add openai@latest together-ai@latest ai@latest @openrouter/ai-sdk-provider@latest '@anthropic-ai/sdk@<1' '@mistralai/mistralai@<3'" + "test:build-latest": "pnpm install && pnpm add openai@latest together-ai@latest ai@latest @openrouter/ai-sdk-provider@latest '@anthropic-ai/sdk@<1' @mistralai/mistralai@latest" }, "dependencies": { "@anthropic-ai/sdk": "0.63.0",