From 84f12e6f6e313a21f5c762faa16b67585044630a Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 28 Oct 2026 17:05:23 +0100 Subject: [PATCH] test(e2e): Check node-openai-decisions through the event proxy Renames node-openai-decisions-send-to-sentry to node-openai-decisions and asserts the gen_ai.decide spans at the local event proxy instead of polling the Sentry API. The app still calls the real Decisions API. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/optional-e2e.yml | 3 +- dev-packages/e2e-tests/.env.example | 2 +- .../playwright.config.ts | 27 --- .../tests/send-to-sentry.test.ts | 224 ------------------ .../.gitignore | 0 .../package.json | 7 +- .../playwright.config.ts | 9 + .../pnpm-workspace.yaml | 0 .../src/app.ts | 6 +- .../src/instrument.ts | 3 +- .../start-event-proxy.mjs | 6 + .../tests/decisions.test.ts | 195 +++++++++++++++ .../tsconfig.json | 0 13 files changed, 220 insertions(+), 262 deletions(-) delete mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/playwright.config.ts delete mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts rename dev-packages/e2e-tests/test-applications/{node-openai-decisions-send-to-sentry => node-openai-decisions}/.gitignore (100%) rename dev-packages/e2e-tests/test-applications/{node-openai-decisions-send-to-sentry => node-openai-decisions}/package.json (73%) create mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions/playwright.config.ts rename dev-packages/e2e-tests/test-applications/{node-openai-decisions-send-to-sentry => node-openai-decisions}/pnpm-workspace.yaml (100%) rename dev-packages/e2e-tests/test-applications/{node-openai-decisions-send-to-sentry => node-openai-decisions}/src/app.ts (94%) rename dev-packages/e2e-tests/test-applications/{node-openai-decisions-send-to-sentry => node-openai-decisions}/src/instrument.ts (63%) create mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions/start-event-proxy.mjs create mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions/tests/decisions.test.ts rename dev-packages/e2e-tests/test-applications/{node-openai-decisions-send-to-sentry => node-openai-decisions}/tsconfig.json (100%) diff --git a/.github/workflows/optional-e2e.yml b/.github/workflows/optional-e2e.yml index f73eaa256fc0..cff2e2437ad1 100644 --- a/.github/workflows/optional-e2e.yml +++ b/.github/workflows/optional-e2e.yml @@ -59,8 +59,7 @@ jobs: E2E_OPENROUTER_API_KEY: ${{ secrets.E2E_OPENROUTER_API_KEY }} # Only this app talks to OpenAI directly (OpenRouter has no Decisions route), so only it gets the key. E2E_OPENAI_DECISIONS_API_KEY: - ${{ matrix.test-application == 'node-openai-decisions-send-to-sentry' && secrets.E2E_OPENAI_DECISIONS_API_KEY || - '' }} + ${{ matrix.test-application == 'node-openai-decisions' && secrets.E2E_OPENAI_DECISIONS_API_KEY || '' }} # Used by test apps that deploy a real Cloudflare Worker, e.g. cloudflare-workers-send-to-sentry CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} diff --git a/dev-packages/e2e-tests/.env.example b/dev-packages/e2e-tests/.env.example index 3815cae556f0..17326f227680 100644 --- a/dev-packages/e2e-tests/.env.example +++ b/dev-packages/e2e-tests/.env.example @@ -17,7 +17,7 @@ E2E_TEST_SENTRY_PROJECT= E2E_OPENROUTER_API_KEY= # An OpenAI API key for the Decisions API (OpenRouter has no Decisions route). -# Only needed to run node-openai-decisions-send-to-sentry locally. +# Only needed to run node-openai-decisions locally. E2E_OPENAI_DECISIONS_API_KEY= # Cloudflare credentials for E2E tests that deploy a real Worker (e.g. cloudflare-workers-send-to-sentry). diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/playwright.config.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/playwright.config.ts deleted file mode 100644 index af5ea8f0b951..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/playwright.config.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { defineConfig } from '@playwright/test'; - -const port = 3030; - -export default defineConfig({ - testDir: './tests', - /* - * Each Sentry poll can take up to 180s. The first poll also waits the ~2min until the span can be - * queried, so the later polls usually pass at once. This leaves room for two full polls. - */ - timeout: 400_000, - fullyParallel: true, - forbidOnly: !!process.env.CI, - retries: 0, - // Every test spends most of its time polling Sentry, so run them all at once. - workers: '100%', - reporter: process.env.CI ? [['list'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list', - use: { - baseURL: `http://localhost:${port}`, - }, - webServer: { - command: 'pnpm start', - port, - stdout: 'pipe', - stderr: 'pipe', - }, -}); diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts deleted file mode 100644 index f15dec0a9e8d..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts +++ /dev/null @@ -1,224 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { - SEARCH_GEN_AI__INPUT__MESSAGES, - SEARCH_GEN_AI__OPERATION__NAME, - SEARCH_GEN_AI__OUTPUT__MESSAGES, - SEARCH_GEN_AI__PROVIDER__NAME, - SEARCH_GEN_AI__REQUEST__MODEL, - SEARCH_GEN_AI__RESPONSE__MODEL, - SEARCH_GEN_AI__USAGE__INPUT_TOKENS, - SEARCH_GEN_AI__USAGE__OUTPUT_TOKENS, - SEARCH_GEN_AI__USAGE__TOTAL_TOKENS, - SEARCH_ORIGIN, - SEARCH_SPAN__OP, - SEARCH_SPAN__STATUS, -} from '@sentry/conventions/attributes/search'; -import { GEN_AI_DECIDE, HTTP_SERVER } from '@sentry/conventions/op'; -import { - EVENT_POLLING_OPTIONS, - fetchSpanAttributes, - fetchTrace, - findSpanInTrace, - flattenTrace, - traceTarget, -} from '@sentry-internal/test-utils/cli'; - -// Mirrors src/app.ts. -const MODEL = 'gpt-6-luna'; - -/** - * What every decide span carries once Sentry has stored it, whatever the model answers. The keys are the - * search names, e.g. `span.op` and `origin` rather than the `sentry.*` attributes the SDK sends. - * `span.description` stays empty: ingest only derives it for ops it knows, and `gen_ai.decide` is new. - */ -const DECIDE_SPAN_ATTRIBUTES = { - 'span.name': `decide ${MODEL}`, - [SEARCH_SPAN__OP]: GEN_AI_DECIDE, - [SEARCH_SPAN__STATUS]: 'ok', - [SEARCH_ORIGIN]: 'auto.ai.openai', - [SEARCH_GEN_AI__OPERATION__NAME]: 'decide', - [SEARCH_GEN_AI__PROVIDER__NAME]: 'openai', - [SEARCH_GEN_AI__REQUEST__MODEL]: MODEL, - [SEARCH_GEN_AI__RESPONSE__MODEL]: MODEL, -}; - -test('records a text decision with its state, questions and answers', async ({ baseURL }) => { - const response = await fetch(`${baseURL}/decide`); - const body = await response.json(); - expect(response.status, JSON.stringify(body)).toBe(200); - const { traceId }: { traceId: string } = body; - - console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`); - - let spanId: string | undefined; - await expect - .poll(() => (spanId = findSpanInTrace(traceId, GEN_AI_DECIDE)?.event_id), EVENT_POLLING_OPTIONS) - .toBeDefined(); - - let attributes: Record | undefined; - await expect - .poll(() => (attributes = fetchSpanAttributes(traceId, spanId!)), EVENT_POLLING_OPTIONS) - .toMatchObject(DECIDE_SPAN_ATTRIBUTES); - - // Integers come back as strings. Decisions bills input tokens only, so there are no output tokens. - expect(Number(attributes![SEARCH_GEN_AI__USAGE__OUTPUT_TOKENS])).toBe(0); - expect(Number(attributes![SEARCH_GEN_AI__USAGE__INPUT_TOKENS])).toBeGreaterThan(0); - expect(Number(attributes![SEARCH_GEN_AI__USAGE__TOTAL_TOKENS])).toBe( - Number(attributes![SEARCH_GEN_AI__USAGE__INPUT_TOKENS]), - ); - - // The request span is the segment; it ends last, so it can land after the decide span. - await expect - .poll(() => { - const requestSpan = flattenTrace(fetchTrace(traceId)).find(item => item.op === HTTP_SERVER); - return flattenTrace(requestSpan?.children ?? []).some(item => item.event_id === spanId); - }, EVENT_POLLING_OPTIONS) - .toBe(true); - - expect(JSON.parse(attributes![SEARCH_GEN_AI__INPUT__MESSAGES] as string)).toEqual([ - { - type: 'evaluation', - state: 'I was charged twice this month and want a refund.', - questions: { - wants_refund: { type: 'predicate', instructions: 'Is a refund requested?' }, - department: { - type: 'choice', - instructions: 'Which team should handle this?', - criteria: { billing: 'Charges and refunds', other: 'Anything else' }, - }, - urgency: { - type: 'score', - instructions: 'How urgent is this ticket?', - criteria: ['low', 'high'], - }, - }, - }, - ]); - - // The answers depend on the model, the shape does not. - expect(JSON.parse(attributes![SEARCH_GEN_AI__OUTPUT__MESSAGES] as string)).toEqual([ - { - type: 'evaluation', - answers: { - wants_refund: { type: 'predicate', probability: expect.any(Number) }, - department: { - type: 'choice', - choice: expect.stringMatching(/^(billing|other)$/), - probabilities: { billing: expect.any(Number), other: expect.any(Number) }, - confidence: expect.any(Number), - }, - urgency: { - type: 'score', - score: expect.any(Number), - probabilities: { 0: expect.any(Number), 1: expect.any(Number) }, - legend: { 0: 'low', 1: 'high' }, - confidence: expect.any(Number), - }, - }, - }, - ]); -}); - -test('records an image decision with its image data', async ({ baseURL }) => { - const response = await fetch(`${baseURL}/decide-image`); - const body = await response.json(); - expect(response.status, JSON.stringify(body)).toBe(200); - const { traceId }: { traceId: string } = body; - - console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`); - - let spanId: string | undefined; - await expect - .poll(() => (spanId = findSpanInTrace(traceId, GEN_AI_DECIDE)?.event_id), EVENT_POLLING_OPTIONS) - .toBeDefined(); - - let attributes: Record | undefined; - await expect - .poll(() => (attributes = fetchSpanAttributes(traceId, spanId!)), EVENT_POLLING_OPTIONS) - .toMatchObject(DECIDE_SPAN_ATTRIBUTES); - - expect(JSON.parse(attributes![SEARCH_GEN_AI__INPUT__MESSAGES] as string)).toEqual([ - { - type: 'evaluation', - state: [ - { - role: 'user', - content: [ - { type: 'input_text', text: 'Is this image mostly red?' }, - { type: 'input_image', image_url: expect.stringMatching(/^data:image\/png;base64,/) }, - ], - }, - ], - questions: { is_red: { type: 'predicate', instructions: 'Is the image mostly red?' } }, - }, - ]); - - expect(JSON.parse(attributes![SEARCH_GEN_AI__OUTPUT__MESSAGES] as string)).toEqual([ - { type: 'evaluation', answers: { is_red: { type: 'predicate', probability: expect.any(Number) } } }, - ]); -}); - -test('records a Vercel AI decision with the confidence of its answers', async ({ baseURL }) => { - const response = await fetch(`${baseURL}/vercel-ai-decide`); - const body = await response.json(); - expect(response.status, JSON.stringify(body)).toBe(200); - const { traceId }: { traceId: string } = body; - - console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`); - - let spanId: string | undefined; - await expect - .poll(() => (spanId = findSpanInTrace(traceId, GEN_AI_DECIDE)?.event_id), EVENT_POLLING_OPTIONS) - .toBeDefined(); - - let attributes: Record | undefined; - await expect - .poll(() => (attributes = fetchSpanAttributes(traceId, spanId!)), EVENT_POLLING_OPTIONS) - .toMatchObject({ - 'span.name': `decide ${MODEL}`, - [SEARCH_SPAN__OP]: GEN_AI_DECIDE, - [SEARCH_SPAN__STATUS]: 'ok', - [SEARCH_ORIGIN]: 'auto.ai.vercelai', - [SEARCH_GEN_AI__OPERATION__NAME]: 'decide', - [SEARCH_GEN_AI__PROVIDER__NAME]: 'openai.decision', - [SEARCH_GEN_AI__REQUEST__MODEL]: MODEL, - }); - - expect(JSON.parse(attributes![SEARCH_GEN_AI__INPUT__MESSAGES] as string)).toEqual([ - { - type: 'evaluation', - state: 'I was charged twice this month and want a refund.', - questions: { - wantsRefund: { type: 'boolean', instructions: 'Is a refund requested?' }, - department: { - type: 'choice', - instructions: 'Which team should handle this?', - criteria: { billing: 'Charges and refunds', other: 'Anything else' }, - }, - urgency: { type: 'score', instructions: 'How urgent is this ticket?', criteria: ['low', 'high'] }, - }, - }, - ]); - - // The OpenAI provider moves `confidence` into its provider metadata; the span must keep it on the answers. - expect(JSON.parse(attributes![SEARCH_GEN_AI__OUTPUT__MESSAGES] as string)).toEqual([ - { - type: 'evaluation', - answers: { - wantsRefund: { type: 'boolean', probability: expect.any(Number) }, - department: { - type: 'choice', - choice: expect.stringMatching(/^(billing|other)$/), - probabilities: { billing: expect.any(Number), other: expect.any(Number) }, - confidence: expect.any(Number), - }, - urgency: { - type: 'score', - score: expect.any(Number), - probabilities: { 0: expect.any(Number), 1: expect.any(Number) }, - confidence: expect.any(Number), - }, - }, - }, - ]); -}); diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/.gitignore b/dev-packages/e2e-tests/test-applications/node-openai-decisions/.gitignore similarity index 100% rename from dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/.gitignore rename to dev-packages/e2e-tests/test-applications/node-openai-decisions/.gitignore diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json b/dev-packages/e2e-tests/test-applications/node-openai-decisions/package.json similarity index 73% rename from dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json rename to dev-packages/e2e-tests/test-applications/node-openai-decisions/package.json index 7785dcfc5b72..41a84ca03d45 100644 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions/package.json @@ -1,6 +1,6 @@ { - "name": "node-openai-decisions-send-to-sentry", - "description": "The OpenAI Decisions API as a user runs it: a plain Sentry.init on an express app, real decisions requests against OpenAI through the openai client and the Vercel AI SDK, and the gen_ai.decide spans read back from a real Sentry project", + "name": "node-openai-decisions", + "description": "The OpenAI Decisions API as a user runs it: real decisions requests against OpenAI through the openai client and the Vercel AI SDK, and the gen_ai.decide spans they record", "version": "1.0.0", "private": true, "type": "module", @@ -27,7 +27,6 @@ "@sentry/conventions": "0.30.0", "@types/express": "^4.17.21", "@types/node": "^24.0.0", - "sentry": "~0.44.1", "typescript": "~5.9.0" }, "sentryTest": { @@ -35,7 +34,7 @@ "optionalVariants": [ { "build-command": "pnpm test:build-latest", - "label": "node-openai-decisions-send-to-sentry (latest)" + "label": "node-openai-decisions (latest)" } ] } diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions/playwright.config.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions/playwright.config.ts new file mode 100644 index 000000000000..7fe69f2eddd2 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions/playwright.config.ts @@ -0,0 +1,9 @@ +import { getPlaywrightConfig } from '@sentry-internal/test-utils'; + +const config = getPlaywrightConfig( + { startCommand: 'pnpm start' }, + // Each test makes a real Decisions API call, which does not reliably fit the default 30s timeout. + { timeout: 90_000 }, +); + +export default config; diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/pnpm-workspace.yaml b/dev-packages/e2e-tests/test-applications/node-openai-decisions/pnpm-workspace.yaml similarity index 100% rename from dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/pnpm-workspace.yaml rename to dev-packages/e2e-tests/test-applications/node-openai-decisions/pnpm-workspace.yaml diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions/src/app.ts similarity index 94% rename from dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts rename to dev-packages/e2e-tests/test-applications/node-openai-decisions/src/app.ts index 20c7ba9a96c7..4a3f15581765 100644 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions/src/app.ts @@ -23,8 +23,8 @@ const RED_PIXEL = const app = express(); -// Each route makes one `decisions.create` call and answers with the trace id, so the test can find -// the request's spans in Sentry. +// Each route makes one decisions call and answers with the trace id, so the test can check it picked the +// spans of its own request. app.get('/decide', async (_req, res, next) => { try { @@ -107,5 +107,5 @@ app.use((error: Error, _req: express.Request, res: express.Response, _next: expr const port = Number(process.env.PORT ?? 3030); app.listen(port, () => { // eslint-disable-next-line no-console - console.log(`node-openai-decisions-send-to-sentry listening on port ${port}`); + console.log(`node-openai-decisions listening on port ${port}`); }); diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/instrument.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions/src/instrument.ts similarity index 63% rename from dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/instrument.ts rename to dev-packages/e2e-tests/test-applications/node-openai-decisions/src/instrument.ts index e0a0c1807f75..9d6f0f49f765 100644 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/instrument.ts +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions/src/instrument.ts @@ -2,6 +2,7 @@ import * as Sentry from '@sentry/node'; Sentry.init({ dsn: process.env.E2E_TEST_DSN, - environment: 'qa', // dynamic sampling bias to keep transactions + environment: 'qa', + tunnel: 'http://localhost:3031/', tracesSampleRate: 1, }); diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions/start-event-proxy.mjs b/dev-packages/e2e-tests/test-applications/node-openai-decisions/start-event-proxy.mjs new file mode 100644 index 000000000000..a40b90989577 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions/start-event-proxy.mjs @@ -0,0 +1,6 @@ +import { startEventProxyServer } from '@sentry-internal/test-utils'; + +startEventProxyServer({ + port: 3031, + proxyServerName: 'node-openai-decisions', +}); diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions/tests/decisions.test.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions/tests/decisions.test.ts new file mode 100644 index 000000000000..2f4647ecbf6e --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions/tests/decisions.test.ts @@ -0,0 +1,195 @@ +import { expect, test } from '@playwright/test'; +import { + GEN_AI_INPUT_MESSAGES, + GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MODEL, + GEN_AI_RESPONSE_MODEL, + GEN_AI_USAGE_INPUT_TOKENS, + GEN_AI_USAGE_OUTPUT_TOKENS, + GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_ORIGIN, +} from '@sentry/conventions/attributes'; +import { GEN_AI_DECIDE, HTTP_SERVER } from '@sentry/conventions/op'; +import type { SerializedStreamedSpan } from '@sentry-internal/test-utils'; +import { collectStreamedSpans, getSpanOp } from '@sentry-internal/test-utils'; + +const APP = 'node-openai-decisions'; + +// Mirrors src/app.ts. +const MODEL = 'gpt-6-luna'; + +const attr = (span: SerializedStreamedSpan, key: string): unknown => span.attributes?.[key]?.value; + +const isDecideSpan = (span: SerializedStreamedSpan, origin: string): boolean => + getSpanOp(span) === GEN_AI_DECIDE && attr(span, SENTRY_ORIGIN) === origin; + +/** + * Calls `path` and returns its decide span once the request span (the segment) has arrived as well. The + * text and image decisions share an origin, so `inputMarker` tells their spans apart. + */ +async function requestDecideSpan( + baseURL: string | undefined, + path: string, + origin: string, + inputMarker: string, +): Promise { + const matches = (span: SerializedStreamedSpan): boolean => + isDecideSpan(span, origin) && String(attr(span, GEN_AI_INPUT_MESSAGES)).includes(inputMarker); + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => spansOfTrace.some(matches) && spansOfTrace.some(span => span.is_segment), + ); + + const response = await fetch(`${baseURL}${path}`); + const body = await response.json(); + expect(response.status, JSON.stringify(body)).toBe(200); + + const spans = await spansPromise; + const decideSpans = spans.filter(span => getSpanOp(span) === GEN_AI_DECIDE); + expect(decideSpans).toHaveLength(1); + const decideSpan = decideSpans[0]!; + expect(decideSpan.trace_id).toBe(body.traceId); + expect(getSpanOp(spans.find(span => span.is_segment)!)).toBe(HTTP_SERVER); + + return decideSpan; +} + +/** What every decide span of the `openai` client carries, whatever the model answers. */ +function expectOpenAiDecideSpan(span: SerializedStreamedSpan): void { + expect(span.name).toBe(`decide ${MODEL}`); + expect(span.status).toBe('ok'); + expect(attr(span, GEN_AI_OPERATION_NAME)).toBe('decide'); + expect(attr(span, GEN_AI_PROVIDER_NAME)).toBe('openai'); + expect(attr(span, GEN_AI_REQUEST_MODEL)).toBe(MODEL); + expect(attr(span, GEN_AI_RESPONSE_MODEL)).toBe(MODEL); +} + +test('records a text decision with its state, questions and answers', async ({ baseURL }) => { + const span = await requestDecideSpan(baseURL, '/decide', 'auto.ai.openai', 'charged twice'); + + expectOpenAiDecideSpan(span); + + // Decisions bills input tokens only, so there are no output tokens. + expect(attr(span, GEN_AI_USAGE_OUTPUT_TOKENS)).toBe(0); + expect(attr(span, GEN_AI_USAGE_INPUT_TOKENS)).toBeGreaterThan(0); + expect(attr(span, GEN_AI_USAGE_TOTAL_TOKENS)).toBe(attr(span, GEN_AI_USAGE_INPUT_TOKENS)); + + expect(JSON.parse(attr(span, GEN_AI_INPUT_MESSAGES) as string)).toEqual([ + { + type: 'evaluation', + state: 'I was charged twice this month and want a refund.', + questions: { + wants_refund: { type: 'predicate', instructions: 'Is a refund requested?' }, + department: { + type: 'choice', + instructions: 'Which team should handle this?', + criteria: { billing: 'Charges and refunds', other: 'Anything else' }, + }, + urgency: { + type: 'score', + instructions: 'How urgent is this ticket?', + criteria: ['low', 'high'], + }, + }, + }, + ]); + + // The answers depend on the model, the shape does not. + expect(JSON.parse(attr(span, GEN_AI_OUTPUT_MESSAGES) as string)).toEqual([ + { + type: 'evaluation', + answers: { + wants_refund: { type: 'predicate', probability: expect.any(Number) }, + department: { + type: 'choice', + choice: expect.stringMatching(/^(billing|other)$/), + probabilities: { billing: expect.any(Number), other: expect.any(Number) }, + confidence: expect.any(Number), + }, + urgency: { + type: 'score', + score: expect.any(Number), + probabilities: { 0: expect.any(Number), 1: expect.any(Number) }, + legend: { 0: 'low', 1: 'high' }, + confidence: expect.any(Number), + }, + }, + }, + ]); +}); + +test('records an image decision with its image data', async ({ baseURL }) => { + const span = await requestDecideSpan(baseURL, '/decide-image', 'auto.ai.openai', 'mostly red'); + + expectOpenAiDecideSpan(span); + + expect(JSON.parse(attr(span, GEN_AI_INPUT_MESSAGES) as string)).toEqual([ + { + type: 'evaluation', + state: [ + { + role: 'user', + content: [ + { type: 'input_text', text: 'Is this image mostly red?' }, + { type: 'input_image', image_url: expect.stringMatching(/^data:image\/png;base64,/) }, + ], + }, + ], + questions: { is_red: { type: 'predicate', instructions: 'Is the image mostly red?' } }, + }, + ]); + + expect(JSON.parse(attr(span, GEN_AI_OUTPUT_MESSAGES) as string)).toEqual([ + { type: 'evaluation', answers: { is_red: { type: 'predicate', probability: expect.any(Number) } } }, + ]); +}); + +test('records a Vercel AI decision with the confidence of its answers', async ({ baseURL }) => { + const span = await requestDecideSpan(baseURL, '/vercel-ai-decide', 'auto.ai.vercelai', 'charged twice'); + + expect(span.name).toBe(`decide ${MODEL}`); + expect(span.status).toBe('ok'); + expect(attr(span, GEN_AI_OPERATION_NAME)).toBe('decide'); + expect(attr(span, GEN_AI_PROVIDER_NAME)).toBe('openai.decision'); + expect(attr(span, GEN_AI_REQUEST_MODEL)).toBe(MODEL); + + expect(JSON.parse(attr(span, GEN_AI_INPUT_MESSAGES) as string)).toEqual([ + { + type: 'evaluation', + state: 'I was charged twice this month and want a refund.', + questions: { + wantsRefund: { type: 'boolean', instructions: 'Is a refund requested?' }, + department: { + type: 'choice', + instructions: 'Which team should handle this?', + criteria: { billing: 'Charges and refunds', other: 'Anything else' }, + }, + urgency: { type: 'score', instructions: 'How urgent is this ticket?', criteria: ['low', 'high'] }, + }, + }, + ]); + + // The OpenAI provider moves `confidence` into its provider metadata; the span must keep it on the answers. + expect(JSON.parse(attr(span, GEN_AI_OUTPUT_MESSAGES) as string)).toEqual([ + { + type: 'evaluation', + answers: { + wantsRefund: { type: 'boolean', probability: expect.any(Number) }, + department: { + type: 'choice', + choice: expect.stringMatching(/^(billing|other)$/), + probabilities: { billing: expect.any(Number), other: expect.any(Number) }, + confidence: expect.any(Number), + }, + urgency: { + type: 'score', + score: expect.any(Number), + probabilities: { 0: expect.any(Number), 1: expect.any(Number) }, + confidence: expect.any(Number), + }, + }, + }, + ]); +}); diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tsconfig.json b/dev-packages/e2e-tests/test-applications/node-openai-decisions/tsconfig.json similarity index 100% rename from dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tsconfig.json rename to dev-packages/e2e-tests/test-applications/node-openai-decisions/tsconfig.json