diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/index.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/index.ts index e48428c9c732..42d2d895fea0 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/index.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/index.ts @@ -34,6 +34,14 @@ export default Sentry.withSentry( return new Response(JSON.stringify(result)); } + if (url.pathname === '/evaluate') { + const result = await ai.run('typesafe/jev', { + state: 'Help! My payouts have been failing for 3 days.', + questions: { is_urgent: { type: 'noul', instructions: 'Does this convey urgency?' } }, + }); + return new Response(JSON.stringify(result)); + } + if (url.pathname === '/stream') { const stream = (await ai.run('@cf/meta/llama-3.1-8b-instruct', { messages: [{ role: 'user', content: 'What is the capital of France?' }], diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/mocks.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/mocks.ts index bce096d4ae75..e0a26d02bf86 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/mocks.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/mocks.ts @@ -23,6 +23,17 @@ export class MockAi { throw error; } + if (model === 'typesafe/jev') { + return { + state: 'Completed', + result: { + model: 'jev-1.13.0', + answers: { is_urgent: { type: 'noul', noul: 0.97 } }, + usage: { input_tokens: 426, output_tokens: 73 }, + }, + }; + } + if (inputs?.stream === true) { return createSseStream([ '{"response":"The capital "}', diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts index 726f91a5e467..4eaf940ef9c7 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts @@ -5,6 +5,7 @@ import { GEN_AI_REQUEST_MAX_TOKENS, GEN_AI_REQUEST_MODEL, GEN_AI_REQUEST_TEMPERATURE, + GEN_AI_RESPONSE_MODEL, GEN_AI_RESPONSE_STREAMING, GEN_AI_RESPONSE_TEXT, GEN_AI_USAGE_INPUT_TOKENS, @@ -132,6 +133,52 @@ it('traces a streaming Workers AI text generation request', async ({ signal }) = await runner.completed(); }); +it('traces a TypeSafe Jev evaluation like the TypeSafe integration', async ({ signal }) => { + const runner = createRunner(__dirname) + .ignore('event') + .expect(envelope => { + const spans = getSpansFromEnvelope(envelope); + const segmentSpan = spans.find(span => span.is_segment); + + const genAiSpans = spans.filter(span => getSpanOp(span)?.startsWith('gen_ai.')); + expect(genAiSpans).toHaveLength(1); + + expect(genAiSpans[0]).toEqual( + expect.objectContaining({ + name: 'evaluate typesafe/jev', + status: 'ok', + is_segment: false, + attributes: { + 'sentry.origin': { value: 'auto.ai.cloudflare.workers_ai', type: 'string' }, + 'sentry.op': { value: 'gen_ai.evaluate', type: 'string' }, + [GEN_AI_PROVIDER_NAME]: { value: 'cloudflare.workers_ai', type: 'string' }, + [GEN_AI_OPERATION_NAME]: { value: 'evaluate', type: 'string' }, + [GEN_AI_REQUEST_MODEL]: { value: 'typesafe/jev', type: 'string' }, + [GEN_AI_RESPONSE_MODEL]: { value: 'jev-1.13.0', type: 'string' }, + [GEN_AI_USAGE_INPUT_TOKENS]: { value: 426, type: 'integer' }, + [GEN_AI_USAGE_OUTPUT_TOKENS]: { value: 73, type: 'integer' }, + [GEN_AI_USAGE_TOTAL_TOKENS]: { value: 499, type: 'integer' }, + // collect only output messages + [GEN_AI_OUTPUT_MESSAGES]: { + type: 'string', + value: '[{"type":"evaluation","answers":{"is_urgent":{"type":"noul","noul":0.97}}}]', + }, + 'sentry.is_localhost': { value: true, type: 'boolean' }, + [SENTRY_TRACE_LIFECYCLE]: { value: 'stream', type: 'string' }, + [SENTRY_SEGMENT_NAME]: { value: segmentSpan!.name, type: 'string' }, + [SENTRY_SEGMENT_ID]: { value: segmentSpan!.span_id, type: 'string' }, + [SENTRY_SDK_NAME]: { value: 'sentry.javascript.cloudflare', type: 'string' }, + [SENTRY_SDK_VERSION]: { value: SDK_VERSION, type: 'string' }, + [SEMANTIC_ATTRIBUTE_SENTRY_ENVIRONMENT]: { value: 'production', type: 'string' }, + }, + }), + ); + }) + .start(signal); + await runner.makeRequest('get', '/evaluate'); + await runner.completed(); +}); + // The Workers AI integration deliberately does not call `captureException` itself. // When a `run` call fails, the error must bubble up out of the fetch handler and be // reported by the top-level Cloudflare instrumentation instead — so it shows up in diff --git a/dev-packages/e2e-tests/test-applications/cloudflare-workers-send-to-sentry/src/index.ts b/dev-packages/e2e-tests/test-applications/cloudflare-workers-send-to-sentry/src/index.ts index 324c9161cf69..87ae6c9cff41 100644 --- a/dev-packages/e2e-tests/test-applications/cloudflare-workers-send-to-sentry/src/index.ts +++ b/dev-packages/e2e-tests/test-applications/cloudflare-workers-send-to-sentry/src/index.ts @@ -31,6 +31,13 @@ export default { await env.AI.run('@cf/meta/llama-3.2-1b-instruct', { prompt: 'Say hi', max_tokens: 5 }); return Response.json({ traceId: spanContext?.traceId }); } + case '/test-workers-ai-jev': { + await env.AI.run('typesafe/jev', { + state: 'Help! My payouts have been failing for 3 days.', + questions: { is_urgent: { type: 'noul', instructions: 'Does this convey urgency?' } }, + }); + return Response.json({ traceId: spanContext?.traceId }); + } case '/test-span': return Response.json({ spanId: spanContext?.spanId, traceId: spanContext?.traceId }); case '/test-workflow-sleep': { diff --git a/dev-packages/e2e-tests/test-applications/cloudflare-workers-send-to-sentry/tests/send-to-sentry.test.ts b/dev-packages/e2e-tests/test-applications/cloudflare-workers-send-to-sentry/tests/send-to-sentry.test.ts index 2397db37089f..b391f2f8244b 100644 --- a/dev-packages/e2e-tests/test-applications/cloudflare-workers-send-to-sentry/tests/send-to-sentry.test.ts +++ b/dev-packages/e2e-tests/test-applications/cloudflare-workers-send-to-sentry/tests/send-to-sentry.test.ts @@ -93,3 +93,24 @@ test('Sends a Workers AI gen_ai span to Sentry', async () => { 'gen_ai.output.messages': expect.stringContaining('"role":"assistant"'), }); }); + +test('Sends a Workers AI gen_ai.evaluate span for a TypeSafe Jev call to Sentry', async () => { + const { traceId }: { traceId: string } = JSON.parse(await fetchFromWorker(`${workerUrl}/test-workers-ai-jev`, 200)); + + console.log(`Polling for gen_ai.evaluate span: sentry trace view ${traceTarget(traceId)}`); + + let spanId: string | undefined; + await expect + .poll(() => (spanId = findSpanInTrace(traceId, 'gen_ai.evaluate')?.event_id), EVENT_POLLING_OPTIONS) + .toBeDefined(); + + await expect + .poll(() => fetchSpanAttributes(traceId, spanId!), EVENT_POLLING_OPTIONS) + .toMatchObject({ + 'gen_ai.operation.name': 'evaluate', + 'gen_ai.request.model': 'typesafe/jev', + 'gen_ai.response.model': expect.stringMatching(/^jev-/), + 'gen_ai.input.messages': expect.stringContaining('My payouts have been failing'), + 'gen_ai.output.messages': expect.stringContaining('"is_urgent"'), + }); +}); diff --git a/packages/server-utils/src/ai/typesafe/index.ts b/packages/server-utils/src/ai/typesafe/index.ts index b8fe48878024..328853cf5b02 100644 --- a/packages/server-utils/src/ai/typesafe/index.ts +++ b/packages/server-utils/src/ai/typesafe/index.ts @@ -49,17 +49,19 @@ function getRequestAttributes( [GEN_AI_OPERATION_NAME]: 'evaluate', [GEN_AI_PROVIDER_NAME]: TYPESAFE_PROVIDER_NAME, ...(model ? { [GEN_AI_REQUEST_MODEL]: model } : {}), - ...(recordInputs - ? { - [GEN_AI_INPUT_MESSAGES]: stringify([ - { type: 'evaluation', state: request.state, questions: request.questions }, - ]), - } - : {}), + ...(recordInputs ? { [GEN_AI_INPUT_MESSAGES]: getEvaluationInputMessages(request) } : {}), }; } -/** Add the response model, token usage and (optionally) the answers of a `systemOne` result. */ +/** Serialize the `state` and `questions` of an evaluation request. Also used for TypeSafe models on Workers AI. */ +export function getEvaluationInputMessages(request: Record): string | undefined { + return stringify([{ type: 'evaluation', state: request.state, questions: request.questions }]); +} + +/** + * Add the response model, token usage and (optionally) the answers of a `systemOne` result. + * Also used for TypeSafe models on Workers AI. + */ export function addResponseAttributes(span: Span, result: unknown, recordOutputs: boolean): void { if (!isObjectLike(result)) { return; diff --git a/packages/server-utils/src/ai/workers-ai/index.ts b/packages/server-utils/src/ai/workers-ai/index.ts index c02222cc1877..eb3a7c5407cb 100644 --- a/packages/server-utils/src/ai/workers-ai/index.ts +++ b/packages/server-utils/src/ai/workers-ai/index.ts @@ -89,7 +89,7 @@ function instrumentRun( } // The model did not actually return a stream — finalize the span eagerly. - addResponseAttributes(span, result, options.recordOutputs); + addResponseAttributes(span, result, options.recordOutputs, operationName); span.end(); return result; }, handleError); @@ -105,7 +105,7 @@ function instrumentRun( return originalResult.then(result => { if (!returnsRawResponse) { - addResponseAttributes(span, result, options.recordOutputs); + addResponseAttributes(span, result, options.recordOutputs, operationName); } return result; }); diff --git a/packages/server-utils/src/ai/workers-ai/utils.ts b/packages/server-utils/src/ai/workers-ai/utils.ts index 33908290be72..67645a456988 100644 --- a/packages/server-utils/src/ai/workers-ai/utils.ts +++ b/packages/server-utils/src/ai/workers-ai/utils.ts @@ -16,10 +16,16 @@ import { GEN_AI_SYSTEM_INSTRUCTIONS, } from '@sentry/conventions/attributes'; import { GEN_AI_CHAT, GEN_AI_EMBEDDINGS } from '@sentry/conventions/op'; -import { SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN, stringify } from '@sentry/core'; +import { isObjectLike, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN, stringify } from '@sentry/core'; import type { Span, SpanAttributeValue } from '@sentry/core'; import { GEN_AI_REQUEST_STREAM_ATTRIBUTE } from '../core/gen-ai-attributes'; -import { extractSystemInstructions, setOutputMessagesAttribute, setTokenUsageAttributes } from '../core/utils'; +import { + extractSystemInstructions, + getGenAiSpanOp, + setOutputMessagesAttribute, + setTokenUsageAttributes, +} from '../core/utils'; +import { addResponseAttributes as addEvaluateResponseAttributes, getEvaluationInputMessages } from '../typesafe'; // Re-exported so `workers-ai/streaming.ts` keeps importing it from this module. export { setOutputMessagesAttribute }; import { WORKERS_AI_ORIGIN, WORKERS_AI_PROVIDER_NAME } from './constants'; @@ -29,15 +35,20 @@ import type { WorkersAiInput, WorkersAiOutput } from './types'; * Determine the gen_ai operation name from the inputs passed to `AI.run`. * Workers AI exposes a single `run` method, so we infer the operation from the input shape. */ -export type WorkersAiOperationName = 'chat' | 'embeddings'; +export type WorkersAiOperationName = 'chat' | 'embeddings' | 'evaluate'; export const WORKERS_AI_OPERATION_SPAN_OPS: Record = { chat: GEN_AI_CHAT, embeddings: GEN_AI_EMBEDDINGS, + evaluate: getGenAiSpanOp('evaluate'), }; export function getOperationName(inputs: unknown): WorkersAiOperationName { if (inputs && typeof inputs === 'object') { + // TypeSafe evaluation models (e.g. `typesafe/jev`) + if ('state' in inputs && 'questions' in inputs) { + return 'evaluate'; + } if ('messages' in inputs || 'prompt' in inputs) { return 'chat'; } @@ -102,6 +113,11 @@ export function addRequestAttributes(span: Span, inputs: unknown, operationName: } const params = inputs as WorkersAiInput; + if (operationName === 'evaluate') { + span.setAttribute(GEN_AI_INPUT_MESSAGES, getEvaluationInputMessages(inputs as Record)); + return; + } + // Store embeddings input on a separate attribute if (operationName === 'embeddings') { const text = params.text; @@ -132,7 +148,19 @@ export function addRequestAttributes(span: Span, inputs: unknown, operationName: /** * Record the response attributes (token usage, response text, tool calls) on the span. */ -export function addResponseAttributes(span: Span, result: unknown, recordOutputs: boolean): void { +export function addResponseAttributes( + span: Span, + result: unknown, + recordOutputs: boolean, + operationName?: WorkersAiOperationName, +): void { + if (operationName === 'evaluate') { + // The binding wraps the TypeSafe response as `{ state, result }`, but the docs show it unwrapped. + const body = isObjectLike(result) && isObjectLike(result.result) ? result.result : result; + addEvaluateResponseAttributes(span, body, recordOutputs); + return; + } + if ( !result || typeof result !== 'object' || diff --git a/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts b/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts index 4baced963ec2..bc282e0348bc 100644 --- a/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts @@ -5,8 +5,12 @@ import { GEN_AI_OUTPUT_MESSAGES, GEN_AI_PROVIDER_NAME, GEN_AI_REQUEST_MODEL, + GEN_AI_RESPONSE_MODEL, GEN_AI_RESPONSE_TEXT, GEN_AI_SYSTEM_INSTRUCTIONS, + GEN_AI_USAGE_INPUT_TOKENS, + GEN_AI_USAGE_OUTPUT_TOKENS, + GEN_AI_USAGE_TOTAL_TOKENS, } from '@sentry/conventions/attributes'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { @@ -160,6 +164,45 @@ describe('instrumentWorkersAiClient', () => { }); }); + it('records TypeSafe models as evaluate spans, like the TypeSafe integration', async () => { + const client = new TestClient( + getDefaultTestClientOptions({ dsn: 'https://public@dsn.ingest.sentry.io/1337', tracesSampleRate: 1 }), + ); + setCurrentClient(client); + client.init(); + const endedSpans: Span[] = []; + client.on('spanEnd', span => endedSpans.push(span)); + + const questions = { is_urgent: { type: 'noul', instructions: 'Does this convey urgency?' } }; + const answers = { is_urgent: { type: 'noul', noul: 0.97 } }; + const ai = { + run: vi.fn().mockResolvedValue({ + state: 'Completed', + result: { model: 'jev-1.13.0', answers, usage: { input_tokens: 426, output_tokens: 73 } }, + }), + }; + + await instrumentWorkersAiClient(ai).run('typesafe/jev', { state: 'Help!', questions }); + + const span = spanToStaticSpanJSON(endedSpans[0]!); + expect(span.description).toBe('evaluate typesafe/jev'); + expect(span.data).toEqual({ + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.cloudflare.workers_ai', + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.evaluate', + [SEMANTIC_ATTRIBUTE_SENTRY_SAMPLE_RATE]: 1, + [SENTRY_SEGMENT_NAME_SOURCE]: 'custom', + [GEN_AI_PROVIDER_NAME]: 'cloudflare.workers_ai', + [GEN_AI_OPERATION_NAME]: 'evaluate', + [GEN_AI_REQUEST_MODEL]: 'typesafe/jev', + [GEN_AI_RESPONSE_MODEL]: 'jev-1.13.0', + [GEN_AI_USAGE_INPUT_TOKENS]: 426, + [GEN_AI_USAGE_OUTPUT_TOKENS]: 73, + [GEN_AI_USAGE_TOTAL_TOKENS]: 499, + [GEN_AI_INPUT_MESSAGES]: JSON.stringify([{ type: 'evaluation', state: 'Help!', questions }]), + [GEN_AI_OUTPUT_MESSAGES]: JSON.stringify([{ type: 'evaluation', answers }]), + }); + }); + describe('span names', () => { function setupClient(traceLifecycle: 'static' | 'stream'): Span[] { const client = new TestClient( diff --git a/packages/server-utils/test/ai/lib/utils/workers-ai-utils.test.ts b/packages/server-utils/test/ai/lib/utils/workers-ai-utils.test.ts index 70878dc1a96e..8d9f9d15b257 100644 --- a/packages/server-utils/test/ai/lib/utils/workers-ai-utils.test.ts +++ b/packages/server-utils/test/ai/lib/utils/workers-ai-utils.test.ts @@ -14,6 +14,7 @@ import { GEN_AI_REQUEST_TOP_K, GEN_AI_REQUEST_TOP_P, GEN_AI_OUTPUT_MESSAGES, + GEN_AI_RESPONSE_MODEL, GEN_AI_RESPONSE_TEXT, GEN_AI_RESPONSE_TOOL_CALLS, GEN_AI_SYSTEM_INSTRUCTIONS, @@ -59,6 +60,10 @@ describe('workers-ai utils', () => { expect(getOperationName({ text: 'embed me' })).toBe('embeddings'); }); + it('returns "evaluate" for TypeSafe state and questions inputs', () => { + expect(getOperationName({ state: 'Help!', questions: {} })).toBe('evaluate'); + }); + it('prefers "chat" when both messages and text are present', () => { expect(getOperationName({ messages: [{ role: 'user', content: 'Hi' }], text: 'embed me' })).toBe('chat'); }); @@ -185,6 +190,29 @@ describe('workers-ai utils', () => { }); describe('addResponseAttributes', () => { + it.each([ + { name: 'wrapped in { state, result }', wrap: (body: object) => ({ state: 'Completed', result: body }) }, + { name: 'unwrapped', wrap: (body: object) => body }, + ])('records evaluate responses $name', ({ wrap }) => { + const { span, attributes } = createMockSpan(); + const answers = { is_urgent: { type: 'noul', noul: 0.95 } }; + + addResponseAttributes( + span, + wrap({ model: 'jev-1.13.0', answers, usage: { input_tokens: 366, output_tokens: 50 } }), + true, + 'evaluate', + ); + + expect(attributes).toEqual({ + [GEN_AI_RESPONSE_MODEL]: 'jev-1.13.0', + [GEN_AI_USAGE_INPUT_TOKENS]: 366, + [GEN_AI_USAGE_OUTPUT_TOKENS]: 50, + [GEN_AI_USAGE_TOTAL_TOKENS]: 416, + [GEN_AI_OUTPUT_MESSAGES]: JSON.stringify([{ type: 'evaluation', answers }]), + }); + }); + it('sets token usage and computes the total from input and output tokens', () => { const { span, attributes } = createMockSpan();