feat(cloudflare): Trace Clef calls in Workers AI as evaluate spans

dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/index.ts

@@ -42,6 +42,15 @@ export default Sentry.withSentry(
         return new Response(JSON.stringify(result));
       }
 
+      if (url.pathname === '/evaluate-clef') {
+        const result = await ai.run('@cf/cloudflare/clef', {
+          model: 'clef',
+          state: 'Checkout has been failing for every customer for the last hour.',
+          questions: { urgent: { type: 'noul', instructions: 'Is this support request urgent?' } },
+        });
+        return new Response(JSON.stringify(result));
+      }
+
       if (url.pathname === '/stream') {
         const stream = (await ai.run('@cf/meta/llama-3.1-8b-instruct', {
           messages: [{ role: 'user', content: 'What is the capital of France?' }],

dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/mocks.ts

@@ -34,6 +34,15 @@ export class MockAi {
       };
     }
 
+    // Unwrapped, as in the documented Clef output schema, so this covers the path without `{ state, result }`.
+    if (model === '@cf/cloudflare/clef') {
+      return {
+        model: 'clef',
+        answers: { urgent: { type: 'noul', noul: 0.98 } },
+        usage: { input_tokens: 412, output_tokens: 1 },
+      };
+    }
+
     if (inputs?.stream === true) {
       return createSseStream([
         '{"response":"The capital "}',

dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts

@@ -17,6 +17,9 @@ import {
   SENTRY_SEGMENT_NAME,
   SENTRY_TRACE_LIFECYCLE,
   SENTRY_ENVIRONMENT,
+  SENTRY_IS_LOCALHOST,
+  SENTRY_OP,
+  SENTRY_ORIGIN,
 } from '@sentry/conventions/attributes';
 import { SDK_VERSION } from '@sentry/core';
 import { expect, it } from 'vitest';
@@ -45,8 +48,8 @@ it('traces a basic Workers AI text generation request', async ({ signal }) => {
           status: 'ok',
           is_segment: false,
           attributes: {
-            'sentry.origin': { value: 'auto.ai.cloudflare.workers_ai', type: 'string' },
-            'sentry.op': { value: 'gen_ai.chat', type: 'string' },
+            [SENTRY_ORIGIN]: { value: 'auto.ai.cloudflare.workers_ai', type: 'string' },
+            [SENTRY_OP]: { value: 'gen_ai.chat', type: 'string' },
             [GEN_AI_PROVIDER_NAME]: { value: 'cloudflare.workers_ai', type: 'string' },
             [GEN_AI_OPERATION_NAME]: { value: 'chat', type: 'string' },
             [GEN_AI_REQUEST_MODEL]: { value: '@cf/meta/llama-3.1-8b-instruct', type: 'string' },
@@ -64,7 +67,7 @@ it('traces a basic Workers AI text generation request', async ({ signal }) => {
               type: 'string',
               value: 'The capital of France is Paris.',
             },
-            'sentry.is_localhost': { value: true, type: 'boolean' },
+            [SENTRY_IS_LOCALHOST]: { value: true, type: 'boolean' },
             [SENTRY_TRACE_LIFECYCLE]: { value: 'stream', type: 'string' },
             [SENTRY_SEGMENT_NAME]: { value: segmentSpan!.name, type: 'string' },
             [SENTRY_SEGMENT_ID]: { value: segmentSpan!.span_id, type: 'string' },
@@ -99,8 +102,8 @@ it('traces a streaming Workers AI text generation request', async ({ signal }) =
           status: 'ok',
           is_segment: false,
           attributes: {
-            'sentry.origin': { value: 'auto.ai.cloudflare.workers_ai', type: 'string' },
-            'sentry.op': { value: 'gen_ai.chat', type: 'string' },
+            [SENTRY_ORIGIN]: { value: 'auto.ai.cloudflare.workers_ai', type: 'string' },
+            [SENTRY_OP]: { value: 'gen_ai.chat', type: 'string' },
             [GEN_AI_PROVIDER_NAME]: { value: 'cloudflare.workers_ai', type: 'string' },
             [GEN_AI_OPERATION_NAME]: { value: 'chat', type: 'string' },
             [GEN_AI_REQUEST_MODEL]: { value: '@cf/meta/llama-3.1-8b-instruct', type: 'string' },
@@ -118,7 +121,7 @@ it('traces a streaming Workers AI text generation request', async ({ signal }) =
               type: 'string',
               value: 'The capital of France is Paris.',
             },
-            'sentry.is_localhost': { value: true, type: 'boolean' },
+            [SENTRY_IS_LOCALHOST]: { value: true, type: 'boolean' },
             [SENTRY_TRACE_LIFECYCLE]: { value: 'stream', type: 'string' },
             [SENTRY_SEGMENT_NAME]: { value: segmentSpan!.name, type: 'string' },
             [SENTRY_SEGMENT_ID]: { value: segmentSpan!.span_id, type: 'string' },
@@ -150,8 +153,8 @@ it('traces a TypeSafe Jev evaluation like the TypeSafe integration', async ({ si
           status: 'ok',
           is_segment: false,
           attributes: {
-            'sentry.origin': { value: 'auto.ai.cloudflare.workers_ai', type: 'string' },
-            'sentry.op': { value: 'gen_ai.evaluate', type: 'string' },
+            [SENTRY_ORIGIN]: { value: 'auto.ai.cloudflare.workers_ai', type: 'string' },
+            [SENTRY_OP]: { value: 'gen_ai.evaluate', type: 'string' },
             [GEN_AI_PROVIDER_NAME]: { value: 'cloudflare.workers_ai', type: 'string' },
             [GEN_AI_OPERATION_NAME]: { value: 'evaluate', type: 'string' },
             [GEN_AI_REQUEST_MODEL]: { value: 'typesafe/jev', type: 'string' },
@@ -164,7 +167,7 @@ it('traces a TypeSafe Jev evaluation like the TypeSafe integration', async ({ si
               type: 'string',
               value: '[{"type":"evaluation","answers":{"is_urgent":{"type":"noul","noul":0.97}}}]',
             },
-            'sentry.is_localhost': { value: true, type: 'boolean' },
+            [SENTRY_IS_LOCALHOST]: { value: true, type: 'boolean' },
             [SENTRY_TRACE_LIFECYCLE]: { value: 'stream', type: 'string' },
             [SENTRY_SEGMENT_NAME]: { value: segmentSpan!.name, type: 'string' },
             [SENTRY_SEGMENT_ID]: { value: segmentSpan!.span_id, type: 'string' },
@@ -180,6 +183,52 @@ it('traces a TypeSafe Jev evaluation like the TypeSafe integration', async ({ si
   await runner.completed();
 });
 
+it('traces a Clef evaluation like a TypeSafe Jev evaluation', async ({ signal }) => {
+  const runner = createRunner(__dirname)
+    .ignore('event')
+    .expect(envelope => {
+      const spans = getSpansFromEnvelope(envelope);
+      const segmentSpan = spans.find(span => span.is_segment);
+
+      const genAiSpans = spans.filter(span => getSpanOp(span)?.startsWith('gen_ai.'));
+      expect(genAiSpans).toHaveLength(1);
+
+      expect(genAiSpans[0]).toEqual(
+        expect.objectContaining({
+          name: 'evaluate @cf/cloudflare/clef',
+          status: 'ok',
+          is_segment: false,
+          attributes: {
+            [SENTRY_ORIGIN]: { value: 'auto.ai.cloudflare.workers_ai', type: 'string' },
+            [SENTRY_OP]: { value: 'gen_ai.evaluate', type: 'string' },
+            [GEN_AI_PROVIDER_NAME]: { value: 'cloudflare.workers_ai', type: 'string' },
+            [GEN_AI_OPERATION_NAME]: { value: 'evaluate', type: 'string' },
+            [GEN_AI_REQUEST_MODEL]: { value: '@cf/cloudflare/clef', type: 'string' },
+            [GEN_AI_RESPONSE_MODEL]: { value: 'clef', type: 'string' },
+            [GEN_AI_USAGE_INPUT_TOKENS]: { value: 412, type: 'integer' },
+            [GEN_AI_USAGE_OUTPUT_TOKENS]: { value: 1, type: 'integer' },
+            [GEN_AI_USAGE_TOTAL_TOKENS]: { value: 413, type: 'integer' },
+            // collect only output messages
+            [GEN_AI_OUTPUT_MESSAGES]: {
+              type: 'string',
+              value: '[{"type":"evaluation","answers":{"urgent":{"type":"noul","noul":0.98}}}]',
+            },
+            [SENTRY_IS_LOCALHOST]: { value: true, type: 'boolean' },
+            [SENTRY_TRACE_LIFECYCLE]: { value: 'stream', type: 'string' },
+            [SENTRY_SEGMENT_NAME]: { value: segmentSpan!.name, type: 'string' },
+            [SENTRY_SEGMENT_ID]: { value: segmentSpan!.span_id, type: 'string' },
+            [SENTRY_SDK_NAME]: { value: 'sentry.javascript.cloudflare', type: 'string' },
+            [SENTRY_SDK_VERSION]: { value: SDK_VERSION, type: 'string' },
+            [SENTRY_ENVIRONMENT]: { value: 'production', type: 'string' },
+          },
+        }),
+      );
+    })
+    .start(signal);
+  await runner.makeRequest('get', '/evaluate-clef');
+  await runner.completed();
+});
+
 // The Workers AI integration deliberately does not call `captureException` itself.
 // When a `run` call fails, the error must bubble up out of the fetch handler and be
 // reported by the top-level Cloudflare instrumentation instead — so it shows up in

dev-packages/e2e-tests/test-applications/cloudflare-workers-send-to-sentry/src/index.ts

@@ -38,6 +38,14 @@ export default {
         });
         return Response.json({ traceId: spanContext?.traceId });
       }
+      case '/test-workers-ai-clef': {
+        await env.AI.run('@cf/cloudflare/clef', {
+          model: 'clef',
+          state: 'Checkout has been failing for every customer for the last hour.',
+          questions: { urgent: { type: 'noul', instructions: 'Is this support request urgent?' } },
+        });
+        return Response.json({ traceId: spanContext?.traceId });
+      }
       case '/test-span':
         return Response.json({ spanId: spanContext?.spanId, traceId: spanContext?.traceId });
       case '/test-workflow-sleep': {

dev-packages/e2e-tests/test-applications/cloudflare-workers-send-to-sentry/tests/send-to-sentry.test.ts

@@ -114,3 +114,24 @@ test('Sends a Workers AI gen_ai.evaluate span for a TypeSafe Jev call to Sentry'
       'gen_ai.output.messages': expect.stringContaining('"is_urgent"'),
     });
 });
+
+test('Sends a Workers AI gen_ai.evaluate span for a Clef call to Sentry', async () => {
+  const { traceId }: { traceId: string } = JSON.parse(await fetchFromWorker(`${workerUrl}/test-workers-ai-clef`, 200));
+
+  console.log(`Polling for gen_ai.evaluate span: sentry trace view ${traceTarget(traceId)}`);
+
+  let spanId: string | undefined;
+  await expect
+    .poll(() => (spanId = findSpanInTrace(traceId, 'gen_ai.evaluate')?.event_id), EVENT_POLLING_OPTIONS)
+    .toBeDefined();
+
+  await expect
+    .poll(() => fetchSpanAttributes(traceId, spanId!), EVENT_POLLING_OPTIONS)
+    .toMatchObject({
+      'gen_ai.operation.name': 'evaluate',
+      'gen_ai.request.model': '@cf/cloudflare/clef',
+      'gen_ai.response.model': expect.stringContaining('clef'),
+      'gen_ai.input.messages': expect.stringContaining('Checkout has been failing'),
+      'gen_ai.output.messages': expect.stringContaining('"urgent"'),
+    });
+});

packages/server-utils/src/ai/workers-ai/utils.ts

@@ -40,12 +40,17 @@ export const WORKERS_AI_OPERATION_SPAN_OPS: Record<WorkersAiOperationName, strin
   evaluate: getGenAiSpanOp('evaluate'),
 };
 
+/**
+ * Model ID prefixes of the evaluation models (TypeSafe Jev and Cloudflare Clef), traced as `evaluate` spans.
+ */
+const EVALUATE_MODEL_PREFIXES = ['typesafe/jev', '@cf/cloudflare/clef'];
+
 /**
  * Determine the gen_ai operation name from the model and inputs passed to `AI.run`.
  * Workers AI exposes a single `run` method, so we infer the operation from the model ID and the input shape.
  */
 export function getOperationName(model: unknown, inputs: unknown): WorkersAiOperationName {
-  if (typeof model === 'string' && model.startsWith('typesafe/jev')) {
+  if (typeof model === 'string' && EVALUATE_MODEL_PREFIXES.some(prefix => model.startsWith(prefix))) {
     return 'evaluate';
   }
   if (inputs && typeof inputs === 'object') {

packages/server-utils/test/ai/lib/utils/workers-ai-utils.test.ts

@@ -65,6 +65,10 @@ describe('workers-ai utils', () => {
       expect(getOperationName('typesafe/jev-1.13', { state: 'Help!', questions: {} })).toBe('evaluate');
     });
 
+    it('returns "evaluate" for Cloudflare Clef models', () => {
+      expect(getOperationName('@cf/cloudflare/clef', { state: 'Help!', questions: {} })).toBe('evaluate');
+    });
+
     it('does not return "evaluate" for other TypeSafe models', () => {
       expect(getOperationName('typesafe/other', { prompt: 'Hello' })).toBe('chat');
     });