From eba47ac941ee7fdd42e59445e035d929896b9e0b Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Wed, 17 Jun 2026 11:18:49 +0200 Subject: [PATCH 1/2] refactor(observability): move backend presets out of core --- apps/cli/src/commands/eval/commands/run.ts | 2 +- apps/cli/src/commands/eval/otel-backends.ts | 164 ++++++++++++++++++ apps/cli/src/commands/eval/run-eval.ts | 23 ++- .../test/commands/eval/otel-backends.test.ts | 134 ++++++++++++++ .../docs/docs/evaluation/running-evals.mdx | 26 ++- .../docs/docs/integrations/langfuse.mdx | 4 +- .../docs/docs/integrations/phoenix.mdx | 4 +- examples/features/langfuse-export/README.md | 2 +- packages/core/src/observability/index.ts | 9 +- .../core/src/observability/otel-exporter.ts | 48 +---- packages/core/src/observability/types.ts | 21 ++- .../test/observability/otel-exporter.test.ts | 88 +--------- packages/phoenix-adapter/README.md | 5 + packages/phoenix-adapter/src/index.ts | 1 + packages/phoenix-adapter/src/otel-backend.ts | 93 ++++++++++ .../phoenix-adapter/test/otel-backend.test.ts | 70 ++++++++ 16 files changed, 543 insertions(+), 151 deletions(-) create mode 100644 apps/cli/src/commands/eval/otel-backends.ts create mode 100644 apps/cli/test/commands/eval/otel-backends.test.ts create mode 100644 packages/phoenix-adapter/src/otel-backend.ts create mode 100644 packages/phoenix-adapter/test/otel-backend.test.ts diff --git a/apps/cli/src/commands/eval/commands/run.ts b/apps/cli/src/commands/eval/commands/run.ts index 6b10cd52a..2b657939b 100644 --- a/apps/cli/src/commands/eval/commands/run.ts +++ b/apps/cli/src/commands/eval/commands/run.ts @@ -142,7 +142,7 @@ export const evalRunCommand = command({ otelBackend: option({ type: optional(string), long: 'otel-backend', - description: 'Use a backend preset (langfuse, braintrust, confident)', + description: 'Use an OTel backend resolver (langfuse, braintrust, confident, or local)', }), otelCaptureContent: flag({ long: 'otel-capture-content', diff --git a/apps/cli/src/commands/eval/otel-backends.ts b/apps/cli/src/commands/eval/otel-backends.ts new file mode 100644 index 000000000..6ee6fa50b --- /dev/null +++ b/apps/cli/src/commands/eval/otel-backends.ts @@ -0,0 +1,164 @@ +/** + * OTel backend resolver loading for the eval CLI. + * + * Core owns generic OTLP export only. This module keeps CLI ergonomics for + * `--otel-backend ` by checking project-local resolver files first, then + * falling back to the small set of resolver names already exposed by the CLI. + * + * To add a local resolver, create `.agentv/otel-backends/.ts` and export + * a resolver object as `default`, `otelBackend`, or `resolver`. + */ + +import { access } from 'node:fs/promises'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import type { + OtelBackendResolution, + OtelBackendResolver, + OtelBackendResolverContext, +} from '@agentv/core'; + +const RESOLVER_EXTENSIONS = ['.ts', '.js', '.mts', '.mjs'] as const; + +const builtinOtelBackendResolvers: readonly OtelBackendResolver[] = [ + { + name: 'langfuse', + resolve: ({ env }) => { + const baseUrl = trimTrailingSlash(env.LANGFUSE_HOST ?? 'https://cloud.langfuse.com'); + const publicKey = env.LANGFUSE_PUBLIC_KEY ?? ''; + const secretKey = env.LANGFUSE_SECRET_KEY ?? ''; + + return { + endpoint: `${baseUrl}/api/public/otel/v1/traces`, + headers: { + Authorization: `Basic ${Buffer.from(`${publicKey}:${secretKey}`).toString('base64')}`, + }, + }; + }, + }, + { + name: 'braintrust', + resolve: ({ env }) => { + const headers: Record = { + Authorization: `Bearer ${env.BRAINTRUST_API_KEY ?? ''}`, + }; + const parent = + env.BRAINTRUST_PARENT ?? + (env.BRAINTRUST_PROJECT_ID ? `project_id:${env.BRAINTRUST_PROJECT_ID}` : undefined) ?? + (env.BRAINTRUST_PROJECT ? `project_name:${env.BRAINTRUST_PROJECT}` : undefined); + + if (parent) { + headers['x-bt-parent'] = parent; + } + + return { + endpoint: 'https://api.braintrust.dev/otel/v1/traces', + headers, + }; + }, + }, + { + name: 'confident', + resolve: ({ env }) => ({ + endpoint: 'https://otel.confident-ai.com/v1/traces', + headers: { + 'x-confident-api-key': env.CONFIDENT_API_KEY ?? '', + }, + }), + }, +]; + +const builtinOtelBackendResolversByName = new Map( + builtinOtelBackendResolvers.map((resolver) => [resolver.name, resolver]), +); + +export async function resolveOtelBackend( + name: string, + context: OtelBackendResolverContext, +): Promise { + const resolver = await loadOtelBackendResolver(name, context.cwd); + return resolver?.resolve(context); +} + +export async function loadOtelBackendResolver( + name: string, + cwd: string, +): Promise { + const localResolverPath = await findLocalOtelBackendResolver(name, cwd); + if (localResolverPath) { + return importOtelBackendResolver(localResolverPath, name); + } + + return builtinOtelBackendResolversByName.get(name); +} + +export function getBuiltinOtelBackendResolverNames(): readonly string[] { + return builtinOtelBackendResolvers.map((resolver) => resolver.name); +} + +async function findLocalOtelBackendResolver( + name: string, + cwd: string, +): Promise { + if (!isSafeResolverName(name)) { + return undefined; + } + + for (const dir of getResolverSearchDirs(cwd)) { + for (const ext of RESOLVER_EXTENSIONS) { + const candidate = path.join(dir, `${name}${ext}`); + try { + await access(candidate); + return candidate; + } catch { + // Candidate does not exist in this directory. + } + } + } + + return undefined; +} + +function getResolverSearchDirs(cwd: string): readonly string[] { + const dirs: string[] = []; + let current = path.resolve(cwd); + const root = path.parse(current).root; + + while (current !== root) { + dirs.push(path.join(current, '.agentv', 'otel-backends')); + current = path.dirname(current); + } + + return dirs; +} + +function isSafeResolverName(name: string): boolean { + return name.length > 0 && !name.includes('/') && !name.includes('\\') && !name.startsWith('.'); +} + +async function importOtelBackendResolver( + filePath: string, + fallbackName: string, +): Promise { + const mod = await import(pathToFileURL(filePath).href); + const candidate = mod.default ?? mod.otelBackend ?? mod.resolver; + + if (!candidate || typeof candidate.resolve !== 'function') { + throw new Error( + `OTel backend resolver '${fallbackName}' from ${filePath} must export a resolver object`, + ); + } + + return { + ...candidate, + name: + typeof candidate.name === 'string' && candidate.name.length > 0 + ? candidate.name + : fallbackName, + } as OtelBackendResolver; +} + +function trimTrailingSlash(value: string): string { + return value.replace(/\/+$/, ''); +} diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index e6eb7f291..080dd1e3f 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -51,6 +51,7 @@ import { writeInitialBenchmarkArtifact, } from './artifact-writer.js'; import { loadEnvFromHierarchy } from './env.js'; +import { resolveOtelBackend } from './otel-backends.js'; import { type OutputWriter, createOutputWriter } from './output-writer.js'; import { ProgressDisplay, type Verdict, type WorkerProgress } from './progress-display.js'; import { buildDefaultRunDir, normalizeExperimentName } from './result-layout.js'; @@ -1267,19 +1268,28 @@ export async function runEvalCommand( if (options.exportOtel || useFileExport) { try { - const { OtelTraceExporter, OTEL_BACKEND_PRESETS } = await import('@agentv/core'); + const { OtelTraceExporter } = await import('@agentv/core'); // Resolve endpoint and headers let endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT; let headers: Record = {}; + let resourceAttributes: Record = {}; if (options.otelBackend) { - const preset = OTEL_BACKEND_PRESETS[options.otelBackend]; - if (preset) { - endpoint = preset.endpoint; - headers = preset.headers(process.env); + const resolvedBackend = await resolveOtelBackend(options.otelBackend, { + env: process.env, + cwd, + }); + + if (resolvedBackend) { + endpoint = resolvedBackend.endpoint; + headers = { ...headers, ...resolvedBackend.headers }; + resourceAttributes = { ...resourceAttributes, ...resolvedBackend.resourceAttributes }; + for (const warning of resolvedBackend.warnings ?? []) { + console.warn(warning); + } } else { - console.warn(`Unknown OTel backend preset: ${options.otelBackend}`); + console.warn(`Unknown OTel backend resolver: ${options.otelBackend}`); } } @@ -1297,6 +1307,7 @@ export async function runEvalCommand( otelExporter = new OtelTraceExporter({ endpoint, headers, + resourceAttributes, captureContent, groupTurns: options.otelGroupTurns, otlpFilePath: options.otelFile ? path.resolve(options.otelFile) : undefined, diff --git a/apps/cli/test/commands/eval/otel-backends.test.ts b/apps/cli/test/commands/eval/otel-backends.test.ts new file mode 100644 index 000000000..1a647eef2 --- /dev/null +++ b/apps/cli/test/commands/eval/otel-backends.test.ts @@ -0,0 +1,134 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +import { + getBuiltinOtelBackendResolverNames, + resolveOtelBackend, +} from '../../../src/commands/eval/otel-backends.js'; + +describe('OTel backend resolvers', () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = await mkdtemp(path.join(tmpdir(), 'agentv-otel-backends-')); + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + it('keeps the existing CLI backend names available outside core', () => { + expect(getBuiltinOtelBackendResolverNames()).toEqual(['langfuse', 'braintrust', 'confident']); + }); + + it('resolves Langfuse endpoint and Basic auth headers', async () => { + const resolved = await resolveOtelBackend('langfuse', { + cwd: tempDir, + env: { + LANGFUSE_HOST: 'https://langfuse.example.com/', + LANGFUSE_PUBLIC_KEY: 'pk-test', + LANGFUSE_SECRET_KEY: 'sk-test', + }, + }); + + expect(resolved).toEqual({ + endpoint: 'https://langfuse.example.com/api/public/otel/v1/traces', + headers: { + Authorization: `Basic ${Buffer.from('pk-test:sk-test').toString('base64')}`, + }, + }); + }); + + it('resolves Braintrust auth and project routing headers', async () => { + const resolved = await resolveOtelBackend('braintrust', { + cwd: tempDir, + env: { + BRAINTRUST_API_KEY: 'bt-key', + BRAINTRUST_PROJECT: 'agentv-evals', + }, + }); + + expect(resolved).toEqual({ + endpoint: 'https://api.braintrust.dev/otel/v1/traces', + headers: { + Authorization: 'Bearer bt-key', + 'x-bt-parent': 'project_name:agentv-evals', + }, + }); + }); + + it('resolves Confident auth headers', async () => { + const resolved = await resolveOtelBackend('confident', { + cwd: tempDir, + env: { CONFIDENT_API_KEY: 'conf-key' }, + }); + + expect(resolved).toEqual({ + endpoint: 'https://otel.confident-ai.com/v1/traces', + headers: { + 'x-confident-api-key': 'conf-key', + }, + }); + }); + + it('discovers a project-local resolver by backend name', async () => { + const nestedDir = path.join(tempDir, 'evals', 'suite'); + const resolverDir = path.join(tempDir, '.agentv', 'otel-backends'); + await mkdir(nestedDir, { recursive: true }); + await mkdir(resolverDir, { recursive: true }); + await writeFile( + path.join(resolverDir, 'local.ts'), + ` + export default { + resolve: ({ env, cwd }) => ({ + endpoint: env.LOCAL_OTEL_ENDPOINT ?? cwd, + headers: { "x-local": "true" }, + resourceAttributes: { "agentv.test": "local" }, + }), + }; + `, + 'utf8', + ); + + const resolved = await resolveOtelBackend('local', { + cwd: nestedDir, + env: { LOCAL_OTEL_ENDPOINT: 'https://otel.example.com/v1/traces' }, + }); + + expect(resolved).toEqual({ + endpoint: 'https://otel.example.com/v1/traces', + headers: { 'x-local': 'true' }, + resourceAttributes: { 'agentv.test': 'local' }, + }); + }); + + it('uses a local resolver before a built-in resolver with the same name', async () => { + const resolverDir = path.join(tempDir, '.agentv', 'otel-backends'); + await mkdir(resolverDir, { recursive: true }); + await writeFile( + path.join(resolverDir, 'langfuse.ts'), + ` + export const resolver = { + name: "langfuse", + resolve: () => ({ endpoint: "https://local.example.com/v1/traces" }), + }; + `, + 'utf8', + ); + + const resolved = await resolveOtelBackend('langfuse', { + cwd: tempDir, + env: {}, + }); + + expect(resolved).toEqual({ endpoint: 'https://local.example.com/v1/traces' }); + }); + + it('returns undefined for unknown backend names', async () => { + const resolved = await resolveOtelBackend('unknown', { cwd: tempDir, env: {} }); + + expect(resolved).toBeUndefined(); + }); +}); diff --git a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx index 10915d5c5..76702001e 100644 --- a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx +++ b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx @@ -159,7 +159,7 @@ OpenTelemetry-compatible backend. Stream traces directly to an observability backend during evaluation using `--export-otel`: ```bash -# Use a backend preset (braintrust, langfuse, confident) +# Use a built-in CLI backend resolver (braintrust, langfuse, confident) agentv eval evals/my-eval.yaml --export-otel --otel-backend braintrust # Include message content and tool I/O in spans (disabled by default for privacy) @@ -213,9 +213,31 @@ export LANGFUSE_SECRET_KEY=sk-... agentv eval evals/my-eval.yaml --export-otel --otel-backend langfuse --otel-capture-content ``` +#### Local Backend Resolvers + +For project-specific backend routing, create `.agentv/otel-backends/.ts` and select it +with `--otel-backend `: + +```ts +export default { + name: 'my-backend', + resolve: ({ env }) => ({ + endpoint: env.MY_OTEL_ENDPOINT ?? 'https://otel.example.com/v1/traces', + headers: { Authorization: `Bearer ${env.MY_OTEL_TOKEN ?? ''}` }, + }), +}; +``` + +```bash +agentv eval evals/my-eval.yaml --export-otel --otel-backend my-backend +``` + +Backend resolvers keep platform-specific endpoint, header, and project-routing logic outside +AgentV core. + #### Custom OTLP Endpoint -For backends not covered by presets, configure via environment variables: +For generic OTLP export without a backend resolver, configure via environment variables: ```bash export OTEL_EXPORTER_OTLP_ENDPOINT=https://your-backend/v1/traces diff --git a/apps/web/src/content/docs/docs/integrations/langfuse.mdx b/apps/web/src/content/docs/docs/integrations/langfuse.mdx index 74eb56a2c..a7906e774 100644 --- a/apps/web/src/content/docs/docs/integrations/langfuse.mdx +++ b/apps/web/src/content/docs/docs/integrations/langfuse.mdx @@ -5,7 +5,7 @@ sidebar: order: 1 --- -AgentV streams evaluation traces to [Langfuse](https://langfuse.com) using standard OTLP/HTTP — no Langfuse SDK required. The `langfuse` backend preset handles endpoint construction and authentication automatically. +AgentV streams evaluation traces to [Langfuse](https://langfuse.com) using standard OTLP/HTTP — no Langfuse SDK required. The `langfuse` backend resolver handles endpoint construction and authentication automatically. ## Quick Start @@ -54,7 +54,7 @@ Langfuse dashboards recognize the `gen_ai.*` semantic conventions and display to | Flag | Description | |------|-------------| | `--export-otel` | Enable live OTel export | -| `--otel-backend langfuse` | Use the Langfuse endpoint and auth preset | +| `--otel-backend langfuse` | Use the Langfuse endpoint and auth resolver | | `--otel-capture-content` | Include message and tool content in spans (disabled by default for privacy) | | `--otel-group-turns` | Add `agentv.turn.N` parent spans that group messages by conversation turn | diff --git a/apps/web/src/content/docs/docs/integrations/phoenix.mdx b/apps/web/src/content/docs/docs/integrations/phoenix.mdx index a32824ab9..0005700d9 100644 --- a/apps/web/src/content/docs/docs/integrations/phoenix.mdx +++ b/apps/web/src/content/docs/docs/integrations/phoenix.mdx @@ -93,7 +93,9 @@ For trace export, use AgentV's standard OTel options: agentv eval evals/my-eval.yaml --otel-file traces/eval.otlp.json ``` -For live OTel export to a configured backend, use the options documented in +For live OTel export to Phoenix, use a local `.agentv/otel-backends/phoenix.ts` +resolver that imports `phoenixOtelBackend` from `@agentv/phoenix-adapter`, then select it +with `--otel-backend phoenix`. General live export options are documented in [Running Evaluations](/docs/evaluation/running-evals/#live-otel-export/). ## Package Docs diff --git a/examples/features/langfuse-export/README.md b/examples/features/langfuse-export/README.md index 964399bca..f424ef17f 100644 --- a/examples/features/langfuse-export/README.md +++ b/examples/features/langfuse-export/README.md @@ -2,7 +2,7 @@ Demonstrates exporting eval traces to [Langfuse](https://langfuse.com) via OpenTelemetry. -AgentV uses OTLP/HTTP — no Langfuse SDK required. The `langfuse` backend preset handles endpoint and auth automatically. +AgentV uses OTLP/HTTP — no Langfuse SDK required. The `langfuse` backend resolver handles endpoint and auth automatically. ## Setup diff --git a/packages/core/src/observability/index.ts b/packages/core/src/observability/index.ts index a36f693f8..dbeaf6db7 100644 --- a/packages/core/src/observability/index.ts +++ b/packages/core/src/observability/index.ts @@ -1,3 +1,8 @@ -export type { OtelExportOptions, OtelBackendPreset } from './types.js'; -export { OTEL_BACKEND_PRESETS, OtelTraceExporter, OtelStreamingObserver } from './otel-exporter.js'; +export type { + OtelBackendResolution, + OtelBackendResolver, + OtelBackendResolverContext, + OtelExportOptions, +} from './types.js'; +export { OtelTraceExporter, OtelStreamingObserver } from './otel-exporter.js'; export { OtlpJsonFileExporter } from './otlp-json-file-exporter.js'; diff --git a/packages/core/src/observability/otel-exporter.ts b/packages/core/src/observability/otel-exporter.ts index 7a39635aa..eccfd389e 100644 --- a/packages/core/src/observability/otel-exporter.ts +++ b/packages/core/src/observability/otel-exporter.ts @@ -4,52 +4,9 @@ import type { ProviderTokenUsage, } from '../evaluation/providers/types.js'; import type { EvaluationResult } from '../evaluation/types.js'; -import type { OtelBackendPreset, OtelExportOptions } from './types.js'; +import type { OtelExportOptions } from './types.js'; -export type { OtelExportOptions, OtelBackendPreset }; - -// --------------------------------------------------------------------------- -// Backend presets -// --------------------------------------------------------------------------- - -export const OTEL_BACKEND_PRESETS: Record = { - langfuse: { - name: 'langfuse', - endpoint: process.env.LANGFUSE_HOST - ? `${process.env.LANGFUSE_HOST}/api/public/otel/v1/traces` - : 'https://cloud.langfuse.com/api/public/otel/v1/traces', - headers: (env) => { - const pub = env.LANGFUSE_PUBLIC_KEY ?? ''; - const secret = env.LANGFUSE_SECRET_KEY ?? ''; - return { Authorization: `Basic ${Buffer.from(`${pub}:${secret}`).toString('base64')}` }; - }, - }, - braintrust: { - name: 'braintrust', - endpoint: 'https://api.braintrust.dev/otel/v1/traces', - headers: (env) => { - const headers: Record = { - Authorization: `Bearer ${env.BRAINTRUST_API_KEY ?? ''}`, - }; - // x-bt-parent is required by Braintrust to associate traces with a project - const parent = - env.BRAINTRUST_PARENT ?? - (env.BRAINTRUST_PROJECT_ID ? `project_id:${env.BRAINTRUST_PROJECT_ID}` : undefined) ?? - (env.BRAINTRUST_PROJECT ? `project_name:${env.BRAINTRUST_PROJECT}` : undefined); - if (parent) { - headers['x-bt-parent'] = parent; - } - return headers; - }, - }, - confident: { - name: 'confident', - endpoint: 'https://otel.confident-ai.com/v1/traces', - headers: (env) => ({ - 'x-confident-api-key': env.CONFIDENT_API_KEY ?? '', - }), - }, -}; +export type { OtelExportOptions }; // --------------------------------------------------------------------------- // OTel type aliases (resolved dynamically at init) @@ -92,6 +49,7 @@ export class OtelTraceExporter { const resource = resourceFromAttributes({ [ATTR_SERVICE_NAME]: this.options.serviceName ?? 'agentv', + ...this.options.resourceAttributes, }); // biome-ignore lint/suspicious/noExplicitAny: OTel processor types loaded dynamically diff --git a/packages/core/src/observability/types.ts b/packages/core/src/observability/types.ts index 84fe94fdd..0526527d0 100644 --- a/packages/core/src/observability/types.ts +++ b/packages/core/src/observability/types.ts @@ -4,6 +4,8 @@ export interface OtelExportOptions { readonly endpoint?: string; /** Custom headers (e.g., auth) */ readonly headers?: Record; + /** Resource attributes to attach to the trace provider */ + readonly resourceAttributes?: Record; /** Whether to include message content in spans */ readonly captureContent?: boolean; /** Service name for OTel resource */ @@ -14,9 +16,20 @@ export interface OtelExportOptions { readonly otlpFilePath?: string; } -/** Preset configuration for a known observability backend. */ -export interface OtelBackendPreset { - readonly name: string; +export interface OtelBackendResolverContext { + readonly env: Record; + readonly cwd: string; +} + +export interface OtelBackendResolution { readonly endpoint: string; - readonly headers: (env: Record) => Record; + readonly headers?: Record; + readonly resourceAttributes?: Record; + readonly warnings?: readonly string[]; +} + +/** Generic resolver contract for OTel backend endpoint/header/resource routing. */ +export interface OtelBackendResolver { + readonly name: string; + resolve(context: OtelBackendResolverContext): OtelBackendResolution; } diff --git a/packages/core/test/observability/otel-exporter.test.ts b/packages/core/test/observability/otel-exporter.test.ts index c10ef1ba0..85741113f 100644 --- a/packages/core/test/observability/otel-exporter.test.ts +++ b/packages/core/test/observability/otel-exporter.test.ts @@ -5,93 +5,7 @@ import { afterEach, describe, expect, it } from 'bun:test'; import { buildTraceFromMessages } from '../../src/evaluation/trace.js'; -import { OTEL_BACKEND_PRESETS, OtelTraceExporter } from '../../src/observability/otel-exporter.js'; - -// --------------------------------------------------------------------------- -// Backend presets -// --------------------------------------------------------------------------- - -describe('OTel backend presets', () => { - describe('OTEL_BACKEND_PRESETS registry', () => { - it('contains langfuse, braintrust, and confident entries', () => { - expect(OTEL_BACKEND_PRESETS).toHaveProperty('langfuse'); - expect(OTEL_BACKEND_PRESETS).toHaveProperty('braintrust'); - expect(OTEL_BACKEND_PRESETS).toHaveProperty('confident'); - }); - - it('each preset has name, endpoint, and headers function', () => { - for (const [key, preset] of Object.entries(OTEL_BACKEND_PRESETS)) { - expect(preset.name).toBe(key); - expect(typeof preset.endpoint).toBe('string'); - expect(typeof preset.headers).toBe('function'); - } - }); - }); - - describe('langfuse preset', () => { - const preset = OTEL_BACKEND_PRESETS.langfuse; - - it('generates Basic auth header from public + secret key env vars', () => { - const env = { - LANGFUSE_PUBLIC_KEY: 'pk-test-123', - LANGFUSE_SECRET_KEY: 'sk-test-456', - }; - const headers = preset.headers(env); - const expected = `Basic ${Buffer.from('pk-test-123:sk-test-456').toString('base64')}`; - expect(headers).toEqual({ Authorization: expected }); - }); - - it('falls back to empty strings when env vars are missing', () => { - const headers = preset.headers({}); - const expected = `Basic ${Buffer.from(':').toString('base64')}`; - expect(headers).toEqual({ Authorization: expected }); - }); - - it('uses default cloud.langfuse.com endpoint when LANGFUSE_HOST is not set', () => { - // The preset endpoint is evaluated at module load time using process.env. - // When LANGFUSE_HOST is not set, the default endpoint is used. - expect(preset.endpoint).toContain('langfuse.com/api/public/otel/v1/traces'); - }); - }); - - describe('braintrust preset', () => { - const preset = OTEL_BACKEND_PRESETS.braintrust; - - it('generates Bearer token from BRAINTRUST_API_KEY env var', () => { - const env = { BRAINTRUST_API_KEY: 'bt-key-789' }; - const headers = preset.headers(env); - expect(headers).toEqual({ Authorization: 'Bearer bt-key-789' }); - }); - - it('falls back to empty Bearer token when env var is missing', () => { - const headers = preset.headers({}); - expect(headers).toEqual({ Authorization: 'Bearer ' }); - }); - - it('uses api.braintrust.dev endpoint', () => { - expect(preset.endpoint).toBe('https://api.braintrust.dev/otel/v1/traces'); - }); - }); - - describe('confident preset', () => { - const preset = OTEL_BACKEND_PRESETS.confident; - - it('generates x-confident-api-key header from CONFIDENT_API_KEY env var', () => { - const env = { CONFIDENT_API_KEY: 'conf-key-abc' }; - const headers = preset.headers(env); - expect(headers).toEqual({ 'x-confident-api-key': 'conf-key-abc' }); - }); - - it('falls back to empty key when env var is missing', () => { - const headers = preset.headers({}); - expect(headers).toEqual({ 'x-confident-api-key': '' }); - }); - - it('uses otel.confident-ai.com endpoint', () => { - expect(preset.endpoint).toBe('https://otel.confident-ai.com/v1/traces'); - }); - }); -}); +import { OtelTraceExporter } from '../../src/observability/otel-exporter.js'; // --------------------------------------------------------------------------- // OtelTraceExporter class diff --git a/packages/phoenix-adapter/README.md b/packages/phoenix-adapter/README.md index 528400be9..54d0e6d4f 100644 --- a/packages/phoenix-adapter/README.md +++ b/packages/phoenix-adapter/README.md @@ -4,6 +4,11 @@ Converts AgentV eval YAML suites into Phoenix datasets and can run Phoenix exper Current adapter support is intentionally small: deterministic `contains`, `regex`, `equals`, and `is-json` assertions run through a Phoenix CODE evaluator. LLM, code, trace, composite, metric, and custom evaluator families are reported as unsupported instead of being silently mapped. +The package also exports `phoenixOtelBackend`, a backend resolver for AgentV's +local `.agentv/otel-backends/phoenix.ts` hook. It resolves Phoenix collector +endpoint, auth headers, and `PHOENIX_PROJECT_NAME` resource routing outside +`@agentv/core`. + ```bash bun --filter @agentv/phoenix-adapter phoenix:assert-smoke bun --filter @agentv/phoenix-adapter phoenix:dry-run diff --git a/packages/phoenix-adapter/src/index.ts b/packages/phoenix-adapter/src/index.ts index ef018a94b..34e0143c6 100644 --- a/packages/phoenix-adapter/src/index.ts +++ b/packages/phoenix-adapter/src/index.ts @@ -1,5 +1,6 @@ export { discoverAgentVEvals } from './agentv/discovery.js'; export { loadAgentVEvalSuite } from './agentv/load-spec.js'; +export { phoenixOtelBackend } from './otel-backend.js'; export { createPhoenixDatasetPayload } from './phoenix/datasets.js'; export { runSuite } from './run/run-suite.js'; diff --git a/packages/phoenix-adapter/src/otel-backend.ts b/packages/phoenix-adapter/src/otel-backend.ts new file mode 100644 index 000000000..7ef96c7aa --- /dev/null +++ b/packages/phoenix-adapter/src/otel-backend.ts @@ -0,0 +1,93 @@ +/** + * Phoenix OTel backend resolver. + * + * This file is the Phoenix-specific boundary for AgentV trace export routing. + * Core receives only generic OTLP endpoint, headers, and resource attributes. + */ + +import type { OtelBackendResolver } from '@agentv/core'; + +const DEFAULT_PHOENIX_COLLECTOR_ENDPOINT = 'http://localhost:6006'; +const OPENINFERENCE_PROJECT_NAME = 'openinference.project.name'; + +export const phoenixOtelBackend: OtelBackendResolver = { + name: 'phoenix', + resolve: ({ env }) => { + const warnings: string[] = []; + const headers = parsePhoenixClientHeaders(env.PHOENIX_CLIENT_HEADERS, warnings); + const apiKey = trimOptional(env.PHOENIX_API_KEY); + + if (apiKey && !hasHeader(headers, 'authorization')) { + headers.authorization = `Bearer ${apiKey}`; + } + + return { + endpoint: normalizePhoenixTraceEndpoint( + trimOptional(env.PHOENIX_COLLECTOR_ENDPOINT) ?? DEFAULT_PHOENIX_COLLECTOR_ENDPOINT, + ), + headers, + resourceAttributes: { + [OPENINFERENCE_PROJECT_NAME]: trimOptional(env.PHOENIX_PROJECT_NAME) ?? 'default', + }, + warnings, + }; + }, +}; + +function normalizePhoenixTraceEndpoint(endpoint: string): string { + const trimmed = endpoint.replace(/\/+$/, ''); + if (trimmed.endsWith('/v1/traces')) { + return trimmed; + } + return `${trimmed}/v1/traces`; +} + +function parsePhoenixClientHeaders( + value: string | undefined, + warnings: string[], +): Record { + const headers: Record = {}; + const raw = trimOptional(value); + + if (!raw) { + return headers; + } + + for (const segment of raw.split(',')) { + const entry = segment.trim(); + if (!entry) { + continue; + } + + const separatorIndex = entry.indexOf('='); + if (separatorIndex <= 0) { + warnings.push(`Ignoring invalid PHOENIX_CLIENT_HEADERS entry: ${entry}`); + continue; + } + + const rawName = entry.slice(0, separatorIndex).trim(); + const rawHeaderValue = entry.slice(separatorIndex + 1).trim(); + + try { + const name = decodeURIComponent(rawName).trim().toLowerCase(); + const headerValue = decodeURIComponent(rawHeaderValue).trim(); + if (name) { + headers[name] = headerValue; + } + } catch { + warnings.push(`Ignoring invalid PHOENIX_CLIENT_HEADERS entry: ${entry}`); + } + } + + return headers; +} + +function hasHeader(headers: Record, name: string): boolean { + const normalized = name.toLowerCase(); + return Object.keys(headers).some((header) => header.toLowerCase() === normalized); +} + +function trimOptional(value: string | undefined): string | undefined { + const trimmed = value?.trim(); + return trimmed ? trimmed : undefined; +} diff --git a/packages/phoenix-adapter/test/otel-backend.test.ts b/packages/phoenix-adapter/test/otel-backend.test.ts new file mode 100644 index 000000000..d6555e487 --- /dev/null +++ b/packages/phoenix-adapter/test/otel-backend.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, test } from 'bun:test'; + +import { phoenixOtelBackend } from '../src/otel-backend.js'; + +describe('phoenixOtelBackend', () => { + test('resolves default local Phoenix endpoint and project resource attribute', () => { + const resolved = phoenixOtelBackend.resolve({ cwd: process.cwd(), env: {} }); + + expect(resolved).toEqual({ + endpoint: 'http://localhost:6006/v1/traces', + headers: {}, + resourceAttributes: { + 'openinference.project.name': 'default', + }, + warnings: [], + }); + }); + + test('normalizes Phoenix endpoint, API key, client headers, and project name', () => { + const resolved = phoenixOtelBackend.resolve({ + cwd: process.cwd(), + env: { + PHOENIX_COLLECTOR_ENDPOINT: 'https://app.phoenix.arize.com/s/my-space/', + PHOENIX_API_KEY: 'px-key', + PHOENIX_PROJECT_NAME: 'agentv-evals', + PHOENIX_CLIENT_HEADERS: 'x-custom=one%20two', + }, + }); + + expect(resolved).toEqual({ + endpoint: 'https://app.phoenix.arize.com/s/my-space/v1/traces', + headers: { + 'x-custom': 'one two', + authorization: 'Bearer px-key', + }, + resourceAttributes: { + 'openinference.project.name': 'agentv-evals', + }, + warnings: [], + }); + }); + + test('does not append duplicate traces path or override explicit auth header', () => { + const resolved = phoenixOtelBackend.resolve({ + cwd: process.cwd(), + env: { + PHOENIX_COLLECTOR_ENDPOINT: 'http://phoenix.example.com/v1/traces', + PHOENIX_API_KEY: 'px-key', + PHOENIX_CLIENT_HEADERS: 'authorization=Bearer%20override', + }, + }); + + expect(resolved.endpoint).toBe('http://phoenix.example.com/v1/traces'); + expect(resolved.headers).toEqual({ authorization: 'Bearer override' }); + }); + + test('reports invalid client header entries as warnings', () => { + const resolved = phoenixOtelBackend.resolve({ + cwd: process.cwd(), + env: { + PHOENIX_CLIENT_HEADERS: 'valid=value,not-a-header', + }, + }); + + expect(resolved.headers).toEqual({ valid: 'value' }); + expect(resolved.warnings).toEqual([ + 'Ignoring invalid PHOENIX_CLIENT_HEADERS entry: not-a-header', + ]); + }); +}); From c4c3de70ddb8bf062aa96b5f18d91e5dcfc31d9a Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Wed, 17 Jun 2026 12:59:22 +0200 Subject: [PATCH 2/2] fix(observability): address backend resolver review --- apps/cli/src/commands/eval/otel-backends.ts | 13 +++-- .../test/commands/eval/otel-backends.test.ts | 49 +++++++++++++++++-- .../docs/docs/evaluation/running-evals.mdx | 7 +-- .../docs/docs/integrations/phoenix.mdx | 2 +- ...026-06-11-phoenix-observability-adapter.md | 6 +-- .../core/src/observability/otel-exporter.ts | 9 ++-- .../observability/otlp-json-file-exporter.ts | 17 ++++++- .../test/observability/file-exporters.test.ts | 33 +++++++++++++ packages/phoenix-adapter/README.md | 2 +- 9 files changed, 117 insertions(+), 21 deletions(-) diff --git a/apps/cli/src/commands/eval/otel-backends.ts b/apps/cli/src/commands/eval/otel-backends.ts index 6ee6fa50b..58c9df718 100644 --- a/apps/cli/src/commands/eval/otel-backends.ts +++ b/apps/cli/src/commands/eval/otel-backends.ts @@ -5,8 +5,9 @@ * `--otel-backend ` by checking project-local resolver files first, then * falling back to the small set of resolver names already exposed by the CLI. * - * To add a local resolver, create `.agentv/otel-backends/.ts` and export - * a resolver object as `default`, `otelBackend`, or `resolver`. + * To add a local resolver, create `.agentv/otel-backends/.mjs` + * (or a Node-loadable `.js`) and export a resolver object as `default`, + * `otelBackend`, or `resolver`. */ import { access } from 'node:fs/promises'; @@ -19,7 +20,7 @@ import type { OtelBackendResolverContext, } from '@agentv/core'; -const RESOLVER_EXTENSIONS = ['.ts', '.js', '.mts', '.mjs'] as const; +const RESOLVER_EXTENSIONS = ['.mjs', '.js'] as const; const builtinOtelBackendResolvers: readonly OtelBackendResolver[] = [ { @@ -142,9 +143,11 @@ async function importOtelBackendResolver( fallbackName: string, ): Promise { const mod = await import(pathToFileURL(filePath).href); - const candidate = mod.default ?? mod.otelBackend ?? mod.resolver; + const candidate = [mod.default, mod.otelBackend, mod.resolver].find( + (value) => value && typeof value.resolve === 'function', + ); - if (!candidate || typeof candidate.resolve !== 'function') { + if (!candidate) { throw new Error( `OTel backend resolver '${fallbackName}' from ${filePath} must export a resolver object`, ); diff --git a/apps/cli/test/commands/eval/otel-backends.test.ts b/apps/cli/test/commands/eval/otel-backends.test.ts index 1a647eef2..728812709 100644 --- a/apps/cli/test/commands/eval/otel-backends.test.ts +++ b/apps/cli/test/commands/eval/otel-backends.test.ts @@ -79,7 +79,7 @@ describe('OTel backend resolvers', () => { await mkdir(nestedDir, { recursive: true }); await mkdir(resolverDir, { recursive: true }); await writeFile( - path.join(resolverDir, 'local.ts'), + path.join(resolverDir, 'local.mjs'), ` export default { resolve: ({ env, cwd }) => ({ @@ -104,11 +104,11 @@ describe('OTel backend resolvers', () => { }); }); - it('uses a local resolver before a built-in resolver with the same name', async () => { + it('uses a local ESM resolver before a built-in resolver with the same name', async () => { const resolverDir = path.join(tempDir, '.agentv', 'otel-backends'); await mkdir(resolverDir, { recursive: true }); await writeFile( - path.join(resolverDir, 'langfuse.ts'), + path.join(resolverDir, 'langfuse.mjs'), ` export const resolver = { name: "langfuse", @@ -126,9 +126,52 @@ describe('OTel backend resolvers', () => { expect(resolved).toEqual({ endpoint: 'https://local.example.com/v1/traces' }); }); + it('loads CommonJS .js resolver files', async () => { + const resolverDir = path.join(tempDir, '.agentv', 'otel-backends'); + await mkdir(resolverDir, { recursive: true }); + await writeFile( + path.join(resolverDir, 'commonjs.js'), + ` + module.exports = { + name: "commonjs", + resolve: () => ({ endpoint: "https://commonjs.example.com/v1/traces" }), + }; + `, + 'utf8', + ); + + const resolved = await resolveOtelBackend('commonjs', { + cwd: tempDir, + env: {}, + }); + + expect(resolved).toEqual({ endpoint: 'https://commonjs.example.com/v1/traces' }); + }); + it('returns undefined for unknown backend names', async () => { const resolved = await resolveOtelBackend('unknown', { cwd: tempDir, env: {} }); expect(resolved).toBeUndefined(); }); + + it('ignores TypeScript resolver files because packaged Node cannot import them', async () => { + const resolverDir = path.join(tempDir, '.agentv', 'otel-backends'); + await mkdir(resolverDir, { recursive: true }); + await writeFile( + path.join(resolverDir, 'typescript-only.ts'), + ` + export default { + resolve: () => ({ endpoint: "https://typescript.example.com/v1/traces" }), + }; + `, + 'utf8', + ); + + const resolved = await resolveOtelBackend('typescript-only', { + cwd: tempDir, + env: {}, + }); + + expect(resolved).toBeUndefined(); + }); }); diff --git a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx index 76702001e..b3fd696f5 100644 --- a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx +++ b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx @@ -215,10 +215,10 @@ agentv eval evals/my-eval.yaml --export-otel --otel-backend langfuse --otel-capt #### Local Backend Resolvers -For project-specific backend routing, create `.agentv/otel-backends/.ts` and select it +For project-specific backend routing, create `.agentv/otel-backends/.mjs` and select it with `--otel-backend `: -```ts +```js export default { name: 'my-backend', resolve: ({ env }) => ({ @@ -233,7 +233,8 @@ agentv eval evals/my-eval.yaml --export-otel --otel-backend my-backend ``` Backend resolvers keep platform-specific endpoint, header, and project-routing logic outside -AgentV core. +AgentV core. AgentV also loads Node-compatible `.js` resolver files when you prefer +CommonJS or your project configures `.js` as ESM. #### Custom OTLP Endpoint diff --git a/apps/web/src/content/docs/docs/integrations/phoenix.mdx b/apps/web/src/content/docs/docs/integrations/phoenix.mdx index 0005700d9..78316feab 100644 --- a/apps/web/src/content/docs/docs/integrations/phoenix.mdx +++ b/apps/web/src/content/docs/docs/integrations/phoenix.mdx @@ -93,7 +93,7 @@ For trace export, use AgentV's standard OTel options: agentv eval evals/my-eval.yaml --otel-file traces/eval.otlp.json ``` -For live OTel export to Phoenix, use a local `.agentv/otel-backends/phoenix.ts` +For live OTel export to Phoenix, use a local `.agentv/otel-backends/phoenix.mjs` resolver that imports `phoenixOtelBackend` from `@agentv/phoenix-adapter`, then select it with `--otel-backend phoenix`. General live export options are documented in [Running Evaluations](/docs/evaluation/running-evals/#live-otel-export/). diff --git a/docs/adr/2026-06-11-phoenix-observability-adapter.md b/docs/adr/2026-06-11-phoenix-observability-adapter.md index 2ce72b370..c866606d3 100644 --- a/docs/adr/2026-06-11-phoenix-observability-adapter.md +++ b/docs/adr/2026-06-11-phoenix-observability-adapter.md @@ -60,11 +60,11 @@ export interface OtelBackendResolver { Registration/discovery should remain boring and local-first. In this ADR, "plugin" should not imply a coding-agent plugin or package marketplace; this is only a backend resolver module seam: - support explicit TypeScript registration for programmatic callers; -- optionally discover `.agentv/otel-backends/*.ts`, where the filename is the backend name; +- optionally discover Node-loadable `.agentv/otel-backends/*.mjs` or `*.js`, where the filename is the backend name; - keep `execution.otel_backend: ` and `--otel-backend ` as the user-facing selectors; - do not add package names, package auto-installation, a remote marketplace, trust prompts, or a general-purpose plugin host for this need. -The Phoenix adapter can then expose a resolver, for example `phoenixOtelBackend`, and users can opt in from project config or a local `.agentv/otel-backends/phoenix.ts` file. Reusable npm packages can come later only if repeated project-local resolver files become real friction. +The Phoenix adapter can then expose a resolver, for example `phoenixOtelBackend`, and users can opt in from project config or a local `.agentv/otel-backends/phoenix.mjs` file. Reusable npm packages can come later only if repeated project-local resolver files become real friction. ## Migration path for Phoenix @@ -100,5 +100,5 @@ Negative: ## Open questions - Should the existing `langfuse`, `braintrust`, and `confident` core presets migrate to resolver modules in a follow-up cleanup? -- Should resolver loading be limited to local `.agentv/otel-backends/*.ts`, or should `agentv.config.ts` support direct resolver imports first? +- Should resolver loading stay limited to local Node-loadable `.agentv/otel-backends/*.mjs`/`*.js`, or should `agentv.config.ts` support direct resolver imports first? - What exact Phoenix project-routing headers should the adapter emit across local Phoenix and hosted Phoenix variants? diff --git a/packages/core/src/observability/otel-exporter.ts b/packages/core/src/observability/otel-exporter.ts index eccfd389e..b457d7ae5 100644 --- a/packages/core/src/observability/otel-exporter.ts +++ b/packages/core/src/observability/otel-exporter.ts @@ -47,10 +47,11 @@ export class OtelTraceExporter { const { resourceFromAttributes } = resourcesMod; const { ATTR_SERVICE_NAME } = semconvMod; - const resource = resourceFromAttributes({ + const resourceAttributes = { [ATTR_SERVICE_NAME]: this.options.serviceName ?? 'agentv', ...this.options.resourceAttributes, - }); + }; + const resource = resourceFromAttributes(resourceAttributes); // biome-ignore lint/suspicious/noExplicitAny: OTel processor types loaded dynamically const processors: any[] = []; @@ -70,7 +71,9 @@ export class OtelTraceExporter { if (this.options.otlpFilePath) { const { OtlpJsonFileExporter } = await import('./otlp-json-file-exporter.js'); processors.push( - new SimpleSpanProcessor(new OtlpJsonFileExporter(this.options.otlpFilePath)), + new SimpleSpanProcessor( + new OtlpJsonFileExporter(this.options.otlpFilePath, resourceAttributes), + ), ); } diff --git a/packages/core/src/observability/otlp-json-file-exporter.ts b/packages/core/src/observability/otlp-json-file-exporter.ts index c41f3321e..67dfdeda7 100644 --- a/packages/core/src/observability/otlp-json-file-exporter.ts +++ b/packages/core/src/observability/otlp-json-file-exporter.ts @@ -12,13 +12,19 @@ export class OtlpJsonFileExporter { // biome-ignore lint/suspicious/noExplicitAny: serialized span data private spans: any[] = []; private filePath: string; + private resourceAttributes: Record; - constructor(filePath: string) { + constructor(filePath: string, resourceAttributes: Record = {}) { this.filePath = filePath; + this.resourceAttributes = { ...resourceAttributes }; } export(spans: ReadableSpan[], resultCallback: (result: { code: number }) => void): void { for (const span of spans) { + this.resourceAttributes = { + ...this.resourceAttributes, + ...getSpanResourceAttributes(span), + }; this.spans.push({ traceId: span.spanContext().traceId, spanId: span.spanContext().spanId, @@ -57,7 +63,7 @@ export class OtlpJsonFileExporter { const otlpJson = { resourceSpans: [ { - resource: { attributes: [] }, + resource: { attributes: convertAttributes(this.resourceAttributes) }, scopeSpans: [ { scope: { name: 'agentv', version: '1.0.0' }, @@ -73,6 +79,13 @@ export class OtlpJsonFileExporter { } } +function getSpanResourceAttributes(span: ReadableSpan): Record { + if (span.resource?.attributes && typeof span.resource.attributes === 'object') { + return span.resource.attributes; + } + return {}; +} + function hrTimeToNanos(hrTime: [number, number]): string { return String(hrTime[0] * 1_000_000_000 + hrTime[1]); } diff --git a/packages/core/test/observability/file-exporters.test.ts b/packages/core/test/observability/file-exporters.test.ts index 3ebc388ce..7b341ea32 100644 --- a/packages/core/test/observability/file-exporters.test.ts +++ b/packages/core/test/observability/file-exporters.test.ts @@ -22,6 +22,7 @@ function makeSpan(overrides: { startTime?: [number, number]; endTime?: [number, number]; attributes?: Record; + resourceAttributes?: Record; status?: { code: number }; events?: Array<{ name: string; time: [number, number]; attributes?: Record }>; }) { @@ -35,6 +36,7 @@ function makeSpan(overrides: { startTime: overrides.startTime ?? [1000, 0], endTime: overrides.endTime ?? [1001, 0], attributes: overrides.attributes ?? {}, + resource: { attributes: overrides.resourceAttributes ?? {} }, status: overrides.status ?? { code: 0 }, events: overrides.events ?? [], }; @@ -156,4 +158,35 @@ describe('OtlpJsonFileExporter', () => { }, }); }); + + it('serializes resource attributes into the OTLP resource span', async () => { + const filePath = path.join(testDir, 'otlp', 'resource-attrs.json'); + const exporter = new OtlpJsonFileExporter(filePath, { + 'service.name': 'agentv', + 'openinference.project.name': 'phoenix-project', + }); + + exporter.export( + [ + makeSpan({ + resourceAttributes: { + 'agentv.resource.flag': true, + }, + }), + ], + () => {}, + ); + + await exporter.shutdown(); + + const parsed = JSON.parse(await readFile(filePath, 'utf8')); + const attrs = parsed.resourceSpans[0].resource.attributes; + const byKey = Object.fromEntries( + attrs.map((a: { key: string; value: unknown }) => [a.key, a.value]), + ); + + expect(byKey['service.name']).toEqual({ stringValue: 'agentv' }); + expect(byKey['openinference.project.name']).toEqual({ stringValue: 'phoenix-project' }); + expect(byKey['agentv.resource.flag']).toEqual({ boolValue: true }); + }); }); diff --git a/packages/phoenix-adapter/README.md b/packages/phoenix-adapter/README.md index 54d0e6d4f..344b3ba7a 100644 --- a/packages/phoenix-adapter/README.md +++ b/packages/phoenix-adapter/README.md @@ -5,7 +5,7 @@ Converts AgentV eval YAML suites into Phoenix datasets and can run Phoenix exper Current adapter support is intentionally small: deterministic `contains`, `regex`, `equals`, and `is-json` assertions run through a Phoenix CODE evaluator. LLM, code, trace, composite, metric, and custom evaluator families are reported as unsupported instead of being silently mapped. The package also exports `phoenixOtelBackend`, a backend resolver for AgentV's -local `.agentv/otel-backends/phoenix.ts` hook. It resolves Phoenix collector +local `.agentv/otel-backends/phoenix.mjs` hook. It resolves Phoenix collector endpoint, auth headers, and `PHOENIX_PROJECT_NAME` resource routing outside `@agentv/core`.