diff --git a/apps/web/src/content/docs/docs/targets/coding-agents.mdx b/apps/web/src/content/docs/docs/targets/coding-agents.mdx index 764ac9b5..efd18f8d 100644 --- a/apps/web/src/content/docs/docs/targets/coding-agents.mdx +++ b/apps/web/src/content/docs/docs/targets/coding-agents.mdx @@ -140,7 +140,12 @@ targets: |-------|----------|-------------| | `model` | No | Model to use (defaults to copilot's default) | | `cwd` | No | Working directory | -| `custom_provider` | No | OpenAI-compatible provider endpoint for `copilot`, `copilot-cli`, or `copilot-sdk` | +| `subprovider` | No | OpenAI-compatible provider type for `copilot`, `copilot-cli`, or `copilot-sdk`, such as `openai` or `azure` | +| `base_url` | No | Provider base URL or Azure resource URL/name | +| `api_key` | No | Provider API key. Prefer `${{ ENV_VAR }}` references. | +| `bearer_token` | No | Provider bearer token. Prefer `${{ ENV_VAR }}` references. Takes precedence over `api_key` when set. | +| `api_version` | No | Provider API version, primarily for Azure endpoints | +| `wire_api` | No | Provider wire API format, such as `responses` | | `grader_target` | Yes | LLM target for evaluation | Route Copilot through an OpenAI-compatible endpoint: @@ -149,15 +154,14 @@ Route Copilot through an OpenAI-compatible endpoint: targets: - name: copilot-openai provider: copilot-cli - custom_provider: - type: openai - base_url: ${{ OPENAI_ENDPOINT }} - api_key: ${{ OPENAI_API_KEY }} - wire_api: ${{ COPILOT_PROVIDER_WIRE_API }} + subprovider: openai + base_url: ${{ OPENAI_ENDPOINT }} + api_key: ${{ OPENAI_API_KEY }} + wire_api: ${{ COPILOT_PROVIDER_WIRE_API }} grader_target: azure-base ``` -Values can come from environment variables through `${{ ... }}` interpolation. For `copilot-cli`, AgentV maps this shared block to Copilot's documented provider environment variables before spawning `copilot`; omitted fields leave existing ambient `COPILOT_PROVIDER_*` values unchanged. +Values can come from environment variables through `${{ ... }}` interpolation. For `copilot-cli`, AgentV maps these flat fields to Copilot's documented provider environment variables before spawning `copilot`; omitted fields leave existing ambient `COPILOT_PROVIDER_*` values unchanged. ## Pi Coding Agent diff --git a/packages/core/src/evaluation/providers/copilot-cli.ts b/packages/core/src/evaluation/providers/copilot-cli.ts index 0c0ffc1b..481f8941 100644 --- a/packages/core/src/evaluation/providers/copilot-cli.ts +++ b/packages/core/src/evaluation/providers/copilot-cli.ts @@ -103,10 +103,6 @@ export class CopilotCliProvider implements Provider { const startTime = new Date().toISOString(); const startMs = Date.now(); - if (this.config.customProvider) { - return await this.invokePromptMode(request, startTime, startMs); - } - const logger = await this.createStreamLogger(request, 'acp').catch(() => undefined); // Build command args @@ -116,6 +112,7 @@ export class CopilotCliProvider implements Provider { // Spawn the CLI process const agentProcess = spawn(executable, args, { env: buildCopilotCliProviderEnv(process.env, this.config.customProvider), + cwd: this.resolveCwd(request.cwd) ?? process.cwd(), stdio: ['pipe', 'pipe', 'inherit'], }); trackChild(agentProcess); @@ -136,6 +133,7 @@ export class CopilotCliProvider implements Provider { const input = Writable.toWeb(agentProcess.stdin); const output = Readable.toWeb(agentProcess.stdout) as ReadableStream; const stream = acp.ndJsonStream(input, output); + const customProvider = this.config.customProvider; const client: acp.Client = { async requestPermission(): Promise { @@ -148,7 +146,7 @@ export class CopilotCliProvider implements Provider { const update = params.update; const sessionUpdate = update.sessionUpdate; - logger?.handleEvent(sessionUpdate, update); + logger?.handleEvent(sessionUpdate, sanitizeSensitiveValue(update, customProvider)); if (sessionUpdate === 'tool_call') { const callId = update.toolCallId ?? randomUUID(); @@ -788,6 +786,30 @@ function sanitizeSensitiveText( return sanitized; } +function sanitizeSensitiveValue( + value: unknown, + customProvider: CopilotCustomProviderConfig | undefined, +): unknown { + if (!customProvider) { + return value; + } + if (typeof value === 'string') { + return sanitizeSensitiveText(value, customProvider); + } + if (Array.isArray(value)) { + return value.map((item) => sanitizeSensitiveValue(item, customProvider)); + } + if (value && typeof value === 'object') { + return Object.fromEntries( + Object.entries(value).map(([key, entry]) => [ + key, + sanitizeSensitiveValue(entry, customProvider), + ]), + ); + } + return value; +} + async function defaultCopilotCliPromptRunner( options: CopilotCliPromptRunOptions, ): Promise { diff --git a/packages/core/src/evaluation/providers/copilot-sdk.ts b/packages/core/src/evaluation/providers/copilot-sdk.ts index 15173d73..06f74783 100644 --- a/packages/core/src/evaluation/providers/copilot-sdk.ts +++ b/packages/core/src/evaluation/providers/copilot-sdk.ts @@ -14,7 +14,7 @@ import { } from './copilot-utils.js'; import { normalizeToolCall } from './normalize-tool-call.js'; import { buildPromptDocument, normalizeInputFiles } from './preread.js'; -import type { CopilotCustomProviderConfig, CopilotSdkResolvedConfig } from './targets.js'; +import type { CopilotSdkResolvedConfig } from './targets.js'; import type { Message, Provider, @@ -36,7 +36,12 @@ async function loadCopilotSdk(): Promise { const message = error instanceof Error ? error.message : String(error); if (message.includes('vscode-jsonrpc')) { throw new Error( - `Failed to load @github/copilot-sdk due to a known ESM compatibility issue with vscode-jsonrpc (https://github.com/github/copilot-sdk/issues/710).\n\nWorkarounds:\n - Use the copilot-cli target instead (recommended): set target type to "copilot-cli" in your eval YAML\n - If running under Node.js 24+: set NODE_OPTIONS="--experimental-specifier-resolution=node"\n - Wait for vscode-jsonrpc@9.0.0 stable to be released upstream`, + '@github/copilot-sdk failed to load: vscode-jsonrpc ESM import specifier mismatch.\n' + + "The package imports 'vscode-jsonrpc/node' but the installed version exposes 'node.js'.\n\n" + + 'Repair (run once in your project root):\n' + + " node -e \"const p=require.resolve('vscode-jsonrpc/package.json').replace('/package.json',''); require('fs').symlinkSync(p+'/node.js',p+'/node','file')\" 2>/dev/null || true\n\n" + + 'Or switch to the copilot-cli target (no SDK dependency):\n' + + ' Set provider: copilot-cli in your eval YAML', ); } throw new Error( @@ -84,7 +89,8 @@ export class CopilotSdkProvider implements Provider { } const sdk = await loadCopilotSdk(); - const client = await this.getOrCreateClient(sdk); + const evalCwd = this.resolveCwd(request.cwd); + const client = await this.getOrCreateClient(sdk, evalCwd ?? undefined); const startTime = new Date().toISOString(); const startMs = Date.now(); @@ -101,12 +107,11 @@ export class CopilotSdkProvider implements Provider { sessionOptions.model = this.config.model; } - const cwd = this.resolveCwd(request.cwd); - if (cwd) { - sessionOptions.workingDirectory = cwd; + if (evalCwd) { + sessionOptions.workingDirectory = evalCwd; // Auto-discover skill directories from the workspace so the SDK loads // SKILL.md files into the session context (see copilot-sdk docs/features/skills.md). - sessionOptions.skillDirectories = resolveSkillDirectories(cwd); + sessionOptions.skillDirectories = resolveSkillDirectories(evalCwd); } const systemPrompt = this.config.systemPrompt; @@ -118,13 +123,13 @@ export class CopilotSdkProvider implements Provider { }; } - const customProvider = resolveCustomProviderConfig(this.config); + const customProvider = this.config.customProvider; if (customProvider) { const providerType = customProvider.type ?? 'openai'; // biome-ignore lint/suspicious/noExplicitAny: SDK provider config shape is dynamic const provider: any = { type: providerType, - baseUrl: normalizeByokBaseUrl(customProvider.baseUrl, providerType), + baseUrl: normalizeProviderBaseUrl(customProvider.baseUrl, providerType), }; if (customProvider.bearerToken) { provider.bearerToken = customProvider.bearerToken; @@ -299,7 +304,7 @@ export class CopilotSdkProvider implements Provider { } // biome-ignore lint/suspicious/noExplicitAny: SDK client type is dynamically loaded - private async getOrCreateClient(sdk: any): Promise { + private async getOrCreateClient(sdk: any, evalCwd?: string): Promise { if (!this.client) { // biome-ignore lint/suspicious/noExplicitAny: SDK constructor options are dynamic const clientOptions: any = {}; @@ -317,6 +322,15 @@ export class CopilotSdkProvider implements Provider { clientOptions.cliPath = nativePath; } } + // Set the subprocess cwd so --plugin-dir ./relative resolves from the eval workspace. + const resolvedCwd = evalCwd ?? process.cwd(); + clientOptions.cwd = resolvedCwd; + + if (this.config.args && this.config.args.length > 0) { + clientOptions.cliArgs = this.config.args.map((arg) => + arg.startsWith('./') || arg.startsWith('../') ? path.resolve(resolvedCwd, arg) : arg, + ); + } if (this.config.githubToken) { clientOptions.githubToken = this.config.githubToken; } @@ -411,25 +425,6 @@ export class CopilotSdkProvider implements Provider { } } -function resolveCustomProviderConfig( - config: CopilotSdkResolvedConfig, -): CopilotCustomProviderConfig | undefined { - if (config.customProvider) { - return config.customProvider; - } - if (!config.byokBaseUrl) { - return undefined; - } - return { - ...(config.byokType ? { type: config.byokType } : {}), - baseUrl: config.byokBaseUrl, - ...(config.byokApiKey ? { apiKey: config.byokApiKey } : {}), - ...(config.byokBearerToken ? { bearerToken: config.byokBearerToken } : {}), - ...(config.byokApiVersion ? { apiVersion: config.byokApiVersion } : {}), - ...(config.byokWireApi ? { wireApi: config.byokWireApi } : {}), - }; -} - /** * Auto-discover skill directories from a workspace. * Checks standard skill directory locations and returns any that exist. @@ -444,12 +439,12 @@ function resolveSkillDirectories(cwd: string): string[] { } /** - * Normalize a BYOK base URL for the Copilot SDK. + * Normalize a provider base URL for the Copilot SDK. * For Azure type, if the value is a bare resource name (no https:// prefix), * construct the full URL: https://{resourceName}.openai.azure.com * This lets users reuse AZURE_OPENAI_ENDPOINT without a separate env var. */ -function normalizeByokBaseUrl(baseUrl: string, type: string): string { +function normalizeProviderBaseUrl(baseUrl: string, type: string): string { const trimmed = baseUrl.trim().replace(/\/+$/, ''); if (/^https?:\/\//i.test(trimmed)) { return trimmed; diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index be94fe82..32e0630d 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -461,6 +461,7 @@ export interface CopilotCustomProviderConfig { export interface CopilotSdkResolvedConfig { readonly cliUrl?: string; readonly cliPath?: string; + readonly args?: readonly string[]; readonly githubToken?: string; readonly model?: string; readonly cwd?: string; @@ -471,18 +472,6 @@ export interface CopilotSdkResolvedConfig { readonly streamLog?: false | 'raw' | 'summary'; readonly systemPrompt?: string; readonly customProvider?: CopilotCustomProviderConfig; - /** BYOK provider type: "azure", "openai", or "anthropic". */ - readonly byokType?: string; - /** BYOK base URL for the provider endpoint. */ - readonly byokBaseUrl?: string; - /** BYOK API key for authenticating with the provider. */ - readonly byokApiKey?: string; - /** BYOK bearer token (takes precedence over apiKey when set). */ - readonly byokBearerToken?: string; - /** BYOK Azure API version (e.g. "2024-10-21"). Only used when byokType is "azure". */ - readonly byokApiVersion?: string; - /** BYOK wire API format: "completions" or "responses". */ - readonly byokWireApi?: string; } export interface CopilotLogResolvedConfig { @@ -1449,6 +1438,7 @@ function resolveCopilotSdkConfig( ): CopilotSdkResolvedConfig { const cliUrlSource = target.cli_url; const cliPathSource = target.cli_path; + const argsSource = target.args ?? target.arguments; const githubTokenSource = target.github_token; const modelSource = target.model; const cwdSource = target.cwd; @@ -1472,6 +1462,8 @@ function resolveCopilotSdkConfig( optionalEnv: true, }); + const args = resolveOptionalStringArray(argsSource, env, `${target.name} copilot-sdk args`); + const githubToken = resolveOptionalString( githubTokenSource, env, @@ -1511,13 +1503,12 @@ function resolveCopilotSdkConfig( ? systemPromptSource.trim() : undefined; - const customProvider = resolveCopilotCustomProviderConfig(target, env, { - includeByokAlias: true, - }); + const customProvider = resolveCopilotFlatProviderConfig(target, env); return { cliUrl, cliPath, + args, githubToken, model, cwd, @@ -1526,100 +1517,58 @@ function resolveCopilotSdkConfig( logFormat, streamLog: streamLogResult.streamLog, systemPrompt, - ...(customProvider - ? { - customProvider, - byokType: customProvider.type, - byokBaseUrl: customProvider.baseUrl, - byokApiKey: customProvider.apiKey, - byokBearerToken: customProvider.bearerToken, - byokApiVersion: customProvider.apiVersion, - byokWireApi: customProvider.wireApi, - } - : {}), + ...(customProvider ? { customProvider } : {}), }; } -function resolveCopilotCustomProviderConfig( +function resolveCopilotFlatProviderConfig( target: z.infer, env: EnvLookup, - options: { readonly includeByokAlias?: boolean } = {}, ): CopilotCustomProviderConfig | undefined { - const hasCustomProvider = target.custom_provider !== undefined; - const hasByokAlias = options.includeByokAlias === true && target.byok !== undefined; - if (!hasCustomProvider && !hasByokAlias) { - return undefined; - } - - const sourceName = hasCustomProvider ? 'custom_provider' : 'byok'; - const raw = - sourceName === 'custom_provider' - ? (target.custom_provider as unknown) - : (target.byok as unknown); + const baseUrlSource = target.base_url; + if (!baseUrlSource) return undefined; - if (raw === null) { - return undefined; - } - if (typeof raw !== 'object' || Array.isArray(raw)) { - throw new Error(`${target.name}: '${sourceName}' must be an object`); - } - - const provider = raw as Record; - const type = resolveOptionalString(provider.type, env, `${target.name} ${sourceName} type`, { + const baseUrl = resolveOptionalString(baseUrlSource, env, `${target.name} copilot base URL`, { allowLiteral: true, optionalEnv: true, }); - const baseUrl = resolveOptionalString( - provider.base_url, + if (!baseUrl) return undefined; + + const type = resolveOptionalString( + target.subprovider, env, - `${target.name} ${sourceName} base URL`, + `${target.name} copilot provider type`, { allowLiteral: true, optionalEnv: true, }, ); - const apiKey = resolveOptionalString( - provider.api_key, - env, - `${target.name} ${sourceName} API key`, - { - allowLiteral: false, - optionalEnv: true, - }, - ); + const apiKey = resolveOptionalString(target.api_key, env, `${target.name} copilot API key`, { + allowLiteral: false, + optionalEnv: true, + }); const bearerToken = resolveOptionalString( - provider.bearer_token, + target.bearer_token, env, - `${target.name} ${sourceName} bearer token`, + `${target.name} copilot bearer token`, { allowLiteral: false, optionalEnv: true, }, ); const apiVersion = resolveOptionalString( - provider.api_version, - env, - `${target.name} ${sourceName} API version`, - { - allowLiteral: true, - optionalEnv: true, - }, - ); - const wireApi = resolveOptionalString( - provider.wire_api, + target.api_version, env, - `${target.name} ${sourceName} wire API`, + `${target.name} copilot API version`, { allowLiteral: true, optionalEnv: true, }, ); - - if (!baseUrl) { - throw new Error( - `${target.name}: '${sourceName}.base_url' is required when '${sourceName}' is specified`, - ); - } + const wireApi = resolveOptionalString(target.wire_api, env, `${target.name} copilot wire API`, { + allowLiteral: true, + optionalEnv: true, + }); return { ...(type ? { type } : {}), @@ -1686,7 +1635,7 @@ function resolveCopilotCliConfig( typeof systemPromptSource === 'string' && systemPromptSource.trim().length > 0 ? systemPromptSource.trim() : undefined; - const customProvider = resolveCopilotCustomProviderConfig(target, env); + const customProvider = resolveCopilotFlatProviderConfig(target, env); return { executable, diff --git a/packages/core/src/evaluation/providers/types.ts b/packages/core/src/evaluation/providers/types.ts index a3270df5..f4dbd85c 100644 --- a/packages/core/src/evaluation/providers/types.ts +++ b/packages/core/src/evaluation/providers/types.ts @@ -434,10 +434,6 @@ export interface TargetDefinition { readonly cli_url?: string | unknown | undefined; readonly cli_path?: string | unknown | undefined; readonly github_token?: string | unknown | undefined; - // Copilot custom provider fields - readonly custom_provider?: Record | undefined; - // Copilot SDK BYOK compatibility alias - readonly byok?: Record | undefined; // Retry configuration fields readonly max_retries?: number | unknown | undefined; readonly retry_initial_delay_ms?: number | unknown | undefined; diff --git a/packages/core/src/evaluation/validation/targets-validator.ts b/packages/core/src/evaluation/validation/targets-validator.ts index 596a538e..f5a7c87b 100644 --- a/packages/core/src/evaluation/validation/targets-validator.ts +++ b/packages/core/src/evaluation/validation/targets-validator.ts @@ -118,6 +118,8 @@ const COPILOT_SDK_SETTINGS = new Set([ ...COMMON_SETTINGS, 'cli_url', 'cli_path', + 'args', + 'arguments', 'github_token', 'model', 'cwd', @@ -126,8 +128,12 @@ const COPILOT_SDK_SETTINGS = new Set([ 'log_format', 'stream_log', 'system_prompt', - 'custom_provider', - 'byok', + 'subprovider', + 'base_url', + 'api_key', + 'bearer_token', + 'api_version', + 'wire_api', ]); const COPILOT_CLI_SETTINGS = new Set([ @@ -144,7 +150,12 @@ const COPILOT_CLI_SETTINGS = new Set([ 'log_format', 'stream_log', 'system_prompt', - 'custom_provider', + 'subprovider', + 'base_url', + 'api_key', + 'bearer_token', + 'api_version', + 'wire_api', ]); const VSCODE_SETTINGS = new Set([ diff --git a/packages/core/test/evaluation/providers/copilot-cli.test.ts b/packages/core/test/evaluation/providers/copilot-cli.test.ts index de2365fd..87483a0c 100644 --- a/packages/core/test/evaluation/providers/copilot-cli.test.ts +++ b/packages/core/test/evaluation/providers/copilot-cli.test.ts @@ -1,9 +1,11 @@ import { afterEach, beforeAll, beforeEach, describe, expect, it, mock } from 'bun:test'; import type { ChildProcess } from 'node:child_process'; +import { EventEmitter } from 'node:events'; import { mkdtemp, readFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import path from 'node:path'; +import { PassThrough } from 'node:stream'; import { DEFAULT_COPILOT_TIMEOUT_MS } from '../../../src/evaluation/providers/copilot-utils.js'; import { extractLastAssistantContent } from '../../../src/evaluation/providers/types.js'; @@ -13,12 +15,64 @@ type CopilotCliModule = typeof import('../../../src/evaluation/providers/copilot let CopilotCliProvider: CopilotCliModule['CopilotCliProvider']; let buildCopilotCliProviderEnv: CopilotCliModule['buildCopilotCliProviderEnv']; let originalLogEnv: string | undefined; +let spawnMock: ReturnType; +let acpSessionUpdates: Array<{ update: { sessionUpdate: string; [key: string]: unknown } }>; +let acpPromptResponse: Record; + +function createMockChildProcess(): ChildProcess { + const child = new EventEmitter() as EventEmitter & { + pid: number; + stdin: PassThrough; + stdout: PassThrough; + stderr: PassThrough; + kill: ReturnType; + }; + child.pid = 12345; + child.stdin = new PassThrough(); + child.stdout = new PassThrough(); + child.stderr = new PassThrough(); + child.kill = mock(() => true); + return child as unknown as ChildProcess; +} beforeAll(async () => { + spawnMock = mock(() => createMockChildProcess()); + mock.module('node:child_process', () => ({ + spawn: spawnMock, + })); mock.module('@agentclientprotocol/sdk', () => ({ PROTOCOL_VERSION: 1, ndJsonStream: mock(() => ({})), - ClientSideConnection: class MockClientSideConnection {}, + ClientSideConnection: class MockClientSideConnection { + private readonly client: { + sessionUpdate?: (params: { + update: { sessionUpdate: string; [key: string]: unknown }; + }) => Promise; + }; + + constructor( + createClient: (_agent: unknown) => { + sessionUpdate?: (params: { + update: { sessionUpdate: string; [key: string]: unknown }; + }) => Promise; + }, + ) { + this.client = createClient({}); + } + + async initialize(): Promise {} + + async newSession(): Promise<{ sessionId: string }> { + return { sessionId: 'session-1' }; + } + + async prompt(): Promise> { + for (const update of acpSessionUpdates) { + await this.client.sessionUpdate?.(update); + } + return acpPromptResponse; + } + }, })); const module = await import('../../../src/evaluation/providers/copilot-cli.js'); CopilotCliProvider = module.CopilotCliProvider; @@ -28,6 +82,16 @@ beforeAll(async () => { beforeEach(() => { originalLogEnv = process.env.AGENTV_COPILOT_CLI_STREAM_LOGS; process.env.AGENTV_COPILOT_CLI_STREAM_LOGS = 'false'; + spawnMock.mockClear(); + acpSessionUpdates = [ + { + update: { + sessionUpdate: 'agent_message_chunk', + content: { type: 'text', text: 'agentv-copilot-gateway-ok' }, + }, + }, + ]; + acpPromptResponse = {}; }); afterEach(() => { @@ -101,19 +165,17 @@ describe('buildCopilotCliProviderEnv', () => { }); }); -describe('CopilotCliProvider custom provider prompt mode', () => { - it('uses non-ACP prompt mode with custom provider env and default long timeout', async () => { - const runner = mock(async () => ({ - stdout: '\u001b[32magentv-copilot-gateway-ok\u001b[0m\n', - stderr: 'warning secret-key', - exitCode: 0, - })); +describe('CopilotCliProvider custom provider ACP mode', () => { + it('uses ACP mode with custom provider env vars when customProvider is resolved', async () => { + const runner = mock(async () => { + throw new Error('prompt mode should not be used'); + }); const provider = new CopilotCliProvider( 'copilot-cli-custom', { executable: '/usr/bin/copilot', model: 'gpt-5-mini', - args: ['--extra-flag'], + args: ['--plugin-dir', './plugins', '--extra-flag'], customProvider: { type: 'openai', baseUrl: 'https://api.openai.example/v1', @@ -130,119 +192,77 @@ describe('CopilotCliProvider custom provider prompt mode', () => { }); expect(extractLastAssistantContent(response.output)).toBe('agentv-copilot-gateway-ok'); - expect(runner).toHaveBeenCalledTimes(1); - const invocation = runner.mock.calls[0][0]; - expect(invocation.executable).toBe('/usr/bin/copilot'); - expect(invocation.cwd).toBe('/tmp/copilot-workspace'); - expect(invocation.timeoutMs).toBe(DEFAULT_COPILOT_TIMEOUT_MS); - expect(invocation.env.COPILOT_PROVIDER_TYPE).toBe('openai'); - expect(invocation.env.COPILOT_PROVIDER_BASE_URL).toBe('https://api.openai.example/v1'); - expect(invocation.env.COPILOT_PROVIDER_API_KEY).toBe('secret-key'); - expect(invocation.env.COPILOT_PROVIDER_WIRE_API).toBe('responses'); - expect(invocation.args.slice(0, 6)).toEqual([ - '-s', + expect(runner).not.toHaveBeenCalled(); + expect(spawnMock).toHaveBeenCalledTimes(1); + + const [executable, args, options] = spawnMock.mock.calls[0] as [ + string, + string[], + { + cwd: string; + env: NodeJS.ProcessEnv; + stdio: string[]; + }, + ]; + expect(executable).toBe('/usr/bin/copilot'); + expect(args.slice(0, 6)).toEqual([ + '--acp', + '--stdio', '--allow-all-tools', - '--no-color', + '--yolo', '--model', 'gpt-5-mini', - '--extra-flag', ]); - expect(invocation.args).not.toContain('--acp'); - expect(invocation.args).not.toContain('--stdio'); - expect(invocation.args.at(-2)).toBe('-p'); - expect(invocation.args.at(-1)).toContain('agentv-copilot-gateway-ok'); - - const raw = response.raw as Record; - expect(raw.stderr).toBe('warning [redacted]'); + expect(args).toContain('--plugin-dir'); + expect(args).toContain('./plugins'); + expect(args).not.toContain('-p'); + expect(options.cwd).toBe('/tmp/copilot-workspace'); + expect(options.stdio).toEqual(['pipe', 'pipe', 'inherit']); + expect(options.env.COPILOT_PROVIDER_TYPE).toBe('openai'); + expect(options.env.COPILOT_PROVIDER_BASE_URL).toBe('https://api.openai.example/v1'); + expect(options.env.COPILOT_PROVIDER_API_KEY).toBe('secret-key'); + expect(options.env.COPILOT_PROVIDER_WIRE_API).toBe('responses'); }); - it('uses explicit timeout for custom provider prompt mode when configured', async () => { - const runner = mock(async () => ({ - stdout: 'done', - stderr: '', - exitCode: 0, - })); - const provider = new CopilotCliProvider( - 'copilot-cli-custom', - { - executable: '/usr/bin/copilot', - timeoutMs: 30_000, - customProvider: { - type: 'openai', - baseUrl: 'https://api.openai.example/v1', - apiKey: 'secret-key', - }, + it('uses configured cwd for ACP spawn when request cwd is omitted', async () => { + const provider = new CopilotCliProvider('copilot-cli-custom', { + executable: '/usr/bin/copilot', + cwd: '/tmp/eval-workspace', + customProvider: { + type: 'openai', + baseUrl: 'https://api.openai.example/v1', + apiKey: 'secret-key', }, - runner, - ); + }); await provider.invoke({ question: 'Return done' }); - expect(runner.mock.calls[0][0].timeoutMs).toBe(30_000); + const options = spawnMock.mock.calls[0][2] as { cwd: string }; + expect(options.cwd).toBe('/tmp/eval-workspace'); }); - it('redacts custom provider credentials from prompt-mode errors', async () => { - const runner = mock(async () => ({ - stdout: '', - stderr: 'upstream rejected secret-key', - exitCode: 1, - })); - const provider = new CopilotCliProvider( - 'copilot-cli-custom', + it('redacts custom provider credentials from ACP stream logs', async () => { + const logDir = await mkdtemp(path.join(tmpdir(), 'agentv-copilot-cli-logs-')); + Reflect.deleteProperty(process.env, 'AGENTV_COPILOT_CLI_STREAM_LOGS'); + acpSessionUpdates = [ { - executable: '/usr/bin/copilot', - customProvider: { - type: 'openai', - baseUrl: 'https://api.openai.example/v1', - apiKey: 'secret-key', + update: { + sessionUpdate: 'agent_message_chunk', + content: { type: 'text', text: 'stdout included test-api-key' }, }, }, - runner, - ); + ]; - let message = ''; try { - await provider.invoke({ question: 'Return done' }); - } catch (error) { - message = error instanceof Error ? error.message : String(error); - } - - expect(message).toContain('[redacted]'); - expect(message).not.toContain('secret-key'); - }); - - it('redacts custom provider credentials from prompt-mode stream logs', async () => { - const logDir = await mkdtemp(path.join(tmpdir(), 'agentv-copilot-cli-logs-')); - Reflect.deleteProperty(process.env, 'AGENTV_COPILOT_CLI_STREAM_LOGS'); - - try { - const runner = mock( - async (options: { - readonly onStdoutChunk?: (chunk: string) => void; - readonly onStderrChunk?: (chunk: string) => void; - }) => { - options.onStdoutChunk?.('stdout included test-api-key'); - options.onStderrChunk?.('stderr included test-api-key'); - return { - stdout: 'done', - stderr: '', - exitCode: 0, - }; - }, - ); - const provider = new CopilotCliProvider( - 'copilot-cli-custom', - { - executable: '/usr/bin/copilot', - logDir, - customProvider: { - type: 'openai', - baseUrl: 'https://api.openai.example/v1', - apiKey: 'test-api-key', - }, + const provider = new CopilotCliProvider('copilot-cli-custom', { + executable: '/usr/bin/copilot', + logDir, + customProvider: { + type: 'openai', + baseUrl: 'https://api.openai.example/v1', + apiKey: 'test-api-key', }, - runner, - ); + }); const response = await provider.invoke({ question: 'Return done' }); const logFile = (response.raw as Record).logFile; diff --git a/packages/core/test/evaluation/providers/copilot-sdk.test.ts b/packages/core/test/evaluation/providers/copilot-sdk.test.ts index 3e1651eb..2167e977 100644 --- a/packages/core/test/evaluation/providers/copilot-sdk.test.ts +++ b/packages/core/test/evaluation/providers/copilot-sdk.test.ts @@ -160,6 +160,64 @@ describe('CopilotSdkProvider', () => { expect(constructorArgs.cliUrl).toBe('http://localhost:9999'); }); + it('passes args as cliArgs to CopilotClient constructor', async () => { + const session = createMockSession({ + events: [{ type: 'assistant.message', data: { content: 'response' } }], + }); + const client = createMockClient(session); + + const CopilotClientMock = mock(function CopilotClient() { + return client; + }); + mock.module('@github/copilot-sdk', () => ({ + CopilotClient: CopilotClientMock, + })); + + const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js'); + + const provider = new CopilotSdkProvider('test-target', { + args: ['--verbose', 'enabled'], + }); + + await provider.invoke({ question: 'Test' }); + + const constructorArgs = CopilotClientMock.mock.calls[0][0]; + expect(constructorArgs.cliArgs).toEqual(['--verbose', 'enabled']); + }); + + it('resolves relative args paths against eval cwd', async () => { + const session = createMockSession({ + events: [{ type: 'assistant.message', data: { content: 'response' } }], + }); + const client = createMockClient(session); + + const CopilotClientMock = mock(function CopilotClient() { + return client; + }); + mock.module('@github/copilot-sdk', () => ({ + CopilotClient: CopilotClientMock, + })); + + const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js'); + + const provider = new CopilotSdkProvider('test-target', { + args: ['--plugin-dir', './plugins', '--shared-dir', '../shared', '--mode', 'agent'], + }); + + await provider.invoke({ question: 'Test', cwd: fixturesRoot }); + + const constructorArgs = CopilotClientMock.mock.calls[0][0]; + expect(constructorArgs.cwd).toBe(path.resolve(fixturesRoot)); + expect(constructorArgs.cliArgs).toEqual([ + '--plugin-dir', + path.resolve(fixturesRoot, './plugins'), + '--shared-dir', + path.resolve(fixturesRoot, '../shared'), + '--mode', + 'agent', + ]); + }); + it('handles timeout', async () => { const session = createMockSession(); // Override sendAndWait to be slow @@ -358,7 +416,7 @@ describe('CopilotSdkProvider', () => { expect(result.kind).toBe('approved'); }); - it('passes byok provider block to createSession for azure', async () => { + it('passes resolved custom provider config to createSession for azure', async () => { const session = createMockSession({ events: [{ type: 'assistant.message', data: { content: 'response' } }], }); @@ -370,10 +428,12 @@ describe('CopilotSdkProvider', () => { const provider = new CopilotSdkProvider('test-target', { model: 'gpt-4o', - byokType: 'azure', - byokBaseUrl: 'https://my-resource.openai.azure.com', - byokApiKey: 'azure-secret', - byokApiVersion: '2024-10-21', + customProvider: { + type: 'azure', + baseUrl: 'https://my-resource.openai.azure.com', + apiKey: 'azure-secret', + apiVersion: '2024-10-21', + }, }); await provider.invoke({ question: 'Test' }); @@ -397,9 +457,11 @@ describe('CopilotSdkProvider', () => { const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js'); const provider = new CopilotSdkProvider('test-target', { - byokType: 'azure', - byokBaseUrl: 'my-resource-eastus2', - byokApiKey: 'key', + customProvider: { + type: 'azure', + baseUrl: 'my-resource-eastus2', + apiKey: 'key', + }, }); await provider.invoke({ question: 'Test' }); @@ -408,7 +470,7 @@ describe('CopilotSdkProvider', () => { expect(sessionOptions.provider.baseUrl).toBe('https://my-resource-eastus2.openai.azure.com'); }); - it('passes full URL through unchanged for azure byok', async () => { + it('passes full URL through unchanged for azure custom provider', async () => { const session = createMockSession({ events: [{ type: 'assistant.message', data: { content: 'response' } }], }); @@ -419,9 +481,11 @@ describe('CopilotSdkProvider', () => { const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js'); const provider = new CopilotSdkProvider('test-target', { - byokType: 'azure', - byokBaseUrl: 'https://my-resource.openai.azure.com', - byokApiKey: 'key', + customProvider: { + type: 'azure', + baseUrl: 'https://my-resource.openai.azure.com', + apiKey: 'key', + }, }); await provider.invoke({ question: 'Test' }); @@ -430,7 +494,7 @@ describe('CopilotSdkProvider', () => { expect(sessionOptions.provider.baseUrl).toBe('https://my-resource.openai.azure.com'); }); - it('passes byok provider block with bearer token', async () => { + it('passes resolved custom provider config with bearer token', async () => { const session = createMockSession({ events: [{ type: 'assistant.message', data: { content: 'response' } }], }); @@ -441,9 +505,11 @@ describe('CopilotSdkProvider', () => { const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js'); const provider = new CopilotSdkProvider('test-target', { - byokType: 'openai', - byokBaseUrl: 'https://custom-endpoint.example.com/v1', - byokBearerToken: 'bearer-secret', + customProvider: { + type: 'openai', + baseUrl: 'https://custom-endpoint.example.com/v1', + bearerToken: 'bearer-secret', + }, }); await provider.invoke({ question: 'Test' }); @@ -454,7 +520,7 @@ describe('CopilotSdkProvider', () => { expect(sessionOptions.provider.apiKey).toBeUndefined(); }); - it('passes byok provider block with wireApi', async () => { + it('passes resolved custom provider config with wireApi', async () => { const session = createMockSession({ events: [{ type: 'assistant.message', data: { content: 'response' } }], }); @@ -465,10 +531,12 @@ describe('CopilotSdkProvider', () => { const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js'); const provider = new CopilotSdkProvider('test-target', { - byokType: 'openai', - byokBaseUrl: 'https://resource.openai.azure.com/openai/v1/', - byokApiKey: 'key', - byokWireApi: 'responses', + customProvider: { + type: 'openai', + baseUrl: 'https://resource.openai.azure.com/openai/v1/', + apiKey: 'key', + wireApi: 'responses', + }, }); await provider.invoke({ question: 'Test' }); @@ -477,7 +545,7 @@ describe('CopilotSdkProvider', () => { expect(sessionOptions.provider.wireApi).toBe('responses'); }); - it('passes customProvider block to createSession for openai-compatible endpoints', async () => { + it('passes resolved custom provider config to createSession for openai-compatible endpoints', async () => { const session = createMockSession({ events: [{ type: 'assistant.message', data: { content: 'response' } }], }); @@ -507,7 +575,7 @@ describe('CopilotSdkProvider', () => { }); }); - it('does not set provider when byok is not configured', async () => { + it('does not set provider when custom provider is not configured', async () => { const session = createMockSession({ events: [{ type: 'assistant.message', data: { content: 'response' } }], }); @@ -527,7 +595,7 @@ describe('CopilotSdkProvider', () => { expect(sessionOptions.provider).toBeUndefined(); }); - it('defaults byok type to openai when not specified', async () => { + it('defaults custom provider type to openai when not specified', async () => { const session = createMockSession({ events: [{ type: 'assistant.message', data: { content: 'response' } }], }); @@ -538,7 +606,9 @@ describe('CopilotSdkProvider', () => { const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js'); const provider = new CopilotSdkProvider('test-target', { - byokBaseUrl: 'http://localhost:11434/v1', + customProvider: { + baseUrl: 'http://localhost:11434/v1', + }, }); await provider.invoke({ question: 'Test' }); @@ -547,7 +617,7 @@ describe('CopilotSdkProvider', () => { expect(sessionOptions.provider.type).toBe('openai'); }); - it('does not set azure block for non-azure byok type', async () => { + it('does not set azure block for non-azure custom provider type', async () => { const session = createMockSession({ events: [{ type: 'assistant.message', data: { content: 'response' } }], }); @@ -558,10 +628,12 @@ describe('CopilotSdkProvider', () => { const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js'); const provider = new CopilotSdkProvider('test-target', { - byokType: 'openai', - byokBaseUrl: 'https://api.openai.com/v1', - byokApiKey: 'key', - byokApiVersion: '2024-10-21', // should be ignored for non-azure + customProvider: { + type: 'openai', + baseUrl: 'https://api.openai.com/v1', + apiKey: 'key', + apiVersion: '2024-10-21', // should be ignored for non-azure + }, }); await provider.invoke({ question: 'Test' }); diff --git a/packages/core/test/evaluation/providers/targets.test.ts b/packages/core/test/evaluation/providers/targets.test.ts index a58bdde7..813e87c9 100644 --- a/packages/core/test/evaluation/providers/targets.test.ts +++ b/packages/core/test/evaluation/providers/targets.test.ts @@ -784,7 +784,7 @@ describe('resolveTargetDefinition', () => { expect(target.config.executable).toBe('copilot'); }); - it('resolves copilot-cli with custom_provider openai config', () => { + it('resolves copilot-cli flat base_url/api_key as custom provider', () => { const env = { OPENAI_ENDPOINT: 'https://api.openai.example/v1', OPENAI_API_KEY: 'openai-secret', @@ -793,16 +793,14 @@ describe('resolveTargetDefinition', () => { const target = resolveTargetDefinition( { - name: 'copilot-cli-openai', + name: 'copilot-cli-openai-flat', provider: 'copilot-cli', - custom_provider: { - type: 'openai', - base_url: '${{ OPENAI_ENDPOINT }}', - api_key: '${{ OPENAI_API_KEY }}', - bearer_token: '${{ OPTIONAL_BEARER_TOKEN }}', - wire_api: 'responses', - api_version: '2024-10-21', - }, + subprovider: 'openai', + base_url: '${{ OPENAI_ENDPOINT }}', + api_key: '${{ OPENAI_API_KEY }}', + bearer_token: '${{ OPTIONAL_BEARER_TOKEN }}', + wire_api: 'responses', + api_version: '2024-10-21', }, env, ); @@ -822,70 +820,7 @@ describe('resolveTargetDefinition', () => { }); }); - it('resolves copilot-sdk with byok azure config', () => { - const env = { - AZURE_OPENAI_ENDPOINT: 'https://my-resource.openai.azure.com', - AZURE_OPENAI_API_KEY: 'azure-secret', - AZURE_DEPLOYMENT_NAME: 'gpt-4o', - } satisfies Record; - - const target = resolveTargetDefinition( - { - name: 'copilot-sdk-azure', - provider: 'copilot-sdk', - model: '${{ AZURE_DEPLOYMENT_NAME }}', - byok: { - type: 'azure', - base_url: '${{ AZURE_OPENAI_ENDPOINT }}', - api_key: '${{ AZURE_OPENAI_API_KEY }}', - api_version: '2024-10-21', - }, - }, - env, - ); - - expect(target.kind).toBe('copilot-sdk'); - if (target.kind !== 'copilot-sdk') { - throw new Error('expected copilot-sdk target'); - } - - expect(target.config.model).toBe('gpt-4o'); - expect(target.config.byokType).toBe('azure'); - expect(target.config.byokBaseUrl).toBe('https://my-resource.openai.azure.com'); - expect(target.config.byokApiKey).toBe('azure-secret'); - expect(target.config.byokApiVersion).toBe('2024-10-21'); - }); - - it('resolves copilot-sdk with byok openai config', () => { - const env = { - OPENAI_API_KEY: 'openai-secret', - } satisfies Record; - - const target = resolveTargetDefinition( - { - name: 'copilot-sdk-openai', - provider: 'copilot-sdk', - model: 'gpt-5', - byok: { - type: 'openai', - base_url: 'https://api.openai.com/v1', - api_key: '${{ OPENAI_API_KEY }}', - }, - }, - env, - ); - - expect(target.kind).toBe('copilot-sdk'); - if (target.kind !== 'copilot-sdk') { - throw new Error('expected copilot-sdk target'); - } - - expect(target.config.byokType).toBe('openai'); - expect(target.config.byokBaseUrl).toBe('https://api.openai.com/v1'); - expect(target.config.byokApiKey).toBe('openai-secret'); - }); - - it('resolves copilot-sdk with custom_provider openai config', () => { + it('resolves copilot-sdk flat base_url/api_key as custom provider', () => { const env = { OPENAI_ENDPOINT: 'https://api.openai.example/v1', OPENAI_API_KEY: 'openai-secret', @@ -893,15 +828,13 @@ describe('resolveTargetDefinition', () => { const target = resolveTargetDefinition( { - name: 'copilot-sdk-openai-custom-provider', + name: 'copilot-sdk-openai-flat', provider: 'copilot-sdk', model: 'gpt-5', - custom_provider: { - type: 'openai', - base_url: '${{ OPENAI_ENDPOINT }}', - api_key: '${{ OPENAI_API_KEY }}', - wire_api: 'responses', - }, + subprovider: 'openai', + base_url: '${{ OPENAI_ENDPOINT }}', + api_key: '${{ OPENAI_API_KEY }}', + wire_api: 'responses', }, env, ); @@ -917,27 +850,16 @@ describe('resolveTargetDefinition', () => { apiKey: 'openai-secret', wireApi: 'responses', }); - expect(target.config.byokType).toBe('openai'); - expect(target.config.byokBaseUrl).toBe('https://api.openai.example/v1'); - expect(target.config.byokApiKey).toBe('openai-secret'); - expect(target.config.byokWireApi).toBe('responses'); }); - it('copilot-sdk byok defaults type to undefined when not specified', () => { - const env = { - MY_KEY: 'secret', - } satisfies Record; - + it('resolves copilot-sdk args field', () => { const target = resolveTargetDefinition( { - name: 'copilot-sdk-byok-minimal', + name: 'copilot-sdk-with-args', provider: 'copilot-sdk', - byok: { - base_url: 'http://localhost:11434/v1', - api_key: '${{ MY_KEY }}', - }, + args: ['--plugin-dir', '${{ COPILOT_PLUGIN_DIR }}'], }, - env, + { COPILOT_PLUGIN_DIR: './plugins' }, ); expect(target.kind).toBe('copilot-sdk'); @@ -945,60 +867,38 @@ describe('resolveTargetDefinition', () => { throw new Error('expected copilot-sdk target'); } - expect(target.config.byokType).toBeUndefined(); - expect(target.config.byokBaseUrl).toBe('http://localhost:11434/v1'); - expect(target.config.byokApiKey).toBe('secret'); + expect(target.config.args).toEqual(['--plugin-dir', './plugins']); }); - it('copilot-sdk byok rejects missing base_url', () => { + it('copilot flat config rejects literal api_key', () => { expect(() => resolveTargetDefinition( { - name: 'copilot-sdk-no-url', + name: 'copilot-literal-key', provider: 'copilot-sdk', - byok: { - type: 'azure', - api_key: '${{ MY_KEY }}', - }, - }, - { MY_KEY: 'secret' }, - ), - ).toThrow(/byok\.base_url.*required/i); - }); - - it('copilot-sdk byok rejects literal api_key', () => { - expect(() => - resolveTargetDefinition( - { - name: 'copilot-sdk-literal-key', - provider: 'copilot-sdk', - byok: { - base_url: 'https://example.com', - api_key: 'plaintext-secret', - }, + base_url: 'https://example.com', + api_key: 'plaintext-secret', }, {}, ), ).toThrow(/must use.*VARIABLE_NAME/i); }); - it('copilot-sdk byok rejects literal bearer_token', () => { + it('copilot flat config rejects literal bearer_token', () => { expect(() => resolveTargetDefinition( { - name: 'copilot-sdk-literal-bearer', + name: 'copilot-literal-bearer', provider: 'copilot-sdk', - byok: { - base_url: 'https://example.com', - bearer_token: 'plaintext-bearer-secret', - }, + base_url: 'https://example.com', + bearer_token: 'plaintext-bearer-secret', }, {}, ), ).toThrow(/must use.*VARIABLE_NAME/i); }); - it('copilot-sdk byok supports bearer_token', () => { + it('copilot-sdk flat config supports bearer_token', () => { const env = { MY_TOKEN: 'bearer-secret', } satisfies Record; @@ -1007,10 +907,8 @@ describe('resolveTargetDefinition', () => { { name: 'copilot-sdk-bearer', provider: 'copilot-sdk', - byok: { - base_url: 'https://custom-endpoint.example.com/v1', - bearer_token: '${{ MY_TOKEN }}', - }, + base_url: 'https://custom-endpoint.example.com/v1', + bearer_token: '${{ MY_TOKEN }}', }, env, ); @@ -1020,11 +918,11 @@ describe('resolveTargetDefinition', () => { throw new Error('expected copilot-sdk target'); } - expect(target.config.byokBearerToken).toBe('bearer-secret'); - expect(target.config.byokApiKey).toBeUndefined(); + expect(target.config.customProvider?.bearerToken).toBe('bearer-secret'); + expect(target.config.customProvider?.apiKey).toBeUndefined(); }); - it('copilot-sdk byok supports wire_api', () => { + it('copilot-sdk flat config supports wire_api', () => { const env = { FOUNDRY_KEY: 'foundry-secret', } satisfies Record; @@ -1034,12 +932,10 @@ describe('resolveTargetDefinition', () => { name: 'copilot-sdk-responses', provider: 'copilot-sdk', model: 'gpt-5', - byok: { - type: 'openai', - base_url: 'https://resource.openai.azure.com/openai/v1/', - api_key: '${{ FOUNDRY_KEY }}', - wire_api: 'responses', - }, + subprovider: 'openai', + base_url: 'https://resource.openai.azure.com/openai/v1/', + api_key: '${{ FOUNDRY_KEY }}', + wire_api: 'responses', }, env, ); @@ -1049,10 +945,10 @@ describe('resolveTargetDefinition', () => { throw new Error('expected copilot-sdk target'); } - expect(target.config.byokWireApi).toBe('responses'); + expect(target.config.customProvider?.wireApi).toBe('responses'); }); - it('copilot-sdk without byok has no byok fields', () => { + it('copilot-sdk without base_url has no custom provider', () => { const target = resolveTargetDefinition( { name: 'copilot-sdk-plain', @@ -1067,9 +963,6 @@ describe('resolveTargetDefinition', () => { throw new Error('expected copilot-sdk target'); } - expect(target.config.byokType).toBeUndefined(); - expect(target.config.byokBaseUrl).toBeUndefined(); - expect(target.config.byokApiKey).toBeUndefined(); expect(target.config.customProvider).toBeUndefined(); }); diff --git a/packages/core/test/evaluation/validation/targets-validator.test.ts b/packages/core/test/evaluation/validation/targets-validator.test.ts index 9cdf33ff..f13994f8 100644 --- a/packages/core/test/evaluation/validation/targets-validator.test.ts +++ b/packages/core/test/evaluation/validation/targets-validator.test.ts @@ -122,8 +122,34 @@ describe('validateTargetsFile', () => { expect(result.valid).toBe(true); }); - it('accepts custom_provider on copilot SDK and CLI targets', async () => { - const filePath = path.join(tempDir, 'copilot-custom-provider.yaml'); + it('accepts flat provider fields on copilot SDK and CLI targets', async () => { + const filePath = path.join(tempDir, 'copilot-flat-provider.yaml'); + await writeFile( + filePath, + `targets: + - name: copilot-sdk-custom-provider + provider: copilot-sdk + subprovider: openai + base_url: \${{ OPENAI_ENDPOINT }} + api_key: \${{ OPENAI_API_KEY }} + wire_api: responses + - name: copilot-cli-custom-provider + provider: copilot-cli + subprovider: openai + base_url: \${{ OPENAI_ENDPOINT }} + api_key: \${{ OPENAI_API_KEY }} + wire_api: responses +`, + ); + + const result = await validateTargetsFile(filePath); + + expect(result.valid).toBe(true); + expect(result.errors.filter((error) => error.severity === 'warning')).toEqual([]); + }); + + it('warns on removed copilot custom_provider and byok fields', async () => { + const filePath = path.join(tempDir, 'copilot-removed-provider-fields.yaml'); await writeFile( filePath, `targets: @@ -133,6 +159,12 @@ describe('validateTargetsFile', () => { type: openai base_url: \${{ OPENAI_ENDPOINT }} api_key: \${{ OPENAI_API_KEY }} + - name: copilot-sdk-byok + provider: copilot-sdk + byok: + type: openai + base_url: \${{ OPENAI_ENDPOINT }} + api_key: \${{ OPENAI_API_KEY }} - name: copilot-cli-custom provider: copilot-cli custom_provider: @@ -150,7 +182,12 @@ describe('validateTargetsFile', () => { error.severity === 'warning' && error.message.includes("Unknown setting 'custom_provider'"), ), - ).toBe(false); + ).toBe(true); + expect( + result.errors.some( + (error) => error.severity === 'warning' && error.message.includes("Unknown setting 'byok'"), + ), + ).toBe(true); }); it('accepts env-templated use_target values without resolving the env during validation', async () => {