feat(copilot): flat pi-style provider config, SDK args, ACP custom providers

apps/web/src/content/docs/docs/targets/coding-agents.mdx

@@ -140,7 +140,12 @@ targets:
 |-------|----------|-------------|
 | `model` | No | Model to use (defaults to copilot's default) |
 | `cwd` | No | Working directory |
-| `custom_provider` | No | OpenAI-compatible provider endpoint for `copilot`, `copilot-cli`, or `copilot-sdk` |
+| `subprovider` | No | OpenAI-compatible provider type for `copilot`, `copilot-cli`, or `copilot-sdk`, such as `openai` or `azure` |
+| `base_url` | No | Provider base URL or Azure resource URL/name |
+| `api_key` | No | Provider API key. Prefer `${{ ENV_VAR }}` references. |
+| `bearer_token` | No | Provider bearer token. Prefer `${{ ENV_VAR }}` references. Takes precedence over `api_key` when set. |
+| `api_version` | No | Provider API version, primarily for Azure endpoints |
+| `wire_api` | No | Provider wire API format, such as `responses` |
 | `grader_target` | Yes | LLM target for evaluation |
 
 Route Copilot through an OpenAI-compatible endpoint:
@@ -149,15 +154,14 @@ Route Copilot through an OpenAI-compatible endpoint:
 targets:
   - name: copilot-openai
     provider: copilot-cli
-    custom_provider:
-      type: openai
-      base_url: ${{ OPENAI_ENDPOINT }}
-      api_key: ${{ OPENAI_API_KEY }}
-      wire_api: ${{ COPILOT_PROVIDER_WIRE_API }}
+    subprovider: openai
+    base_url: ${{ OPENAI_ENDPOINT }}
+    api_key: ${{ OPENAI_API_KEY }}
+    wire_api: ${{ COPILOT_PROVIDER_WIRE_API }}
     grader_target: azure-base
 ```
 
-Values can come from environment variables through `${{ ... }}` interpolation. For `copilot-cli`, AgentV maps this shared block to Copilot's documented provider environment variables before spawning `copilot`; omitted fields leave existing ambient `COPILOT_PROVIDER_*` values unchanged.
+Values can come from environment variables through `${{ ... }}` interpolation. For `copilot-cli`, AgentV maps these flat fields to Copilot's documented provider environment variables before spawning `copilot`; omitted fields leave existing ambient `COPILOT_PROVIDER_*` values unchanged.
 
 ## Pi Coding Agent
 

packages/core/src/evaluation/providers/copilot-cli.ts

@@ -103,10 +103,6 @@ export class CopilotCliProvider implements Provider {
     const startTime = new Date().toISOString();
     const startMs = Date.now();
 
-    if (this.config.customProvider) {
-      return await this.invokePromptMode(request, startTime, startMs);
-    }
-
     const logger = await this.createStreamLogger(request, 'acp').catch(() => undefined);
 
     // Build command args
@@ -116,6 +112,7 @@ export class CopilotCliProvider implements Provider {
     // Spawn the CLI process
     const agentProcess = spawn(executable, args, {
       env: buildCopilotCliProviderEnv(process.env, this.config.customProvider),
+      cwd: this.resolveCwd(request.cwd) ?? process.cwd(),
       stdio: ['pipe', 'pipe', 'inherit'],
     });
     trackChild(agentProcess);
@@ -136,6 +133,7 @@ export class CopilotCliProvider implements Provider {
     const input = Writable.toWeb(agentProcess.stdin);
     const output = Readable.toWeb(agentProcess.stdout) as ReadableStream<Uint8Array>;
     const stream = acp.ndJsonStream(input, output);
+    const customProvider = this.config.customProvider;
 
     const client: acp.Client = {
       async requestPermission(): Promise<acp.RequestPermissionResponse> {
@@ -148,7 +146,7 @@ export class CopilotCliProvider implements Provider {
         const update = params.update;
         const sessionUpdate = update.sessionUpdate;
 
-        logger?.handleEvent(sessionUpdate, update);
+        logger?.handleEvent(sessionUpdate, sanitizeSensitiveValue(update, customProvider));
 
         if (sessionUpdate === 'tool_call') {
           const callId = update.toolCallId ?? randomUUID();
@@ -788,6 +786,30 @@ function sanitizeSensitiveText(
   return sanitized;
 }
 
+function sanitizeSensitiveValue(
+  value: unknown,
+  customProvider: CopilotCustomProviderConfig | undefined,
+): unknown {
+  if (!customProvider) {
+    return value;
+  }
+  if (typeof value === 'string') {
+    return sanitizeSensitiveText(value, customProvider);
+  }
+  if (Array.isArray(value)) {
+    return value.map((item) => sanitizeSensitiveValue(item, customProvider));
+  }
+  if (value && typeof value === 'object') {
+    return Object.fromEntries(
+      Object.entries(value).map(([key, entry]) => [
+        key,
+        sanitizeSensitiveValue(entry, customProvider),
+      ]),
+    );
+  }
+  return value;
+}
+
 async function defaultCopilotCliPromptRunner(
   options: CopilotCliPromptRunOptions,
 ): Promise<CopilotCliPromptRunResult> {

packages/core/src/evaluation/providers/copilot-sdk.ts

@@ -14,7 +14,7 @@ import {
 } from './copilot-utils.js';
 import { normalizeToolCall } from './normalize-tool-call.js';
 import { buildPromptDocument, normalizeInputFiles } from './preread.js';
-import type { CopilotCustomProviderConfig, CopilotSdkResolvedConfig } from './targets.js';
+import type { CopilotSdkResolvedConfig } from './targets.js';
 import type {
   Message,
   Provider,
@@ -36,7 +36,12 @@ async function loadCopilotSdk(): Promise<typeof import('@github/copilot-sdk')> {
       const message = error instanceof Error ? error.message : String(error);
       if (message.includes('vscode-jsonrpc')) {
         throw new Error(
-          `Failed to load @github/copilot-sdk due to a known ESM compatibility issue with vscode-jsonrpc (https://github.com/github/copilot-sdk/issues/710).\n\nWorkarounds:\n  - Use the copilot-cli target instead (recommended): set target type to "copilot-cli" in your eval YAML\n  - If running under Node.js 24+: set NODE_OPTIONS="--experimental-specifier-resolution=node"\n  - Wait for vscode-jsonrpc@9.0.0 stable to be released upstream`,
+          '@github/copilot-sdk failed to load: vscode-jsonrpc ESM import specifier mismatch.\n' +
+            "The package imports 'vscode-jsonrpc/node' but the installed version exposes 'node.js'.\n\n" +
+            'Repair (run once in your project root):\n' +
+            "  node -e \"const p=require.resolve('vscode-jsonrpc/package.json').replace('/package.json',''); require('fs').symlinkSync(p+'/node.js',p+'/node','file')\" 2>/dev/null || true\n\n" +
+            'Or switch to the copilot-cli target (no SDK dependency):\n' +
+            '  Set provider: copilot-cli in your eval YAML',
         );
       }
       throw new Error(
@@ -84,7 +89,8 @@ export class CopilotSdkProvider implements Provider {
     }
 
     const sdk = await loadCopilotSdk();
-    const client = await this.getOrCreateClient(sdk);
+    const evalCwd = this.resolveCwd(request.cwd);
+    const client = await this.getOrCreateClient(sdk, evalCwd ?? undefined);
 
     const startTime = new Date().toISOString();
     const startMs = Date.now();
@@ -101,12 +107,11 @@ export class CopilotSdkProvider implements Provider {
       sessionOptions.model = this.config.model;
     }
 
-    const cwd = this.resolveCwd(request.cwd);
-    if (cwd) {
-      sessionOptions.workingDirectory = cwd;
+    if (evalCwd) {
+      sessionOptions.workingDirectory = evalCwd;
       // Auto-discover skill directories from the workspace so the SDK loads
       // SKILL.md files into the session context (see copilot-sdk docs/features/skills.md).
-      sessionOptions.skillDirectories = resolveSkillDirectories(cwd);
+      sessionOptions.skillDirectories = resolveSkillDirectories(evalCwd);
     }
 
     const systemPrompt = this.config.systemPrompt;
@@ -118,13 +123,13 @@ export class CopilotSdkProvider implements Provider {
       };
     }
 
-    const customProvider = resolveCustomProviderConfig(this.config);
+    const customProvider = this.config.customProvider;
     if (customProvider) {
       const providerType = customProvider.type ?? 'openai';
       // biome-ignore lint/suspicious/noExplicitAny: SDK provider config shape is dynamic
       const provider: any = {
         type: providerType,
-        baseUrl: normalizeByokBaseUrl(customProvider.baseUrl, providerType),
+        baseUrl: normalizeProviderBaseUrl(customProvider.baseUrl, providerType),
       };
       if (customProvider.bearerToken) {
         provider.bearerToken = customProvider.bearerToken;
@@ -299,7 +304,7 @@ export class CopilotSdkProvider implements Provider {
   }
 
   // biome-ignore lint/suspicious/noExplicitAny: SDK client type is dynamically loaded
-  private async getOrCreateClient(sdk: any): Promise<any> {
+  private async getOrCreateClient(sdk: any, evalCwd?: string): Promise<any> {
     if (!this.client) {
       // biome-ignore lint/suspicious/noExplicitAny: SDK constructor options are dynamic
       const clientOptions: any = {};
@@ -317,6 +322,15 @@ export class CopilotSdkProvider implements Provider {
           clientOptions.cliPath = nativePath;
         }
       }
+      // Set the subprocess cwd so --plugin-dir ./relative resolves from the eval workspace.
+      const resolvedCwd = evalCwd ?? process.cwd();
+      clientOptions.cwd = resolvedCwd;
+
+      if (this.config.args && this.config.args.length > 0) {
+        clientOptions.cliArgs = this.config.args.map((arg) =>
+          arg.startsWith('./') || arg.startsWith('../') ? path.resolve(resolvedCwd, arg) : arg,
+        );
+      }
       if (this.config.githubToken) {
         clientOptions.githubToken = this.config.githubToken;
       }
@@ -411,25 +425,6 @@ export class CopilotSdkProvider implements Provider {
   }
 }
 
-function resolveCustomProviderConfig(
-  config: CopilotSdkResolvedConfig,
-): CopilotCustomProviderConfig | undefined {
-  if (config.customProvider) {
-    return config.customProvider;
-  }
-  if (!config.byokBaseUrl) {
-    return undefined;
-  }
-  return {
-    ...(config.byokType ? { type: config.byokType } : {}),
-    baseUrl: config.byokBaseUrl,
-    ...(config.byokApiKey ? { apiKey: config.byokApiKey } : {}),
-    ...(config.byokBearerToken ? { bearerToken: config.byokBearerToken } : {}),
-    ...(config.byokApiVersion ? { apiVersion: config.byokApiVersion } : {}),
-    ...(config.byokWireApi ? { wireApi: config.byokWireApi } : {}),
-  };
-}
-
 /**
  * Auto-discover skill directories from a workspace.
  * Checks standard skill directory locations and returns any that exist.
@@ -444,12 +439,12 @@ function resolveSkillDirectories(cwd: string): string[] {
 }
 
 /**
- * Normalize a BYOK base URL for the Copilot SDK.
+ * Normalize a provider base URL for the Copilot SDK.
  * For Azure type, if the value is a bare resource name (no https:// prefix),
  * construct the full URL: https://{resourceName}.openai.azure.com
  * This lets users reuse AZURE_OPENAI_ENDPOINT without a separate env var.
  */
-function normalizeByokBaseUrl(baseUrl: string, type: string): string {
+function normalizeProviderBaseUrl(baseUrl: string, type: string): string {
   const trimmed = baseUrl.trim().replace(/\/+$/, '');
   if (/^https?:\/\//i.test(trimmed)) {
     return trimmed;